mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-13 21:09:43 +00:00
Compare commits
49
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
36be02a1b8 | ||
|
|
76c6b155c2 | ||
|
|
d0b597ff49 | ||
|
|
e19b0eba85 | ||
|
|
df75468a8b | ||
|
|
0a55717283 | ||
|
|
bcab31bf72 | ||
|
|
6ae62ce535 | ||
|
|
6e2f79c0ad | ||
|
|
b0862f7a41 | ||
|
|
45991eca09 | ||
|
|
76c42556a2 | ||
|
|
6f8378a31c | ||
|
|
56115ee0a4 | ||
|
|
321d1cf913 | ||
|
|
790a713ba9 | ||
|
|
cd470dffbe | ||
|
|
7ff89d8607 | ||
|
|
493849f2e3 | ||
|
|
87ac61748c | ||
|
|
26de38f4b6 | ||
|
|
d2d4c504e8 | ||
|
|
b37e8e2da2 | ||
|
|
45d8116880 | ||
|
|
f68e98c376 | ||
|
|
f3d5d1e021 | ||
|
|
02004c5912 | ||
|
|
ef30031e2e | ||
|
|
67511ed6a7 | ||
|
|
b35b473508 | ||
|
|
d4f2c3ac66 | ||
|
|
37300bf5c8 | ||
|
|
cb8a2e15c8 | ||
|
|
aac038abb9 | ||
|
|
588fba226d | ||
|
|
c670742589 | ||
|
|
f897ce1b26 | ||
|
|
cbfce41d7e | ||
|
|
1e1c4cb91f | ||
|
|
0c340e1eb6 | ||
|
|
ae38cdefb4 | ||
|
|
a97fa52f10 | ||
|
|
43673e86f2 | ||
|
|
81ef51cc5c | ||
|
|
ddce26ff2d | ||
|
|
ee29cc627f | ||
|
|
f3caf42be4 | ||
|
|
c34744247a | ||
|
|
10f712f9d7 |
@@ -236,7 +236,7 @@ jobs:
|
||||
build:
|
||||
name: "🏗️ Build Docker Image"
|
||||
needs: [e2e-test]
|
||||
runs-on: [self-hosted, Linux]
|
||||
runs-on: [self-hosted, meshcore-vm]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
@@ -271,7 +271,7 @@ jobs:
|
||||
name: "🚀 Deploy Staging"
|
||||
if: github.event_name == 'push'
|
||||
needs: [build]
|
||||
runs-on: [self-hosted, Linux]
|
||||
runs-on: [self-hosted, meshcore-vm]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -428,6 +429,49 @@ func TestMaxTransmissionID(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// --- MaxTransmissionID incremental tracking ---
|
||||
|
||||
func TestMaxTransmissionIDIncremental(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
maxTx := store.MaxTransmissionID()
|
||||
maxObs := store.MaxObservationID()
|
||||
|
||||
if maxTx <= 0 {
|
||||
t.Fatalf("expected maxTx > 0 after Load, got %d", maxTx)
|
||||
}
|
||||
if maxObs <= 0 {
|
||||
t.Fatalf("expected maxObs > 0 after Load, got %d", maxObs)
|
||||
}
|
||||
|
||||
// Verify incremental field matches brute-force iteration
|
||||
store.mu.RLock()
|
||||
bruteMaxTx := 0
|
||||
for id := range store.byTxID {
|
||||
if id > bruteMaxTx {
|
||||
bruteMaxTx = id
|
||||
}
|
||||
}
|
||||
bruteMaxObs := 0
|
||||
for id := range store.byObsID {
|
||||
if id > bruteMaxObs {
|
||||
bruteMaxObs = id
|
||||
}
|
||||
}
|
||||
store.mu.RUnlock()
|
||||
|
||||
if maxTx != bruteMaxTx {
|
||||
t.Errorf("maxTxID mismatch: incremental=%d brute=%d", maxTx, bruteMaxTx)
|
||||
}
|
||||
if maxObs != bruteMaxObs {
|
||||
t.Errorf("maxObsID mismatch: incremental=%d brute=%d", maxObs, bruteMaxObs)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Route handler DB fallback (no store) ---
|
||||
|
||||
func TestHandleBulkHealthNoStore(t *testing.T) {
|
||||
@@ -770,6 +814,56 @@ func TestPrefixMapResolve(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestPrefixMapCap(t *testing.T) {
|
||||
// 16-char pubkey — longer than maxPrefixLen
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aabbccdd11223344", Name: "LongKey"},
|
||||
{PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
t.Run("short prefixes still work", func(t *testing.T) {
|
||||
n := pm.resolve("aabb")
|
||||
if n == nil || n.Name != "LongKey" {
|
||||
t.Errorf("expected LongKey for short prefix, got %v", n)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("full pubkey exact match works", func(t *testing.T) {
|
||||
n := pm.resolve("aabbccdd11223344")
|
||||
if n == nil || n.Name != "LongKey" {
|
||||
t.Errorf("expected LongKey for full key, got %v", n)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("intermediate prefix beyond cap returns nil", func(t *testing.T) {
|
||||
// 10-char prefix — beyond maxPrefixLen but not full key
|
||||
n := pm.resolve("aabbccdd11")
|
||||
if n != nil {
|
||||
t.Errorf("expected nil for intermediate prefix beyond cap, got %v", n.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("short key within cap has all prefixes", func(t *testing.T) {
|
||||
for l := 2; l <= 8; l++ {
|
||||
pfx := "eeff0011"[:l]
|
||||
n := pm.resolve(pfx)
|
||||
if n == nil || n.Name != "ShortKey" {
|
||||
t.Errorf("prefix %q: expected ShortKey, got %v", pfx, n)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("map size is capped", func(t *testing.T) {
|
||||
// LongKey: 7 prefix entries (2..8) + 1 full key = 8
|
||||
// ShortKey: 7 prefix entries (2..8), no full key entry (len == maxPrefixLen) = 7
|
||||
// No overlapping prefixes between the two nodes → 8 + 7 = 15 unique map keys
|
||||
if len(pm.m) != 15 {
|
||||
t.Errorf("expected 15 map entries (8 for LongKey + 7 for ShortKey), got %d", len(pm.m))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --- pathLen ---
|
||||
|
||||
func TestPathLen(t *testing.T) {
|
||||
@@ -1333,6 +1427,40 @@ func TestGetNodeLocations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- GetNodeLocationsByKeys ---
|
||||
|
||||
func TestGetNodeLocationsByKeys(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// Query with a known key
|
||||
pk := "aabbccdd11223344"
|
||||
locs := db.GetNodeLocationsByKeys([]string{pk})
|
||||
if len(locs) != 1 {
|
||||
t.Errorf("expected 1 location, got %d", len(locs))
|
||||
}
|
||||
if entry, ok := locs[strings.ToLower(pk)]; ok {
|
||||
if entry["lat"] == nil {
|
||||
t.Error("expected non-nil lat")
|
||||
}
|
||||
} else {
|
||||
t.Error("expected node location for test repeater")
|
||||
}
|
||||
|
||||
// Query with no keys returns empty map
|
||||
empty := db.GetNodeLocationsByKeys([]string{})
|
||||
if len(empty) != 0 {
|
||||
t.Errorf("expected 0 locations for empty keys, got %d", len(empty))
|
||||
}
|
||||
|
||||
// Query with unknown key returns empty map
|
||||
unknown := db.GetNodeLocationsByKeys([]string{"nonexistent"})
|
||||
if len(unknown) != 0 {
|
||||
t.Errorf("expected 0 locations for unknown key, got %d", len(unknown))
|
||||
}
|
||||
}
|
||||
|
||||
// --- Store edge cases ---
|
||||
|
||||
func TestStoreQueryPacketsEdgeCases(t *testing.T) {
|
||||
@@ -1906,6 +2034,48 @@ func TestTxToMap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTxToMapLazyObservations(t *testing.T) {
|
||||
snr := 10.5
|
||||
rssi := -90.0
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc",
|
||||
Observations: []*StoreObs{
|
||||
{ID: 10, ObserverID: "obs1", ObserverName: "O1", SNR: &snr, RSSI: &rssi, Timestamp: "2025-01-01"},
|
||||
{ID: 11, ObserverID: "obs2", ObserverName: "O2", SNR: &snr, RSSI: &rssi, Timestamp: "2025-01-02"},
|
||||
},
|
||||
}
|
||||
|
||||
// Without flag: no observations key
|
||||
m := txToMap(tx)
|
||||
if _, ok := m["observations"]; ok {
|
||||
t.Error("txToMap without includeObservations should not include observations key")
|
||||
}
|
||||
|
||||
// With false: no observations key
|
||||
m = txToMap(tx, false)
|
||||
if _, ok := m["observations"]; ok {
|
||||
t.Error("txToMap(tx, false) should not include observations key")
|
||||
}
|
||||
|
||||
// With true: observations included
|
||||
m = txToMap(tx, true)
|
||||
obs, ok := m["observations"]
|
||||
if !ok {
|
||||
t.Fatal("txToMap(tx, true) should include observations key")
|
||||
}
|
||||
obsList, ok := obs.([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("observations should be []map[string]interface{}")
|
||||
}
|
||||
if len(obsList) != 2 {
|
||||
t.Errorf("expected 2 observations, got %d", len(obsList))
|
||||
}
|
||||
if obsList[0]["observer_id"] != "obs1" {
|
||||
t.Errorf("expected observer_id obs1, got %v", obsList[0]["observer_id"])
|
||||
}
|
||||
}
|
||||
|
||||
// --- filterTxSlice ---
|
||||
|
||||
func TestFilterTxSlice(t *testing.T) {
|
||||
@@ -2099,6 +2269,84 @@ func TestSubpathPrecomputedIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubpathTxIndexPopulated(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
// spTxIndex must be populated alongside spIndex
|
||||
if len(store.spTxIndex) == 0 {
|
||||
t.Fatal("expected spTxIndex to be populated after Load()")
|
||||
}
|
||||
|
||||
// Every key in spIndex must also exist in spTxIndex with matching count
|
||||
for key, count := range store.spIndex {
|
||||
txs, ok := store.spTxIndex[key]
|
||||
if !ok {
|
||||
t.Errorf("spTxIndex missing key %q that exists in spIndex", key)
|
||||
continue
|
||||
}
|
||||
if len(txs) != count {
|
||||
t.Errorf("spTxIndex[%q] has %d txs, spIndex count is %d", key, len(txs), count)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSubpathDetail should return correct match count via indexed lookup
|
||||
detail := store.GetSubpathDetail([]string{"eeff", "0011"})
|
||||
if detail == nil {
|
||||
t.Fatal("expected non-nil detail for existing subpath")
|
||||
}
|
||||
matches, _ := detail["totalMatches"].(int)
|
||||
if matches != 1 {
|
||||
t.Errorf("totalMatches = %d, want 1", matches)
|
||||
}
|
||||
|
||||
// Non-existent subpath should return 0 matches
|
||||
detail2 := store.GetSubpathDetail([]string{"zzzz", "yyyy"})
|
||||
if detail2 == nil {
|
||||
t.Fatal("expected non-nil result even for non-existent subpath")
|
||||
}
|
||||
matches2, _ := detail2["totalMatches"].(int)
|
||||
if matches2 != 0 {
|
||||
t.Errorf("totalMatches for non-existent subpath = %d, want 0", matches2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubpathDetailMixedCaseHops(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
// Query with lowercase hops to establish baseline
|
||||
lower := store.GetSubpathDetail([]string{"eeff", "0011"})
|
||||
if lower == nil {
|
||||
t.Fatal("expected non-nil detail for lowercase subpath")
|
||||
}
|
||||
lowerMatches, _ := lower["totalMatches"].(int)
|
||||
if lowerMatches == 0 {
|
||||
t.Fatal("expected >0 matches for lowercase subpath")
|
||||
}
|
||||
|
||||
// Query with mixed-case hops — must return the same results (case-insensitive)
|
||||
mixed := store.GetSubpathDetail([]string{"EEFF", "0011"})
|
||||
if mixed == nil {
|
||||
t.Fatal("expected non-nil detail for mixed-case subpath")
|
||||
}
|
||||
mixedMatches, _ := mixed["totalMatches"].(int)
|
||||
if mixedMatches != lowerMatches {
|
||||
t.Errorf("mixed-case totalMatches = %d, want %d (same as lowercase)", mixedMatches, lowerMatches)
|
||||
}
|
||||
|
||||
// All-uppercase should also match
|
||||
upper := store.GetSubpathDetail([]string{"EEFF", "0011"})
|
||||
upperMatches, _ := upper["totalMatches"].(int)
|
||||
if upperMatches != lowerMatches {
|
||||
t.Errorf("uppercase totalMatches = %d, want %d", upperMatches, lowerMatches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreGetAnalyticsRFCacheHit(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
@@ -3716,6 +3964,71 @@ func TestGetChannelMessagesAfterIngest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// --- resolveRegionObservers caching ---
|
||||
|
||||
func TestResolveRegionObserversCaching(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
store := &PacketStore{db: db}
|
||||
|
||||
// First call should populate cache.
|
||||
obs1 := store.resolveRegionObservers("SJC")
|
||||
if obs1 == nil || len(obs1) == 0 {
|
||||
t.Fatal("expected observer IDs for SJC on first call")
|
||||
}
|
||||
|
||||
// Second call should return cached result (same pointer).
|
||||
obs2 := store.resolveRegionObservers("SJC")
|
||||
if len(obs2) != len(obs1) {
|
||||
t.Errorf("cached result differs: got %d, want %d", len(obs2), len(obs1))
|
||||
}
|
||||
|
||||
// Non-existent region should return nil even from cache.
|
||||
obs3 := store.resolveRegionObservers("NONEXIST")
|
||||
if obs3 != nil {
|
||||
t.Errorf("expected nil for NONEXIST, got %v", obs3)
|
||||
}
|
||||
|
||||
// Verify cache fields are set.
|
||||
if store.regionObsCache == nil {
|
||||
t.Error("regionObsCache should be non-nil after calls")
|
||||
}
|
||||
if store.regionObsCacheTime.IsZero() {
|
||||
t.Error("regionObsCacheTime should be set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRegionObserversCacheMissNewRegion(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
store := &PacketStore{db: db}
|
||||
|
||||
// Populate cache with SJC.
|
||||
obs1 := store.resolveRegionObservers("SJC")
|
||||
if obs1 == nil || len(obs1) == 0 {
|
||||
t.Fatal("expected observer IDs for SJC on first call")
|
||||
}
|
||||
|
||||
// Cache is now valid. Request a different region that exists in DB.
|
||||
// Before the fix, this would return nil from the map lookup instead of
|
||||
// fetching from DB, silently returning "no observers" for up to 30s.
|
||||
obs2 := store.resolveRegionObservers("LAX")
|
||||
// LAX may or may not have data in the test DB, but the key point is:
|
||||
// a non-existent region should be fetched (not just nil-returned).
|
||||
// Verify the region key was cached (even if empty).
|
||||
store.regionObsMu.Lock()
|
||||
_, cached := store.regionObsCache["LAX"]
|
||||
store.regionObsMu.Unlock()
|
||||
if !cached {
|
||||
t.Error("LAX should be cached after resolveRegionObservers call, even if empty")
|
||||
}
|
||||
_ = obs2
|
||||
}
|
||||
|
||||
func TestIndexByNodePreCheck(t *testing.T) {
|
||||
store := &PacketStore{
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
@@ -3913,3 +4226,116 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --- Distance index incremental update (#365, replaces debounce #557) ---
|
||||
|
||||
func TestDistanceIncrementalUpdate(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
// Record initial distance index size.
|
||||
initialHops := len(store.distHops)
|
||||
initialPaths := len(store.distPaths)
|
||||
|
||||
// Insert a new observation with a different path to trigger an incremental update.
|
||||
maxObsID := db.GetMaxObservationID()
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix())
|
||||
|
||||
store.IngestNewObservations(maxObsID, 500)
|
||||
|
||||
// Distance index should have been updated incrementally (sizes may differ
|
||||
// if the new path resolves differently, but should not panic or corrupt).
|
||||
_ = len(store.distHops)
|
||||
_ = len(store.distPaths)
|
||||
|
||||
// Insert another observation with yet another path.
|
||||
maxObsID = db.GetMaxObservationID()
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix())
|
||||
|
||||
store.IngestNewObservations(maxObsID, 500)
|
||||
|
||||
// Verify the index is still coherent (no duplicates for the same tx).
|
||||
txSeen := make(map[int]int)
|
||||
for _, r := range store.distPaths {
|
||||
if r.tx != nil {
|
||||
txSeen[r.tx.ID]++
|
||||
}
|
||||
}
|
||||
for txID, count := range txSeen {
|
||||
if count > 1 {
|
||||
t.Errorf("distPaths has %d entries for tx %d (expected at most 1)", count, txID)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Distance index: %d→%d hops, %d→%d paths (incremental)",
|
||||
initialHops, len(store.distHops), initialPaths, len(store.distPaths))
|
||||
}
|
||||
|
||||
func TestHandleBatchObservations(t *testing.T) {
|
||||
_, router := setupNoStoreServer(t)
|
||||
|
||||
t.Run("empty hashes returns empty results", func(t *testing.T) {
|
||||
body := strings.NewReader(`{"hashes":[]}`)
|
||||
req := httptest.NewRequest("POST", "/api/packets/observations", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
results, ok := resp["results"].(map[string]interface{})
|
||||
if !ok || len(results) != 0 {
|
||||
t.Fatalf("expected empty results map, got %v", resp)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid JSON returns 400", func(t *testing.T) {
|
||||
body := strings.NewReader(`not json`)
|
||||
req := httptest.NewRequest("POST", "/api/packets/observations", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("expected 400, got %d", w.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("too many hashes returns 400", func(t *testing.T) {
|
||||
hashes := make([]string, 201)
|
||||
for i := range hashes {
|
||||
hashes[i] = fmt.Sprintf("hash%d", i)
|
||||
}
|
||||
data, _ := json.Marshal(map[string][]string{"hashes": hashes})
|
||||
req := httptest.NewRequest("POST", "/api/packets/observations", bytes.NewReader(data))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 400 {
|
||||
t.Fatalf("expected 400, got %d", w.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid hashes with no store returns empty results", func(t *testing.T) {
|
||||
body := strings.NewReader(`{"hashes":["abc123","def456"]}`)
|
||||
req := httptest.NewRequest("POST", "/api/packets/observations", body)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
_, ok := resp["results"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected results map, got %v", resp)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+46
-7
@@ -15,9 +15,10 @@ import (
|
||||
|
||||
// DB wraps a read-only connection to the MeshCore SQLite database.
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
path string // filesystem path to the database file
|
||||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||||
conn *sql.DB
|
||||
path string // filesystem path to the database file
|
||||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||||
hasResolvedPath bool // observations table has resolved_path column
|
||||
}
|
||||
|
||||
// OpenDB opens a read-only SQLite connection with WAL mode.
|
||||
@@ -61,9 +62,13 @@ func (db *DB) detectSchema() {
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil && colName == "observer_idx" {
|
||||
db.isV3 = true
|
||||
return
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||||
if colName == "observer_idx" {
|
||||
db.isV3 = true
|
||||
}
|
||||
if colName == "resolved_path" {
|
||||
db.hasResolvedPath = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -372,7 +377,8 @@ type PacketQuery struct {
|
||||
Until string
|
||||
Region string
|
||||
Node string
|
||||
Order string // ASC or DESC
|
||||
Order string // ASC or DESC
|
||||
ExpandObservations bool // when true, include observation sub-maps in txToMap output
|
||||
}
|
||||
|
||||
// PacketResult wraps paginated packet list.
|
||||
@@ -1492,6 +1498,39 @@ func (db *DB) GetNodeLocations() map[string]map[string]interface{} {
|
||||
return result
|
||||
}
|
||||
|
||||
// GetNodeLocationsByKeys returns location data only for the given public keys.
|
||||
// This avoids fetching ALL nodes when only a few keys need to be matched.
|
||||
func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interface{} {
|
||||
result := make(map[string]map[string]interface{})
|
||||
if len(keys) == 0 {
|
||||
return result
|
||||
}
|
||||
placeholders := make([]string, len(keys))
|
||||
args := make([]interface{}, len(keys))
|
||||
for i, k := range keys {
|
||||
placeholders[i] = "?"
|
||||
args[i] = strings.ToLower(k)
|
||||
}
|
||||
query := "SELECT public_key, lat, lon, role FROM nodes WHERE LOWER(public_key) IN (" + strings.Join(placeholders, ",") + ")"
|
||||
rows, err := db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var pk string
|
||||
var role sql.NullString
|
||||
var lat, lon sql.NullFloat64
|
||||
rows.Scan(&pk, &lat, &lon, &role)
|
||||
result[strings.ToLower(pk)] = map[string]interface{}{
|
||||
"lat": nullFloat(lat),
|
||||
"lon": nullFloat(lon),
|
||||
"role": nullStr(role),
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// QueryMultiNodePackets returns transmissions referencing any of the given pubkeys.
|
||||
func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) {
|
||||
if len(pubkeys) == 0 {
|
||||
|
||||
@@ -162,24 +162,50 @@ func TestEvictStale_NoEvictionWhenDisabled(t *testing.T) {
|
||||
|
||||
func TestEvictStale_MemoryBasedEviction(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
// Create enough packets to exceed a small memory limit
|
||||
// 1000 packets * 5KB + 2000 obs * 500B ≈ 6MB
|
||||
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
|
||||
// All packets are recent (1h old) so time-based won't trigger
|
||||
// All packets are recent (1h old) so time-based won't trigger.
|
||||
store.retentionHours = 24
|
||||
store.maxMemoryMB = 3 // ~3MB limit, should evict roughly half
|
||||
store.maxMemoryMB = 3
|
||||
// Inject deterministic estimator: simulates 6MB (over 3MB limit).
|
||||
// Uses packet count so it scales correctly after eviction.
|
||||
store.memoryEstimator = func() float64 {
|
||||
return float64(len(store.packets)*5120+store.totalObs*500) / 1048576.0
|
||||
}
|
||||
|
||||
evicted := store.EvictStale()
|
||||
if evicted == 0 {
|
||||
t.Fatal("expected some evictions for memory cap")
|
||||
}
|
||||
// After eviction, estimated memory should be <= 3MB
|
||||
estMB := store.estimatedMemoryMB()
|
||||
if estMB > 3.5 { // small tolerance
|
||||
if estMB > 3.5 {
|
||||
t.Fatalf("expected <=3.5MB after eviction, got %.1fMB", estMB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictStale_MemoryBasedEviction_UnderestimatedHeap verifies that eviction
|
||||
// fires correctly when actual heap is much larger than a formula-based estimate
|
||||
// would report — the scenario that caused OOM kills in production.
|
||||
func TestEvictStale_MemoryBasedEviction_UnderestimatedHeap(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
|
||||
store.retentionHours = 24
|
||||
store.maxMemoryMB = 500
|
||||
// Simulate actual heap 5x over budget (like production: ~5GB actual vs ~1GB limit).
|
||||
store.memoryEstimator = func() float64 {
|
||||
return 2500.0 // 2500MB actual vs 500MB limit
|
||||
}
|
||||
|
||||
evicted := store.EvictStale()
|
||||
if evicted == 0 {
|
||||
t.Fatal("expected evictions when heap is 5x over limit")
|
||||
}
|
||||
// Should keep roughly 500/2500 * 0.9 = 18% of packets → ~180 of 1000.
|
||||
remaining := len(store.packets)
|
||||
if remaining > 250 {
|
||||
t.Fatalf("expected most packets evicted (heap 5x over), but %d of 1000 remain", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvictStale_CleansNodeIndexes(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
store := makeTestStore(10, now.Add(-48*time.Hour), 0)
|
||||
|
||||
+66
-5
@@ -144,6 +144,50 @@ func main() {
|
||||
log.Fatalf("[store] failed to load: %v", err)
|
||||
}
|
||||
|
||||
// Initialize persisted neighbor graph
|
||||
dbPath = database.path
|
||||
if err := ensureNeighborEdgesTable(dbPath); err != nil {
|
||||
log.Printf("[neighbor] warning: could not create neighbor_edges table: %v", err)
|
||||
}
|
||||
// Add resolved_path column if missing.
|
||||
// NOTE on startup ordering (review item #10): ensureResolvedPathColumn runs AFTER
|
||||
// OpenDB/detectSchema, so db.hasResolvedPath will be false on first run with a
|
||||
// pre-existing DB. This means Load() won't SELECT resolved_path from SQLite.
|
||||
// That's OK: backfillResolvedPaths (below) computes and persists them in-memory
|
||||
// AND to SQLite. On next restart, detectSchema finds the column and Load() reads it.
|
||||
if err := ensureResolvedPathColumn(dbPath); err != nil {
|
||||
log.Printf("[store] warning: could not add resolved_path column: %v", err)
|
||||
} else {
|
||||
database.hasResolvedPath = true // detectSchema ran before column was added; fix the flag
|
||||
}
|
||||
|
||||
// Load or build neighbor graph
|
||||
if neighborEdgesTableExists(database.conn) {
|
||||
store.graph = loadNeighborEdgesFromDB(database.conn)
|
||||
log.Printf("[neighbor] loaded persisted neighbor graph")
|
||||
} else {
|
||||
log.Printf("[neighbor] no persisted edges found, building from store...")
|
||||
rw, rwErr := openRW(dbPath)
|
||||
if rwErr == nil {
|
||||
edgeCount := buildAndPersistEdges(store, rw)
|
||||
rw.Close()
|
||||
log.Printf("[neighbor] persisted %d edges", edgeCount)
|
||||
}
|
||||
store.graph = BuildFromStore(store)
|
||||
}
|
||||
|
||||
// Backfill resolved_path for observations that don't have it yet
|
||||
if backfilled := backfillResolvedPaths(store, dbPath); backfilled > 0 {
|
||||
log.Printf("[store] backfilled resolved_path for %d observations", backfilled)
|
||||
}
|
||||
|
||||
// Re-pick best observation now that resolved paths are populated
|
||||
store.mu.Lock()
|
||||
for _, tx := range store.packets {
|
||||
pickBestObservation(tx)
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
// WebSocket hub
|
||||
hub := NewHub()
|
||||
|
||||
@@ -180,8 +224,15 @@ func main() {
|
||||
defer stopEviction()
|
||||
|
||||
// Auto-prune old packets if retention.packetDays is configured
|
||||
var stopPrune func()
|
||||
if cfg.Retention != nil && cfg.Retention.PacketDays > 0 {
|
||||
days := cfg.Retention.PacketDays
|
||||
pruneTicker := time.NewTicker(24 * time.Hour)
|
||||
pruneDone := make(chan struct{})
|
||||
stopPrune = func() {
|
||||
pruneTicker.Stop()
|
||||
close(pruneDone)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(1 * time.Minute)
|
||||
if n, err := database.PruneOldPackets(days); err != nil {
|
||||
@@ -189,11 +240,16 @@ func main() {
|
||||
} else {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
}
|
||||
for range time.Tick(24 * time.Hour) {
|
||||
if n, err := database.PruneOldPackets(days); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
for {
|
||||
select {
|
||||
case <-pruneTicker.C:
|
||||
if n, err := database.PruneOldPackets(days); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
}
|
||||
case <-pruneDone:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -218,6 +274,11 @@ func main() {
|
||||
// 1. Stop accepting new WebSocket/poll data
|
||||
poller.Stop()
|
||||
|
||||
// 1b. Stop auto-prune ticker
|
||||
if stopPrune != nil {
|
||||
stopPrune()
|
||||
}
|
||||
|
||||
// 2. Gracefully drain HTTP connections (up to 15s)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -18,7 +18,7 @@ const (
|
||||
// Time-decay half-life: 7 days.
|
||||
affinityHalfLifeHours = 168.0
|
||||
// Cache TTL for the built graph.
|
||||
neighborGraphTTL = 60 * time.Second
|
||||
neighborGraphTTL = 5 * time.Minute
|
||||
// Auto-resolve confidence: best must be >= this factor × second-best.
|
||||
affinityConfidenceRatio = 3.0
|
||||
// Minimum observation count to auto-resolve.
|
||||
@@ -130,6 +130,17 @@ func BuildFromStore(store *PacketStore) *NeighborGraph {
|
||||
return BuildFromStoreWithLog(store, false)
|
||||
}
|
||||
|
||||
// cachedToLower returns strings.ToLower(s), caching results to avoid
|
||||
// repeated allocations for the same pubkey string.
|
||||
func cachedToLower(cache map[string]string, s string) string {
|
||||
if v, ok := cache[s]; ok {
|
||||
return v
|
||||
}
|
||||
v := strings.ToLower(s)
|
||||
cache[s] = v
|
||||
return v
|
||||
}
|
||||
|
||||
// BuildFromStoreWithLog constructs the neighbor graph, optionally logging disambiguation decisions.
|
||||
func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
|
||||
g := NewNeighborGraph()
|
||||
@@ -149,30 +160,27 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
|
||||
// Use cached nodes+PM (avoids DB call if cache is fresh).
|
||||
_, pm := store.getCachedNodesAndPM()
|
||||
|
||||
// Local cache for strings.ToLower — pubkeys are immutable and repeat
|
||||
// across hundreds of thousands of observations.
|
||||
lowerCache := make(map[string]string, 256)
|
||||
|
||||
// Phase 1: Extract edges from every transmission + observation.
|
||||
for _, tx := range packets {
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
|
||||
fromNode := "" // originator pubkey (from byNode index key)
|
||||
// Find the originator pubkey — it's the key in store.byNode.
|
||||
// StoreTx doesn't store from_node directly; we find it via decoded JSON
|
||||
// or the byNode index. However, iterating byNode is expensive.
|
||||
// The originator pubkey is in the decoded JSON "from_node" field,
|
||||
// but parsing JSON per tx is expensive too.
|
||||
// Actually, let's look at how byNode is keyed.
|
||||
// Looking at store.go, byNode maps pubkey → transmissions where that
|
||||
// pubkey is the "from" node. We need the reverse: tx → from_node.
|
||||
// The from_node is embedded in DecodedJSON.
|
||||
// For efficiency, let's extract it once.
|
||||
fromNode = extractFromNode(tx)
|
||||
fromNode := extractFromNode(tx)
|
||||
// Pre-compute lowered originator once per tx (not per observation).
|
||||
fromLower := ""
|
||||
if fromNode != "" {
|
||||
fromLower = cachedToLower(lowerCache, fromNode)
|
||||
}
|
||||
|
||||
for _, obs := range tx.Observations {
|
||||
path := parsePathJSON(obs.PathJSON)
|
||||
observerPK := strings.ToLower(obs.ObserverID)
|
||||
observerPK := cachedToLower(lowerCache, obs.ObserverID)
|
||||
|
||||
if len(path) == 0 {
|
||||
// Zero-hop
|
||||
if isAdvert && fromNode != "" {
|
||||
fromLower := strings.ToLower(fromNode)
|
||||
if isAdvert && fromLower != "" {
|
||||
if fromLower != observerPK { // self-edge guard
|
||||
g.upsertEdge(fromLower, observerPK, "", observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
|
||||
}
|
||||
@@ -181,20 +189,19 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
|
||||
}
|
||||
|
||||
// Edge 1: originator ↔ path[0] — ADVERTs only
|
||||
if isAdvert && fromNode != "" {
|
||||
firstHop := strings.ToLower(path[0])
|
||||
fromLower := strings.ToLower(fromNode)
|
||||
if isAdvert && fromLower != "" {
|
||||
firstHop := cachedToLower(lowerCache, path[0])
|
||||
if fromLower != firstHop { // self-edge guard (shouldn't happen but spec says check)
|
||||
candidates := pm.m[firstHop]
|
||||
g.upsertEdgeWithCandidates(fromLower, firstHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
|
||||
g.upsertEdgeWithCandidates(fromLower, firstHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp), lowerCache)
|
||||
}
|
||||
}
|
||||
|
||||
// Edge 2: observer ↔ path[last] — ALL packet types
|
||||
lastHop := strings.ToLower(path[len(path)-1])
|
||||
lastHop := cachedToLower(lowerCache, path[len(path)-1])
|
||||
if observerPK != lastHop { // self-edge guard
|
||||
candidates := pm.m[lastHop]
|
||||
g.upsertEdgeWithCandidates(observerPK, lastHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp))
|
||||
g.upsertEdgeWithCandidates(observerPK, lastHop, candidates, observerPK, obs.SNR, parseTimestamp(obs.Timestamp), lowerCache)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,12 +218,10 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
|
||||
|
||||
// extractFromNode pulls the originator pubkey from a StoreTx's DecodedJSON.
|
||||
// ADVERTs use "pubKey", other packets may use "from_node" or "from".
|
||||
// Uses the cached ParsedDecoded() accessor to avoid repeated json.Unmarshal.
|
||||
func extractFromNode(tx *StoreTx) string {
|
||||
if tx.DecodedJSON == "" {
|
||||
return ""
|
||||
}
|
||||
var decoded map[string]interface{}
|
||||
if err := jsonUnmarshalFast(tx.DecodedJSON, &decoded); err != nil {
|
||||
decoded := tx.ParsedDecoded()
|
||||
if decoded == nil {
|
||||
return ""
|
||||
}
|
||||
// ADVERTs store the originator pubkey as "pubKey"; other packets may use
|
||||
@@ -275,9 +280,9 @@ func (g *NeighborGraph) upsertEdge(pubkeyA, pubkeyB, prefix, observer string, sn
|
||||
}
|
||||
|
||||
// upsertEdgeWithCandidates handles prefix-based edges that may be ambiguous.
|
||||
func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candidates []nodeInfo, observer string, snr *float64, ts time.Time) {
|
||||
func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candidates []nodeInfo, observer string, snr *float64, ts time.Time, lc map[string]string) {
|
||||
if len(candidates) == 1 {
|
||||
resolved := strings.ToLower(candidates[0].PublicKey)
|
||||
resolved := cachedToLower(lc, candidates[0].PublicKey)
|
||||
if resolved == knownPK {
|
||||
return // self-edge guard
|
||||
}
|
||||
@@ -288,7 +293,7 @@ func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candida
|
||||
// Filter out self from candidates
|
||||
filtered := make([]string, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
pk := strings.ToLower(c.PublicKey)
|
||||
pk := cachedToLower(lc, c.PublicKey)
|
||||
if pk != knownPK {
|
||||
filtered = append(filtered, pk)
|
||||
}
|
||||
|
||||
@@ -717,3 +717,120 @@ func TestNeighborGraph_CacheTTL(t *testing.T) {
|
||||
t.Error("old graph should be stale")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeighborGraph_TTLIsReasonable(t *testing.T) {
|
||||
// TTL must be long enough to avoid rebuild storms on busy meshes,
|
||||
// but short enough to reflect topology changes within minutes.
|
||||
if neighborGraphTTL < 1*time.Minute {
|
||||
t.Errorf("neighborGraphTTL too short (%v), will cause rebuild storms", neighborGraphTTL)
|
||||
}
|
||||
if neighborGraphTTL > 10*time.Minute {
|
||||
t.Errorf("neighborGraphTTL too long (%v), topology changes will be stale", neighborGraphTTL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedToLower(t *testing.T) {
|
||||
cache := make(map[string]string)
|
||||
// Basic lowercasing
|
||||
if got := cachedToLower(cache, "AABB"); got != "aabb" {
|
||||
t.Errorf("expected 'aabb', got %q", got)
|
||||
}
|
||||
// Verify it was cached
|
||||
if _, ok := cache["AABB"]; !ok {
|
||||
t.Error("expected 'AABB' to be in cache")
|
||||
}
|
||||
// Same input returns cached result
|
||||
if got := cachedToLower(cache, "AABB"); got != "aabb" {
|
||||
t.Errorf("expected cached 'aabb', got %q", got)
|
||||
}
|
||||
// Already lowercase stays the same
|
||||
if got := cachedToLower(cache, "aabb"); got != "aabb" {
|
||||
t.Errorf("expected 'aabb', got %q", got)
|
||||
}
|
||||
// Empty string
|
||||
if got := cachedToLower(cache, ""); got != "" {
|
||||
t.Errorf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedDecoded_Caching(t *testing.T) {
|
||||
tx := &StoreTx{DecodedJSON: `{"pubKey":"abc123","name":"test"}`}
|
||||
// First call parses
|
||||
d1 := tx.ParsedDecoded()
|
||||
if d1 == nil {
|
||||
t.Fatal("expected non-nil parsed result")
|
||||
}
|
||||
if d1["pubKey"] != "abc123" {
|
||||
t.Errorf("expected pubKey=abc123, got %v", d1["pubKey"])
|
||||
}
|
||||
// Second call must return the exact same map (pointer equality proves caching)
|
||||
d2 := tx.ParsedDecoded()
|
||||
if &d1 == nil || &d2 == nil {
|
||||
t.Fatal("unexpected nil")
|
||||
}
|
||||
// Mutate d1 and verify d2 sees the mutation — proves same underlying map
|
||||
d1["_sentinel"] = true
|
||||
if d2["_sentinel"] != true {
|
||||
t.Error("expected same map instance from second call (caching broken)")
|
||||
}
|
||||
delete(d1, "_sentinel") // clean up
|
||||
}
|
||||
|
||||
func TestParsedDecoded_EmptyJSON(t *testing.T) {
|
||||
tx := &StoreTx{DecodedJSON: ""}
|
||||
d := tx.ParsedDecoded()
|
||||
if d != nil {
|
||||
t.Errorf("expected nil for empty DecodedJSON, got %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsedDecoded_InvalidJSON(t *testing.T) {
|
||||
tx := &StoreTx{DecodedJSON: "not json"}
|
||||
d := tx.ParsedDecoded()
|
||||
if d != nil {
|
||||
t.Errorf("expected nil for invalid JSON, got %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFromNode_UsesCachedParse(t *testing.T) {
|
||||
tx := &StoreTx{DecodedJSON: `{"pubKey":"aabb1122"}`}
|
||||
// First call to extractFromNode should use ParsedDecoded
|
||||
from := extractFromNode(tx)
|
||||
if from != "aabb1122" {
|
||||
t.Errorf("expected aabb1122, got %q", from)
|
||||
}
|
||||
// ParsedDecoded should now be cached
|
||||
d := tx.ParsedDecoded()
|
||||
if d == nil || d["pubKey"] != "aabb1122" {
|
||||
t.Error("expected ParsedDecoded to return cached result")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBuildFromStore(b *testing.B) {
|
||||
// Simulate a dataset with many packets and repeated pubkeys
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aaaa1111", Name: "NodeA"},
|
||||
{PublicKey: "bbbb2222", Name: "NodeB"},
|
||||
{PublicKey: "cccc3333", Name: "NodeC"},
|
||||
{PublicKey: "dddd4444", Name: "NodeD"},
|
||||
}
|
||||
const numPackets = 1000
|
||||
packets := make([]*StoreTx, 0, numPackets)
|
||||
for i := 0; i < numPackets; i++ {
|
||||
pt := 4 // ADVERT
|
||||
packets = append(packets, &StoreTx{
|
||||
ID: i,
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"pubKey":"aaaa1111"}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "bbbb2222", PathJSON: `["cccc"]`, Timestamp: nowStr, SNR: ngFloatPtr(-5.0)},
|
||||
},
|
||||
})
|
||||
}
|
||||
store := ngTestStore(nodes, packets)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
BuildFromStore(store)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// persistSem limits concurrent async persistence goroutines to 1.
|
||||
// Without this, each ingest cycle spawns a goroutine that opens a new
|
||||
// SQLite RW connection; under sustained load goroutines pile up with
|
||||
// no backpressure, causing contention and busy-timeout cascades.
|
||||
var persistSem = make(chan struct{}, 1)
|
||||
|
||||
// ─── neighbor_edges table ──────────────────────────────────────────────────────
|
||||
|
||||
// ensureNeighborEdgesTable creates the neighbor_edges table if it doesn't exist.
|
||||
// Uses a separate read-write connection since the main DB is read-only.
|
||||
func ensureNeighborEdgesTable(dbPath string) error {
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open rw for neighbor_edges: %w", err)
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
_, err = rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges (
|
||||
node_a TEXT NOT NULL,
|
||||
node_b TEXT NOT NULL,
|
||||
count INTEGER DEFAULT 1,
|
||||
last_seen TEXT,
|
||||
PRIMARY KEY (node_a, node_b)
|
||||
)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadNeighborEdgesFromDB loads all edges from the neighbor_edges table
|
||||
// and builds an in-memory NeighborGraph.
|
||||
func loadNeighborEdgesFromDB(conn *sql.DB) *NeighborGraph {
|
||||
g := NewNeighborGraph()
|
||||
|
||||
rows, err := conn.Query("SELECT node_a, node_b, count, last_seen FROM neighbor_edges")
|
||||
if err != nil {
|
||||
log.Printf("[neighbor] failed to load neighbor_edges: %v", err)
|
||||
return g
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
var a, b string
|
||||
var cnt int
|
||||
var lastSeen sql.NullString
|
||||
if err := rows.Scan(&a, &b, &cnt, &lastSeen); err != nil {
|
||||
continue
|
||||
}
|
||||
ts := time.Time{}
|
||||
if lastSeen.Valid {
|
||||
ts = parseTimestamp(lastSeen.String)
|
||||
}
|
||||
// Build edge directly (both nodes are full pubkeys from persisted data)
|
||||
key := makeEdgeKey(a, b)
|
||||
g.mu.Lock()
|
||||
e, exists := g.edges[key]
|
||||
if !exists {
|
||||
e = &NeighborEdge{
|
||||
NodeA: key.A,
|
||||
NodeB: key.B,
|
||||
Observers: make(map[string]bool),
|
||||
FirstSeen: ts,
|
||||
LastSeen: ts,
|
||||
Count: cnt,
|
||||
}
|
||||
g.edges[key] = e
|
||||
g.byNode[key.A] = append(g.byNode[key.A], e)
|
||||
g.byNode[key.B] = append(g.byNode[key.B], e)
|
||||
} else {
|
||||
e.Count += cnt
|
||||
if ts.After(e.LastSeen) {
|
||||
e.LastSeen = ts
|
||||
}
|
||||
}
|
||||
g.mu.Unlock()
|
||||
count++
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
g.mu.Lock()
|
||||
g.builtAt = time.Now()
|
||||
g.mu.Unlock()
|
||||
log.Printf("[neighbor] loaded %d edges from neighbor_edges table", count)
|
||||
}
|
||||
|
||||
return g
|
||||
}
|
||||
|
||||
// ─── shared async persistence helper ───────────────────────────────────────────
|
||||
|
||||
// persistObsUpdate holds data for a resolved_path SQLite update.
|
||||
type persistObsUpdate struct {
|
||||
obsID int
|
||||
resolvedPath string
|
||||
}
|
||||
|
||||
// persistEdgeUpdate holds data for a neighbor_edges SQLite upsert.
|
||||
type persistEdgeUpdate struct {
|
||||
a, b, ts string
|
||||
}
|
||||
|
||||
// asyncPersistResolvedPathsAndEdges writes resolved_path updates and neighbor
|
||||
// edge upserts to SQLite in a background goroutine. Shared between
|
||||
// IngestNewFromDB and IngestNewObservations to avoid DRY violation.
|
||||
func asyncPersistResolvedPathsAndEdges(dbPath string, obsUpdates []persistObsUpdate, edgeUpdates []persistEdgeUpdate, logPrefix string) {
|
||||
if len(obsUpdates) == 0 && len(edgeUpdates) == 0 {
|
||||
return
|
||||
}
|
||||
// Try-acquire semaphore BEFORE spawning goroutine. If another
|
||||
// persistence operation is already running, drop this batch —
|
||||
// data lives in memory and will be backfilled on restart.
|
||||
select {
|
||||
case persistSem <- struct{}{}:
|
||||
// Acquired — spawn goroutine to do the work.
|
||||
default:
|
||||
log.Printf("[store] %s skipped: persistence already in progress", logPrefix)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { <-persistSem }()
|
||||
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[store] %s rw open error: %v", logPrefix, err)
|
||||
return
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
if len(obsUpdates) > 0 {
|
||||
sqlTx, err := rw.Begin()
|
||||
if err == nil {
|
||||
stmt, err := sqlTx.Prepare("UPDATE observations SET resolved_path = ? WHERE id = ?")
|
||||
if err == nil {
|
||||
var firstErr error
|
||||
for _, u := range obsUpdates {
|
||||
if _, err := stmt.Exec(u.resolvedPath, u.obsID); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
stmt.Close()
|
||||
if firstErr != nil {
|
||||
log.Printf("[store] %s resolved_path error (first): %v", logPrefix, firstErr)
|
||||
}
|
||||
} else {
|
||||
log.Printf("[store] %s resolved_path prepare error: %v", logPrefix, err)
|
||||
}
|
||||
sqlTx.Commit()
|
||||
}
|
||||
}
|
||||
|
||||
if len(edgeUpdates) > 0 {
|
||||
sqlTx, err := rw.Begin()
|
||||
if err == nil {
|
||||
stmt, err := sqlTx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(node_a, node_b) DO UPDATE SET
|
||||
count = count + 1, last_seen = MAX(last_seen, excluded.last_seen)`)
|
||||
if err == nil {
|
||||
var firstErr error
|
||||
for _, e := range edgeUpdates {
|
||||
if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
stmt.Close()
|
||||
if firstErr != nil {
|
||||
log.Printf("[store] %s edge error (first): %v", logPrefix, firstErr)
|
||||
}
|
||||
} else {
|
||||
log.Printf("[store] %s edge prepare error: %v", logPrefix, err)
|
||||
}
|
||||
sqlTx.Commit()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// neighborEdgesTableExists checks if the neighbor_edges table has any data.
|
||||
func neighborEdgesTableExists(conn *sql.DB) bool {
|
||||
var cnt int
|
||||
err := conn.QueryRow("SELECT COUNT(*) FROM neighbor_edges").Scan(&cnt)
|
||||
if err != nil {
|
||||
return false // table doesn't exist
|
||||
}
|
||||
return cnt > 0
|
||||
}
|
||||
|
||||
// buildAndPersistEdges scans all packets in the store, extracts edges per
|
||||
// ADVERT/non-ADVERT rules, and persists them to SQLite.
|
||||
func buildAndPersistEdges(store *PacketStore, rw *sql.DB) int {
|
||||
store.mu.RLock()
|
||||
packets := make([]*StoreTx, len(store.packets))
|
||||
copy(packets, store.packets)
|
||||
store.mu.RUnlock()
|
||||
|
||||
_, pm := store.getCachedNodesAndPM()
|
||||
|
||||
tx, err := rw.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[neighbor] begin tx error: %v", err)
|
||||
return 0
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(node_a, node_b) DO UPDATE SET
|
||||
count = count + 1, last_seen = MAX(last_seen, excluded.last_seen)`)
|
||||
if err != nil {
|
||||
log.Printf("[neighbor] prepare stmt error: %v", err)
|
||||
return 0
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
edgeCount := 0
|
||||
var firstErr error
|
||||
for _, pkt := range packets {
|
||||
for _, obs := range pkt.Observations {
|
||||
for _, ec := range extractEdgesFromObs(obs, pkt, pm) {
|
||||
if _, err := stmt.Exec(ec.A, ec.B, ec.Timestamp); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
edgeCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
log.Printf("[neighbor] edge exec error (first): %v", firstErr)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Printf("[neighbor] commit error: %v", err)
|
||||
return 0
|
||||
}
|
||||
return edgeCount
|
||||
}
|
||||
|
||||
// ─── resolved_path column ──────────────────────────────────────────────────────
|
||||
|
||||
// ensureResolvedPathColumn adds the resolved_path column to observations if missing.
|
||||
func ensureResolvedPathColumn(dbPath string) error {
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
// Check if column already exists
|
||||
rows, err := rw.Query("PRAGMA table_info(observations)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil && colName == "resolved_path" {
|
||||
return nil // already exists
|
||||
}
|
||||
}
|
||||
|
||||
_, err = rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT")
|
||||
if err != nil {
|
||||
return fmt.Errorf("add resolved_path column: %w", err)
|
||||
}
|
||||
log.Println("[store] Added resolved_path column to observations")
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolvePathForObs resolves hop prefixes to full pubkeys for an observation.
|
||||
// Returns nil if path is empty.
|
||||
func resolvePathForObs(pathJSON, observerID string, tx *StoreTx, pm *prefixMap, graph *NeighborGraph) []*string {
|
||||
hops := parsePathJSON(pathJSON)
|
||||
if len(hops) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build context pubkeys: observer + originator (if known)
|
||||
contextPKs := make([]string, 0, 3)
|
||||
if observerID != "" {
|
||||
contextPKs = append(contextPKs, strings.ToLower(observerID))
|
||||
}
|
||||
fromNode := extractFromNode(tx)
|
||||
if fromNode != "" {
|
||||
contextPKs = append(contextPKs, strings.ToLower(fromNode))
|
||||
}
|
||||
|
||||
resolved := make([]*string, len(hops))
|
||||
for i, hop := range hops {
|
||||
// Add adjacent hops as context for disambiguation
|
||||
ctx := make([]string, len(contextPKs), len(contextPKs)+2)
|
||||
copy(ctx, contextPKs)
|
||||
// Add previously resolved hops as context
|
||||
if i > 0 && resolved[i-1] != nil {
|
||||
ctx = append(ctx, *resolved[i-1])
|
||||
}
|
||||
|
||||
node, _, _ := pm.resolveWithContext(hop, ctx, graph)
|
||||
if node != nil {
|
||||
pk := strings.ToLower(node.PublicKey)
|
||||
resolved[i] = &pk
|
||||
}
|
||||
}
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
// marshalResolvedPath converts []*string to JSON for storage.
|
||||
func marshalResolvedPath(rp []*string) string {
|
||||
if len(rp) == 0 {
|
||||
return ""
|
||||
}
|
||||
b, err := json.Marshal(rp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// unmarshalResolvedPath parses a resolved_path JSON string.
|
||||
func unmarshalResolvedPath(s string) []*string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
var result []*string
|
||||
if json.Unmarshal([]byte(s), &result) != nil {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// backfillResolvedPaths resolves paths for all observations that have NULL resolved_path.
|
||||
func backfillResolvedPaths(store *PacketStore, dbPath string) int {
|
||||
// Collect pending observations and snapshot immutable fields under read lock.
|
||||
// graph is set in main.go before backfill is called; nil-safe throughout (review item #6).
|
||||
type obsRef struct {
|
||||
obsID int
|
||||
pathJSON string
|
||||
observerID string
|
||||
txJSON string // snapshot of DecodedJSON for extractFromNode
|
||||
payloadType *int
|
||||
}
|
||||
store.mu.RLock()
|
||||
pm := store.nodePM
|
||||
graph := store.graph
|
||||
var pending []obsRef
|
||||
for _, tx := range store.packets {
|
||||
for _, obs := range tx.Observations {
|
||||
if obs.ResolvedPath == nil && obs.PathJSON != "" && obs.PathJSON != "[]" {
|
||||
pending = append(pending, obsRef{
|
||||
obsID: obs.ID,
|
||||
pathJSON: obs.PathJSON,
|
||||
observerID: obs.ObserverID,
|
||||
txJSON: tx.DecodedJSON,
|
||||
payloadType: tx.PayloadType,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
store.mu.RUnlock()
|
||||
|
||||
if len(pending) == 0 || pm == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Resolve paths outside the lock — resolvePathForObs only reads pm and graph.
|
||||
type resolved struct {
|
||||
obsID int
|
||||
rp []*string
|
||||
rpJSON string
|
||||
}
|
||||
var results []resolved
|
||||
for _, ref := range pending {
|
||||
// Build a minimal StoreTx for extractFromNode (only needs DecodedJSON + PayloadType).
|
||||
fakeTx := &StoreTx{DecodedJSON: ref.txJSON, PayloadType: ref.payloadType}
|
||||
rp := resolvePathForObs(ref.pathJSON, ref.observerID, fakeTx, pm, graph)
|
||||
if len(rp) > 0 {
|
||||
rpJSON := marshalResolvedPath(rp)
|
||||
if rpJSON != "" {
|
||||
results = append(results, resolved{ref.obsID, rp, rpJSON})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Persist to SQLite (no lock needed — separate RW connection).
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[store] backfill: open rw error: %v", err)
|
||||
return 0
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
sqlTx, err := rw.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[store] backfill: begin tx error: %v", err)
|
||||
return 0
|
||||
}
|
||||
defer sqlTx.Rollback()
|
||||
|
||||
stmt, err := sqlTx.Prepare("UPDATE observations SET resolved_path = ? WHERE id = ?")
|
||||
if err != nil {
|
||||
log.Printf("[store] backfill: prepare error: %v", err)
|
||||
return 0
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
var firstErr error
|
||||
for _, r := range results {
|
||||
if _, err := stmt.Exec(r.rpJSON, r.obsID); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
log.Printf("[store] backfill resolved_path exec error (first): %v", firstErr)
|
||||
}
|
||||
|
||||
if err := sqlTx.Commit(); err != nil {
|
||||
log.Printf("[store] backfill: commit error: %v", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
// Update in-memory state under write lock.
|
||||
store.mu.Lock()
|
||||
count := 0
|
||||
for _, r := range results {
|
||||
if obs, ok := store.byObsID[r.obsID]; ok {
|
||||
obs.ResolvedPath = r.rp
|
||||
count++
|
||||
}
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
// ─── Shared helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
// edgeCandidate represents an extracted edge to be persisted.
|
||||
type edgeCandidate struct {
|
||||
A, B, Timestamp string
|
||||
}
|
||||
|
||||
// extractEdgesFromObs extracts neighbor edge candidates from a single observation.
|
||||
// For ADVERTs: originator↔path[0] (if unambiguous). For ALL types: observer↔path[last] (if unambiguous).
|
||||
// Also handles zero-hop ADVERTs (originator↔observer direct link).
|
||||
func extractEdgesFromObs(obs *StoreObs, tx *StoreTx, pm *prefixMap) []edgeCandidate {
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
|
||||
fromNode := extractFromNode(tx)
|
||||
path := parsePathJSON(obs.PathJSON)
|
||||
observerPK := strings.ToLower(obs.ObserverID)
|
||||
ts := obs.Timestamp
|
||||
var edges []edgeCandidate
|
||||
|
||||
if len(path) == 0 {
|
||||
if isAdvert && fromNode != "" {
|
||||
fromLower := strings.ToLower(fromNode)
|
||||
if fromLower != observerPK {
|
||||
a, b := fromLower, observerPK
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
edges = append(edges, edgeCandidate{a, b, ts})
|
||||
}
|
||||
}
|
||||
return edges
|
||||
}
|
||||
|
||||
// Edge 1: originator ↔ path[0] — ADVERTs only (resolve prefix to full pubkey)
|
||||
if isAdvert && fromNode != "" && pm != nil {
|
||||
firstHop := strings.ToLower(path[0])
|
||||
fromLower := strings.ToLower(fromNode)
|
||||
candidates := pm.m[firstHop]
|
||||
if len(candidates) == 1 {
|
||||
resolved := strings.ToLower(candidates[0].PublicKey)
|
||||
if resolved != fromLower {
|
||||
a, b := fromLower, resolved
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
edges = append(edges, edgeCandidate{a, b, ts})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Edge 2: observer ↔ path[last] — ALL packet types
|
||||
if pm != nil {
|
||||
lastHop := strings.ToLower(path[len(path)-1])
|
||||
candidates := pm.m[lastHop]
|
||||
if len(candidates) == 1 {
|
||||
resolved := strings.ToLower(candidates[0].PublicKey)
|
||||
if resolved != observerPK {
|
||||
a, b := observerPK, resolved
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
edges = append(edges, edgeCandidate{a, b, ts})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// openRW opens a read-write SQLite connection (same pattern as PruneOldPackets).
|
||||
func openRW(dbPath string) (*sql.DB, error) {
|
||||
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=10000", dbPath)
|
||||
rw, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rw.SetMaxOpenConns(1)
|
||||
return rw, nil
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// createTestDBWithSchema creates a temp SQLite DB with the standard schema + resolved_path column.
|
||||
func createTestDBWithSchema(t *testing.T) (*DB, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
conn, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create tables
|
||||
conn.Exec(`CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT, hash TEXT UNIQUE, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
|
||||
decoded_json TEXT
|
||||
)`)
|
||||
conn.Exec(`CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY, name TEXT, iata TEXT
|
||||
)`)
|
||||
conn.Exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_id TEXT, observer_name TEXT, direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT,
|
||||
resolved_path TEXT
|
||||
)`)
|
||||
conn.Exec(`CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
|
||||
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT,
|
||||
advert_count INTEGER DEFAULT 0
|
||||
)`)
|
||||
|
||||
conn.Close()
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, dbPath
|
||||
}
|
||||
|
||||
func TestResolvePathForObs(t *testing.T) {
|
||||
// Build a prefix map with known nodes
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
graph := NewNeighborGraph()
|
||||
|
||||
tx := &StoreTx{
|
||||
DecodedJSON: `{"pubKey": "originator1234567890"}`,
|
||||
PayloadType: intPtr(4),
|
||||
}
|
||||
|
||||
// Unambiguous prefixes should resolve
|
||||
rp := resolvePathForObs(`["aa","bb"]`, "observer1", tx, pm, graph)
|
||||
if len(rp) != 2 {
|
||||
t.Fatalf("expected 2 resolved hops, got %d", len(rp))
|
||||
}
|
||||
if rp[0] == nil || !strings.HasPrefix(*rp[0], "aabbcc") {
|
||||
t.Errorf("expected first hop to resolve to Node-AA, got %v", rp[0])
|
||||
}
|
||||
if rp[1] == nil || !strings.HasPrefix(*rp[1], "bbccdd") {
|
||||
t.Errorf("expected second hop to resolve to Node-BB, got %v", rp[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePathForObs_EmptyPath(t *testing.T) {
|
||||
pm := buildPrefixMap(nil)
|
||||
rp := resolvePathForObs(`[]`, "", &StoreTx{}, pm, nil)
|
||||
if rp != nil {
|
||||
t.Errorf("expected nil for empty path, got %v", rp)
|
||||
}
|
||||
|
||||
rp = resolvePathForObs("", "", &StoreTx{}, pm, nil)
|
||||
if rp != nil {
|
||||
t.Errorf("expected nil for empty string, got %v", rp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePathForObs_Unresolvable(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
// "zz" prefix doesn't match any node
|
||||
rp := resolvePathForObs(`["zz"]`, "", &StoreTx{}, pm, nil)
|
||||
if len(rp) != 1 {
|
||||
t.Fatalf("expected 1 hop, got %d", len(rp))
|
||||
}
|
||||
if rp[0] != nil {
|
||||
t.Errorf("expected nil for unresolvable hop, got %v", *rp[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalResolvedPath(t *testing.T) {
|
||||
pk1 := "aabbccdd"
|
||||
var rp []*string
|
||||
rp = append(rp, &pk1, nil)
|
||||
|
||||
j := marshalResolvedPath(rp)
|
||||
if j == "" {
|
||||
t.Fatal("expected non-empty JSON")
|
||||
}
|
||||
|
||||
parsed := unmarshalResolvedPath(j)
|
||||
if len(parsed) != 2 {
|
||||
t.Fatalf("expected 2 elements, got %d", len(parsed))
|
||||
}
|
||||
if parsed[0] == nil || *parsed[0] != "aabbccdd" {
|
||||
t.Errorf("first element wrong: %v", parsed[0])
|
||||
}
|
||||
if parsed[1] != nil {
|
||||
t.Errorf("second element should be nil, got %v", *parsed[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalResolvedPath_Empty(t *testing.T) {
|
||||
if marshalResolvedPath(nil) != "" {
|
||||
t.Error("expected empty for nil")
|
||||
}
|
||||
if marshalResolvedPath([]*string{}) != "" {
|
||||
t.Error("expected empty for empty slice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnmarshalResolvedPath_Invalid(t *testing.T) {
|
||||
if unmarshalResolvedPath("") != nil {
|
||||
t.Error("expected nil for empty string")
|
||||
}
|
||||
if unmarshalResolvedPath("not json") != nil {
|
||||
t.Error("expected nil for invalid JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureNeighborEdgesTable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create initial DB
|
||||
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY)")
|
||||
conn.Close()
|
||||
|
||||
if err := ensureNeighborEdgesTable(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify table exists
|
||||
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?mode=ro")
|
||||
defer conn.Close()
|
||||
var cnt int
|
||||
if err := conn.QueryRow("SELECT COUNT(*) FROM neighbor_edges").Scan(&cnt); err != nil {
|
||||
t.Fatalf("neighbor_edges table not created: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadNeighborEdgesFromDB(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec(`CREATE TABLE neighbor_edges (
|
||||
node_a TEXT NOT NULL, node_b TEXT NOT NULL,
|
||||
count INTEGER DEFAULT 1, last_seen TEXT,
|
||||
PRIMARY KEY (node_a, node_b)
|
||||
)`)
|
||||
conn.Exec("INSERT INTO neighbor_edges VALUES ('aaa', 'bbb', 5, '2024-01-01T00:00:00Z')")
|
||||
conn.Exec("INSERT INTO neighbor_edges VALUES ('ccc', 'ddd', 3, '2024-01-02T00:00:00Z')")
|
||||
|
||||
g := loadNeighborEdgesFromDB(conn)
|
||||
conn.Close()
|
||||
|
||||
// Should have 2 edges
|
||||
edges := g.AllEdges()
|
||||
if len(edges) != 2 {
|
||||
t.Errorf("expected 2 edges, got %d", len(edges))
|
||||
}
|
||||
|
||||
// Check neighbors
|
||||
n := g.Neighbors("aaa")
|
||||
if len(n) != 1 {
|
||||
t.Errorf("expected 1 neighbor for aaa, got %d", len(n))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
|
||||
// Verify resolved_path appears in broadcast maps
|
||||
pk := "aabbccdd"
|
||||
obs := &StoreObs{
|
||||
ID: 1,
|
||||
ObserverID: "obs1",
|
||||
ObserverName: "Observer 1",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
Observations: []*StoreObs{obs},
|
||||
}
|
||||
pickBestObservation(tx)
|
||||
|
||||
if tx.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be set on tx after pickBestObservation")
|
||||
}
|
||||
if *tx.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("expected resolved path to be aabbccdd, got %s", *tx.ResolvedPath[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInTxToMap(t *testing.T) {
|
||||
pk := "aabbccdd"
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
obsKeys: make(map[string]bool),
|
||||
}
|
||||
|
||||
m := txToMap(tx)
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path not in txToMap output")
|
||||
}
|
||||
rpSlice, ok := rp.([]*string)
|
||||
if !ok || len(rpSlice) != 1 || *rpSlice[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected resolved_path: %v", rp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathOmittedWhenNil(t *testing.T) {
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
obsKeys: make(map[string]bool),
|
||||
}
|
||||
|
||||
m := txToMap(tx)
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should not be in map when nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureResolvedPathColumn(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER,
|
||||
observer_id TEXT, path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
conn.Close()
|
||||
|
||||
if err := ensureResolvedPathColumn(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify column exists
|
||||
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?mode=ro")
|
||||
defer conn.Close()
|
||||
rows, _ := conn.Query("PRAGMA table_info(observations)")
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk)
|
||||
if colName == "resolved_path" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if !found {
|
||||
t.Error("resolved_path column not added")
|
||||
}
|
||||
|
||||
// Running again should be idempotent
|
||||
if err := ensureResolvedPathColumn(dbPath); err != nil {
|
||||
t.Fatal("second call should be idempotent:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBDetectsResolvedPathColumn(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create DB without resolved_path
|
||||
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`)
|
||||
conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY)`)
|
||||
conn.Close()
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if db.hasResolvedPath {
|
||||
t.Error("should not detect resolved_path when column missing")
|
||||
}
|
||||
db.Close()
|
||||
|
||||
// Add resolved_path column
|
||||
conn, _ = sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT")
|
||||
conn.Close()
|
||||
|
||||
db, err = OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !db.hasResolvedPath {
|
||||
t.Error("should detect resolved_path when column exists")
|
||||
}
|
||||
db.Close()
|
||||
}
|
||||
|
||||
func TestLoadWithResolvedPath(t *testing.T) {
|
||||
db, dbPath := createTestDBWithSchema(t)
|
||||
defer db.Close()
|
||||
|
||||
// Insert test data
|
||||
rw, _ := openRW(dbPath)
|
||||
rw.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, decoded_json)
|
||||
VALUES (1, 'hash1', '2024-01-01T00:00:00Z', 4, '{"pubKey":"origpk"}')`)
|
||||
rw.Exec(`INSERT INTO observations (id, transmission_id, observer_id, observer_name, path_json, timestamp, resolved_path)
|
||||
VALUES (1, 1, 'obs1', 'Observer1', '["aa"]', '2024-01-01T00:00:00Z', '["aabbccdd"]')`)
|
||||
rw.Close()
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(store.packets) != 1 {
|
||||
t.Fatalf("expected 1 packet, got %d", len(store.packets))
|
||||
}
|
||||
|
||||
tx := store.packets[0]
|
||||
if len(tx.Observations) != 1 {
|
||||
t.Fatalf("expected 1 observation, got %d", len(tx.Observations))
|
||||
}
|
||||
|
||||
obs := tx.Observations[0]
|
||||
if obs.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be loaded")
|
||||
}
|
||||
if len(obs.ResolvedPath) != 1 || *obs.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected ResolvedPath: %v", obs.ResolvedPath)
|
||||
}
|
||||
|
||||
// Check that pickBestObservation propagated resolved_path to tx
|
||||
if tx.ResolvedPath == nil || len(tx.ResolvedPath) != 1 {
|
||||
t.Error("expected ResolvedPath to be propagated to tx")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInAPIResponse(t *testing.T) {
|
||||
// Test that TransmissionResp properly marshals resolved_path
|
||||
pk := "aabbccddee"
|
||||
resp := TransmissionResp{
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
ResolvedPath: []*string{&pk, nil},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path missing from JSON")
|
||||
}
|
||||
rpArr, ok := rp.([]interface{})
|
||||
if !ok || len(rpArr) != 2 {
|
||||
t.Fatalf("unexpected resolved_path shape: %v", rp)
|
||||
}
|
||||
if rpArr[0] != "aabbccddee" {
|
||||
t.Errorf("first element wrong: %v", rpArr[0])
|
||||
}
|
||||
if rpArr[1] != nil {
|
||||
t.Errorf("second element should be null: %v", rpArr[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathOmittedWhenEmpty(t *testing.T) {
|
||||
resp := TransmissionResp{
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(resp)
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should be omitted when nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEdgesFromObs_AdvertNoPath(t *testing.T) {
|
||||
tx := &StoreTx{
|
||||
DecodedJSON: `{"pubKey":"aaaa1111"}`,
|
||||
PayloadType: intPtr(4),
|
||||
}
|
||||
obs := &StoreObs{
|
||||
ObserverID: "bbbb2222",
|
||||
PathJSON: "",
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
edges := extractEdgesFromObs(obs, tx, nil)
|
||||
if len(edges) != 1 {
|
||||
t.Fatalf("expected 1 edge for zero-hop advert, got %d", len(edges))
|
||||
}
|
||||
// Canonical ordering: aaaa < bbbb
|
||||
if edges[0].A != "aaaa1111" || edges[0].B != "bbbb2222" {
|
||||
t.Errorf("unexpected edge: %+v", edges[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEdgesFromObs_NonAdvertNoPath(t *testing.T) {
|
||||
tx := &StoreTx{PayloadType: intPtr(1)}
|
||||
obs := &StoreObs{ObserverID: "obs1", PathJSON: ""}
|
||||
edges := extractEdgesFromObs(obs, tx, nil)
|
||||
if len(edges) != 0 {
|
||||
t.Errorf("expected 0 edges for non-advert without path, got %d", len(edges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEdgesFromObs_WithPath(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
tx := &StoreTx{
|
||||
DecodedJSON: `{"pubKey":"originator00"}`,
|
||||
PayloadType: intPtr(4),
|
||||
}
|
||||
obs := &StoreObs{
|
||||
ObserverID: "observer00",
|
||||
PathJSON: `["aa","ff"]`,
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
edges := extractEdgesFromObs(obs, tx, pm)
|
||||
// Should get: originator↔aa (advert), observer↔ff (last hop)
|
||||
if len(edges) != 2 {
|
||||
t.Fatalf("expected 2 edges, got %d", len(edges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractEdgesFromObs_SameNodeNoEdge(t *testing.T) {
|
||||
tx := &StoreTx{
|
||||
DecodedJSON: `{"pubKey":"same1234"}`,
|
||||
PayloadType: intPtr(4),
|
||||
}
|
||||
obs := &StoreObs{
|
||||
ObserverID: "same1234",
|
||||
PathJSON: "",
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
edges := extractEdgesFromObs(obs, tx, nil)
|
||||
if len(edges) != 0 {
|
||||
t.Errorf("expected 0 edges when originator == observer, got %d", len(edges))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
func TestPersistSemaphoreTryAcquireSkipsBatch(t *testing.T) {
|
||||
// Verify that persistSem is a buffered channel of size 1.
|
||||
if cap(persistSem) != 1 {
|
||||
t.Errorf("persistSem capacity = %d, want 1", cap(persistSem))
|
||||
}
|
||||
// Acquire the semaphore to simulate an in-progress persistence.
|
||||
persistSem <- struct{}{}
|
||||
|
||||
// asyncPersistResolvedPathsAndEdges should skip (not block, not
|
||||
// spawn a goroutine) when the semaphore is already held.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
asyncPersistResolvedPathsAndEdges(
|
||||
"/nonexistent/path.db",
|
||||
[]persistObsUpdate{{obsID: 1, resolvedPath: "x"}},
|
||||
nil,
|
||||
"test",
|
||||
)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// If the function blocks on the semaphore instead of skipping,
|
||||
// this select will hit the timeout.
|
||||
select {
|
||||
case <-done:
|
||||
// Expected: returned immediately because semaphore was busy.
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
<-persistSem
|
||||
t.Fatal("asyncPersistResolvedPathsAndEdges blocked instead of skipping when semaphore was held")
|
||||
}
|
||||
|
||||
<-persistSem // release
|
||||
}
|
||||
@@ -166,6 +166,7 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) {
|
||||
// Insert a unique node
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"ff11223344", "UniqueNode", 37.0, -122.0)
|
||||
srv.store.InvalidateNodeCache()
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ff11223344", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -192,6 +193,7 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) {
|
||||
"ee1aaaaaaa", "Node-E1", 37.0, -122.0)
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"ee1bbbbbbb", "Node-E2", 38.0, -121.0)
|
||||
srv.store.InvalidateNodeCache()
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ee1", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
@@ -204,8 +206,10 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) {
|
||||
if hr == nil {
|
||||
t.Fatal("expected hop in resolved map")
|
||||
}
|
||||
if hr.Confidence != "ambiguous" {
|
||||
t.Fatalf("expected ambiguous, got %s", hr.Confidence)
|
||||
// With both candidates having GPS and no affinity context, the resolver
|
||||
// picks the GPS-preferred candidate → confidence is "gps_preference".
|
||||
if hr.Confidence != "gps_preference" {
|
||||
t.Fatalf("expected gps_preference, got %s", hr.Confidence)
|
||||
}
|
||||
if len(hr.Candidates) != 2 {
|
||||
t.Fatalf("expected 2 candidates, got %d", len(hr.Candidates))
|
||||
|
||||
+151
-48
@@ -118,6 +118,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET")
|
||||
|
||||
// Packet endpoints
|
||||
r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")
|
||||
r.HandleFunc("/api/packets/timestamps", s.handlePacketTimestamps).Methods("GET")
|
||||
r.HandleFunc("/api/packets/{id}", s.handlePacketDetail).Methods("GET")
|
||||
r.HandleFunc("/api/packets", s.handlePackets).Methods("GET")
|
||||
@@ -145,6 +146,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/analytics/hash-sizes", s.handleAnalyticsHashSizes).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/hash-collisions", s.handleAnalyticsHashCollisions).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/subpaths", s.handleAnalyticsSubpaths).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/subpaths-bulk", s.handleAnalyticsSubpathsBulk).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/subpath-detail", s.handleAnalyticsSubpathDetail).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/neighbor-graph", s.handleNeighborGraph).Methods("GET")
|
||||
|
||||
@@ -718,7 +720,8 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
|
||||
Until: r.URL.Query().Get("until"),
|
||||
Region: r.URL.Query().Get("region"),
|
||||
Node: r.URL.Query().Get("node"),
|
||||
Order: "DESC",
|
||||
Order: "DESC",
|
||||
ExpandObservations: r.URL.Query().Get("expand") == "observations",
|
||||
}
|
||||
if r.URL.Query().Get("order") == "asc" {
|
||||
q.Order = "ASC"
|
||||
@@ -760,13 +763,6 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Strip observations from default response
|
||||
if r.URL.Query().Get("expand") != "observations" {
|
||||
for _, p := range result.Packets {
|
||||
delete(p, "observations")
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, result)
|
||||
}
|
||||
|
||||
@@ -791,6 +787,38 @@ var muxBraceParam = regexp.MustCompile(`\{([^}]+)\}`)
|
||||
// perfHexFallback matches hex IDs for perf path normalization fallback.
|
||||
var perfHexFallback = regexp.MustCompile(`[0-9a-f]{8,}`)
|
||||
|
||||
// handleBatchObservations returns observations for multiple hashes in a single request.
|
||||
// POST /api/packets/observations with JSON body: {"hashes": ["abc123", "def456", ...]}
|
||||
// Response: {"results": {"abc123": [...observations...], "def456": [...], ...}}
|
||||
// Limited to 200 hashes per request to prevent abuse.
|
||||
func (s *Server) handleBatchObservations(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Hashes []string `json:"hashes"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, 400, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
const maxHashes = 200
|
||||
if len(body.Hashes) > maxHashes {
|
||||
writeError(w, 400, fmt.Sprintf("too many hashes (max %d)", maxHashes))
|
||||
return
|
||||
}
|
||||
if len(body.Hashes) == 0 {
|
||||
writeJSON(w, map[string]interface{}{"results": map[string]interface{}{}})
|
||||
return
|
||||
}
|
||||
|
||||
results := make(map[string][]ObservationResp, len(body.Hashes))
|
||||
if s.store != nil {
|
||||
for _, hash := range body.Hashes {
|
||||
obs := s.store.GetObservationsForHash(hash)
|
||||
results[hash] = mapSliceToObservations(obs)
|
||||
}
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{"results": results})
|
||||
}
|
||||
|
||||
func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
param := mux.Vars(r)["id"]
|
||||
var packet map[string]interface{}
|
||||
@@ -1065,16 +1093,44 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
prefix1 := strings.ToLower(pubkey)
|
||||
if len(prefix1) > 2 {
|
||||
prefix1 = prefix1[:2]
|
||||
}
|
||||
prefix2 := strings.ToLower(pubkey)
|
||||
// Use the precomputed byPathHop index instead of scanning all packets.
|
||||
// Look up by full pubkey (resolved hops) and by short prefixes (raw hops).
|
||||
lowerPK := strings.ToLower(pubkey)
|
||||
prefix2 := lowerPK
|
||||
if len(prefix2) > 4 {
|
||||
prefix2 = prefix2[:4]
|
||||
}
|
||||
prefix1 := lowerPK
|
||||
if len(prefix1) > 2 {
|
||||
prefix1 = prefix1[:2]
|
||||
}
|
||||
|
||||
s.store.mu.RLock()
|
||||
_, pm := s.store.getCachedNodesAndPM()
|
||||
|
||||
// Collect candidate transmissions from the index, deduplicating by tx ID.
|
||||
seen := make(map[int]bool)
|
||||
var candidates []*StoreTx
|
||||
addCandidates := func(key string) {
|
||||
for _, tx := range s.store.byPathHop[key] {
|
||||
if !seen[tx.ID] {
|
||||
seen[tx.ID] = true
|
||||
candidates = append(candidates, tx)
|
||||
}
|
||||
}
|
||||
}
|
||||
addCandidates(lowerPK) // full pubkey match (from resolved_path)
|
||||
addCandidates(prefix1) // 2-char raw hop match
|
||||
addCandidates(prefix2) // 4-char raw hop match
|
||||
// Also check any raw hops that start with prefix2 (longer prefixes).
|
||||
// Raw hops are typically 2 chars, so iterate only keys with HasPrefix
|
||||
// on the small set of index keys rather than all packets.
|
||||
for key := range s.store.byPathHop {
|
||||
if len(key) > 4 && len(key) < len(lowerPK) && strings.HasPrefix(key, prefix2) {
|
||||
addCandidates(key)
|
||||
}
|
||||
}
|
||||
|
||||
type pathAgg struct {
|
||||
Hops []PathHopResp
|
||||
Count int
|
||||
@@ -1088,28 +1144,13 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
if cached, ok := hopCache[hop]; ok {
|
||||
return cached
|
||||
}
|
||||
r := pm.resolve(hop)
|
||||
r, _, _ := pm.resolveWithContext(hop, nil, s.store.graph)
|
||||
hopCache[hop] = r
|
||||
return r
|
||||
}
|
||||
for _, tx := range s.store.packets {
|
||||
hops := txGetParsedPath(tx)
|
||||
if len(hops) == 0 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, hop := range hops {
|
||||
hl := strings.ToLower(hop)
|
||||
if hl == prefix1 || hl == prefix2 || strings.HasPrefix(hl, prefix2) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, tx := range candidates {
|
||||
totalTransmissions++
|
||||
hops := txGetParsedPath(tx)
|
||||
resolvedHops := make([]PathHopResp, len(hops))
|
||||
sigParts := make([]string, len(hops))
|
||||
for i, hop := range hops {
|
||||
@@ -1337,6 +1378,57 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
|
||||
})
|
||||
}
|
||||
|
||||
// handleAnalyticsSubpathsBulk returns multiple length-range buckets in a single
|
||||
// response, avoiding repeated scans of the same packet data. Query format:
|
||||
// ?groups=2-2:50,3-3:30,4-4:20,5-8:15 (minLen-maxLen:limit per group)
|
||||
func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Request) {
|
||||
region := r.URL.Query().Get("region")
|
||||
groupsParam := r.URL.Query().Get("groups")
|
||||
if groupsParam == "" {
|
||||
writeJSON(w, ErrorResp{Error: "groups parameter required (e.g. groups=2-2:50,3-3:30)"})
|
||||
return
|
||||
}
|
||||
|
||||
var groups []subpathGroup
|
||||
for _, g := range strings.Split(groupsParam, ",") {
|
||||
parts := strings.SplitN(g, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
writeJSON(w, ErrorResp{Error: "invalid group format: " + g})
|
||||
return
|
||||
}
|
||||
rangeParts := strings.SplitN(parts[0], "-", 2)
|
||||
if len(rangeParts) != 2 {
|
||||
writeJSON(w, ErrorResp{Error: "invalid range format: " + parts[0]})
|
||||
return
|
||||
}
|
||||
mn, err1 := strconv.Atoi(rangeParts[0])
|
||||
mx, err2 := strconv.Atoi(rangeParts[1])
|
||||
lim, err3 := strconv.Atoi(parts[1])
|
||||
if err1 != nil || err2 != nil || err3 != nil || mn < 2 || mx < mn || lim < 1 {
|
||||
writeJSON(w, ErrorResp{Error: "invalid group: " + g})
|
||||
return
|
||||
}
|
||||
groups = append(groups, subpathGroup{mn, mx, lim})
|
||||
}
|
||||
|
||||
if s.store == nil {
|
||||
results := make([]map[string]interface{}, len(groups))
|
||||
for i := range groups {
|
||||
results[i] = map[string]interface{}{"subpaths": []interface{}{}, "totalPaths": 0}
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{"results": results})
|
||||
return
|
||||
}
|
||||
|
||||
results := s.store.GetAnalyticsSubpathsBulk(region, groups)
|
||||
writeJSON(w, map[string]interface{}{"results": results})
|
||||
}
|
||||
|
||||
// subpathGroup defines a length-range + limit for the bulk subpaths endpoint.
|
||||
type subpathGroup struct {
|
||||
MinLen, MaxLen, Limit int
|
||||
}
|
||||
|
||||
func (s *Server) handleAnalyticsSubpathDetail(w http.ResponseWriter, r *http.Request) {
|
||||
hops := r.URL.Query().Get("hops")
|
||||
if hops == "" {
|
||||
@@ -1406,24 +1498,25 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
hopLower := strings.ToLower(hop)
|
||||
rows, err := s.db.conn.Query("SELECT public_key, name, lat, lon FROM nodes WHERE LOWER(public_key) LIKE ?", hopLower+"%")
|
||||
if err != nil {
|
||||
resolved[hop] = &HopResolution{Name: nil, Candidates: []HopCandidate{}, Conflicts: []interface{}{}, Confidence: "ambiguous"}
|
||||
continue
|
||||
}
|
||||
|
||||
// Resolve candidates from the in-memory prefix map instead of
|
||||
// issuing per-hop DB queries (fixes N+1 pattern, see #369).
|
||||
var candidates []HopCandidate
|
||||
for rows.Next() {
|
||||
var pk string
|
||||
var name sql.NullString
|
||||
var lat, lon sql.NullFloat64
|
||||
rows.Scan(&pk, &name, &lat, &lon)
|
||||
candidates = append(candidates, HopCandidate{
|
||||
Name: nullStr(name), Pubkey: pk,
|
||||
Lat: nullFloat(lat), Lon: nullFloat(lon),
|
||||
})
|
||||
if pm != nil {
|
||||
if matched, ok := pm.m[hopLower]; ok {
|
||||
for _, ni := range matched {
|
||||
c := HopCandidate{Pubkey: ni.PublicKey}
|
||||
if ni.Name != "" {
|
||||
c.Name = ni.Name
|
||||
}
|
||||
if ni.HasGPS {
|
||||
c.Lat = ni.Lat
|
||||
c.Lon = ni.Lon
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if len(candidates) == 0 {
|
||||
resolved[hop] = &HopResolution{Name: nil, Candidates: []HopCandidate{}, Conflicts: []interface{}{}, Confidence: "no_match"}
|
||||
@@ -1546,8 +1639,12 @@ func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
|
||||
oneHourAgo := time.Now().Add(-1 * time.Hour).Unix()
|
||||
pktCounts := s.db.GetObserverPacketCounts(oneHourAgo)
|
||||
|
||||
// Batch lookup: node locations (observer ID may match a node public_key)
|
||||
nodeLocations := s.db.GetNodeLocations()
|
||||
// Batch lookup: node locations only for observer IDs (not all nodes)
|
||||
observerIDs := make([]string, len(observers))
|
||||
for i, o := range observers {
|
||||
observerIDs[i] = o.ID
|
||||
}
|
||||
nodeLocations := s.db.GetNodeLocationsByKeys(observerIDs)
|
||||
|
||||
result := make([]ObserverResp, 0, len(observers))
|
||||
for _, o := range observers {
|
||||
@@ -1997,6 +2094,9 @@ func mapSliceToTransmissions(maps []map[string]interface{}) []TransmissionResp {
|
||||
tx.PathJSON = m["path_json"]
|
||||
tx.Direction = m["direction"]
|
||||
tx.Score = m["score"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
tx.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, tx)
|
||||
}
|
||||
return result
|
||||
@@ -2018,6 +2118,9 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
|
||||
obs.RSSI = m["rssi"]
|
||||
obs.PathJSON = m["path_json"]
|
||||
obs.Timestamp = m["timestamp"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
obs.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, obs)
|
||||
}
|
||||
return result
|
||||
|
||||
+315
-9
@@ -1105,6 +1105,63 @@ func TestAnalyticsSubpaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyticsSubpathsBulk(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
// Valid request with multiple groups.
|
||||
req := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk?groups=2-2:50,3-3:30,5-8:15", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
results, ok := body["results"].([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected results array")
|
||||
}
|
||||
if len(results) != 3 {
|
||||
t.Errorf("expected 3 result groups, got %d", len(results))
|
||||
}
|
||||
// Each result should have subpaths and totalPaths.
|
||||
for i, r := range results {
|
||||
rm, ok := r.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("result %d not a map", i)
|
||||
}
|
||||
if _, ok := rm["subpaths"]; !ok {
|
||||
t.Errorf("result %d missing subpaths", i)
|
||||
}
|
||||
if _, ok := rm["totalPaths"]; !ok {
|
||||
t.Errorf("result %d missing totalPaths", i)
|
||||
}
|
||||
}
|
||||
|
||||
// Missing groups param → error.
|
||||
req2 := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk", nil)
|
||||
w2 := httptest.NewRecorder()
|
||||
router.ServeHTTP(w2, req2)
|
||||
if w2.Code != 200 {
|
||||
t.Fatalf("expected 200 with error body, got %d", w2.Code)
|
||||
}
|
||||
var errBody map[string]interface{}
|
||||
json.Unmarshal(w2.Body.Bytes(), &errBody)
|
||||
if _, ok := errBody["error"]; !ok {
|
||||
t.Error("expected error field for missing groups param")
|
||||
}
|
||||
|
||||
// Invalid group format.
|
||||
req3 := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk?groups=bad", nil)
|
||||
w3 := httptest.NewRecorder()
|
||||
router.ServeHTTP(w3, req3)
|
||||
var errBody3 map[string]interface{}
|
||||
json.Unmarshal(w3.Body.Bytes(), &errBody3)
|
||||
if _, ok := errBody3["error"]; !ok {
|
||||
t.Error("expected error for invalid group format")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyticsSubpathDetailWithHops(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/analytics/subpath-detail?hops=aa,bb", nil)
|
||||
@@ -1170,6 +1227,11 @@ func TestResolveHopsAmbiguous(t *testing.T) {
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load failed: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
@@ -2105,7 +2167,7 @@ tx := &StoreTx{
|
||||
ID: 9000 + i,
|
||||
RawHex: rawHex,
|
||||
Hash: "testhash" + strconv.Itoa(i),
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2151,7 +2213,7 @@ for i, raw := range raws {
|
||||
ID: 8000 + i,
|
||||
RawHex: raw,
|
||||
Hash: "dominant" + strconv.Itoa(i),
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2190,12 +2252,13 @@ func TestGetNodeHashSizeInfoLatestWins(t *testing.T) {
|
||||
// 4 historical 1-byte adverts, then 1 recent 2-byte advert (latest).
|
||||
// Mode would pick 1 (majority), but latest-wins should pick 2.
|
||||
raws := []string{raw1byte, raw1byte, raw1byte, raw1byte, raw2byte}
|
||||
baseTime := time.Now().UTC().Add(-1 * time.Hour)
|
||||
for i, raw := range raws {
|
||||
tx := &StoreTx{
|
||||
ID: 7000 + i,
|
||||
RawHex: raw,
|
||||
Hash: "latest" + strconv.Itoa(i),
|
||||
FirstSeen: "2024-01-01T0" + strconv.Itoa(i) + ":00:00Z",
|
||||
FirstSeen: baseTime.Add(time.Duration(i) * time.Minute).Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2236,12 +2299,13 @@ func TestGetNodeHashSizeInfoIgnoreDirectZeroHop(t *testing.T) {
|
||||
|
||||
payloadType := 4
|
||||
raws := []string{rawFlood2B, rawDirect0, rawFlood2B, rawDirect0, rawFlood2B}
|
||||
baseTime2 := time.Now().UTC().Add(-1 * time.Hour)
|
||||
for i, raw := range raws {
|
||||
tx := &StoreTx{
|
||||
ID: 9150 + i,
|
||||
RawHex: raw,
|
||||
Hash: "dirignore" + strconv.Itoa(i),
|
||||
FirstSeen: "2024-01-01T0" + strconv.Itoa(i) + ":00:00Z",
|
||||
FirstSeen: baseTime2.Add(time.Duration(i) * time.Minute).Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2284,7 +2348,7 @@ func TestGetNodeHashSizeInfoOnlyDirectZeroHopIgnored(t *testing.T) {
|
||||
ID: 9160,
|
||||
RawHex: rawDirect0,
|
||||
Hash: "onlydirect0",
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2320,7 +2384,7 @@ func TestGetNodeHashSizeInfoDirectNonZeroHopCounted(t *testing.T) {
|
||||
ID: 9170,
|
||||
RawHex: rawDirectNonZero,
|
||||
Hash: "dirnonzero0",
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
@@ -2355,7 +2419,7 @@ func TestGetNodeHashSizeInfoNoAdverts(t *testing.T) {
|
||||
ID: 6000,
|
||||
RawHex: "0440aabb",
|
||||
Hash: "noadverts0",
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: `{"pubKey":"` + pk + `"}`,
|
||||
}
|
||||
@@ -2397,7 +2461,7 @@ func TestHashAnalyticsZeroHopAdvert(t *testing.T) {
|
||||
ID: 8000,
|
||||
RawHex: raw,
|
||||
Hash: "zerohop0",
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
// No PathJSON → txGetParsedPath returns nil (zero hops)
|
||||
@@ -2451,7 +2515,7 @@ func TestAnalyticsHashSizeSameNameDifferentPubkey(t *testing.T) {
|
||||
ID: 6100 + i,
|
||||
RawHex: raw2byte,
|
||||
Hash: "samename" + strconv.Itoa(i),
|
||||
FirstSeen: "2024-01-01T00:00:00Z",
|
||||
FirstSeen: time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
PathJSON: `["AABB"]`,
|
||||
@@ -2491,6 +2555,158 @@ t.Errorf("field %q is null, expected []", field)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestInconsistentNodesExcludesCompanions(t *testing.T) {
|
||||
// Issue #566: inconsistentNodes should only include repeaters and room servers.
|
||||
db := setupTestDB(t)
|
||||
seedTestData(t, db)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load failed: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
payloadType := 4
|
||||
|
||||
// Create three nodes: repeater, room_server, companion — all with inconsistent hash sizes
|
||||
nodes := []struct {
|
||||
pk string
|
||||
role string
|
||||
}{
|
||||
{"aa11111111111111111111111111111111111111111111111111111111111111", "repeater"},
|
||||
{"bb22222222222222222222222222222222222222222222222222222222222222", "room_server"},
|
||||
{"cc33333333333333333333333333333333333333333333333333333333333333", "companion"},
|
||||
}
|
||||
|
||||
for ni, n := range nodes {
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, ?, ?)", n.pk, "Node-"+n.role, n.role)
|
||||
decoded := `{"name":"Node-` + n.role + `","pubKey":"` + n.pk + `"}`
|
||||
// Create flip-flop pattern: 1-byte, 2-byte, 1-byte (transitions=2 → inconsistent)
|
||||
// Use header 0x11 (routeType=FLOOD, payloadType=4) and pathByte 0x41/0x81
|
||||
// (non-zero hop count) so packets aren't skipped by direct zero-hop filter.
|
||||
raws := []string{"11" + "41" + "aabb", "11" + "81" + "aabb", "11" + "41" + "aabb"}
|
||||
for i, raw := range raws {
|
||||
tx := &StoreTx{
|
||||
ID: 7000 + ni*10 + i,
|
||||
RawHex: raw,
|
||||
Hash: "incon-" + n.role + strconv.Itoa(i),
|
||||
FirstSeen: now,
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
store.packets = append(store.packets, tx)
|
||||
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
|
||||
}
|
||||
}
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/hash-collisions", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
|
||||
incon := body["inconsistent_nodes"].([]interface{})
|
||||
for _, item := range incon {
|
||||
node := item.(map[string]interface{})
|
||||
role := node["role"].(string)
|
||||
if role == "companion" {
|
||||
t.Error("companion node should be excluded from inconsistent_nodes")
|
||||
}
|
||||
}
|
||||
|
||||
// Repeater and room_server should be present
|
||||
roles := make(map[string]bool)
|
||||
for _, item := range incon {
|
||||
node := item.(map[string]interface{})
|
||||
roles[node["role"].(string)] = true
|
||||
}
|
||||
if !roles["repeater"] {
|
||||
t.Error("expected repeater in inconsistent_nodes")
|
||||
}
|
||||
if !roles["room_server"] {
|
||||
t.Error("expected room_server in inconsistent_nodes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashSizeInfoTimeWindow(t *testing.T) {
|
||||
// Issue #566: adverts older than 7 days should be excluded from hash size computation.
|
||||
db := setupTestDB(t)
|
||||
seedTestData(t, db)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load failed: %v", err)
|
||||
}
|
||||
|
||||
pk := "dd44444444444444444444444444444444444444444444444444444444444444"
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'OldNode', 'repeater')", pk)
|
||||
|
||||
decoded := `{"name":"OldNode","pubKey":"` + pk + `"}`
|
||||
payloadType := 4
|
||||
|
||||
// Old adverts (>7 days ago) with flip-flop pattern
|
||||
// Use header 0x11 (routeType=FLOOD) and pathByte 0x41/0x81 (non-zero hop count)
|
||||
// so packets aren't skipped by direct zero-hop filter.
|
||||
oldTime := time.Now().UTC().Add(-10 * 24 * time.Hour).Format("2006-01-02T15:04:05.000Z")
|
||||
oldRaws := []string{"11" + "41" + "aabb", "11" + "81" + "aabb", "11" + "41" + "aabb"}
|
||||
for i, raw := range oldRaws {
|
||||
tx := &StoreTx{
|
||||
ID: 6000 + i,
|
||||
RawHex: raw,
|
||||
Hash: "old-" + strconv.Itoa(i),
|
||||
FirstSeen: oldTime,
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded,
|
||||
}
|
||||
store.packets = append(store.packets, tx)
|
||||
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
|
||||
}
|
||||
|
||||
info := store.GetNodeHashSizeInfo()
|
||||
ni := info[pk]
|
||||
if ni != nil && ni.Inconsistent {
|
||||
t.Error("old adverts (>7 days) should be excluded; node should not be flagged as inconsistent")
|
||||
}
|
||||
|
||||
// Now add recent adverts with consistent hash size — should appear in info
|
||||
pk2 := "ee55555555555555555555555555555555555555555555555555555555555555"
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'NewNode', 'repeater')", pk2)
|
||||
decoded2 := `{"name":"NewNode","pubKey":"` + pk2 + `"}`
|
||||
recentTime := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
for i := 0; i < 3; i++ {
|
||||
tx := &StoreTx{
|
||||
ID: 6100 + i,
|
||||
RawHex: "11" + "41" + "aabb",
|
||||
Hash: "new-" + strconv.Itoa(i),
|
||||
FirstSeen: recentTime,
|
||||
PayloadType: &payloadType,
|
||||
DecodedJSON: decoded2,
|
||||
}
|
||||
store.packets = append(store.packets, tx)
|
||||
store.byPayloadType[4] = append(store.byPayloadType[4], tx)
|
||||
}
|
||||
|
||||
// Invalidate cache before second call
|
||||
store.hashSizeInfoMu.Lock()
|
||||
store.hashSizeInfoCache = nil
|
||||
store.hashSizeInfoMu.Unlock()
|
||||
|
||||
info2 := store.GetNodeHashSizeInfo()
|
||||
ni2 := info2[pk2]
|
||||
if ni2 == nil {
|
||||
t.Error("recent adverts should be included in hash size info")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverAnalyticsNoStore(t *testing.T) {
|
||||
_, router := setupNoStoreServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/observers/obs1/analytics", nil)
|
||||
@@ -3277,3 +3493,93 @@ func TestHashCollisionsOnlyRepeaters(t *testing.T) {
|
||||
t.Errorf("expected 2 nodes in collision, got %d", len(collisions[0].Nodes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodePathsEndpointUsesIndex(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
// Verify byPathHop index was built during Load
|
||||
srv.store.mu.RLock()
|
||||
hopKeys := len(srv.store.byPathHop)
|
||||
srv.store.mu.RUnlock()
|
||||
if hopKeys == 0 {
|
||||
t.Fatal("byPathHop index is empty after Load")
|
||||
}
|
||||
|
||||
// Query paths for TestRepeater (pubkey aabbccdd11223344, prefix "aa")
|
||||
// Should find transmissions with hop "aa" in path
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/paths", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Paths []json.RawMessage `json:"paths"`
|
||||
TotalTransmissions int `json:"totalTransmissions"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("bad JSON: %v", err)
|
||||
}
|
||||
|
||||
// Transmission 1 has path ["aa","bb"] which contains "aa" matching prefix of aabbccdd11223344
|
||||
if resp.TotalTransmissions == 0 {
|
||||
t.Error("expected at least 1 transmission matching node paths")
|
||||
}
|
||||
if len(resp.Paths) == 0 {
|
||||
t.Error("expected at least 1 path group")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathHopIndexIncrementalUpdate(t *testing.T) {
|
||||
// Test that addTxToPathHopIndex and removeTxFromPathHopIndex work correctly
|
||||
idx := make(map[string][]*StoreTx)
|
||||
|
||||
pk1 := "fullpubkey1"
|
||||
tx1 := &StoreTx{
|
||||
ID: 1,
|
||||
PathJSON: `["ab","cd"]`,
|
||||
ResolvedPath: []*string{&pk1, nil},
|
||||
}
|
||||
|
||||
addTxToPathHopIndex(idx, tx1)
|
||||
|
||||
// Should be indexed under "ab", "cd", and "fullpubkey1"
|
||||
if len(idx["ab"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'ab', got %d", len(idx["ab"]))
|
||||
}
|
||||
if len(idx["cd"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'cd', got %d", len(idx["cd"]))
|
||||
}
|
||||
if len(idx["fullpubkey1"]) != 1 {
|
||||
t.Errorf("expected 1 entry for resolved pubkey, got %d", len(idx["fullpubkey1"]))
|
||||
}
|
||||
|
||||
// Add another tx with overlapping hop
|
||||
tx2 := &StoreTx{
|
||||
ID: 2,
|
||||
PathJSON: `["ab","ef"]`,
|
||||
}
|
||||
addTxToPathHopIndex(idx, tx2)
|
||||
|
||||
if len(idx["ab"]) != 2 {
|
||||
t.Errorf("expected 2 entries for 'ab', got %d", len(idx["ab"]))
|
||||
}
|
||||
if len(idx["ef"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'ef', got %d", len(idx["ef"]))
|
||||
}
|
||||
|
||||
// Remove tx1
|
||||
removeTxFromPathHopIndex(idx, tx1)
|
||||
|
||||
if len(idx["ab"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'ab' after removal, got %d", len(idx["ab"]))
|
||||
}
|
||||
if _, ok := idx["cd"]; ok {
|
||||
t.Error("expected 'cd' key to be deleted after removal")
|
||||
}
|
||||
if _, ok := idx["fullpubkey1"]; ok {
|
||||
t.Error("expected resolved pubkey key to be deleted after removal")
|
||||
}
|
||||
}
|
||||
|
||||
+920
-335
File diff suppressed because it is too large
Load Diff
@@ -240,6 +240,7 @@ type TransmissionResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
Direction interface{} `json:"direction"`
|
||||
Score interface{} `json:"score,omitempty"`
|
||||
Observations []ObservationResp `json:"observations,omitempty"`
|
||||
@@ -254,6 +255,7 @@ type ObservationResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
Timestamp interface{} `json:"timestamp"`
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
# Security Analysis: MeshCore Channel Encryption
|
||||
|
||||
## Scope
|
||||
|
||||
This analysis covers MeshCore's encryption vulnerabilities in order of practical severity. Section 1 addresses PSK brute-force (the highest-priority practical threat). Sections 2–9 cover AES-128-ECB structural weaknesses. Section 8 covers TXT_MSG. All claims are derived from firmware source (`BaseChatMesh.cpp`, `Utils.cpp`, `Mesh.cpp`, `MeshCore.h`) unless explicitly marked as conjecture.
|
||||
|
||||
## 1. PSK Brute-Force with Timestamp Oracle
|
||||
|
||||
### 1.1 The No-KDF Design
|
||||
|
||||
MeshCore channel PSKs are base64-decoded directly into AES-128 keys with no key derivation function (from `BaseChatMesh::addChannel()`):
|
||||
|
||||
```cpp
|
||||
int len = decode_base64((unsigned char *) psk_base64, strlen(psk_base64), dest->channel.secret);
|
||||
```
|
||||
|
||||
No PBKDF2, scrypt, argon2, or HKDF is applied. The base64-decoded bytes ARE the AES key. This means:
|
||||
|
||||
1. **Human-memorable passphrases have drastically reduced entropy.** If a user types "SecretChannel" as their PSK, the base64-decoded output is ~10 bytes of ASCII-range values. The key space is determined by the passphrase complexity, not by AES-128's theoretical 2^128 key space.
|
||||
|
||||
2. **Short passphrases produce short keys.** `decode_base64` maps every 4 base64 characters to 3 bytes. A passphrase shorter than ~22 base64 characters produces fewer than 16 bytes, and the remainder of the 16-byte key buffer depends on whatever was previously in memory (likely zeros from initialization). An 8-character passphrase decodes to only 6 bytes — the effective key space may be as low as 2^48.
|
||||
|
||||
3. **No salt.** Identical passphrases on different meshes produce identical keys. A single precomputed dictionary attack works globally against all MeshCore deployments.
|
||||
|
||||
### 1.2 Timestamp as Known-Plaintext Oracle
|
||||
|
||||
Every GRP_TXT plaintext begins with a structured, largely predictable header:
|
||||
|
||||
```
|
||||
Block 0: [TS₀][TS₁][TS₂][TS₃][0x00][sender_name][: ][message_start...]
|
||||
```
|
||||
|
||||
An attacker who captures a single packet can verify a candidate PSK by:
|
||||
1. Decrypting block 0 with the candidate key
|
||||
2. Checking if bytes 0–3 produce a plausible Unix timestamp (within a reasonable window of the capture time)
|
||||
3. Checking if byte 4 is 0x00 (TXT_TYPE_PLAIN)
|
||||
4. Optionally checking if bytes 5+ are valid ASCII (sender name)
|
||||
|
||||
The timestamp alone constrains the search: a ±1-hour window around capture time yields ~7,200 valid timestamps out of 2^32 possibilities — a false-positive rate of ~1.7×10^-6. Combined with the type byte and ASCII sender-name check, false positives are effectively zero. **One captured packet is sufficient for definitive key verification.**
|
||||
|
||||
### 1.3 Attack Cost Estimates
|
||||
|
||||
Hardware assumption: commodity GPU (e.g., RTX 4090) performing ~10 billion AES-128-ECB block encryptions per second. This is conservative — optimized implementations achieve higher throughput.
|
||||
|
||||
| Passphrase style | Search space | Time at 10^10 AES/sec |
|
||||
|---|---|---|
|
||||
| Single common English word (10K-word list) | ~10^4 | microseconds |
|
||||
| Single English word (170K full dictionary) | ~1.7×10^5 | microseconds |
|
||||
| Two concatenated common words | ~10^8 | ~10 milliseconds |
|
||||
| Three concatenated common words | ~10^12 | ~100 seconds (~2 min) |
|
||||
| Four random common words (Diceware-style) | ~10^16 | ~10^6 seconds (~12 days) |
|
||||
| Random 8-char alphanumeric (62^8) | ~2.2×10^14 | ~22,000 seconds (~6 hours) |
|
||||
| Random 12-char alphanumeric (62^12) | ~3.2×10^21 | ~10^11 seconds (infeasible) |
|
||||
| Full random 16-byte key (2^128) | ~3.4×10^38 | infeasible |
|
||||
|
||||
**Important caveats on search space:**
|
||||
- Dictionary sizes vary: "common English words" ≈ 3K–10K; full dictionary ≈ 170K. Estimates above use 10K for "common" lists.
|
||||
- Humans do not choose words uniformly. Zipf's law applies — a small fraction of words account for most selections. The effective entropy is **lower** than the uniform assumption, making attacks faster.
|
||||
- Concatenation without separators creates ambiguity ("therapist" = "therapist" or "the"+"rapist"), but this marginally reduces search space rather than increasing it.
|
||||
- Multi-channel amortization: an attacker can test each candidate against ALL captured channels simultaneously, paying the AES cost once per candidate.
|
||||
|
||||
### 1.4 Attack Properties
|
||||
|
||||
- **Offline attack.** No rate limiting, no lockout, no detection. The attacker works entirely on captured ciphertext.
|
||||
- **Single-packet verification.** One GRP_TXT packet is sufficient. No need to collect multiple messages.
|
||||
- **No KDF stretching.** Each candidate requires exactly one AES-128 block decryption (16 bytes), not thousands of hash iterations.
|
||||
- **Global applicability.** No salt means precomputed tables work across all MeshCore deployments using the same passphrase.
|
||||
- **Side-channel exposure.** Since the PSK IS the key (no KDF), any AES key-schedule side-channel directly reveals the passphrase. PSK reuse across systems (e.g., same passphrase for MeshCore and WiFi) means compromise of one compromises both.
|
||||
|
||||
### 1.5 Severity Assessment
|
||||
|
||||
**PSK brute-force is the #1 practical threat to MeshCore channel confidentiality.** Unlike ECB frequency analysis (§5), which requires hundreds of captured messages with repeated content, PSK brute-force requires a single captured packet and succeeds whenever users choose human-memorable passphrases — which is the common case for manually-configured channels.
|
||||
|
||||
Any channel using a passphrase of 3 or fewer common words, or any alphanumeric string shorter than 12 characters, should be considered **vulnerable to offline brute-force within hours to days** using commodity hardware.
|
||||
|
||||
### 1.6 Recommended Mitigations
|
||||
|
||||
**Priority 0 (Critical):** Apply a memory-hard KDF (argon2id preferred; scrypt or PBKDF2 with ≥100K iterations as fallback) to derive the AES key from the passphrase. This transforms each candidate test from ~1 nanosecond to ~100 milliseconds, increasing attack cost by a factor of ~10^8.
|
||||
|
||||
**Priority 0a:** Add a per-channel salt (random bytes stored alongside the channel config) to prevent precomputed/global attacks.
|
||||
|
||||
**Priority 0b:** Document that channel PSKs should be random 16-byte keys (e.g., generated with `openssl rand -base64 22`), not human-memorable passphrases. This is a stopgap until KDF support is added.
|
||||
|
||||
## 2. How Encryption Works
|
||||
|
||||
### Constants (from `MeshCore.h`)
|
||||
- `CIPHER_KEY_SIZE = 16` (AES-128)
|
||||
- `PUB_KEY_SIZE = 32`
|
||||
- `CIPHER_MAC_SIZE` = HMAC-SHA256 truncated output size
|
||||
|
||||
### encrypt() (from `Utils.cpp`)
|
||||
AES-128-ECB, block-by-block. No IV, no counter, no chaining:
|
||||
```cpp
|
||||
aes.setKey(shared_secret, CIPHER_KEY_SIZE); // first 16 bytes of shared_secret
|
||||
while (src_len >= 16) {
|
||||
aes.encryptBlock(dp, src); // each 16-byte block independently
|
||||
dp += 16; src += 16; src_len -= 16;
|
||||
}
|
||||
if (src_len > 0) { // partial final block
|
||||
uint8_t tmp[16];
|
||||
memset(tmp, 0, 16); // zero-fill
|
||||
memcpy(tmp, src, src_len); // copy remaining bytes
|
||||
aes.encryptBlock(dp, tmp);
|
||||
}
|
||||
```
|
||||
|
||||
### encryptThenMAC() (from `Utils.cpp`)
|
||||
```cpp
|
||||
int enc_len = encrypt(shared_secret, dest + CIPHER_MAC_SIZE, src, src_len);
|
||||
SHA256 sha;
|
||||
sha.resetHMAC(shared_secret, PUB_KEY_SIZE); // HMAC uses full 32 bytes
|
||||
sha.update(dest + CIPHER_MAC_SIZE, enc_len);
|
||||
sha.finalizeHMAC(shared_secret, PUB_KEY_SIZE, dest, CIPHER_MAC_SIZE);
|
||||
```
|
||||
|
||||
**Key reuse flaw:** The same `shared_secret` buffer serves both AES and HMAC. AES uses `shared_secret[0..15]` (first 16 bytes). HMAC uses `shared_secret[0..31]` (full 32 bytes). The AES key is a prefix of the HMAC key. See §7 for implications.
|
||||
|
||||
### GRP_TXT Plaintext Construction (from `BaseChatMesh::sendGroupMessage()`)
|
||||
|
||||
```cpp
|
||||
memcpy(temp, ×tamp, 4); // bytes 0-3: Unix timestamp (seconds)
|
||||
temp[4] = 0; // byte 4: TXT_TYPE_PLAIN
|
||||
sprintf((char *)&temp[5], "%s: ", sender_name); // bytes 5+: "SenderName: "
|
||||
char *ep = strchr((char *)&temp[5], 0);
|
||||
int prefix_len = ep - (char *)&temp[5]; // length of "SenderName: "
|
||||
memcpy(ep, text, text_len); // message text (no null terminator)
|
||||
ep[text_len] = 0; // null written AFTER data boundary
|
||||
// data_len passed to encrypt = 5 + prefix_len + text_len
|
||||
```
|
||||
|
||||
**The null terminator is NOT part of the encrypted data length.** The call to `createGroupDatagram` passes length `5 + prefix_len + text_len`. The null at `ep[text_len]` is written to the buffer but is beyond `data_len`. In the final partial block, `encrypt()` zero-fills with `memset(tmp, 0, 16)` before copying the remaining bytes — so a zero byte appears at the position where the null would be, but this is an artifact of zero-padding, not an explicit null in the plaintext.
|
||||
|
||||
On the receiving side, this is confirmed:
|
||||
```cpp
|
||||
data[len] = 0; // need to make a C string again, with null terminator
|
||||
```
|
||||
The receiver must re-add the null after decryption.
|
||||
|
||||
## 3. Block Layout Analysis
|
||||
|
||||
### Notation
|
||||
|
||||
Let `N` = length of sender name. Then:
|
||||
- `prefix_len` = N + 2 (for ": " suffix from `sprintf("%s: ", sender_name)`)
|
||||
- Header overhead = 4 (timestamp) + 1 (type) + prefix_len = N + 7 bytes
|
||||
- Message text begins at byte offset N + 7
|
||||
|
||||
### Block 0
|
||||
|
||||
Block 0 = bytes 0–15 of plaintext:
|
||||
```
|
||||
[TS₀][TS₁][TS₂][TS₃][0x00][sender_name: ][...message start...]
|
||||
```
|
||||
|
||||
The first 9 − N bytes of message text fit in block 0 (when N < 9). For N ≥ 9, no message text fits in block 0.
|
||||
|
||||
### Boundary Condition: Sender Name ≥ 12 Characters
|
||||
|
||||
When N ≥ 12, the header overhead (N + 7 ≥ 19) exceeds 16 bytes. The header itself spills into block 1:
|
||||
|
||||
**Example: sender name "LongUserName1" (N = 13), message "hi":**
|
||||
```
|
||||
Header = 13 + 7 = 20 bytes. Total plaintext = 20 + 2 = 22 bytes.
|
||||
|
||||
Block 0 (bytes 0-15): [TS₀][TS₁][TS₂][TS₃][0x00][L][o][n][g][U][s][e][r][N][a][m]
|
||||
Block 1 (bytes 16-31): [e][1][:][space][h][i][0x00 ×10] ← zero-padded partial block
|
||||
```
|
||||
|
||||
Block 1 here contains the tail of the sender name, the ": " separator, message text, AND zero-padding. For sender names of length 12–15, block 1 is a mix of header and message — **it is NOT "pure message text."**
|
||||
|
||||
For sender names ≥ 16, blocks 0 and 1 are both pure header, and message text doesn't begin until block 1 or later.
|
||||
|
||||
### General Block Content Table
|
||||
|
||||
| Sender name length N | Header bytes | Message starts at byte | Block 0 content | Block 1+ content |
|
||||
|---|---|---|---|---|
|
||||
| 1–8 | 8–15 | 8–15 | timestamp + header + message start | message text + zero-pad |
|
||||
| 9–11 | 16–18 | 16–18 | timestamp + header (no message) | header tail + message + zero-pad |
|
||||
| 12–15 | 19–22 | 19–22 | timestamp + partial header | header tail + message + zero-pad |
|
||||
| ≥16 | ≥23 | ≥23 | timestamp + partial header | header continuation, then message |
|
||||
|
||||
### Typical Case (N = 5, e.g. "Alice")
|
||||
|
||||
Header = 12 bytes. Message starts at byte 12. Block 0 holds 4 bytes of message text.
|
||||
|
||||
```
|
||||
Message "hello world" (11 chars). Total plaintext = 12 + 11 = 23 bytes.
|
||||
|
||||
Block 0 (bytes 0-15): [TS₀][TS₁][TS₂][TS₃][0x00][A][l][i][c][e][:][space][h][e][l][l]
|
||||
Block 1 (bytes 16-22): [o][space][w][o][r][l][d] → padded to: [o][space][w][o][r][l][d][0×9]
|
||||
```
|
||||
|
||||
Block 1 contains 7 bytes of message text and 9 bytes of zero-padding.
|
||||
|
||||
## 4. Attack Surface by Block Position
|
||||
|
||||
### Block 0: Accidental Nonce from Timestamp
|
||||
|
||||
The 4-byte Unix timestamp in bytes 0–3 acts as an **accidental nonce** — it was included "mostly as an extra blob to help make packet_hash unique" (per firmware comment), not as a cryptographic countermeasure against ECB determinism. Nevertheless, it has the effect of making block 0's plaintext vary per message.
|
||||
|
||||
**Precision on uniqueness:** Block 0 is unique per (sender, timestamp-second) pair, not per message. Two messages from the same sender within the same second, on the same channel, with the same type byte, produce identical block 0 plaintext and therefore identical block 0 ciphertext. At typical mesh chat rates, same-second collisions are rare but not impossible for automated/scripted senders.
|
||||
|
||||
**Known-plaintext observation:** Bytes 4–15 of block 0 are largely predictable per sender (type byte is always 0x00 for plain text; sender name and ": " are static). The timestamp is predictable within a window (Unix seconds). An attacker who knows the sender name and approximate time can compute all 16 plaintext bytes of block 0. However, **AES-128 is resistant to known-plaintext attacks** — knowing plaintext-ciphertext pairs for block 0 does not help recover the key or decrypt other blocks.
|
||||
|
||||
### Blocks 1+: Deterministic ECB (for short sender names)
|
||||
|
||||
When the sender name is short enough that the header fits in block 0 (N ≤ 8), blocks 1+ contain **only message text and zero-padding.** No timestamp, no nonce, no per-message varying data. Identical message text at the same block offset produces identical ciphertext, always.
|
||||
|
||||
When N ≥ 9, block 1 contains header spillover, which includes static sender name bytes — these vary per sender but not per message, so block 1 is still deterministic for a given sender once the header portion is fixed.
|
||||
|
||||
**The fundamental ECB property:** For any block beyond the timestamp's reach, `E_K(P) = E_K(P)`. Same plaintext block → same ciphertext block, regardless of when or how many times it's sent.
|
||||
|
||||
### Partial Final Block: Strongest Attack Target
|
||||
|
||||
The final block of every message is zero-padded by `encrypt()` to 16 bytes. The padding bytes are deterministic and known (always 0x00). For a message whose final block contains `B` bytes of actual content:
|
||||
|
||||
- `B` bytes are unknown message text
|
||||
- `16 - B` bytes are known zeros
|
||||
|
||||
When B is small (short final fragment), most of the block is known plaintext. For B = 1, the attacker knows 15 of 16 bytes — only 256 possible plaintext blocks exist. This means:
|
||||
|
||||
- **The final block has at most 2^(8B) possible plaintexts** (versus 2^128 for a full unknown block)
|
||||
- For B ≤ 4, there are ≤ 2^32 possibilities — a small enough space for dictionary attacks given enough ciphertext samples
|
||||
- The attacker can precompute all possible final-block plaintexts for small B values and match against observed ciphertext blocks
|
||||
|
||||
This makes the partial final block a **stronger frequency analysis target** than interior blocks, where all 16 bytes may be unknown text.
|
||||
|
||||
## 5. Feasible Attack Scenarios
|
||||
|
||||
### 4.1 Block Frequency Analysis on Blocks 1+
|
||||
|
||||
**Preconditions (all must hold):**
|
||||
1. Attacker can observe encrypted GRP_TXT packets (passive radio capture)
|
||||
2. Messages from the same sender (or senders with identical name lengths — same block alignment)
|
||||
3. Messages long enough to produce blocks beyond block 0 (text > 9 − N chars)
|
||||
4. Sufficient message volume with repeated content at the same block positions
|
||||
|
||||
**Method:**
|
||||
1. Collect GRP_TXT packets, group by sender hash
|
||||
2. Decompose encrypted payloads into 16-byte blocks (after stripping HMAC prefix)
|
||||
3. Discard block 0 (timestamp-varying)
|
||||
4. Build frequency tables for blocks 1, 2, 3, etc., per sender
|
||||
5. Match high-frequency ciphertext blocks against expected plaintext distributions
|
||||
|
||||
**Practical constraints limiting this attack:**
|
||||
- LoRa bandwidth severely limits message length. Most mesh chat messages are short — many fit entirely within block 0 (≤ 9 − N chars of text), yielding zero analyzable blocks.
|
||||
- Messages that spill into block 1+ tend to be longer and more varied — fewer repeated patterns.
|
||||
- The attack requires repeated identical 16-byte-aligned text fragments from the same sender over time.
|
||||
|
||||
**Conditions under which this attack succeeds:** Automated or scripted senders transmitting repetitive messages longer than block 0 capacity, on a channel with a static PSK, over an extended collection period. For human-typed conversational messages with typical length and variety, the number of repeated block 1+ patterns is likely too low for meaningful frequency analysis. (This is an empirical claim that depends on actual traffic patterns — no formal bound is established here.)
|
||||
|
||||
### 4.2 Partial Final Block Dictionary Attack
|
||||
|
||||
**Preconditions:**
|
||||
1. Attacker knows (or can estimate) the message length modulo 16
|
||||
2. Final block has few content bytes (B ≤ 4)
|
||||
|
||||
**Method:** Enumerate all 2^(8B) candidate plaintexts for the final block. Since AES-ECB is deterministic with a fixed key, the attacker can build a lookup table: if they ever observe a ciphertext block matching one of the candidates in a known-plaintext scenario (e.g., from a leaked or guessed message), they can identify which final-block value corresponds to which ciphertext.
|
||||
|
||||
**Limitation:** Without the key, the attacker cannot compute E_K(candidate) directly. The attack requires collecting enough ciphertext final blocks to perform frequency analysis within the reduced plaintext space. With only 256 possibilities (B=1), convergence is fast given sufficient samples.
|
||||
|
||||
### 4.3 Cross-Sender Correlation
|
||||
|
||||
Senders with identical name lengths produce identical block alignments. Messages from "Alice" (N=5) and "Bobby" (N=5) place message text at the same byte offsets. If both send the same message, their blocks 1+ are identical ciphertext — **but only if they share the same channel PSK** (same AES key). On the same channel, this enables cross-sender frequency analysis within same-name-length groups.
|
||||
|
||||
### 4.4 Message Length Leakage
|
||||
|
||||
Ciphertext length = ⌈(5 + prefix_len + text_len) / 16⌉ × 16 bytes. This reveals the message text length within a 16-byte window (not 15, because the block count is the observable quantity). Not ECB-specific — any block cipher without constant-length padding leaks this.
|
||||
|
||||
### 4.5 Replay Attacks
|
||||
|
||||
`encryptThenMAC()` authenticates the ciphertext, but if the mesh doesn't track previously-seen packet MACs, captured packets can be replayed. The embedded timestamp may be checked for staleness — this requires firmware verification beyond the scope of this analysis.
|
||||
|
||||
### 4.6 No Forward Secrecy
|
||||
|
||||
Channel PSKs are static and shared among all participants. ECDH shared secrets for direct messages are also static (no ephemeral key exchange). Compromise of any key decrypts all past and future traffic encrypted under that key.
|
||||
|
||||
## 6. What Known-Plaintext Does NOT Achieve
|
||||
|
||||
AES-128 is designed to resist known-plaintext attacks. An attacker who knows the full plaintext and ciphertext of block 0 (or any block) **cannot**:
|
||||
- Recover the AES key
|
||||
- Decrypt other blocks encrypted under the same key
|
||||
- Derive any information about other plaintexts from their ciphertexts
|
||||
|
||||
The ECB weakness is **determinism** (identical plaintext → identical ciphertext), not key recovery. The attacks in §5 exploit pattern matching and frequency analysis, not cryptanalysis of AES itself.
|
||||
|
||||
## 7. HMAC Key Reuse: Cryptographic Design Flaw
|
||||
|
||||
From `encryptThenMAC()`:
|
||||
- AES key: `shared_secret[0..15]` (CIPHER_KEY_SIZE = 16)
|
||||
- HMAC key: `shared_secret[0..31]` (PUB_KEY_SIZE = 32)
|
||||
|
||||
The AES key is the first half of the HMAC key. Both are derived from the same `shared_secret` — for channels, this is the PSK; for direct messages, the ECDH shared secret.
|
||||
|
||||
**Why this matters:**
|
||||
1. **Violated key separation principle.** Standard practice dictates that encryption and authentication keys must be independent. Using overlapping portions of the same secret means a weakness in one mechanism could leak information relevant to the other.
|
||||
2. **HMAC key reveals AES key.** If an attacker recovers the 32-byte HMAC key (e.g., through a side-channel attack on the HMAC computation), they automatically obtain the 16-byte AES key as a prefix.
|
||||
3. **No key derivation function.** The shared_secret is used directly — no HKDF or similar KDF is applied to derive independent subkeys. This is a departure from cryptographic best practice (cf. RFC 5869).
|
||||
|
||||
**Practical impact:** In the current threat model (passive radio capture of LoRa packets), this is unlikely to be directly exploitable — HMAC-SHA256 does not leak its key through normal operation. However, it represents a structural weakness that compounds with any future vulnerability in either the AES or HMAC implementation.
|
||||
|
||||
## 8. TXT_MSG (Direct Message) Block Layout
|
||||
|
||||
Direct messages use a different plaintext structure (from `BaseChatMesh::composeMsgPacket()`):
|
||||
|
||||
```cpp
|
||||
memcpy(temp, ×tamp, 4); // bytes 0-3: timestamp
|
||||
temp[4] = (attempt & 3); // byte 4: attempt counter (0-3)
|
||||
memcpy(&temp[5], text, text_len + 1); // bytes 5+: message text
|
||||
// data_len = 5 + text_len (null terminator copied but not counted in length)
|
||||
```
|
||||
|
||||
**Block layout for TXT_MSG:**
|
||||
```
|
||||
Block 0: [TS₀][TS₁][TS₂][TS₃][attempt][text bytes 0-10]
|
||||
Block 1: [text bytes 11-26] (if message long enough)
|
||||
```
|
||||
|
||||
Key differences from GRP_TXT:
|
||||
- **No sender name in plaintext** — the sender is identified by the source hash in the unencrypted packet header, not in the encrypted payload.
|
||||
- **Header is exactly 5 bytes** (4 timestamp + 1 attempt), always. No variable-length field.
|
||||
- **11 bytes of message text fit in block 0** (vs. 9 − N for GRP_TXT).
|
||||
- **Encrypted with per-pair ECDH shared secret**, not a group PSK. Each sender-recipient pair has a unique key.
|
||||
|
||||
**ECB implications for TXT_MSG:**
|
||||
- Block 0 is still protected by the timestamp accidental nonce.
|
||||
- Blocks 1+ are deterministic, same as GRP_TXT — identical message text at the same offset produces identical ciphertext.
|
||||
- However, frequency analysis is harder: each sender-recipient pair uses a different key, so the attacker can only correlate messages within a single pair. The message volume for any given pair is typically much lower than for a group channel.
|
||||
- The fixed 5-byte header means block alignment is consistent across ALL direct messages (unlike GRP_TXT where alignment varies by sender name length). An attacker who compromises one ECDH key can build block frequency tables, but only for that specific pair.
|
||||
|
||||
## 9. Mitigations
|
||||
|
||||
### Priority 1: Switch to AES-128-CTR
|
||||
|
||||
Replace ECB with CTR mode. Use the existing 4-byte timestamp + a 4-byte per-message counter as the 8-byte nonce (padded to 16 bytes for the CTR block). Each byte of plaintext gets XORed with a unique keystream byte — eliminates all block-level determinism.
|
||||
|
||||
**Wire format change:** None if the nonce is derived from header fields already present. If an explicit counter is added, 4 bytes of overhead per message.
|
||||
|
||||
### Priority 2: Derive Independent Subkeys
|
||||
|
||||
Apply HKDF (or at minimum, two distinct SHA-256 hashes) to the shared_secret to produce independent AES and HMAC keys. This is a minimal code change:
|
||||
```
|
||||
aes_key = SHA256(shared_secret || "encrypt")[0..15]
|
||||
hmac_key = SHA256(shared_secret || "authenticate")
|
||||
```
|
||||
|
||||
### Priority 3: Constant-Length Padding
|
||||
|
||||
Pad all messages to a fixed block count (e.g., 4 blocks = 64 bytes) to eliminate length leakage. Expensive on LoRa — should be configurable per channel as a security-vs-bandwidth tradeoff.
|
||||
|
||||
### Priority 4: Replay Protection
|
||||
|
||||
Track seen packet HMACs within a time window. Reject messages with timestamps older than N minutes.
|
||||
|
||||
### Priority 5: Channel Key Rotation
|
||||
|
||||
Manual or automated periodic rotation of channel PSKs. Even monthly rotation limits the exposure window.
|
||||
|
||||
### Priority 6: Forward Secrecy
|
||||
|
||||
Ephemeral ECDH for direct messages. Significant protocol change but prevents retroactive decryption on key compromise.
|
||||
|
||||
## 10. Speculative: LLM-Assisted Analysis
|
||||
|
||||
> **This section is speculation, not formal analysis.** The claims below are plausible but unvalidated. They do not affect the formal findings in §1–9.
|
||||
|
||||
An LLM could reduce the sample size needed for block frequency analysis:
|
||||
|
||||
1. **Context-aware candidate generation:** Given a sender's known patterns (the sender name is recoverable from block 0's predictable prefix), an LLM could generate likely message continuations and predict which plaintext blocks to look for in the frequency tables.
|
||||
2. **Conversational inference:** Timestamps + sender IDs + partially decoded messages could let an LLM reconstruct probable conversation flow, narrowing the search space for unknown blocks.
|
||||
3. **Community-specific vocabulary:** Training on public mesh chat logs could yield common phrases and greeting patterns, further reducing the candidate plaintext space.
|
||||
|
||||
This does not change the fundamental requirement (blocks 1+ must repeat, or the final block must be in a small enough space for dictionary matching). It potentially reduces the number of captured messages needed for convergence, but no quantitative bound is established.
|
||||
|
||||
## 11. Conclusion
|
||||
|
||||
MeshCore's encryption has four vulnerabilities, ranked by practical exploitability:
|
||||
|
||||
### Vulnerability #1: PSK Brute-Force (Critical)
|
||||
|
||||
**No KDF + known-plaintext oracle = offline key recovery from a single packet.** Any channel using a human-memorable passphrase of ≤3 common words or ≤11 alphanumeric characters is recoverable in minutes to hours on commodity GPU hardware. This is the highest-priority threat because it requires minimal attacker capability (one captured packet), succeeds against the most common deployment pattern (human-chosen passphrases), and completely compromises channel confidentiality. See §1.
|
||||
|
||||
### Vulnerability #2: ECB Determinism (Medium)
|
||||
|
||||
**Blocks beyond the timestamp's reach are deterministic.** Identical plaintext at the same block offset always produces identical ciphertext. For GRP_TXT messages longer than ~9 − N characters (where N is sender name length), this enables frequency analysis on blocks 1+. The partial final block, with its known zero-padding, is the strongest individual target. Exploitation requires hundreds of captured messages with repeated content — a higher bar than PSK brute-force. See §4–§5.
|
||||
|
||||
### Vulnerability #3: Key Material Reuse (Medium)
|
||||
|
||||
**AES and HMAC share the same key material** without a key derivation function. The AES key is a prefix of the HMAC key. This violates key separation and creates a structural dependency between the encryption and authentication mechanisms. See §7.
|
||||
|
||||
### Vulnerability #4: No Forward Secrecy (Low–Medium)
|
||||
|
||||
**No forward secrecy, no key rotation, no replay protection.** These are independent of the above but compound the risk: a single key compromise (whether via brute-force or other means) exposes all past and future traffic encrypted under that key. See §9.
|
||||
|
||||
**Summary of recommended mitigations (in priority order):**
|
||||
1. **(Critical)** Apply a memory-hard KDF (argon2id) to channel PSKs — §1.6
|
||||
2. **(Critical)** Add per-channel salt — §1.6
|
||||
3. **(High)** Switch from AES-128-ECB to AES-128-CTR — §9
|
||||
4. **(High)** Derive independent AES and HMAC subkeys via HKDF — §9
|
||||
5. **(Medium)** Constant-length padding, replay protection, key rotation — §9
|
||||
6. **(Low)** Forward secrecy via ephemeral ECDH — §9
|
||||
|
||||
The timestamp in block 0 was not designed as a nonce and should not be relied upon as one.
|
||||
+366
-13
@@ -86,6 +86,7 @@
|
||||
<button class="tab-btn" data-tab="nodes">Nodes</button>
|
||||
<button class="tab-btn" data-tab="distance">Distance</button>
|
||||
<button class="tab-btn" data-tab="neighbor-graph">Neighbor Graph</button>
|
||||
<button class="tab-btn" data-tab="prefix-tool">Prefix Tool</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="analyticsContent" class="analytics-content">
|
||||
@@ -173,6 +174,7 @@
|
||||
case 'nodes': await renderNodesTab(el); break;
|
||||
case 'distance': await renderDistanceTab(el); break;
|
||||
case 'neighbor-graph': await renderNeighborGraphTab(el); break;
|
||||
case 'prefix-tool': await renderPrefixTool(el); break;
|
||||
}
|
||||
// Auto-apply column resizing to all analytics tables
|
||||
requestAnimationFrame(() => {
|
||||
@@ -985,11 +987,13 @@
|
||||
<a href="#/analytics?tab=collisions§ion=hashMatrixSection" style="color:var(--accent)">🔢 Hash Matrix</a>
|
||||
<span style="color:var(--border)">|</span>
|
||||
<a href="#/analytics?tab=collisions§ion=collisionRiskSection" style="color:var(--accent)">💥 Collision Risk</a>
|
||||
<span style="color:var(--border)">|</span>
|
||||
<a href="#/analytics?tab=prefix-tool" style="color:var(--accent)">🔎 Check a prefix →</a>
|
||||
</nav>
|
||||
|
||||
<div class="analytics-card" id="inconsistentHashSection">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center"><h3 style="margin:0">⚠️ Inconsistent Hash Sizes</h3><a href="#/analytics?tab=collisions" style="font-size:11px;color:var(--text-muted)">↑ top</a></div>
|
||||
<p class="text-muted" style="margin:4px 0 8px;font-size:0.8em">Nodes sending adverts with varying hash sizes. Caused by a <a href="https://github.com/meshcore-dev/MeshCore/commit/fcfdc5f" target="_blank" style="color:var(--accent)">bug</a> where automatic adverts ignored the configured multibyte path setting. Fixed in <a href="https://github.com/meshcore-dev/MeshCore/releases/tag/repeater-v1.14.1" target="_blank" style="color:var(--accent)">repeater v1.14.1</a>.</p>
|
||||
<p class="text-muted" style="margin:4px 0 8px;font-size:0.8em">Repeaters and room servers sending adverts with varying hash sizes in the last 7 days. Originally caused by a <a href="https://github.com/meshcore-dev/MeshCore/commit/fcfdc5f" target="_blank" style="color:var(--accent)">firmware bug</a> where automatic adverts ignored the configured multibyte path setting, fixed in <a href="https://github.com/meshcore-dev/MeshCore/releases/tag/repeater-v1.14.1" target="_blank" style="color:var(--accent)">repeater v1.14.1</a>. Companion nodes are excluded.</p>
|
||||
<div id="inconsistentHashList"><div class="text-muted" style="padding:8px"><span class="spinner"></span> Loading…</div></div>
|
||||
</div>
|
||||
|
||||
@@ -1398,12 +1402,8 @@
|
||||
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">Analyzing route patterns…</div>';
|
||||
try {
|
||||
const rq = RegionFilter.regionQueryString();
|
||||
const [d2, d3, d4, d5] = await Promise.all([
|
||||
api('/analytics/subpaths?minLen=2&maxLen=2&limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/subpaths?minLen=3&maxLen=3&limit=30' + rq, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/subpaths?minLen=4&maxLen=4&limit=20' + rq, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/subpaths?minLen=5&maxLen=8&limit=15' + rq, { ttl: CLIENT_TTL.analyticsRF })
|
||||
]);
|
||||
const bulk = await api('/analytics/subpaths-bulk?groups=2-2:50,3-3:30,4-4:20,5-8:15' + rq, { ttl: CLIENT_TTL.analyticsRF });
|
||||
const [d2, d3, d4, d5] = bulk.results;
|
||||
|
||||
function renderTable(data, title) {
|
||||
if (!data.subpaths.length) return `<h4>${title}</h4><div class="text-muted">No data</div>`;
|
||||
@@ -1602,10 +1602,9 @@
|
||||
el.innerHTML = '<div style="padding:40px;text-align:center;color:var(--text-muted)">Loading node analytics…</div>';
|
||||
try {
|
||||
const rq = RegionFilter.regionQueryString();
|
||||
const [nodesResp, bulkHealth, netStatus] = await Promise.all([
|
||||
api('/nodes?limit=200&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }),
|
||||
api('/nodes/bulk-health?limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/nodes/network-status' + (rq ? '?' + rq.slice(1) : ''), { ttl: CLIENT_TTL.analyticsRF })
|
||||
const [nodesResp, bulkHealth] = await Promise.all([
|
||||
api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }),
|
||||
api('/nodes/bulk-health?limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF })
|
||||
]);
|
||||
const nodes = nodesResp.nodes || nodesResp;
|
||||
const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]');
|
||||
@@ -1622,8 +1621,22 @@
|
||||
const byObservers = [...enriched].sort((a, b) => (b.health.observers?.length || 0) - (a.health.observers?.length || 0));
|
||||
const byRecent = [...enriched].filter(n => n.health.stats.lastHeard).sort((a, b) => new Date(b.health.stats.lastHeard) - new Date(a.health.stats.lastHeard));
|
||||
|
||||
// Use server-computed status across ALL nodes
|
||||
const { active, degraded, silent, total: totalNodes, roleCounts } = netStatus;
|
||||
// Compute network status client-side from loaded nodes using shared getHealthThresholds()
|
||||
const now = Date.now();
|
||||
let active = 0, degraded = 0, silent = 0;
|
||||
nodes.forEach(function(n) {
|
||||
const role = n.role || 'unknown';
|
||||
const th = getHealthThresholds(role);
|
||||
const lastMs = n.last_heard ? new Date(n.last_heard).getTime()
|
||||
: n.last_seen ? new Date(n.last_seen).getTime()
|
||||
: 0;
|
||||
const age = lastMs ? (now - lastMs) : Infinity;
|
||||
if (age < th.degradedMs) active++;
|
||||
else if (age < th.silentMs) degraded++;
|
||||
else silent++;
|
||||
});
|
||||
const totalNodes = nodesResp.total || nodes.length;
|
||||
const roleCounts = nodesResp.counts || {};
|
||||
|
||||
function nodeLink(n) {
|
||||
return `<a href="#/nodes/${encodeURIComponent(n.public_key)}/analytics" class="analytics-link">${esc(n.name || n.public_key.slice(0, 12))}</a>`;
|
||||
@@ -2293,5 +2306,345 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
_ngState.animId = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
// --- Prefix Tool: Pure logic (exported for testing via _prefixToolExports) ---
|
||||
const PREFIX_SPACE_SIZES = { 1: 256, 2: 65536, 3: 16777216 };
|
||||
|
||||
/** Build 3-tier prefix indexes from deduplicated nodes. Returns { 1: Map, 2: Map, 3: Map } */
|
||||
function buildPrefixIndex(nodes) {
|
||||
const idx = { 1: new Map(), 2: new Map(), 3: new Map() };
|
||||
nodes.forEach(n => {
|
||||
const pk = n.public_key.toUpperCase();
|
||||
[1, 2, 3].forEach(b => {
|
||||
const p = pk.slice(0, b * 2);
|
||||
if (!idx[b].has(p)) idx[b].set(p, []);
|
||||
idx[b].get(p).push(n);
|
||||
});
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
/** Compute collision stats per tier */
|
||||
function computePrefixStats(idx) {
|
||||
const stats = {};
|
||||
[1, 2, 3].forEach(b => {
|
||||
stats[b] = {
|
||||
usedPrefixes: idx[b].size,
|
||||
collidingPrefixes: [...idx[b].values()].filter(arr => arr.length > 1).length,
|
||||
};
|
||||
});
|
||||
return stats;
|
||||
}
|
||||
|
||||
/** Recommend prefix byte size based on network size */
|
||||
function recommendPrefixSize(totalNodes) {
|
||||
if (totalNodes < 20) {
|
||||
return { rec: '1-byte', detail: `With only ${totalNodes} nodes, 1-byte prefixes have low collision risk.` };
|
||||
} else if (totalNodes < 500) {
|
||||
return { rec: '2-byte', detail: `With ${totalNodes} nodes, 2-byte prefixes are recommended to avoid collisions.` };
|
||||
} else {
|
||||
return { rec: '3-byte', detail: `With ${totalNodes} nodes, 3-byte prefixes are recommended for collision-free operation.` };
|
||||
}
|
||||
}
|
||||
|
||||
/** Validate prefix input. Returns { valid, error, input, isFullKey, tiers } */
|
||||
function validatePrefixInput(raw) {
|
||||
const input = raw.trim().toUpperCase();
|
||||
if (!input) return { valid: false, error: null, input, isEmpty: true };
|
||||
if (!/^[0-9A-F]+$/.test(input)) {
|
||||
return { valid: false, error: 'Invalid input — hex characters only (0-9, A-F).', input };
|
||||
}
|
||||
if (input.length % 2 !== 0 || (input.length !== 2 && input.length !== 4 && input.length !== 6 && input.length < 8)) {
|
||||
return { valid: false, error: 'Prefix must be 2, 4, or 6 hex characters. For a full public key, use 64 characters.', input };
|
||||
}
|
||||
const isFullKey = input.length >= 8;
|
||||
const tiers = isFullKey
|
||||
? [{ b: 1, prefix: input.slice(0, 2) }, { b: 2, prefix: input.slice(0, 4) }, { b: 3, prefix: input.slice(0, 6) }]
|
||||
: [{ b: input.length / 2, prefix: input }];
|
||||
return { valid: true, input, isFullKey, tiers };
|
||||
}
|
||||
|
||||
/** Check a prefix against the index. Returns collision info per tier. */
|
||||
function checkPrefix(raw, idx, nodes) {
|
||||
const v = validatePrefixInput(raw);
|
||||
if (!v.valid) return v;
|
||||
const results = v.tiers.map(({ b, prefix }) => {
|
||||
const matches = idx[b].get(prefix) || [];
|
||||
const colliders = v.isFullKey ? matches.filter(n => n.public_key.toUpperCase() !== v.input) : matches;
|
||||
return { b, prefix, colliders, count: colliders.length };
|
||||
});
|
||||
const inNetwork = v.isFullKey ? nodes.some(n => n.public_key.toUpperCase() === v.input) : null;
|
||||
return { valid: true, input: v.input, isFullKey: v.isFullKey, results, inNetwork };
|
||||
}
|
||||
|
||||
/** Generate a collision-free prefix of the given byte size. Returns null if none available. */
|
||||
function generatePrefix(b, idx, randFn) {
|
||||
const hexLen = b * 2;
|
||||
const totalSpace = PREFIX_SPACE_SIZES[b];
|
||||
const available = totalSpace - idx[b].size;
|
||||
if (available === 0) return null;
|
||||
|
||||
const _rand = randFn || Math.random;
|
||||
if (b === 1) {
|
||||
const free = [];
|
||||
for (let i = 0; i < totalSpace; i++) {
|
||||
const p = i.toString(16).toUpperCase().padStart(hexLen, '0');
|
||||
if (!idx[b].has(p)) free.push(p);
|
||||
}
|
||||
return free[Math.floor(_rand() * free.length)];
|
||||
}
|
||||
// Random sampling with fallback
|
||||
let attempts = 0, prefix;
|
||||
do {
|
||||
prefix = Math.floor(_rand() * totalSpace).toString(16).toUpperCase().padStart(hexLen, '0');
|
||||
} while (idx[b].has(prefix) && ++attempts < 500);
|
||||
if (idx[b].has(prefix)) {
|
||||
for (let i = 0; i < totalSpace; i++) {
|
||||
const p = i.toString(16).toUpperCase().padStart(hexLen, '0');
|
||||
if (!idx[b].has(p)) return p;
|
||||
}
|
||||
}
|
||||
return prefix;
|
||||
}
|
||||
|
||||
// --- Prefix Tool: HTML helpers ---
|
||||
function renderNodeEntry(n, escFn) {
|
||||
const name = escFn(n.name || n.public_key.slice(0, 12));
|
||||
const role = n.role ? `<span class="text-muted" style="font-size:0.82em">${escFn(n.role)}</span>` : '';
|
||||
const when = n.last_seen ? ` <span class="text-muted" style="font-size:0.8em">${new Date(n.last_seen).toLocaleDateString()}</span>` : '';
|
||||
return `<div style="padding:3px 0"><a href="#/nodes/${encodeURIComponent(n.public_key)}" class="analytics-link">${name}</a> ${role}${when}</div>`;
|
||||
}
|
||||
|
||||
function renderSeverityBadge(count) {
|
||||
if (count === 0) return '<span style="color:var(--status-green)">✅ Unique</span>';
|
||||
if (count <= 2) return `<span style="color:var(--status-yellow)">⚠️ ${count} collision${count !== 1 ? 's' : ''}</span>`;
|
||||
return `<span style="color:var(--status-red)">🔴 ${count} collisions</span>`;
|
||||
}
|
||||
|
||||
function renderPrefixStatCard(b, stat, spaceSize) {
|
||||
const hasCollisions = stat.collidingPrefixes > 0;
|
||||
return `<div class="analytics-stat-card" style="flex:1;min-width:150px;border-color:${hasCollisions ? 'var(--status-red)' : 'var(--border)'}">
|
||||
<div class="analytics-stat-label">${b}-byte prefixes</div>
|
||||
<div class="analytics-stat-value" style="font-size:1em">
|
||||
${stat.usedPrefixes.toLocaleString()}
|
||||
<span class="text-muted" style="font-size:0.7em"> / ${spaceSize.toLocaleString()}</span>
|
||||
</div>
|
||||
<div style="font-size:0.82em;margin-top:4px;color:${hasCollisions ? 'var(--status-red)' : 'var(--status-green)'}">
|
||||
${stat.collidingPrefixes === 0
|
||||
? '✅ No collisions'
|
||||
: `⚠️ ${stat.collidingPrefixes} prefix${stat.collidingPrefixes !== 1 ? 'es' : ''} collide`}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderNetworkOverview(totalNodes, stats, rec, recDetail, regionLabel) {
|
||||
const regionNote = regionLabel
|
||||
? `<p class="text-muted" style="font-size:0.85em;margin:4px 0 0">Showing data for region: <strong>${esc(regionLabel)}</strong>. <a href="#/analytics?tab=prefix-tool" style="color:var(--accent)">Check all nodes →</a></p>`
|
||||
: '';
|
||||
return `<div class="analytics-card" id="ptOverview">
|
||||
<div style="display:flex;align-items:center;gap:8px;cursor:pointer;user-select:none" id="ptOverviewToggle">
|
||||
<span id="ptOverviewChevron" style="font-size:0.75em;color:var(--text-muted);transition:transform 0.2s">▶</span>
|
||||
<h3 style="margin:0">Network Overview</h3>
|
||||
</div>
|
||||
<div id="ptOverviewBody" style="display:none">
|
||||
${regionNote}
|
||||
<div style="display:flex;gap:12px;flex-wrap:wrap;margin:12px 0 16px">
|
||||
<div class="analytics-stat-card" style="flex:1;min-width:110px">
|
||||
<div class="analytics-stat-label">Total nodes</div>
|
||||
<div class="analytics-stat-value">${totalNodes.toLocaleString()}</div>
|
||||
</div>
|
||||
${[1, 2, 3].map(b => renderPrefixStatCard(b, stats[b], PREFIX_SPACE_SIZES[b])).join('')}
|
||||
</div>
|
||||
<div style="background:var(--bg-secondary,var(--bg));border:1px solid var(--border);border-radius:6px;padding:10px 14px">
|
||||
<strong>Recommendation: ${rec} prefixes</strong> — ${recDetail}
|
||||
<span class="text-muted" style="font-size:0.8em;display:block;margin-top:4px">Hash size is configured per-node in firmware. Changing requires reflashing.</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderPrefixChecker(initPrefix) {
|
||||
return `<div class="analytics-card" id="ptChecker">
|
||||
<h3 style="margin-top:0">Check a Prefix</h3>
|
||||
<p class="text-muted" style="margin-top:0;font-size:0.9em">Enter a 1-byte (2 hex chars), 2-byte (4 hex chars), or 3-byte (6 hex chars) prefix — or paste a full public key.</p>
|
||||
<div style="display:flex;gap:8px;align-items:flex-start;flex-wrap:wrap">
|
||||
<input id="ptPrefixInput" type="text" placeholder="e.g. A3F1" maxlength="64"
|
||||
style="font-family:var(--mono);font-size:1em;padding:6px 10px;background:var(--bg);color:var(--text);border:1px solid var(--border);border-radius:4px;min-width:180px;flex:1"
|
||||
value="${esc(initPrefix)}">
|
||||
<button id="ptCheckBtn" style="padding:6px 16px;background:var(--accent);color:#fff;border:none;border-radius:4px;cursor:pointer;font-size:0.95em">Check</button>
|
||||
</div>
|
||||
<div id="ptCheckerResults" style="margin-top:14px"></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderPrefixGenerator(initGenerate) {
|
||||
return `<div class="analytics-card" id="ptGenerator">
|
||||
<h3 style="margin-top:0">Generate Available Prefix</h3>
|
||||
<p class="text-muted" style="margin-top:0;font-size:0.9em">Find a prefix with zero current collisions.</p>
|
||||
<div style="display:flex;gap:16px;align-items:center;flex-wrap:wrap;margin-bottom:12px">
|
||||
<label style="display:flex;align-items:center;gap:6px;cursor:pointer">
|
||||
<input type="radio" name="ptGenSize" value="1" ${initGenerate === '1' ? 'checked' : ''}> 1-byte
|
||||
</label>
|
||||
<label style="display:flex;align-items:center;gap:6px;cursor:pointer">
|
||||
<input type="radio" name="ptGenSize" value="2" ${initGenerate !== '1' && initGenerate !== '3' ? 'checked' : ''}> 2-byte
|
||||
<span class="text-muted" style="font-size:0.8em">(recommended)</span>
|
||||
</label>
|
||||
<label style="display:flex;align-items:center;gap:6px;cursor:pointer">
|
||||
<input type="radio" name="ptGenSize" value="3" ${initGenerate === '3' ? 'checked' : ''}> 3-byte
|
||||
</label>
|
||||
<button id="ptGenBtn" style="padding:6px 16px;background:var(--accent);color:#fff;border:none;border-radius:4px;cursor:pointer;font-size:0.95em">Generate</button>
|
||||
</div>
|
||||
<div id="ptGenResult"></div>
|
||||
<div style="margin-top:14px;padding:10px 14px;border:1px solid var(--accent);border-radius:6px;background:var(--bg-secondary,var(--bg));font-size:0.88em">
|
||||
📖 <strong>New to multi-byte prefixes?</strong>
|
||||
<a href="https://github.com/meshcore-dev/MeshCore/blob/main/docs/faq.md#39-q-what-is-multi-byte-support--what-do-1-byte-2-byte-3-byte-adverts-and-messages-mean"
|
||||
target="_blank" rel="noopener noreferrer" style="color:var(--accent);margin-left:4px">
|
||||
Read the MeshCore FAQ on multi-byte support →
|
||||
</a>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderCheckerResults(checkResult, escFn) {
|
||||
if (!checkResult.valid) {
|
||||
return checkResult.error
|
||||
? `<p style="color:var(--status-red);margin:0">${checkResult.error}</p>`
|
||||
: '';
|
||||
}
|
||||
let html = '';
|
||||
if (checkResult.isFullKey) {
|
||||
const inp = checkResult.input;
|
||||
html += `<p class="text-muted" style="font-size:0.85em;margin:0 0 10px">Derived prefixes: <code class="mono">${inp.slice(0,2)}</code> / <code class="mono">${inp.slice(0,4)}</code> / <code class="mono">${inp.slice(0,6)}</code>${checkResult.inNetwork === false ? ' — <em>this node is not yet in the network</em>' : ''}</p>`;
|
||||
}
|
||||
checkResult.results.forEach(({ b, prefix, colliders, count }) => {
|
||||
html += `<div style="margin-bottom:10px;padding:10px 14px;border:1px solid var(--border);border-radius:6px;background:var(--bg-secondary,var(--bg))">
|
||||
<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px">
|
||||
<code class="mono" style="font-weight:700">${prefix}</code>
|
||||
<span class="text-muted" style="font-size:0.82em">${b}-byte</span>
|
||||
${renderSeverityBadge(count)}
|
||||
</div>
|
||||
${count === 0
|
||||
? '<div class="text-muted" style="font-size:0.85em">No existing nodes use this prefix.</div>'
|
||||
: `<div style="font-size:0.85em;max-height:140px;overflow-y:auto">${colliders.map(n => renderNodeEntry(n, escFn)).join('')}</div>`}
|
||||
</div>`;
|
||||
});
|
||||
return html;
|
||||
}
|
||||
|
||||
// --- Prefix Tool: main render (orchestrates the above) ---
|
||||
async function renderPrefixTool(el) {
|
||||
el.innerHTML = '<div style="padding:40px;text-align:center;color:var(--text-muted)">Loading prefix data…</div>';
|
||||
|
||||
const rq = RegionFilter.regionQueryString();
|
||||
const regionLabel = rq ? (new URLSearchParams(rq.slice(1)).get('region') || '') : '';
|
||||
|
||||
let nodesResp;
|
||||
try {
|
||||
nodesResp = await api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList });
|
||||
} catch (e) {
|
||||
el.innerHTML = `<div class="text-muted" role="alert" style="padding:40px">Failed to load: ${esc(e.message)}</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const nodeMap = new Map();
|
||||
(nodesResp.nodes || nodesResp).forEach(n => {
|
||||
if (n.public_key && n.public_key.length >= 6 && !nodeMap.has(n.public_key)) {
|
||||
nodeMap.set(n.public_key, n);
|
||||
}
|
||||
});
|
||||
const nodes = [...nodeMap.values()];
|
||||
|
||||
if (nodes.length === 0) {
|
||||
el.innerHTML = `<div class="analytics-card"><p class="text-muted">No nodes in the network yet. Any prefix is available!</p></div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const stats = computePrefixStats(idx);
|
||||
const totalNodes = nodes.length;
|
||||
const { rec, detail: recDetail } = recommendPrefixSize(totalNodes);
|
||||
|
||||
const hashParams = new URLSearchParams((location.hash.split('?')[1] || ''));
|
||||
const initPrefix = hashParams.get('prefix') || '';
|
||||
const initGenerate = hashParams.get('generate') || '';
|
||||
|
||||
el.innerHTML = renderNetworkOverview(totalNodes, stats, rec, recDetail, regionLabel)
|
||||
+ renderPrefixChecker(initPrefix)
|
||||
+ renderPrefixGenerator(initGenerate);
|
||||
|
||||
// --- Wire up checker ---
|
||||
const doCheck = (raw) => {
|
||||
const resultsEl = document.getElementById('ptCheckerResults');
|
||||
if (!resultsEl) return;
|
||||
const result = checkPrefix(raw, idx, nodes);
|
||||
resultsEl.innerHTML = renderCheckerResults(result, esc);
|
||||
};
|
||||
|
||||
document.getElementById('ptCheckBtn').addEventListener('click', () => doCheck(document.getElementById('ptPrefixInput').value));
|
||||
document.getElementById('ptPrefixInput').addEventListener('keydown', e => { if (e.key === 'Enter') doCheck(e.target.value); });
|
||||
|
||||
// --- Wire up generator ---
|
||||
const doGenerate = () => {
|
||||
const genResultEl = document.getElementById('ptGenResult');
|
||||
if (!genResultEl) return;
|
||||
const sizeInput = el.querySelector('input[name="ptGenSize"]:checked');
|
||||
const b = sizeInput ? parseInt(sizeInput.value) : 2;
|
||||
const prefix = generatePrefix(b, idx);
|
||||
|
||||
if (!prefix) {
|
||||
const next = b < 3 ? (b + 1) + '-byte' : 'a different size';
|
||||
genResultEl.innerHTML = `<p style="color:var(--status-red);margin:0">No collision-free ${b}-byte prefixes available. Try ${next}.</p>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const totalSpace = PREFIX_SPACE_SIZES[b];
|
||||
const available = totalSpace - idx[b].size;
|
||||
genResultEl.innerHTML = `
|
||||
<div style="padding:12px 16px;border:1px solid var(--status-green);border-radius:6px;background:var(--bg-secondary,var(--bg))">
|
||||
<div style="display:flex;align-items:center;gap:10px;flex-wrap:wrap">
|
||||
<code class="mono" style="font-size:1.3em;font-weight:700;color:var(--status-green)">${prefix}</code>
|
||||
<span style="color:var(--status-green)">✅ No existing nodes use this prefix</span>
|
||||
</div>
|
||||
<div class="text-muted" style="font-size:0.85em;margin-top:6px">${available.toLocaleString()} of ${totalSpace.toLocaleString()} ${b}-byte prefixes are available.</div>
|
||||
<div style="margin-top:10px;display:flex;gap:8px;flex-wrap:wrap;align-items:center">
|
||||
<button id="ptRegenBtn" style="padding:5px 14px;background:var(--bg);color:var(--text);border:1px solid var(--border);border-radius:4px;cursor:pointer;font-size:0.9em">Try another</button>
|
||||
<a href="https://agessaman.github.io/meshcore-web-keygen/?prefix=${prefix}" target="_blank" rel="noopener noreferrer"
|
||||
style="padding:5px 14px;background:var(--bg);color:var(--accent);border:1px solid var(--border);border-radius:4px;text-decoration:none;font-size:0.9em">
|
||||
Generate key with this prefix →
|
||||
</a>
|
||||
</div>
|
||||
</div>`;
|
||||
document.getElementById('ptRegenBtn').addEventListener('click', doGenerate);
|
||||
};
|
||||
|
||||
document.getElementById('ptGenBtn').addEventListener('click', doGenerate);
|
||||
|
||||
// Network Overview toggle
|
||||
document.getElementById('ptOverviewToggle').addEventListener('click', () => {
|
||||
const body = document.getElementById('ptOverviewBody');
|
||||
const chevron = document.getElementById('ptOverviewChevron');
|
||||
const open = body.style.display === 'none';
|
||||
body.style.display = open ? '' : 'none';
|
||||
chevron.style.transform = open ? 'rotate(90deg)' : '';
|
||||
});
|
||||
|
||||
// Auto-run from URL params
|
||||
if (initPrefix) {
|
||||
doCheck(initPrefix);
|
||||
setTimeout(() => { document.getElementById('ptChecker')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150);
|
||||
} else if (initGenerate) {
|
||||
doGenerate();
|
||||
setTimeout(() => { document.getElementById('ptGenerator')?.scrollIntoView({ behavior: 'smooth', block: 'start' }); }, 150);
|
||||
}
|
||||
}
|
||||
|
||||
// Export pure functions for testing
|
||||
if (typeof window !== 'undefined') {
|
||||
window._prefixToolExports = {
|
||||
buildPrefixIndex, computePrefixStats, recommendPrefixSize,
|
||||
validatePrefixInput, checkPrefix, generatePrefix,
|
||||
renderSeverityBadge, PREFIX_SPACE_SIZES
|
||||
};
|
||||
}
|
||||
|
||||
registerPage('analytics', { init, destroy });
|
||||
})();
|
||||
|
||||
@@ -463,6 +463,9 @@ function navigate() {
|
||||
currentPage = basePage;
|
||||
|
||||
const app = document.getElementById('app');
|
||||
// Pages with fixed-height containers (maps, virtual-scroll, split-panels)
|
||||
const fixedPages = { packets: 1, nodes: 1, map: 1, live: 1, channels: 1, 'audio-lab': 1 };
|
||||
app.classList.toggle('app-fixed', basePage in fixedPages);
|
||||
if (pages[basePage]?.init) {
|
||||
const t0 = performance.now();
|
||||
pages[basePage].init(app, routeParam);
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ if (typeof window !== 'undefined') window.comparePacketSets = comparePacketSets;
|
||||
packetsB = [];
|
||||
currentView = 'summary';
|
||||
|
||||
app.innerHTML = '<div class="compare-page" style="overflow-y:auto;height:calc(100vh - 56px);padding:16px">' +
|
||||
app.innerHTML = '<div class="compare-page" style="padding:16px">' +
|
||||
'<div class="page-header" style="display:flex;align-items:center;gap:12px;margin-bottom:16px">' +
|
||||
'<a href="#/observers" class="btn-icon" title="Back to Observers" aria-label="Back">\u2190</a>' +
|
||||
'<h2 style="margin:0">\uD83D\uDD0D Observer Comparison</h2>' +
|
||||
|
||||
+1
-2
@@ -1,7 +1,6 @@
|
||||
/* === CoreScope — home.css === */
|
||||
|
||||
/* Override #app overflow:hidden for home page scrolling */
|
||||
#app:has(.home-hero), #app:has(.home-chooser) { overflow-y: auto; }
|
||||
/* Home page now uses body scroll (no #app override needed — see style.css) */
|
||||
|
||||
/* Chooser */
|
||||
.home-chooser {
|
||||
|
||||
+29
-1
@@ -8,6 +8,7 @@ window.HopResolver = (function() {
|
||||
const MAX_HOP_DIST = 1.8; // ~200km in degrees
|
||||
const REGION_RADIUS_KM = 300;
|
||||
let prefixIdx = {}; // lowercase hex prefix → [node, ...]
|
||||
let pubkeyIdx = {}; // full lowercase pubkey → node (O(1) lookup)
|
||||
let nodesList = [];
|
||||
let observerIataMap = {}; // observer_id → iata
|
||||
let iataCoords = {}; // iata → {lat, lon}
|
||||
@@ -35,9 +36,11 @@ window.HopResolver = (function() {
|
||||
function init(nodes, opts) {
|
||||
nodesList = nodes || [];
|
||||
prefixIdx = {};
|
||||
pubkeyIdx = {};
|
||||
for (const n of nodesList) {
|
||||
if (!n.public_key) continue;
|
||||
const pk = n.public_key.toLowerCase();
|
||||
pubkeyIdx[pk] = n;
|
||||
for (let len = 1; len <= 3; len++) {
|
||||
const p = pk.slice(0, len * 2);
|
||||
if (!prefixIdx[p]) prefixIdx[p] = [];
|
||||
@@ -265,5 +268,30 @@ window.HopResolver = (function() {
|
||||
return affinityMap[pubkeyA][pubkeyB] || 0;
|
||||
}
|
||||
|
||||
return { init: init, resolve: resolve, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity };
|
||||
/**
|
||||
* Resolve hops using server-provided resolved_path (full pubkeys).
|
||||
* Returns the same format as resolve() — { [hop]: { name, pubkey, ... } }.
|
||||
* resolved_path is an array aligned with path_json: each element is a
|
||||
* 64-char lowercase hex pubkey or null. Skips entries that are null.
|
||||
*/
|
||||
function resolveFromServer(hops, resolvedPath) {
|
||||
if (!hops || !resolvedPath || hops.length !== resolvedPath.length) return {};
|
||||
var result = {};
|
||||
for (var i = 0; i < hops.length; i++) {
|
||||
var hop = hops[i];
|
||||
var pubkey = resolvedPath[i];
|
||||
if (!pubkey) continue; // null = unresolved, leave for client-side fallback
|
||||
// O(1) lookup via pubkeyIdx built during init()
|
||||
var node = pubkeyIdx[pubkey.toLowerCase()] || null;
|
||||
result[hop] = {
|
||||
name: node ? node.name : pubkey.slice(0, 8),
|
||||
pubkey: pubkey,
|
||||
candidates: node ? [{ name: node.name, pubkey: pubkey, lat: node.lat, lon: node.lon }] : [],
|
||||
conflicts: []
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return { init: init, resolve: resolve, resolveFromServer: resolveFromServer, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity };
|
||||
})();
|
||||
|
||||
+42
-27
@@ -457,6 +457,7 @@
|
||||
id: pkt.id, hash: pkt.hash,
|
||||
raw: pkt.raw_hex,
|
||||
path_json: pkt.path_json,
|
||||
resolved_path: pkt.resolved_path,
|
||||
_ts: new Date(pkt.timestamp || pkt.created_at).getTime(),
|
||||
decoded: { header: { payloadTypeName: typeName }, payload: raw, path: { hops } },
|
||||
snr: pkt.snr, rssi: pkt.rssi, observer: pkt.observer_name
|
||||
@@ -539,6 +540,8 @@
|
||||
clearTimeout(entry.timer);
|
||||
}
|
||||
propagationBuffer.clear();
|
||||
// Batch-update timeline once on restore instead of per-packet while hidden
|
||||
updateTimeline();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -563,7 +566,6 @@
|
||||
if (VCR.mode === 'LIVE') {
|
||||
// Skip animations when tab is backgrounded — just buffer for VCR timeline
|
||||
if (_tabHidden) {
|
||||
updateTimeline();
|
||||
return;
|
||||
}
|
||||
if (realisticPropagation && pkt.hash) {
|
||||
@@ -1343,7 +1345,7 @@
|
||||
html += `<h4 style="font-size:12px;margin:12px 0 6px;color:var(--text-muted);">Recent Packets</h4>
|
||||
<div style="font-size:11px;max-height:200px;overflow-y:auto;">` +
|
||||
recent.slice(0, 10).map(p => `<div style="padding:2px 0;display:flex;justify-content:space-between;">
|
||||
<a href="#/packets/${encodeURIComponent(p.hash || '')}" style="color:var(--accent);text-decoration:none;">${escapeHtml(p.payload_type || '?')}${p.observation_count > 1 ? ' <span class="badge badge-obs" style="font-size:9px">👁 ' + p.observation_count + '</span>' : ''}</a>
|
||||
<a href="#/packets/${encodeURIComponent(p.hash || '')}" style="color:var(--accent);text-decoration:none;">${escapeHtml(p.payload_type || '?')}${transportBadge(p.route_type)}${p.observation_count > 1 ? ' <span class="badge badge-obs" style="font-size:9px">👁 ' + p.observation_count + '</span>' : ''}</a>
|
||||
<span style="color:var(--text-muted)">${formatLiveTimestampHtml(p.timestamp)}</span>
|
||||
</div>`).join('') +
|
||||
'</div>';
|
||||
@@ -1548,7 +1550,7 @@
|
||||
item.innerHTML = `
|
||||
<span class="feed-icon" style="color:${color}">${icon}</span>
|
||||
<span class="feed-type" style="color:${color}">${typeName}</span>
|
||||
${hopStr}${obsBadge}
|
||||
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
|
||||
<span class="feed-text">${escapeHtml(preview)}</span>
|
||||
<span class="feed-time">${formatLiveTimestampHtml(group.latestTs || Date.now())}</span>
|
||||
`;
|
||||
@@ -1650,6 +1652,7 @@
|
||||
}
|
||||
delete nodeMarkers[key];
|
||||
delete nodeData[key];
|
||||
delete nodeActivity[key];
|
||||
pruned = true;
|
||||
}
|
||||
} else if (marker && marker._staleDimmed) {
|
||||
@@ -1665,12 +1668,17 @@
|
||||
if (_el2) _el2.textContent = Object.keys(nodeMarkers).length;
|
||||
if (window.HopResolver) HopResolver.init(Object.values(nodeData));
|
||||
}
|
||||
// Prune orphaned nodeActivity entries (nodes removed above or never tracked)
|
||||
for (var aKey in nodeActivity) {
|
||||
if (!(aKey in nodeData)) delete nodeActivity[aKey];
|
||||
}
|
||||
}
|
||||
|
||||
// Expose for testing
|
||||
window._livePruneStaleNodes = pruneStaleNodes;
|
||||
window._liveNodeMarkers = function() { return nodeMarkers; };
|
||||
window._liveNodeData = function() { return nodeData; };
|
||||
window._liveNodeActivity = function() { return nodeActivity; };
|
||||
window._vcrFormatTime = vcrFormatTime;
|
||||
window._liveDbPacketToLive = dbPacketToLive;
|
||||
window._liveExpandToBufferEntries = expandToBufferEntries;
|
||||
@@ -1690,20 +1698,13 @@
|
||||
|
||||
async function replayRecent() {
|
||||
try {
|
||||
const resp = await fetch('/api/packets?limit=8&groupByHash=true');
|
||||
// Single bulk fetch with expand=observations — no N+1 calls
|
||||
const resp = await fetch('/api/packets?limit=8&expand=observations');
|
||||
const data = await resp.json();
|
||||
const groups = (data.packets || []).reverse();
|
||||
|
||||
// Fetch all observations first, then stagger rendering
|
||||
const allGroups = [];
|
||||
for (let i = 0; i < groups.length; i++) {
|
||||
const group = groups[i];
|
||||
let observations = [];
|
||||
try {
|
||||
const detail = await fetch('/api/packets/' + encodeURIComponent(group.hash));
|
||||
const detailData = await detail.json();
|
||||
observations = detailData.observations || [];
|
||||
} catch {}
|
||||
const allGroups = groups.map((group) => {
|
||||
const observations = group.observations || [];
|
||||
|
||||
const livePackets = observations.map(obs => {
|
||||
const livePkt = dbPacketToLive(Object.assign({}, group, obs, {
|
||||
@@ -1722,8 +1723,8 @@
|
||||
}
|
||||
|
||||
livePackets.forEach(lp => VCR.buffer.push({ ts: lp._ts, pkt: lp }));
|
||||
allGroups.push(livePackets);
|
||||
}
|
||||
return livePackets;
|
||||
});
|
||||
|
||||
// Render with real timing gaps between packets
|
||||
// Sort by earliest timestamp
|
||||
@@ -1855,7 +1856,7 @@
|
||||
var pathKey = hops.join(',');
|
||||
if (seenPathKeys.has(pathKey)) continue;
|
||||
seenPathKeys.add(pathKey);
|
||||
var hopPositions = resolveHopPositions(hops, qp);
|
||||
var hopPositions = resolveHopPositions(hops, qp, window.getResolvedPath ? getResolvedPath(qpkt) : null);
|
||||
if (hopPositions.length >= 2) {
|
||||
allPaths.push({ hopPositions: hopPositions, raw: qpkt.raw || first.raw });
|
||||
} else if (hopPositions.length === 1) {
|
||||
@@ -1892,15 +1893,29 @@
|
||||
}
|
||||
}
|
||||
|
||||
function resolveHopPositions(hops, payload) {
|
||||
// Delegate to shared HopResolver (from hop-resolver.js) instead of reimplementing
|
||||
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
|
||||
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
|
||||
function resolveHopPositions(hops, payload, resolvedPath) {
|
||||
// Prefer server-side resolved_path when available
|
||||
var resolvedMap;
|
||||
if (resolvedPath && resolvedPath.length === hops.length && window.HopResolver && HopResolver.ready()) {
|
||||
resolvedMap = HopResolver.resolveFromServer(hops, resolvedPath);
|
||||
// Fill in any null entries from client-side fallback, preserving sender GPS context
|
||||
var nullHops = hops.filter(function(h, i) { return !resolvedPath[i] && !resolvedMap[h]; });
|
||||
if (nullHops.length) {
|
||||
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
|
||||
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
|
||||
var fallback = HopResolver.resolve(nullHops, originLat, originLon, null, null, null);
|
||||
for (var k in fallback) resolvedMap[k] = fallback[k];
|
||||
}
|
||||
} else {
|
||||
// Delegate to shared HopResolver (from hop-resolver.js) instead of reimplementing
|
||||
const originLat = payload.lat != null && !(payload.lat === 0 && payload.lon === 0) ? payload.lat : null;
|
||||
const originLon = payload.lon != null && !(payload.lon === 0 && payload.lon === 0) ? payload.lon : null;
|
||||
|
||||
// Use HopResolver if available and initialized, otherwise fall back to simple lookup
|
||||
const resolvedMap = (window.HopResolver && HopResolver.ready())
|
||||
? HopResolver.resolve(hops, originLat, originLon, null, null, null)
|
||||
: {};
|
||||
// Use HopResolver if available and initialized, otherwise fall back to simple lookup
|
||||
resolvedMap = (window.HopResolver && HopResolver.ready())
|
||||
? HopResolver.resolve(hops, originLat, originLon, null, null, null)
|
||||
: {};
|
||||
}
|
||||
|
||||
// Convert HopResolver's map format to the array format live.js expects: {key, pos, name, known}
|
||||
const raw = hops.map(hop => {
|
||||
@@ -2484,7 +2499,7 @@
|
||||
item.innerHTML = `
|
||||
<span class="feed-icon" style="color:${color}">${icon}</span>
|
||||
<span class="feed-type" style="color:${color}">${typeName}</span>
|
||||
${hopStr}${obsBadge}
|
||||
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
|
||||
<span class="feed-text">${escapeHtml(preview)}</span>
|
||||
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
|
||||
`;
|
||||
@@ -2552,7 +2567,7 @@
|
||||
item.innerHTML = `
|
||||
<span class="feed-icon" style="color:${color}">${icon}</span>
|
||||
<span class="feed-type" style="color:${color}">${typeName}</span>
|
||||
${hopStr}${obsBadge}
|
||||
${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
|
||||
<span class="feed-text">${escapeHtml(preview)}</span>
|
||||
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
|
||||
`;
|
||||
|
||||
+100
-14
@@ -9,7 +9,7 @@
|
||||
let nodes = [];
|
||||
let targetNodeKey = null;
|
||||
let observers = [];
|
||||
let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clusters: false, hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all' };
|
||||
let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clusters: false, hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all', byteSize: localStorage.getItem('meshcore-map-byte-filter') || 'all' };
|
||||
let selectedReferenceNode = null; // pubkey of the reference node for neighbor filtering
|
||||
let neighborPubkeys = null; // Set of pubkeys that are direct neighbors of selected node
|
||||
let wsHandler = null;
|
||||
@@ -94,6 +94,15 @@
|
||||
<legend class="mc-label">Node Types</legend>
|
||||
<div id="mcRoleChecks"></div>
|
||||
</fieldset>
|
||||
<fieldset class="mc-section">
|
||||
<legend class="mc-label">Byte Size</legend>
|
||||
<div class="filter-group" id="mcByteFilter">
|
||||
<button class="btn ${filters.byteSize==='all'?'active':''}" data-byte="all">All</button>
|
||||
<button class="btn ${filters.byteSize==='1'?'active':''}" data-byte="1">1-byte</button>
|
||||
<button class="btn ${filters.byteSize==='2'?'active':''}" data-byte="2">2-byte</button>
|
||||
<button class="btn ${filters.byteSize==='3'?'active':''}" data-byte="3">3-byte</button>
|
||||
</div>
|
||||
</fieldset>
|
||||
<fieldset class="mc-section">
|
||||
<legend class="mc-label">Display</legend>
|
||||
<label for="mcClusters"><input type="checkbox" id="mcClusters"> Show clusters</label>
|
||||
@@ -181,11 +190,17 @@
|
||||
});
|
||||
|
||||
map.on('zoomend', () => {
|
||||
if (!_renderingMarkers) renderMarkers();
|
||||
clearTimeout(_zoomResizeTimer);
|
||||
_zoomResizeTimer = setTimeout(() => {
|
||||
if (!_renderingMarkers) _repositionMarkers();
|
||||
}, 150);
|
||||
});
|
||||
|
||||
map.on('resize', () => {
|
||||
if (!_renderingMarkers) renderMarkers();
|
||||
clearTimeout(_zoomResizeTimer);
|
||||
_zoomResizeTimer = setTimeout(() => {
|
||||
if (!_renderingMarkers) _repositionMarkers();
|
||||
}, 150);
|
||||
});
|
||||
|
||||
markerLayer = L.layerGroup().addTo(map);
|
||||
@@ -262,6 +277,16 @@
|
||||
});
|
||||
});
|
||||
|
||||
// Byte size filter buttons
|
||||
document.querySelectorAll('#mcByteFilter .btn').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
filters.byteSize = btn.dataset.byte;
|
||||
localStorage.setItem('meshcore-map-byte-filter', filters.byteSize);
|
||||
document.querySelectorAll('#mcByteFilter .btn').forEach(b => b.classList.toggle('active', b.dataset.byte === filters.byteSize));
|
||||
renderMarkers();
|
||||
});
|
||||
});
|
||||
|
||||
// Geo filter overlay
|
||||
(async function () {
|
||||
try {
|
||||
@@ -612,6 +637,8 @@
|
||||
|
||||
var _renderingMarkers = false;
|
||||
var _lastDeconflictZoom = null;
|
||||
var _currentMarkerData = []; // stored marker data for zoom-only repositioning
|
||||
var _zoomResizeTimer = null;
|
||||
|
||||
function deconflictLabels(markers, mapRef) {
|
||||
const placed = [];
|
||||
@@ -662,6 +689,62 @@
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create, update, or remove the offset indicator (dashed line + dot at true GPS position)
|
||||
* for a deconflicted marker. Shared by _renderMarkersInner and _repositionMarkers.
|
||||
* @param {Object} m - marker data object with latLng, adjustedLatLng, offset, _leafletLine, _leafletDot
|
||||
* @param {L.LayerGroup} layer - layer group to add/remove indicators from
|
||||
*/
|
||||
function _updateOffsetIndicator(m, layer) {
|
||||
var pos = m.adjustedLatLng || m.latLng;
|
||||
var redColor = getComputedStyle(document.documentElement).getPropertyValue('--status-red').trim() || '#ef4444';
|
||||
|
||||
if (m.offset > 10) {
|
||||
// Line from true position to adjusted position
|
||||
if (m._leafletLine) {
|
||||
m._leafletLine.setLatLngs([m.latLng, pos]);
|
||||
} else {
|
||||
m._leafletLine = L.polyline([m.latLng, pos], {
|
||||
color: redColor, weight: 2, dashArray: '6,4', opacity: 0.85
|
||||
});
|
||||
layer.addLayer(m._leafletLine);
|
||||
}
|
||||
// Dot at true GPS position
|
||||
if (!m._leafletDot) {
|
||||
m._leafletDot = L.circleMarker(m.latLng, {
|
||||
radius: 3, fillColor: redColor, fillOpacity: 0.9, stroke: true, color: '#fff', weight: 1
|
||||
});
|
||||
layer.addLayer(m._leafletDot);
|
||||
}
|
||||
} else {
|
||||
// No offset — remove indicator if it existed
|
||||
if (m._leafletLine) { layer.removeLayer(m._leafletLine); m._leafletLine = null; }
|
||||
if (m._leafletDot) { layer.removeLayer(m._leafletDot); m._leafletDot = null; }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reposition existing markers by re-running deconfliction at the current zoom.
|
||||
* Avoids clearing and rebuilding all markers — eliminates flicker on zoom/resize.
|
||||
*/
|
||||
function _repositionMarkers() {
|
||||
if (!map || _currentMarkerData.length === 0) return;
|
||||
map.invalidateSize({ animate: false });
|
||||
|
||||
// Re-run deconfliction with current zoom pixel coordinates
|
||||
deconflictLabels(_currentMarkerData, map);
|
||||
|
||||
for (var i = 0; i < _currentMarkerData.length; i++) {
|
||||
var m = _currentMarkerData[i];
|
||||
var pos = m.adjustedLatLng || m.latLng;
|
||||
|
||||
// Update marker position
|
||||
if (m._leafletMarker) m._leafletMarker.setLatLng(pos);
|
||||
|
||||
_updateOffsetIndicator(m, markerLayer);
|
||||
}
|
||||
}
|
||||
|
||||
function renderMarkers() {
|
||||
if (_renderingMarkers) return;
|
||||
_renderingMarkers = true;
|
||||
@@ -670,10 +753,16 @@
|
||||
|
||||
function _renderMarkersInner() {
|
||||
markerLayer.clearLayers();
|
||||
_currentMarkerData = [];
|
||||
|
||||
const filtered = nodes.filter(n => {
|
||||
if (!n.lat || !n.lon) return false;
|
||||
if (!filters[n.role || 'companion']) return false;
|
||||
// Byte size filter (applies only to repeaters)
|
||||
if (filters.byteSize !== 'all' && (n.role || 'companion') === 'repeater') {
|
||||
const hs = n.hash_size || 1;
|
||||
if (String(hs) !== filters.byteSize) return false;
|
||||
}
|
||||
// Status filter
|
||||
if (filters.statusFilter !== 'all') {
|
||||
const role = (n.role || 'companion').toLowerCase();
|
||||
@@ -719,24 +808,20 @@
|
||||
deconflictLabels(allMarkers, map);
|
||||
}
|
||||
|
||||
// Store marker data for zoom/resize repositioning (avoids full rebuild)
|
||||
_currentMarkerData = allMarkers;
|
||||
|
||||
for (const m of allMarkers) {
|
||||
const pos = m.adjustedLatLng || m.latLng;
|
||||
const marker = L.marker(pos, { icon: m.icon, alt: m.alt });
|
||||
marker._nodeKey = m.node.public_key || m.node.id || null;
|
||||
marker.bindPopup(m.popupFn(), { maxWidth: 280 });
|
||||
markerLayer.addLayer(marker);
|
||||
m._leafletMarker = marker;
|
||||
m._leafletLine = null;
|
||||
m._leafletDot = null;
|
||||
|
||||
if (m.offset > 10) {
|
||||
const line = L.polyline([m.latLng, pos], {
|
||||
color: getComputedStyle(document.documentElement).getPropertyValue('--status-red').trim() || '#ef4444', weight: 2, dashArray: '6,4', opacity: 0.85
|
||||
});
|
||||
markerLayer.addLayer(line);
|
||||
// Small dot at true GPS position
|
||||
const dot = L.circleMarker(m.latLng, {
|
||||
radius: 3, fillColor: getComputedStyle(document.documentElement).getPropertyValue('--status-red').trim() || '#ef4444', fillOpacity: 0.9, stroke: true, color: '#fff', weight: 1
|
||||
});
|
||||
markerLayer.addLayer(dot);
|
||||
}
|
||||
_updateOffsetIndicator(m, markerLayer);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -870,6 +955,7 @@
|
||||
map = null;
|
||||
}
|
||||
markerLayer = null;
|
||||
_currentMarkerData = [];
|
||||
routeLayer = null;
|
||||
if (heatLayer) { heatLayer = null; }
|
||||
geoFilterLayer = null;
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
const nodeName = escapeHtml(n.name || n.public_key.slice(0, 12));
|
||||
|
||||
container.innerHTML = `
|
||||
<div style="max-width:1000px;margin:0 auto;padding:12px 16px;height:100%;overflow-y:auto">
|
||||
<div style="max-width:1000px;margin:0 auto;padding:12px 16px">
|
||||
<div style="margin-bottom:12px">
|
||||
<a href="#/nodes/${encodeURIComponent(n.public_key)}" style="color:var(--accent);text-decoration:none;font-size:12px">← Back to ${nodeName}</a>
|
||||
<h2 style="margin:4px 0 2px;font-size:18px">📊 ${nodeName} — Analytics</h2>
|
||||
|
||||
+17
-9
@@ -372,13 +372,25 @@
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch node detail + health data in parallel.
|
||||
* Both selectNode() and loadFullNode() need the same data —
|
||||
* this shared helper avoids duplicating the fetch logic (fixes #391).
|
||||
*/
|
||||
async function fetchNodeDetail(pubkey) {
|
||||
const [nodeData, healthData] = await Promise.all([
|
||||
api('/nodes/' + encodeURIComponent(pubkey), { ttl: CLIENT_TTL.nodeDetail }),
|
||||
api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null)
|
||||
]);
|
||||
nodeData.healthData = healthData;
|
||||
return nodeData;
|
||||
}
|
||||
|
||||
async function loadFullNode(pubkey) {
|
||||
const body = document.getElementById('nodeFullBody');
|
||||
try {
|
||||
const [nodeData, healthData] = await Promise.all([
|
||||
api('/nodes/' + encodeURIComponent(pubkey), { ttl: CLIENT_TTL.nodeDetail }),
|
||||
api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null)
|
||||
]);
|
||||
const nodeData = await fetchNodeDetail(pubkey);
|
||||
const healthData = nodeData.healthData;
|
||||
const n = nodeData.node;
|
||||
const adverts = (nodeData.recentAdverts || []).sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
||||
const title = document.querySelector('.node-full-title');
|
||||
@@ -963,11 +975,7 @@
|
||||
panel.innerHTML = '<div class="text-center text-muted" style="padding:40px">Loading…</div>';
|
||||
|
||||
try {
|
||||
const [data, healthData] = await Promise.all([
|
||||
api('/nodes/' + encodeURIComponent(pubkey), { ttl: CLIENT_TTL.nodeDetail }),
|
||||
api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null)
|
||||
]);
|
||||
data.healthData = healthData;
|
||||
const data = await fetchNodeDetail(pubkey);
|
||||
renderDetail(panel, data);
|
||||
} catch (e) {
|
||||
panel.innerHTML = `<div class="text-muted">Error: ${e.message}</div>`;
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
}
|
||||
|
||||
app.innerHTML = `
|
||||
<div class="observer-detail-page" style="overflow-y:auto;height:calc(100vh - 56px);padding:16px">
|
||||
<div class="observer-detail-page" style="padding:16px">
|
||||
<div class="page-header" style="display:flex;align-items:center;gap:12px;margin-bottom:16px">
|
||||
<a href="#/observers" class="btn-icon" title="Back to Observers" aria-label="Back">←</a>
|
||||
<h2 style="margin:0" id="obsTitle">Observer Detail</h2>
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB After Width: | Height: | Size: 229 KiB |
@@ -28,9 +28,27 @@ window.getParsedPath = function getParsedPath(p) {
|
||||
window.clearParsedCache = function clearParsedCache(p) {
|
||||
delete p._parsedPath;
|
||||
delete p._parsedDecoded;
|
||||
delete p._parsedResolvedPath;
|
||||
return p;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse resolved_path (server-side resolved full pubkeys).
|
||||
* Returns array of pubkey strings (or null entries) if present, or null if absent.
|
||||
* Cached as _parsedResolvedPath on the packet object.
|
||||
*/
|
||||
window.getResolvedPath = function getResolvedPath(p) {
|
||||
if (p._parsedResolvedPath !== undefined) return p._parsedResolvedPath;
|
||||
var raw = p.resolved_path;
|
||||
if (!raw) { p._parsedResolvedPath = null; return null; }
|
||||
if (typeof raw !== 'string') {
|
||||
p._parsedResolvedPath = Array.isArray(raw) ? raw : null;
|
||||
return p._parsedResolvedPath;
|
||||
}
|
||||
try { p._parsedResolvedPath = JSON.parse(raw) || null; } catch (e) { p._parsedResolvedPath = null; }
|
||||
return p._parsedResolvedPath;
|
||||
};
|
||||
|
||||
window.getParsedDecoded = function getParsedDecoded(p) {
|
||||
if (p._parsedDecoded !== undefined) return p._parsedDecoded || {};
|
||||
var raw = p.decoded_json;
|
||||
|
||||
+133
-58
@@ -40,6 +40,21 @@
|
||||
clearTimeout(_renderTimer);
|
||||
_renderTimer = setTimeout(() => renderTableRows(), 200);
|
||||
}
|
||||
|
||||
// Coalesce WS-triggered renders into one per animation frame (#396).
|
||||
// Multiple WS batches arriving within the same frame only trigger a single
|
||||
// renderTableRows() call on the next rAF, preventing rapid full rebuilds.
|
||||
function scheduleWSRender() {
|
||||
_wsRenderDirty = true;
|
||||
if (_wsRafId) return; // already scheduled
|
||||
_wsRafId = requestAnimationFrame(function () {
|
||||
_wsRafId = null;
|
||||
if (_wsRenderDirty) {
|
||||
_wsRenderDirty = false;
|
||||
renderTableRows();
|
||||
}
|
||||
});
|
||||
}
|
||||
const PANEL_WIDTH_KEY = 'meshcore-panel-width';
|
||||
const PANEL_CLOSE_HTML = '<button class="panel-close-btn" title="Close detail pane (Esc)">✕</button>';
|
||||
|
||||
@@ -59,6 +74,8 @@
|
||||
let _lastVisibleEnd = -1; // last rendered end index (for dirty checking)
|
||||
let _vsScrollHandler = null; // scroll listener reference
|
||||
let _wsRenderTimer = null; // debounce timer for WS-triggered renders
|
||||
let _wsRafId = null; // rAF id for coalescing WS-triggered renders (#396)
|
||||
let _wsRenderDirty = false; // dirty flag for rAF render coalescing (#396)
|
||||
let _observerFilterSet = null; // cached Set from filters.observer, hoisted above loops (#427)
|
||||
|
||||
function closeDetailPanel() {
|
||||
@@ -171,6 +188,29 @@
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-populate hopNameCache from server-side resolved_path on packets.
|
||||
* Packets with resolved_path skip client-side HopResolver entirely.
|
||||
* Must call ensureHopResolver() first so nodesList is available for name lookup.
|
||||
*/
|
||||
async function cacheResolvedPaths(packets) {
|
||||
if (!packets || !packets.length) return;
|
||||
let needsInit = false;
|
||||
for (const p of packets) {
|
||||
const rp = getResolvedPath(p);
|
||||
if (rp) { needsInit = true; break; }
|
||||
}
|
||||
if (!needsInit) return;
|
||||
await ensureHopResolver();
|
||||
for (const p of packets) {
|
||||
const rp = getResolvedPath(p);
|
||||
if (!rp) continue;
|
||||
const hops = getParsedPath(p);
|
||||
const resolved = HopResolver.resolveFromServer(hops, rp);
|
||||
Object.assign(hopNameCache, resolved);
|
||||
}
|
||||
}
|
||||
|
||||
function renderHop(h, observerId) {
|
||||
// Use per-packet cache key if observer context available (ambiguous hops differ by region)
|
||||
const cacheKey = observerId ? h + ':' + observerId : h;
|
||||
@@ -269,7 +309,7 @@
|
||||
const obs = data.observations.find(o => String(o.id) === String(obsTarget));
|
||||
if (obs) {
|
||||
expandedHashes.add(h);
|
||||
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, timestamp: obs.timestamp, first_seen: obs.timestamp};
|
||||
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, timestamp: obs.timestamp, first_seen: obs.timestamp};
|
||||
clearParsedCache(obsPacket);
|
||||
selectPacket(obs.id, h, {packet: obsPacket, breakdown: data.breakdown, observations: data.observations}, obs.id);
|
||||
} else {
|
||||
@@ -371,9 +411,16 @@
|
||||
if (!filtered.length) return;
|
||||
|
||||
// Resolve any new hops, then update and re-render
|
||||
// Pre-populate from server-side resolved_path, then fall back for remaining
|
||||
const newHops = new Set();
|
||||
for (const p of filtered) {
|
||||
try { getParsedPath(p).forEach(h => { if (!(h in hopNameCache)) newHops.add(h); }); } catch {}
|
||||
const rp = getResolvedPath(p);
|
||||
const hops = getParsedPath(p);
|
||||
if (rp && rp.length === hops.length && window.HopResolver && HopResolver.ready()) {
|
||||
const resolved = HopResolver.resolveFromServer(hops, rp);
|
||||
Object.assign(hopNameCache, resolved);
|
||||
}
|
||||
try { hops.forEach(h => { if (!(h in hopNameCache)) newHops.add(h); }); } catch {}
|
||||
}
|
||||
(newHops.size ? resolveHops([...newHops]) : Promise.resolve()).then(() => {
|
||||
if (groupByHash) {
|
||||
@@ -431,9 +478,8 @@
|
||||
if (packets.length > PACKET_LIMIT) packets.length = PACKET_LIMIT;
|
||||
}
|
||||
totalCount += filtered.length;
|
||||
// Debounce WS-triggered renders to avoid rapid full rebuilds
|
||||
clearTimeout(_wsRenderTimer);
|
||||
_wsRenderTimer = setTimeout(function () { renderTableRows(); }, 200);
|
||||
// Coalesce WS-triggered renders via rAF (#396)
|
||||
scheduleWSRender();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -444,6 +490,8 @@
|
||||
wsHandler = null;
|
||||
detachVScrollListener();
|
||||
clearTimeout(_wsRenderTimer);
|
||||
if (_wsRafId) { cancelAnimationFrame(_wsRafId); _wsRafId = null; }
|
||||
_wsRenderDirty = false;
|
||||
_displayPackets = [];
|
||||
_rowCounts = [];
|
||||
_rowCountsDirty = false;
|
||||
@@ -494,7 +542,11 @@
|
||||
if (filters.hash) params.set('hash', filters.hash);
|
||||
if (filters.node) params.set('node', filters.node);
|
||||
if (filters.observer) params.set('observer', filters.observer);
|
||||
params.set('groupByHash', 'true'); // always fetch grouped
|
||||
if (groupByHash) {
|
||||
params.set('groupByHash', 'true');
|
||||
} else {
|
||||
params.set('expand', 'observations');
|
||||
}
|
||||
|
||||
const data = await api('/packets?' + params.toString());
|
||||
packets = data.packets || [];
|
||||
@@ -502,20 +554,14 @@
|
||||
for (const p of packets) { if (p.hash) hashIndex.set(p.hash, p); }
|
||||
totalCount = data.total || packets.length;
|
||||
|
||||
// When ungrouped, fetch observations for all multi-obs packets and flatten
|
||||
// When ungrouped, flatten observations inline (single API call, no N+1)
|
||||
if (!groupByHash) {
|
||||
const multiObs = packets.filter(p => (p.observation_count || p.count || 1) > 1);
|
||||
await Promise.all(multiObs.map(async (p) => {
|
||||
try {
|
||||
const d = await api(`/packets/${p.hash}`);
|
||||
if (d?.observations) p._children = d.observations.map(o => clearParsedCache({...d.packet, ...o, _isObservation: true}));
|
||||
} catch {}
|
||||
}));
|
||||
// Flatten: replace grouped packets with individual observations
|
||||
const flat = [];
|
||||
for (const p of packets) {
|
||||
if (p._children && p._children.length > 1) {
|
||||
for (const c of p._children) flat.push(c);
|
||||
if (p.observations && p.observations.length > 1) {
|
||||
for (const o of p.observations) {
|
||||
flat.push(clearParsedCache({...p, ...o, _isObservation: true, observations: undefined}));
|
||||
}
|
||||
} else {
|
||||
flat.push(p);
|
||||
}
|
||||
@@ -524,7 +570,10 @@
|
||||
totalCount = flat.length;
|
||||
}
|
||||
|
||||
// Pre-resolve all path hops to node names
|
||||
// Pre-resolve from server-side resolved_path (preferred, no client-side disambiguation needed)
|
||||
await cacheResolvedPaths(packets);
|
||||
|
||||
// Pre-resolve all path hops to node names (fallback for packets without resolved_path)
|
||||
const allHops = new Set();
|
||||
for (const p of packets) {
|
||||
try { getParsedPath(p).forEach(h => allHops.add(h)); } catch {}
|
||||
@@ -547,19 +596,22 @@
|
||||
// Ambiguous hops are already resolved by HopResolver client-side
|
||||
// No need for per-observer server API calls
|
||||
|
||||
// Restore expanded group children
|
||||
// Restore expanded group children (parallel fetch, Map lookup)
|
||||
if (groupByHash && expandedHashes.size > 0) {
|
||||
for (const hash of expandedHashes) {
|
||||
const group = packets.find(p => p.hash === hash);
|
||||
if (group) {
|
||||
try {
|
||||
const childData = await api(`/packets?hash=${hash}&limit=20`);
|
||||
group._children = childData.packets || [];
|
||||
sortGroupChildren(group);
|
||||
} catch {}
|
||||
} else {
|
||||
// Group no longer in results — remove from expanded
|
||||
const expandedArr = [...expandedHashes];
|
||||
const results = await Promise.all(expandedArr.map(hash => {
|
||||
const group = hashIndex.get(hash);
|
||||
if (!group) return { hash, group: null, data: null };
|
||||
return api(`/packets?hash=${hash}&limit=20`)
|
||||
.then(data => ({ hash, group, data }))
|
||||
.catch(() => ({ hash, group, data: null }));
|
||||
}));
|
||||
for (const { hash, group, data } of results) {
|
||||
if (!group) {
|
||||
expandedHashes.delete(hash);
|
||||
} else if (data) {
|
||||
group._children = data.packets || [];
|
||||
sortGroupChildren(group);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -837,18 +889,30 @@
|
||||
obsSortSel.addEventListener('change', async function () {
|
||||
obsSortMode = this.value;
|
||||
localStorage.setItem('meshcore-obs-sort', obsSortMode);
|
||||
// For non-observer sorts, fetch children for visible groups that don't have them yet
|
||||
// For non-observer sorts, batch-fetch children for visible groups that don't have them yet
|
||||
if (obsSortMode !== SORT_OBSERVER && groupByHash) {
|
||||
const toFetch = packets.filter(p => p.hash && !p._children && (p.observation_count || 0) > 1);
|
||||
await Promise.all(toFetch.map(async (p) => {
|
||||
if (toFetch.length > 0) {
|
||||
const hashes = toFetch.map(p => p.hash);
|
||||
try {
|
||||
const data = await api(`/packets/${p.hash}`);
|
||||
if (data?.packet && data.observations) {
|
||||
p._children = data.observations.map(o => clearParsedCache({...data.packet, ...o, _isObservation: true}));
|
||||
p._fetchedData = data;
|
||||
const resp = await fetch('/api/packets/observations', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({hashes})
|
||||
});
|
||||
if (resp.ok) {
|
||||
const data = await resp.json();
|
||||
const results = data.results || {};
|
||||
for (const p of toFetch) {
|
||||
const obs = results[p.hash];
|
||||
if (obs && obs.length) {
|
||||
p._children = obs.map(o => clearParsedCache({...p, ...o, _isObservation: true}));
|
||||
p._fetchedData = {packet: p, observations: obs};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}));
|
||||
}
|
||||
}
|
||||
// Re-sort all groups with children
|
||||
for (const p of packets) {
|
||||
@@ -1012,11 +1076,11 @@
|
||||
}
|
||||
else if (action === 'select-observation') {
|
||||
const parentHash = row.dataset.parentHash;
|
||||
const group = packets.find(p => p.hash === parentHash);
|
||||
const group = hashIndex.get(parentHash);
|
||||
const child = group?._children?.find(c => String(c.id) === String(value));
|
||||
if (child) {
|
||||
const parentData = group._fetchedData;
|
||||
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, timestamp: child.timestamp, first_seen: child.timestamp} : child;
|
||||
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, timestamp: child.timestamp, first_seen: child.timestamp} : child;
|
||||
if (parentData) { clearParsedCache(obsPacket); }
|
||||
selectPacket(child.id, parentHash, {packet: obsPacket, breakdown: parentData?.breakdown, observations: parentData?.observations}, child.id);
|
||||
}
|
||||
@@ -1489,11 +1553,18 @@
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Re-resolve hops using client-side HopResolver with sender GPS context
|
||||
// Resolve hops: prefer server-side resolved_path, fall back to client-side HopResolver
|
||||
if (pathHops.length) {
|
||||
try {
|
||||
await ensureHopResolver();
|
||||
const resolved = HopResolver.resolve(pathHops);
|
||||
const serverResolved = getResolvedPath(pkt);
|
||||
let resolved;
|
||||
if (serverResolved && serverResolved.length === pathHops.length) {
|
||||
await ensureHopResolver();
|
||||
resolved = HopResolver.resolveFromServer(pathHops, serverResolved);
|
||||
} else {
|
||||
await ensureHopResolver();
|
||||
resolved = HopResolver.resolve(pathHops);
|
||||
}
|
||||
if (resolved) {
|
||||
for (const [k, v] of Object.entries(resolved)) {
|
||||
hopNameCache[k] = v;
|
||||
@@ -1670,22 +1741,25 @@
|
||||
if (routeBtn && pathHops.length) {
|
||||
routeBtn.addEventListener('click', async () => {
|
||||
try {
|
||||
// Anchor disambiguation from sender's location if known (e.g. ADVERT lat/lon)
|
||||
const senderLat = decoded.lat || decoded.latitude;
|
||||
const senderLon = decoded.lon || decoded.longitude;
|
||||
// Resolve observer position for backward-pass anchor
|
||||
let obsLat = null, obsLon = null;
|
||||
const obsId = obsName(pkt.observer_id);
|
||||
if (obsId && HopResolver.ready()) {
|
||||
// Try to find observer in nodes list by name — best effort
|
||||
// Prefer server-side resolved_path if available
|
||||
const serverResolved = getResolvedPath(pkt);
|
||||
let resolvedKeys;
|
||||
if (serverResolved && serverResolved.length === pathHops.length) {
|
||||
// Use server-resolved pubkeys, fall back to short prefix for null entries
|
||||
resolvedKeys = pathHops.map((h, i) => serverResolved[i] || h);
|
||||
} else {
|
||||
// Fall back to client-side HopResolver
|
||||
const senderLat = decoded.lat || decoded.latitude;
|
||||
const senderLon = decoded.lon || decoded.longitude;
|
||||
let obsLat = null, obsLon = null;
|
||||
const obsId = obsName(pkt.observer_id);
|
||||
await ensureHopResolver();
|
||||
const data = { resolved: HopResolver.resolve(pathHops, senderLat || null, senderLon || null, obsLat, obsLon, pkt.observer_id) };
|
||||
resolvedKeys = pathHops.map(h => {
|
||||
const r = data.resolved?.[h];
|
||||
return r?.pubkey || h;
|
||||
});
|
||||
}
|
||||
await ensureHopResolver();
|
||||
const data = { resolved: HopResolver.resolve(pathHops, senderLat || null, senderLon || null, obsLat, obsLon, pkt.observer_id) };
|
||||
// Pass full pubkeys (client-disambiguated) to map, falling back to short prefix
|
||||
const resolvedKeys = pathHops.map(h => {
|
||||
const r = data.resolved?.[h];
|
||||
return r?.pubkey || h;
|
||||
});
|
||||
// Build origin info for the sender node
|
||||
const origin = {};
|
||||
if (decoded.pubKey) origin.pubkey = decoded.pubKey;
|
||||
@@ -2009,14 +2083,15 @@
|
||||
const data = await api(`/packets/${hash}`);
|
||||
const pkt = data.packet;
|
||||
if (!pkt) return;
|
||||
const group = packets.find(p => p.hash === hash);
|
||||
const group = hashIndex.get(hash);
|
||||
if (group && data.observations) {
|
||||
group._children = data.observations.map(o => clearParsedCache({...pkt, ...o, _isObservation: true}));
|
||||
group._fetchedData = data;
|
||||
// Sort children based on current sort mode
|
||||
sortGroupChildren(group);
|
||||
}
|
||||
// Resolve any new hops from children
|
||||
// Resolve hops from children: prefer server-side resolved_path
|
||||
await cacheResolvedPaths(group?._children || []);
|
||||
const childHops = new Set();
|
||||
for (const c of (group?._children || [])) {
|
||||
try { getParsedPath(c).forEach(h => childHops.add(h)); } catch {}
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
let interval = null;
|
||||
|
||||
async function render(app) {
|
||||
app.innerHTML = '<div id="perfWrapper" style="height:100%;overflow-y:auto;padding:16px 24px;"><h2>⚡ Performance Dashboard</h2><div id="perfContent">Loading...</div></div>';
|
||||
app.innerHTML = '<div id="perfWrapper" style="padding:16px 24px;"><h2>⚡ Performance Dashboard</h2><div id="perfContent">Loading...</div></div>';
|
||||
await refresh();
|
||||
}
|
||||
|
||||
|
||||
+9
-4
@@ -181,7 +181,12 @@ a:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible
|
||||
}
|
||||
|
||||
/* === Layout === */
|
||||
#app { height: calc(100vh - 52px); height: calc(100dvh - 52px); overflow: hidden; }
|
||||
/* Default: body-scroll mode — content pushes beyond viewport, iOS status-bar
|
||||
tap-to-scroll works because <body> is the scroll container. Pages that need
|
||||
a fixed-height container (maps, virtual-scroll, split-panels) add
|
||||
.app-fixed via the router so their children can use height:100%. */
|
||||
#app { min-height: calc(100vh - 52px); min-height: calc(100dvh - 52px); }
|
||||
#app.app-fixed { height: calc(100vh - 52px); height: calc(100dvh - 52px); min-height: 0; overflow: hidden; }
|
||||
|
||||
.split-layout {
|
||||
display: flex; height: 100%; overflow: hidden;
|
||||
@@ -674,7 +679,7 @@ button.ch-item.selected { background: var(--selected-bg); }
|
||||
.advert-info { font-size: 12px; line-height: 1.5; }
|
||||
|
||||
/* === Traces Page === */
|
||||
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; overflow-y: auto; height: 100%; }
|
||||
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; }
|
||||
.trace-search {
|
||||
display: flex; gap: 8px; margin-bottom: 20px;
|
||||
}
|
||||
@@ -746,7 +751,7 @@ button.ch-item.selected { background: var(--selected-bg); }
|
||||
::-webkit-scrollbar-thumb:hover { background: var(--text-muted); }
|
||||
|
||||
/* === Observers Page === */
|
||||
.observers-page { padding: 20px; max-width: 1200px; margin: 0 auto; overflow-y: auto; height: calc(100vh - 56px); }
|
||||
.observers-page { padding: 20px; max-width: 1200px; margin: 0 auto; }
|
||||
.obs-summary { display: flex; gap: 20px; margin-bottom: 16px; flex-wrap: wrap; }
|
||||
.obs-stat { display: flex; align-items: center; gap: 6px; font-size: 14px; color: var(--text-muted); }
|
||||
.health-dot { width: 10px; height: 10px; border-radius: 50%; display: inline-block; flex-shrink: 0; }
|
||||
@@ -1138,7 +1143,7 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
|
||||
.node-activity-time { color: var(--text-muted); white-space: nowrap; min-width: 70px; font-size: 12px; }
|
||||
|
||||
/* Analytics page */
|
||||
.analytics-page { padding: 16px 24px; max-width: 1600px; margin: 0 auto; overflow-y: auto; height: 100%; }
|
||||
.analytics-page { padding: 16px 24px; max-width: 1600px; margin: 0 auto; }
|
||||
.analytics-header { margin-bottom: 20px; }
|
||||
.analytics-header h2 { margin: 0 0 4px; }
|
||||
.analytics-card { background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; padding: 16px; margin-bottom: 16px; }
|
||||
|
||||
+152
-11
@@ -564,6 +564,93 @@ console.log('\n=== hop-resolver.js ===');
|
||||
});
|
||||
}
|
||||
|
||||
// ===== resolveFromServer (hop-resolver.js, M4 #555) =====
|
||||
console.log('\n=== resolveFromServer (hop-resolver.js) ===');
|
||||
{
|
||||
const ctx = makeSandbox();
|
||||
ctx.IATA_COORDS_GEO = {};
|
||||
loadInCtx(ctx, 'public/hop-resolver.js');
|
||||
const HR = ctx.window.HopResolver;
|
||||
|
||||
test('resolveFromServer works without init (uses pubkey prefix as name)', () => {
|
||||
const pk = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
|
||||
const result = HR.resolveFromServer(['AB'], [pk]);
|
||||
assert.strictEqual(result['AB'].name, pk.slice(0, 8));
|
||||
assert.strictEqual(result['AB'].pubkey, pk);
|
||||
});
|
||||
|
||||
test('resolveFromServer with matching node', () => {
|
||||
const pubkey = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
|
||||
HR.init([{ public_key: pubkey, name: 'NodeA', lat: 37.3, lon: -122.0 }]);
|
||||
const result = HR.resolveFromServer(['AB'], [pubkey]);
|
||||
assert.strictEqual(result['AB'].name, 'NodeA');
|
||||
assert.strictEqual(result['AB'].pubkey, pubkey);
|
||||
assert.ok(!result['AB'].ambiguous);
|
||||
});
|
||||
|
||||
test('resolveFromServer with null entry skips it', () => {
|
||||
const pubkey = 'abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890';
|
||||
HR.init([{ public_key: pubkey, name: 'NodeA', lat: 37.3, lon: -122.0 }]);
|
||||
const result = HR.resolveFromServer(['AB', 'CD'], [pubkey, null]);
|
||||
assert.strictEqual(result['AB'].name, 'NodeA');
|
||||
assert.ok(!('CD' in result)); // null entries are skipped
|
||||
});
|
||||
|
||||
test('resolveFromServer with unknown pubkey uses prefix', () => {
|
||||
HR.init([{ public_key: 'aaaa0000', name: 'Other' }]);
|
||||
const unknownPk = '1111111111111111111111111111111111111111111111111111111111111111';
|
||||
const result = HR.resolveFromServer(['AB'], [unknownPk]);
|
||||
assert.strictEqual(result['AB'].name, unknownPk.slice(0, 8));
|
||||
assert.strictEqual(result['AB'].pubkey, unknownPk);
|
||||
});
|
||||
|
||||
test('resolveFromServer mismatched lengths returns empty', () => {
|
||||
HR.init([{ public_key: 'abcdef1234567890', name: 'NodeA' }]);
|
||||
const result = HR.resolveFromServer(['AB', 'CD'], ['abcdef1234567890']);
|
||||
assert.strictEqual(Object.keys(result).length, 0);
|
||||
});
|
||||
}
|
||||
|
||||
// ===== getResolvedPath (packet-helpers.js, M4 #555) =====
|
||||
console.log('\n=== getResolvedPath (packet-helpers.js) ===');
|
||||
{
|
||||
const ctx = makeSandbox();
|
||||
loadInCtx(ctx, 'public/packet-helpers.js');
|
||||
const getResolvedPath = ctx.window.getResolvedPath;
|
||||
|
||||
test('getResolvedPath returns null when absent', () => {
|
||||
assert.strictEqual(getResolvedPath({}), null);
|
||||
});
|
||||
|
||||
test('getResolvedPath parses JSON string', () => {
|
||||
const pkt = { resolved_path: '["aabb","ccdd",null]' };
|
||||
const result = getResolvedPath(pkt);
|
||||
assert.deepStrictEqual(result, ['aabb', 'ccdd', null]);
|
||||
});
|
||||
|
||||
test('getResolvedPath returns array as-is', () => {
|
||||
const arr = ['aabb', null];
|
||||
const pkt = { resolved_path: arr };
|
||||
assert.strictEqual(getResolvedPath(pkt), arr);
|
||||
});
|
||||
|
||||
test('getResolvedPath caches result', () => {
|
||||
const pkt = { resolved_path: '["aabb"]' };
|
||||
const r1 = getResolvedPath(pkt);
|
||||
const r2 = getResolvedPath(pkt);
|
||||
assert.strictEqual(r1, r2); // same reference
|
||||
});
|
||||
|
||||
test('clearParsedCache clears resolved path cache', () => {
|
||||
const clearParsedCache = ctx.window.clearParsedCache;
|
||||
const pkt = { resolved_path: '["aabb"]' };
|
||||
getResolvedPath(pkt);
|
||||
assert.ok(pkt._parsedResolvedPath !== undefined);
|
||||
clearParsedCache(pkt);
|
||||
assert.strictEqual(pkt._parsedResolvedPath, undefined);
|
||||
});
|
||||
}
|
||||
|
||||
// ===== haversineKm exposed from HopResolver (issue #433) =====
|
||||
console.log('\n=== haversineKm (hop-resolver.js) ===');
|
||||
{
|
||||
@@ -998,6 +1085,56 @@ console.log('\n=== live.js: pruneStaleNodes ===');
|
||||
assert.ok(markers['apiNode'], 'API stale node should NOT be removed');
|
||||
assert.ok(data['apiNode'], 'API stale node data should NOT be removed');
|
||||
});
|
||||
|
||||
test('pruneStaleNodes cleans up nodeActivity for removed nodes', () => {
|
||||
const { ctx } = makeLiveSandbox();
|
||||
const prune = ctx.window._livePruneStaleNodes;
|
||||
const markers = ctx.window._liveNodeMarkers();
|
||||
const data = ctx.window._liveNodeData();
|
||||
const activity = ctx.window._liveNodeActivity();
|
||||
|
||||
// WS-only stale node
|
||||
markers['staleNode'] = { _glowMarker: null };
|
||||
data['staleNode'] = { public_key: 'staleNode', role: 'companion', _liveSeen: Date.now() - 48 * 3600000 };
|
||||
activity['staleNode'] = 5;
|
||||
|
||||
// Active node
|
||||
markers['activeNode'] = { setStyle: function() {}, _glowMarker: null };
|
||||
data['activeNode'] = { public_key: 'activeNode', role: 'companion', _liveSeen: Date.now() };
|
||||
activity['activeNode'] = 3;
|
||||
|
||||
prune();
|
||||
|
||||
assert.ok(!markers['staleNode'], 'stale node marker removed');
|
||||
assert.ok(!data['staleNode'], 'stale node data removed');
|
||||
assert.ok(!activity['staleNode'], 'stale node activity removed');
|
||||
assert.ok(markers['activeNode'], 'active node marker preserved');
|
||||
assert.ok(data['activeNode'], 'active node data preserved');
|
||||
assert.strictEqual(activity['activeNode'], 3, 'active node activity preserved');
|
||||
});
|
||||
|
||||
test('pruneStaleNodes removes orphaned nodeActivity entries', () => {
|
||||
const { ctx } = makeLiveSandbox();
|
||||
const prune = ctx.window._livePruneStaleNodes;
|
||||
const markers = ctx.window._liveNodeMarkers();
|
||||
const data = ctx.window._liveNodeData();
|
||||
const activity = ctx.window._liveNodeActivity();
|
||||
|
||||
// Add an active node
|
||||
markers['existingNode'] = { setStyle: function() {}, _glowMarker: null };
|
||||
data['existingNode'] = { public_key: 'existingNode', role: 'companion', _liveSeen: Date.now() };
|
||||
activity['existingNode'] = 2;
|
||||
|
||||
// Add orphaned activity (no corresponding nodeData)
|
||||
activity['ghostNode'] = 10;
|
||||
|
||||
prune();
|
||||
|
||||
assert.ok(markers['existingNode'], 'existing node preserved');
|
||||
assert.ok(data['existingNode'], 'existing node data preserved');
|
||||
assert.strictEqual(activity['existingNode'], 2, 'existing node activity preserved');
|
||||
assert.ok(!activity['ghostNode'], 'orphaned activity entry removed');
|
||||
});
|
||||
}
|
||||
|
||||
// ===== live.js: vcrFormatTime respects UTC/local setting =====
|
||||
@@ -3056,20 +3193,24 @@ console.log('\n=== channels.js: formatHashHex (issue #465) ===');
|
||||
'destroy must reset observerMap to empty Map');
|
||||
});
|
||||
|
||||
test('WS handler debounces render via _wsRenderTimer', () => {
|
||||
test('WS handler coalesces render via rAF (#396)', () => {
|
||||
const wsBlock = src.slice(src.indexOf('wsHandler = debouncedOnWS'), src.indexOf('function destroy()'));
|
||||
assert.ok(wsBlock.includes('_wsRenderTimer'),
|
||||
'WS handler must debounce renders via _wsRenderTimer');
|
||||
assert.ok(wsBlock.includes('clearTimeout(_wsRenderTimer)'),
|
||||
'WS handler must clear pending timer before scheduling new render');
|
||||
assert.ok(/setTimeout\(function \(\) \{ renderTableRows\(\); \}/.test(wsBlock),
|
||||
'WS handler must schedule renderTableRows via setTimeout');
|
||||
assert.ok(wsBlock.includes('scheduleWSRender()'),
|
||||
'WS handler must coalesce renders via scheduleWSRender()');
|
||||
// Verify scheduleWSRender uses requestAnimationFrame
|
||||
const schedFn = src.slice(src.indexOf('function scheduleWSRender()'), src.indexOf('function scheduleWSRender()') + 300);
|
||||
assert.ok(schedFn.includes('requestAnimationFrame'),
|
||||
'scheduleWSRender must use requestAnimationFrame for coalescing');
|
||||
assert.ok(schedFn.includes('_wsRenderDirty'),
|
||||
'scheduleWSRender must use dirty flag pattern');
|
||||
});
|
||||
|
||||
test('destroy clears _wsRenderTimer', () => {
|
||||
const destroyBlock = src.slice(src.indexOf('function destroy()'), src.indexOf('function destroy()') + 500);
|
||||
assert.ok(destroyBlock.includes('clearTimeout(_wsRenderTimer)'),
|
||||
'destroy must clear _wsRenderTimer to prevent stale renders after navigation');
|
||||
test('destroy clears rAF and dirty flag (#396)', () => {
|
||||
const destroyBlock = src.slice(src.indexOf('function destroy()'), src.indexOf('function destroy()') + 600);
|
||||
assert.ok(destroyBlock.includes('cancelAnimationFrame(_wsRafId)'),
|
||||
'destroy must cancel pending rAF to prevent stale renders after navigation');
|
||||
assert.ok(destroyBlock.includes('_wsRenderDirty = false'),
|
||||
'destroy must reset dirty flag');
|
||||
});
|
||||
}
|
||||
// ===== NODES.JS: shared sandbox factory =====
|
||||
|
||||
@@ -881,6 +881,17 @@ console.log('\n=== live.js: source-level safety checks ===');
|
||||
assert.ok(src.includes('const existingIds = new Set(VCR.buffer.map(b => b.pkt.id)'),
|
||||
'vcrRewind should dedup by packet ID');
|
||||
});
|
||||
|
||||
test('feed items include transport badge', () => {
|
||||
const count = (src.match(/transportBadge\(pkt\.route_type\)/g) || []).length;
|
||||
assert.ok(count >= 3,
|
||||
`feed rendering should call transportBadge(pkt.route_type) in at least 3 places (found ${count})`);
|
||||
});
|
||||
|
||||
test('node detail recent packets include transport badge', () => {
|
||||
assert.ok(src.includes('transportBadge(p.route_type)'),
|
||||
'node detail recent packets should call transportBadge(p.route_type)');
|
||||
});
|
||||
}
|
||||
|
||||
// ===== SUMMARY =====
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
/* Unit tests for prefix tool logic (analytics.js _prefixToolExports) */
|
||||
'use strict';
|
||||
const vm = require('vm');
|
||||
const fs = require('fs');
|
||||
const assert = require('assert');
|
||||
|
||||
let passed = 0, failed = 0;
|
||||
function test(name, fn) {
|
||||
try { fn(); passed++; console.log(` ✅ ${name}`); }
|
||||
catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); }
|
||||
}
|
||||
|
||||
// Load analytics.js in a VM sandbox with minimal stubs
|
||||
const code = fs.readFileSync(__dirname + '/public/analytics.js', 'utf8');
|
||||
const sandbox = {
|
||||
window: {},
|
||||
document: { addEventListener() {} },
|
||||
location: { hash: '' },
|
||||
setTimeout: () => {},
|
||||
requestAnimationFrame: () => {},
|
||||
console,
|
||||
Map, Set, Array, Object, Number, Math, Date, JSON,
|
||||
encodeURIComponent,
|
||||
URLSearchParams,
|
||||
parseInt, parseFloat, isNaN, isFinite,
|
||||
RegExp, Error, TypeError, RangeError,
|
||||
Promise: { resolve: () => ({ then: () => ({}) }) },
|
||||
};
|
||||
sandbox.window = sandbox;
|
||||
sandbox.self = sandbox;
|
||||
|
||||
try {
|
||||
vm.runInNewContext(code, sandbox, { filename: 'analytics.js', timeout: 5000 });
|
||||
} catch (e) {
|
||||
// IIFE may throw due to missing DOM — that's fine, we just need the exports
|
||||
}
|
||||
|
||||
const ex = sandbox.window._prefixToolExports;
|
||||
if (!ex) {
|
||||
console.log('❌ _prefixToolExports not found on window');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { buildPrefixIndex, computePrefixStats, recommendPrefixSize,
|
||||
validatePrefixInput, checkPrefix, generatePrefix,
|
||||
renderSeverityBadge, PREFIX_SPACE_SIZES } = ex;
|
||||
|
||||
console.log('\n--- buildPrefixIndex ---');
|
||||
|
||||
test('builds 3-tier index from nodes', () => {
|
||||
const nodes = [
|
||||
{ public_key: 'A1B2C3D4E5F6' },
|
||||
{ public_key: 'A1B2FFFFFF00' },
|
||||
{ public_key: 'FF00112233AA' },
|
||||
];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
assert.strictEqual(idx[1].size, 2); // A1, FF
|
||||
assert.strictEqual(idx[2].size, 2); // A1B2, FF00
|
||||
assert.strictEqual(idx[3].size, 3); // A1B2C3, A1B2FF, FF0011
|
||||
assert.strictEqual(idx[1].get('A1').length, 2);
|
||||
assert.strictEqual(idx[2].get('A1B2').length, 2);
|
||||
assert.strictEqual(idx[1].get('FF').length, 1);
|
||||
});
|
||||
|
||||
test('handles empty node list', () => {
|
||||
const idx = buildPrefixIndex([]);
|
||||
assert.strictEqual(idx[1].size, 0);
|
||||
assert.strictEqual(idx[2].size, 0);
|
||||
assert.strictEqual(idx[3].size, 0);
|
||||
});
|
||||
|
||||
console.log('\n--- computePrefixStats ---');
|
||||
|
||||
test('detects collisions', () => {
|
||||
const nodes = [
|
||||
{ public_key: 'A1B2C3D4E5F6' },
|
||||
{ public_key: 'A1B2FFFFFF00' },
|
||||
];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const stats = computePrefixStats(idx);
|
||||
assert.strictEqual(stats[1].collidingPrefixes, 1); // A1 collides
|
||||
assert.strictEqual(stats[2].collidingPrefixes, 1); // A1B2 collides
|
||||
assert.strictEqual(stats[3].collidingPrefixes, 0); // no 3-byte collision
|
||||
});
|
||||
|
||||
test('no collisions when all unique', () => {
|
||||
const nodes = [
|
||||
{ public_key: 'A1B2C3D4E5F6' },
|
||||
{ public_key: 'B1B2C3D4E5F6' },
|
||||
];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const stats = computePrefixStats(idx);
|
||||
assert.strictEqual(stats[1].collidingPrefixes, 0);
|
||||
});
|
||||
|
||||
console.log('\n--- recommendPrefixSize ---');
|
||||
|
||||
test('recommends 1-byte for small networks (<20)', () => {
|
||||
const r = recommendPrefixSize(5);
|
||||
assert.strictEqual(r.rec, '1-byte');
|
||||
});
|
||||
|
||||
test('recommends 2-byte for medium networks (20-499)', () => {
|
||||
const r = recommendPrefixSize(100);
|
||||
assert.strictEqual(r.rec, '2-byte');
|
||||
});
|
||||
|
||||
test('recommends 3-byte for large networks (>=500)', () => {
|
||||
const r = recommendPrefixSize(500);
|
||||
assert.strictEqual(r.rec, '3-byte');
|
||||
});
|
||||
|
||||
test('recommends 3-byte for very large networks', () => {
|
||||
const r = recommendPrefixSize(5000);
|
||||
assert.strictEqual(r.rec, '3-byte');
|
||||
});
|
||||
|
||||
test('boundary: 19 nodes = 1-byte', () => {
|
||||
assert.strictEqual(recommendPrefixSize(19).rec, '1-byte');
|
||||
});
|
||||
|
||||
test('boundary: 20 nodes = 2-byte', () => {
|
||||
assert.strictEqual(recommendPrefixSize(20).rec, '2-byte');
|
||||
});
|
||||
|
||||
test('boundary: 499 nodes = 2-byte', () => {
|
||||
assert.strictEqual(recommendPrefixSize(499).rec, '2-byte');
|
||||
});
|
||||
|
||||
console.log('\n--- validatePrefixInput ---');
|
||||
|
||||
test('empty input', () => {
|
||||
const r = validatePrefixInput('');
|
||||
assert.strictEqual(r.valid, false);
|
||||
assert.strictEqual(r.isEmpty, true);
|
||||
});
|
||||
|
||||
test('valid 1-byte prefix', () => {
|
||||
const r = validatePrefixInput('A1');
|
||||
assert.strictEqual(r.valid, true);
|
||||
assert.strictEqual(r.tiers.length, 1);
|
||||
assert.strictEqual(r.tiers[0].b, 1);
|
||||
assert.strictEqual(r.tiers[0].prefix, 'A1');
|
||||
});
|
||||
|
||||
test('valid 2-byte prefix', () => {
|
||||
const r = validatePrefixInput('a1b2');
|
||||
assert.strictEqual(r.valid, true);
|
||||
assert.strictEqual(r.tiers[0].prefix, 'A1B2');
|
||||
assert.strictEqual(r.isFullKey, false);
|
||||
});
|
||||
|
||||
test('valid 3-byte prefix', () => {
|
||||
const r = validatePrefixInput('A1B2C3');
|
||||
assert.strictEqual(r.valid, true);
|
||||
assert.strictEqual(r.tiers[0].b, 3);
|
||||
});
|
||||
|
||||
test('full public key (64 chars) derives 3 tiers', () => {
|
||||
const pk = 'A1B2C3D4' + '0'.repeat(56);
|
||||
const r = validatePrefixInput(pk);
|
||||
assert.strictEqual(r.valid, true);
|
||||
assert.strictEqual(r.isFullKey, true);
|
||||
assert.strictEqual(r.tiers.length, 3);
|
||||
assert.strictEqual(r.tiers[0].prefix, 'A1');
|
||||
assert.strictEqual(r.tiers[1].prefix, 'A1B2');
|
||||
assert.strictEqual(r.tiers[2].prefix, 'A1B2C3');
|
||||
});
|
||||
|
||||
test('rejects non-hex', () => {
|
||||
const r = validatePrefixInput('ZZZZ');
|
||||
assert.strictEqual(r.valid, false);
|
||||
assert(r.error.includes('hex'));
|
||||
});
|
||||
|
||||
test('rejects odd-length input', () => {
|
||||
const r = validatePrefixInput('A1B');
|
||||
assert.strictEqual(r.valid, false);
|
||||
assert(r.error.includes('2, 4, or 6'));
|
||||
});
|
||||
|
||||
console.log('\n--- checkPrefix ---');
|
||||
|
||||
test('detects collision on 1-byte', () => {
|
||||
const nodes = [{ public_key: 'A1B2C3D4E5F6' }, { public_key: 'A1FFFFFF0000' }];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const r = checkPrefix('A1', idx, nodes);
|
||||
assert.strictEqual(r.valid, true);
|
||||
assert.strictEqual(r.results[0].count, 2);
|
||||
});
|
||||
|
||||
test('no collision for unused prefix', () => {
|
||||
const nodes = [{ public_key: 'A1B2C3D4E5F6' }];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const r = checkPrefix('FF', idx, nodes);
|
||||
assert.strictEqual(r.results[0].count, 0);
|
||||
});
|
||||
|
||||
test('full key excludes self from colliders', () => {
|
||||
const pk = 'A1B2C3D4E5F60000';
|
||||
const nodes = [{ public_key: pk }, { public_key: 'A1B2FFFFFF000000' }];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const r = checkPrefix(pk, idx, nodes);
|
||||
assert.strictEqual(r.isFullKey, true);
|
||||
// 1-byte tier: A1 has both nodes, but self excluded = 1 collider
|
||||
assert.strictEqual(r.results[0].count, 1);
|
||||
});
|
||||
|
||||
console.log('\n--- generatePrefix ---');
|
||||
|
||||
test('generates a collision-free 1-byte prefix', () => {
|
||||
const nodes = [];
|
||||
// Fill all but one 1-byte prefix
|
||||
for (let i = 0; i < 255; i++) {
|
||||
nodes.push({ public_key: i.toString(16).toUpperCase().padStart(2, '0') + '0000000000' });
|
||||
}
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const prefix = generatePrefix(1, idx, () => 0.5);
|
||||
assert.strictEqual(prefix, 'FF'); // only FF is free
|
||||
assert(!idx[1].has(prefix));
|
||||
});
|
||||
|
||||
test('returns null when no prefix available', () => {
|
||||
const nodes = [];
|
||||
for (let i = 0; i < 256; i++) {
|
||||
nodes.push({ public_key: i.toString(16).toUpperCase().padStart(2, '0') + '0000000000' });
|
||||
}
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const prefix = generatePrefix(1, idx);
|
||||
assert.strictEqual(prefix, null);
|
||||
});
|
||||
|
||||
test('generates 2-byte prefix not in index', () => {
|
||||
const nodes = [{ public_key: 'A1B2C3D4E5F6' }];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const prefix = generatePrefix(2, idx, () => 0.5);
|
||||
assert.strictEqual(typeof prefix, 'string');
|
||||
assert.strictEqual(prefix.length, 4);
|
||||
assert(!idx[2].has(prefix));
|
||||
});
|
||||
|
||||
test('uses deterministic random function', () => {
|
||||
const nodes = [{ public_key: 'A1B2C3D4E5F6' }];
|
||||
const idx = buildPrefixIndex(nodes);
|
||||
const p1 = generatePrefix(2, idx, () => 0.1);
|
||||
const p2 = generatePrefix(2, idx, () => 0.1);
|
||||
assert.strictEqual(p1, p2);
|
||||
});
|
||||
|
||||
console.log('\n--- renderSeverityBadge ---');
|
||||
|
||||
test('unique badge for 0', () => {
|
||||
assert(renderSeverityBadge(0).includes('Unique'));
|
||||
});
|
||||
|
||||
test('warning badge for 1-2', () => {
|
||||
assert(renderSeverityBadge(1).includes('1 collision'));
|
||||
assert(renderSeverityBadge(2).includes('2 collisions'));
|
||||
});
|
||||
|
||||
test('red badge for 3+', () => {
|
||||
assert(renderSeverityBadge(5).includes('5 collisions'));
|
||||
assert(renderSeverityBadge(5).includes('status-red'));
|
||||
});
|
||||
|
||||
// --- Summary ---
|
||||
console.log(`\n${passed} passed, ${failed} failed`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
Reference in New Issue
Block a user