mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-27 23:54:10 +00:00
@ ## What this PR does Implements region-scoped transport-route packet tracking with two sub-features: ### Feature 1 — Scope statistics (`scope_name`) - At ingest, transport-route packets (route_type 0/3) with Code1 != `0000` are HMAC-matched against configured `hashRegions` keys (mirroring the `hashChannels` pattern). Matched region name (or `""` for unknown) stored in new `transmissions.scope_name` column via migration `scope_name_v1`. - New `GET /api/scope-stats?window=` endpoint (1h/24h/7d, 30s server-side TTL) returning transport totals, scoped/unscoped counts, per-region breakdown, and time-series. - New **Scopes** tab in Analytics with summary cards, per-region table, and two-line SVG chart. Auto-refreshes every 60s. ### Feature 2 — Node default scope (`default_scope`) - Per-node `default_scope` column on `nodes`/`inactive_nodes` (migration `nodes_default_scope_v1`) tracks the most recently matched region for each node, derived from transport-scoped ADVERT packets. - `GET /api/nodes` response includes `default_scope` field when column is present. - Node detail panel displays the default scope badge. - Async startup backfill (`BackfillDefaultScopeAsync`) populates the column for nodes with pre-existing ADVERT data. ### Config Add `hashRegions` to `config.json` (see `config.example.json`). One entry per region name (with or without leading `#`). @ --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Kpa-clawbot <kpaclawbot@outlook.com> Co-authored-by: openclaw-bot <bot@openclaw.local>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
Kpa-clawbot
openclaw-bot
parent
ac7d3dd72c
commit
2329639f45
@@ -31,3 +31,5 @@ cmd/ingestor/ingestor.exe
|
||||
!test-fixtures/e2e-fixture.db
|
||||
corescope-server
|
||||
cmd/server/server
|
||||
# Local-only planning and design files
|
||||
docs/superpowers/
|
||||
|
||||
@@ -22,6 +22,7 @@ COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
COPY internal/dbschema/ ../../internal/dbschema/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
COPY internal/perfio/ ../../internal/perfio/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
RUN go mod download
|
||||
COPY cmd/server/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
@@ -37,6 +38,7 @@ COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
COPY internal/dbschema/ ../../internal/dbschema/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
COPY internal/perfio/ ../../internal/perfio/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
RUN go mod download
|
||||
COPY cmd/ingestor/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
|
||||
@@ -50,6 +50,7 @@ type Config struct {
|
||||
ChannelKeysPath string `json:"channelKeysPath,omitempty"`
|
||||
ChannelKeys map[string]string `json:"channelKeys,omitempty"`
|
||||
HashChannels []string `json:"hashChannels,omitempty"`
|
||||
HashRegions []string `json:"hashRegions,omitempty"`
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -158,7 +160,7 @@ func TestHandleMessageChannelMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Alice: Hello everyone","channel_idx":3,"SNR":5.0,"RSSI":-95,"score":10,"direction":"rx","sender_timestamp":1700000000}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/2", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -218,7 +220,7 @@ func TestHandleMessageChannelMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -233,7 +235,7 @@ func TestHandleMessageChannelNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":"no sender here"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -250,7 +252,7 @@ func TestHandleMessageDirectMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Bob: Hey there","sender_timestamp":1700000000,"SNR":3.0,"rssi":-100,"Score":8,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc123", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -294,7 +296,7 @@ func TestHandleMessageDirectMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -309,7 +311,7 @@ func TestHandleMessageDirectNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: []byte(`{"text":"message with no colon"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -328,7 +330,7 @@ func TestHandleMessageUppercaseScoreDirection(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","Score":9.0,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var score *float64
|
||||
var direction *string
|
||||
@@ -349,7 +351,7 @@ func TestHandleMessageChannelLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":3.0,"rssi":-90,"Score":5,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/0", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -365,7 +367,7 @@ func TestHandleMessageDirectLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":2.0,"rssi":-85,"score":7,"direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -388,7 +390,7 @@ func TestHandleMessageAdvertWithTelemetry(t *testing.T) {
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
// Should have created transmission, node, and observer
|
||||
var txCount, nodeCount, obsCount int
|
||||
@@ -430,7 +432,7 @@ func TestHandleMessageAdvertGeoFiltered(t *testing.T) {
|
||||
}
|
||||
// Legacy silent-drop behavior is now opt-in via ForeignAdverts.Mode="drop"
|
||||
// (#730). The new default — flag — is covered by foreign_advert_test.go.
|
||||
handleMessage(store, "test", source, msg, nil, &Config{
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{
|
||||
GeoFilter: gf,
|
||||
ForeignAdverts: &ForeignAdvertConfig{Mode: "drop"},
|
||||
})
|
||||
@@ -670,7 +672,7 @@ func TestHandleMessageCorruptedAdvertNoNode(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -692,7 +694,7 @@ func TestHandleMessageNonAdvertPacket(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -753,8 +755,13 @@ func TestDecodeAdvertSensorNoName(t *testing.T) {
|
||||
// --- db.go: OpenStore error path (invalid dir) ---
|
||||
|
||||
func TestOpenStoreInvalidPath(t *testing.T) {
|
||||
// Path under /dev/null can't create directory
|
||||
_, err := OpenStore("/dev/null/impossible/path/db.sqlite")
|
||||
// Create a regular file then try to open a DB inside it — impossible on all platforms.
|
||||
f, err := os.CreateTemp(t.TempDir(), "not-a-dir")
|
||||
if err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
f.Close()
|
||||
_, err = OpenStore(filepath.Join(f.Name(), "db.sqlite"))
|
||||
if err == nil {
|
||||
t.Error("should error on impossible path")
|
||||
}
|
||||
@@ -869,7 +876,7 @@ func TestHandleMessageChannelLongSender(t *testing.T) {
|
||||
longText := "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -888,7 +895,7 @@ func TestHandleMessageDirectLongSender(t *testing.T) {
|
||||
longText := "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -905,7 +912,7 @@ func TestHandleMessageDirectUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"X: hi","Score":6,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/d1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -935,7 +942,7 @@ func TestHandleMessageChannelUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Y: hi","Score":4,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/5", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -966,7 +973,7 @@ func TestHandleMessageRawLowercaseScore(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","score":3.5}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var score *float64
|
||||
if err := store.db.QueryRow("SELECT score FROM observations LIMIT 1").Scan(&score); err != nil {
|
||||
@@ -985,7 +992,7 @@ func TestHandleMessageStatusNoOrigin(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs5/status",
|
||||
payload: []byte(`{"model":"L1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id = 'obs5'").Scan(&count); err != nil {
|
||||
|
||||
+150
-28
@@ -66,13 +66,13 @@ type Store struct {
|
||||
path string // filesystem path to the SQLite DB (used to resolve queue dirs)
|
||||
Stats DBStats
|
||||
|
||||
stmtGetTxByHash *sql.Stmt
|
||||
stmtInsertTransmission *sql.Stmt
|
||||
stmtUpdateTxFirstSeen *sql.Stmt
|
||||
stmtInsertObservation *sql.Stmt
|
||||
stmtUpsertNode *sql.Stmt
|
||||
stmtIncrementAdvertCount *sql.Stmt
|
||||
stmtUpsertObserver *sql.Stmt
|
||||
stmtGetTxByHash *sql.Stmt
|
||||
stmtInsertTransmission *sql.Stmt
|
||||
stmtUpdateTxFirstSeen *sql.Stmt
|
||||
stmtInsertObservation *sql.Stmt
|
||||
stmtUpsertNode *sql.Stmt
|
||||
stmtIncrementAdvertCount *sql.Stmt
|
||||
stmtUpsertObserver *sql.Stmt
|
||||
stmtGetObserverRowid *sql.Stmt
|
||||
stmtUpdateObserverLastSeen *sql.Stmt
|
||||
stmtUpdateNodeTelemetry *sql.Stmt
|
||||
@@ -475,6 +475,22 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] observations.raw_hex column added")
|
||||
}
|
||||
|
||||
// Migration: add scope_name column to transmissions (#899)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'scope_name_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding scope_name column to transmissions...")
|
||||
if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil {
|
||||
log.Printf("[migration] transmissions.scope_name: %v (may already exist)", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL`); err != nil {
|
||||
log.Printf("[migration] idx_tx_scope_name: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('scope_name_v1')`); err != nil {
|
||||
return fmt.Errorf("recording scope_name_v1 migration: %w", err)
|
||||
}
|
||||
log.Println("[migration] scope_name column added")
|
||||
}
|
||||
|
||||
// Migration: add last_packet_at column to observers (#last-packet-at)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
@@ -551,6 +567,22 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] from_pubkey column + index added")
|
||||
}
|
||||
|
||||
// Migration: add default_scope column to nodes (#899 Feature 3)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'nodes_default_scope_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding default_scope column to nodes/inactive_nodes...")
|
||||
if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN default_scope TEXT DEFAULT NULL`); err != nil {
|
||||
log.Printf("[migration] nodes.default_scope: %v (may already exist)", err)
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN default_scope TEXT DEFAULT NULL`); err != nil {
|
||||
log.Printf("[migration] inactive_nodes.default_scope: %v (may already exist)", err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('nodes_default_scope_v1')`); err != nil {
|
||||
return fmt.Errorf("recording nodes_default_scope_v1 migration: %w", err)
|
||||
}
|
||||
log.Println("[migration] default_scope column added to nodes/inactive_nodes")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -563,8 +595,8 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtInsertTransmission, err = s.db.Prepare(`
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, from_pubkey)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -693,6 +725,7 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
data.RawHex, hash, now,
|
||||
data.RouteType, data.PayloadType, data.PayloadVersion,
|
||||
data.DecodedJSON, nilIfEmpty(data.ChannelHash),
|
||||
scopeNameForDB(data),
|
||||
nilIfEmpty(data.FromPubkey),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -1095,6 +1128,58 @@ func (s *Store) BackfillPathJSONAsync() {
|
||||
}()
|
||||
}
|
||||
|
||||
// BackfillDefaultScopeAsync populates default_scope for existing nodes that have
|
||||
// transport-scoped ADVERT rows (scope_name IS NOT NULL AND scope_name != “).
|
||||
// Runs in a background goroutine so it does not block MQTT startup.
|
||||
// Uses the from_pubkey index — O(nodes × indexed lookup), not a full table scan.
|
||||
//
|
||||
// Concurrency: the store uses SetMaxOpenConns(1) so all DB writes — including
|
||||
// MQTT packet inserts and any concurrent backfill goroutines — serialize through
|
||||
// the single connection pool. busy_timeout(5000) handles transient cross-process
|
||||
// contention with the read-only server process. No additional locking is needed.
|
||||
func (s *Store) BackfillDefaultScopeAsync(regionKeys map[string][]byte) {
|
||||
// No region keys configured — all scope_name values will be NULL, nothing to backfill.
|
||||
if len(regionKeys) == 0 {
|
||||
return
|
||||
}
|
||||
s.backfillWg.Add(1)
|
||||
go func() {
|
||||
defer s.backfillWg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[backfill] default_scope async panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
var done int
|
||||
if s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'backfill_default_scope_v1'").Scan(&done) == nil {
|
||||
return // already ran
|
||||
}
|
||||
|
||||
res, err := s.db.Exec(`
|
||||
UPDATE nodes SET default_scope = (
|
||||
SELECT t.scope_name FROM transmissions t
|
||||
WHERE t.from_pubkey = nodes.public_key
|
||||
AND t.payload_type = 4
|
||||
AND t.scope_name IS NOT NULL AND t.scope_name != ''
|
||||
ORDER BY t.first_seen DESC LIMIT 1 -- most-recently observed scope wins; first_seen is insertion time
|
||||
) WHERE EXISTS (
|
||||
SELECT 1 FROM transmissions t
|
||||
WHERE t.from_pubkey = nodes.public_key
|
||||
AND t.payload_type = 4
|
||||
AND t.scope_name IS NOT NULL AND t.scope_name != ''
|
||||
)`)
|
||||
if err != nil {
|
||||
log.Printf("[backfill] default_scope: %v", err)
|
||||
return
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
s.Stats.IncBackfill("default_scope")
|
||||
log.Printf("[backfill] default_scope populated for %d nodes", n)
|
||||
s.db.Exec(`INSERT INTO _migrations (name) VALUES ('backfill_default_scope_v1')`)
|
||||
}()
|
||||
}
|
||||
|
||||
// LogStats logs current operational metrics.
|
||||
func (s *Store) LogStats() {
|
||||
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d sig_drops=%d",
|
||||
@@ -1203,24 +1288,26 @@ func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) {
|
||||
|
||||
// PacketData holds the data needed to insert a packet into the DB.
|
||||
type PacketData struct {
|
||||
RawHex string
|
||||
Timestamp string
|
||||
ObserverID string
|
||||
ObserverName string
|
||||
SNR *float64
|
||||
RSSI *float64
|
||||
Score *float64
|
||||
Direction *string
|
||||
Hash string
|
||||
RouteType int
|
||||
PayloadType int
|
||||
PayloadVersion int
|
||||
PathJSON string
|
||||
DecodedJSON string
|
||||
ChannelHash string // grouping key for channel queries (#762)
|
||||
Region string // observer region: payload > topic > source config (#788)
|
||||
Foreign bool // true when ADVERT GPS lies outside configured geofilter (#730)
|
||||
FromPubkey string // pubkey of the originating node, for exact-match attribution (#1143)
|
||||
RawHex string
|
||||
Timestamp string
|
||||
ObserverID string
|
||||
ObserverName string
|
||||
SNR *float64
|
||||
RSSI *float64
|
||||
Score *float64
|
||||
Direction *string
|
||||
Hash string
|
||||
RouteType int
|
||||
PayloadType int
|
||||
PayloadVersion int
|
||||
PathJSON string
|
||||
DecodedJSON string
|
||||
ChannelHash string // grouping key for channel queries (#762)
|
||||
ScopeName string // matched region name, or "" for unknown-scoped
|
||||
IsTransportScoped bool // true when route_type IN (0,3) AND Code1 ≠ "0000"
|
||||
Region string // observer region: payload > topic > source config (#788)
|
||||
Foreign bool // true when ADVERT GPS lies outside configured geofilter (#730)
|
||||
FromPubkey string // pubkey of the originating node, for exact-match attribution (#1143)
|
||||
}
|
||||
|
||||
// nilIfEmpty returns nil for empty strings (for nullable DB columns).
|
||||
@@ -1231,6 +1318,36 @@ func nilIfEmpty(s string) interface{} {
|
||||
return s
|
||||
}
|
||||
|
||||
// scopeNameForDB encodes PacketData scope semantics for DB storage:
|
||||
// non-transport-scoped → nil (SQL NULL); transport-scoped → pointer to ScopeName
|
||||
// (may be "" for unknown region, "#name" for matched region).
|
||||
func scopeNameForDB(data *PacketData) *string {
|
||||
if !data.IsTransportScoped {
|
||||
return nil
|
||||
}
|
||||
s := data.ScopeName
|
||||
return &s
|
||||
}
|
||||
|
||||
// UpdateNodeDefaultScope records the most-recently observed region scope for a
|
||||
// node. Skips the UPDATE when the stored value already matches to avoid
|
||||
// redundant writes on the hot MQTT ingest path. Updates both nodes and
|
||||
// inactive_nodes to stay consistent.
|
||||
func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error {
|
||||
// Short-circuit: skip if already stored.
|
||||
var cur sql.NullString
|
||||
row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey)
|
||||
if row.Scan(&cur) == nil && cur.Valid && cur.String == scope {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.db.Exec(`UPDATE nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey); err != nil {
|
||||
return err
|
||||
}
|
||||
// Mirror to inactive_nodes (node may be there if recently moved by retention).
|
||||
_, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey)
|
||||
return err
|
||||
}
|
||||
|
||||
// MQTTPacketMessage is the JSON payload from an MQTT raw packet message.
|
||||
type MQTTPacketMessage struct {
|
||||
Raw string `json:"raw"`
|
||||
@@ -1246,7 +1363,7 @@ type MQTTPacketMessage struct {
|
||||
// path_json is derived directly from raw_hex header bytes (not decoded.Path.Hops)
|
||||
// to guarantee the stored path always matches the raw bytes. This matters for
|
||||
// TRACE packets where decoded.Path.Hops is overwritten with payload hops (#886).
|
||||
func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string) *PacketData {
|
||||
func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionKeys map[string][]byte) *PacketData {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
pathJSON := "[]"
|
||||
// For TRACE packets, path_json must be the payload-decoded route hops
|
||||
@@ -1295,6 +1412,11 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
|
||||
}
|
||||
}
|
||||
|
||||
if decoded.TransportCodes != nil && decoded.TransportCodes.Code1 != "0000" {
|
||||
pd.IsTransportScoped = true
|
||||
pd.ScopeName = matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1)
|
||||
}
|
||||
|
||||
// Populate from_pubkey at write time (#1143). ADVERTs carry the
|
||||
// originating node's pubkey directly; other packet types stay NULL
|
||||
// (downstream attribution queries handle NULL gracefully).
|
||||
|
||||
+135
-10
@@ -642,7 +642,7 @@ func TestEndToEndIngest(t *testing.T) {
|
||||
msg := &MQTTPacketMessage{
|
||||
Raw: rawHex,
|
||||
}
|
||||
pktData := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
pktData := BuildPacketData(msg, decoded, "obs1", "SJC", nil)
|
||||
if _, err := s.InsertTransmission(pktData); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -836,7 +836,7 @@ func TestBuildPacketData(t *testing.T) {
|
||||
Origin: "test-observer",
|
||||
}
|
||||
|
||||
pkt := BuildPacketData(msg, decoded, "obs123", "SJC")
|
||||
pkt := BuildPacketData(msg, decoded, "obs123", "SJC", nil)
|
||||
|
||||
if pkt.RawHex != rawHex {
|
||||
t.Errorf("rawHex mismatch")
|
||||
@@ -881,7 +881,7 @@ func TestBuildPacketDataWithHops(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg := &MQTTPacketMessage{Raw: raw}
|
||||
pkt := BuildPacketData(msg, decoded, "", "")
|
||||
pkt := BuildPacketData(msg, decoded, "", "", nil)
|
||||
|
||||
if pkt.PathJSON == "[]" {
|
||||
t.Error("pathJSON should contain hops")
|
||||
@@ -894,7 +894,7 @@ func TestBuildPacketDataWithHops(t *testing.T) {
|
||||
func TestBuildPacketDataNilSNRRSSI(t *testing.T) {
|
||||
decoded, _ := DecodePacket("0A00"+strings.Repeat("00", 10), nil, false)
|
||||
msg := &MQTTPacketMessage{Raw: "0A00" + strings.Repeat("00", 10)}
|
||||
pkt := BuildPacketData(msg, decoded, "", "")
|
||||
pkt := BuildPacketData(msg, decoded, "", "", nil)
|
||||
|
||||
if pkt.SNR != nil {
|
||||
t.Errorf("SNR should be nil")
|
||||
@@ -1695,7 +1695,7 @@ func TestBuildPacketDataScoreAndDirection(t *testing.T) {
|
||||
Direction: &dir,
|
||||
}
|
||||
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC", nil)
|
||||
if pkt.Score == nil || *pkt.Score != 42.0 {
|
||||
t.Errorf("Score=%v, want 42.0", pkt.Score)
|
||||
}
|
||||
@@ -1707,7 +1707,7 @@ func TestBuildPacketDataScoreAndDirection(t *testing.T) {
|
||||
func TestBuildPacketDataNilScoreDirection(t *testing.T) {
|
||||
decoded, _ := DecodePacket("0A00"+strings.Repeat("00", 10), nil, false)
|
||||
msg := &MQTTPacketMessage{Raw: "0A00" + strings.Repeat("00", 10)}
|
||||
pkt := BuildPacketData(msg, decoded, "", "")
|
||||
pkt := BuildPacketData(msg, decoded, "", "", nil)
|
||||
|
||||
if pkt.Score != nil {
|
||||
t.Errorf("Score should be nil, got %v", *pkt.Score)
|
||||
@@ -2139,7 +2139,7 @@ func TestBuildPacketData_TraceUsesPayloadHops(t *testing.T) {
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pd := BuildPacketData(msg, decoded, "test-obs", "TST")
|
||||
pd := BuildPacketData(msg, decoded, "test-obs", "TST", nil)
|
||||
|
||||
// For TRACE: path_json MUST be the payload-decoded route hops, NOT the SNR bytes
|
||||
expectedPathJSON := `["67","33","D6","33","67"]`
|
||||
@@ -2171,7 +2171,7 @@ func TestBuildPacketData_NonTracePathJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pd := BuildPacketData(msg, decoded, "obs1", "TST")
|
||||
pd := BuildPacketData(msg, decoded, "obs1", "TST", nil)
|
||||
|
||||
expectedPathJSON := `["AA","BB"]`
|
||||
if pd.PathJSON != expectedPathJSON {
|
||||
@@ -2179,6 +2179,131 @@ func TestBuildPacketData_NonTracePathJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopeNameMigration(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Verify column exists
|
||||
rows, err := store.db.Query("PRAGMA table_info(transmissions)")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA: %v", err)
|
||||
}
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName, colType string
|
||||
var notNull, pk int
|
||||
var dflt interface{}
|
||||
if err := rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk); err == nil && colName == "scope_name" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if !found {
|
||||
t.Fatal("scope_name column not found in transmissions")
|
||||
}
|
||||
|
||||
// Verify column actually stores and retrieves values (NULL and non-NULL).
|
||||
_, err = store.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name)
|
||||
VALUES ('aabb', 'hash1', '2026-01-01T00:00:00Z', 0, 5, '#belgium')`)
|
||||
if err != nil {
|
||||
t.Fatalf("insert scoped row: %v", err)
|
||||
}
|
||||
_, err = store.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name)
|
||||
VALUES ('ccdd', 'hash2', '2026-01-01T00:00:01Z', 0, 5, NULL)`)
|
||||
if err != nil {
|
||||
t.Fatalf("insert unscoped row: %v", err)
|
||||
}
|
||||
|
||||
var name string
|
||||
if err := store.db.QueryRow(`SELECT scope_name FROM transmissions WHERE hash = 'hash1'`).Scan(&name); err != nil {
|
||||
t.Fatalf("read scope_name: %v", err)
|
||||
}
|
||||
if name != "#belgium" {
|
||||
t.Errorf("scope_name = %q, want #belgium", name)
|
||||
}
|
||||
|
||||
var nullScope interface{}
|
||||
if err := store.db.QueryRow(`SELECT scope_name FROM transmissions WHERE hash = 'hash2'`).Scan(&nullScope); err != nil {
|
||||
t.Fatalf("read null scope_name: %v", err)
|
||||
}
|
||||
if nullScope != nil {
|
||||
t.Errorf("scope_name for unscoped = %v, want nil", nullScope)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Feature 3: default_scope column on nodes (#899) ---
|
||||
|
||||
func TestUpdateNodeDefaultScope(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Insert a node into nodes and inactive_nodes so both tables can be updated.
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES ('pk1', 'Node1')`); err != nil {
|
||||
t.Fatalf("insert node: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES ('pk1', 'Node1')`); err != nil {
|
||||
t.Fatalf("insert inactive node: %v", err)
|
||||
}
|
||||
|
||||
// First call: writes scope to both tables.
|
||||
if err := store.UpdateNodeDefaultScope("pk1", "#belgium"); err != nil {
|
||||
t.Fatalf("UpdateNodeDefaultScope: %v", err)
|
||||
}
|
||||
var got string
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = 'pk1'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read nodes.default_scope: %v", err)
|
||||
}
|
||||
if got != "#belgium" {
|
||||
t.Errorf("nodes.default_scope = %q, want #belgium", got)
|
||||
}
|
||||
var gotInactive string
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM inactive_nodes WHERE public_key = 'pk1'`).Scan(&gotInactive); err != nil {
|
||||
t.Fatalf("read inactive_nodes.default_scope: %v", err)
|
||||
}
|
||||
if gotInactive != "#belgium" {
|
||||
t.Errorf("inactive_nodes.default_scope = %q, want #belgium", gotInactive)
|
||||
}
|
||||
|
||||
// Second call with same value: short-circuit, no redundant UPDATE (verify no error and value stable).
|
||||
if err := store.UpdateNodeDefaultScope("pk1", "#belgium"); err != nil {
|
||||
t.Fatalf("UpdateNodeDefaultScope short-circuit: %v", err)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = 'pk1'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read after short-circuit: %v", err)
|
||||
}
|
||||
if got != "#belgium" {
|
||||
t.Errorf("after short-circuit nodes.default_scope = %q, want #belgium", got)
|
||||
}
|
||||
|
||||
// Third call with different value: updates both tables.
|
||||
if err := store.UpdateNodeDefaultScope("pk1", "#eu"); err != nil {
|
||||
t.Fatalf("UpdateNodeDefaultScope update: %v", err)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = 'pk1'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read after update: %v", err)
|
||||
}
|
||||
if got != "#eu" {
|
||||
t.Errorf("after update nodes.default_scope = %q, want #eu", got)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM inactive_nodes WHERE public_key = 'pk1'`).Scan(&gotInactive); err != nil {
|
||||
t.Fatalf("read inactive after update: %v", err)
|
||||
}
|
||||
if gotInactive != "#eu" {
|
||||
t.Errorf("after update inactive_nodes.default_scope = %q, want #eu", gotInactive)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Issue #888: Backfill path_json from raw_hex ---
|
||||
|
||||
func TestBackfillPathJsonFromRawHex(t *testing.T) {
|
||||
@@ -2369,7 +2494,7 @@ func TestBuildPacketDataRegionFromPayload(t *testing.T) {
|
||||
decoded := &DecodedPacket{
|
||||
Header: Header{RouteType: 1, PayloadType: 3},
|
||||
}
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC", nil)
|
||||
// When payload has region, it should override the topic-derived region
|
||||
if pkt.Region != "PDX" {
|
||||
t.Fatalf("expected region PDX from payload, got %q", pkt.Region)
|
||||
@@ -2381,7 +2506,7 @@ func TestBuildPacketDataRegionFallsBackToTopic(t *testing.T) {
|
||||
decoded := &DecodedPacket{
|
||||
Header: Header{RouteType: 1, PayloadType: 3},
|
||||
}
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC", nil)
|
||||
if pkt.Region != "SJC" {
|
||||
t.Fatalf("expected region SJC from topic, got %q", pkt.Region)
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestHandleMessageDecodeErrorLog_PII_Issue1211(t *testing.T) {
|
||||
log.SetOutput(&buf)
|
||||
defer log.SetOutput(orig)
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "decode error") {
|
||||
|
||||
@@ -169,6 +169,7 @@ type DecodedPacket struct {
|
||||
Payload Payload `json:"payload"`
|
||||
Raw string `json:"raw"`
|
||||
Anomaly string `json:"anomaly,omitempty"`
|
||||
payloadRaw []byte
|
||||
}
|
||||
|
||||
func decodeHeader(b byte) Header {
|
||||
@@ -927,6 +928,7 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
Payload: payload,
|
||||
Raw: strings.ToUpper(hexString),
|
||||
Anomaly: anomaly,
|
||||
payloadRaw: payloadBuf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -447,6 +447,28 @@ func TestValidateAdvert(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacketPayloadRaw(t *testing.T) {
|
||||
// Build a minimal TRANSPORT_FLOOD packet (route_type=0):
|
||||
// header(1) + transport_codes(4) + path_len(1) + payload(N)
|
||||
// Header 0x00 = route_type=TRANSPORT_FLOOD, payload_type=0, version=0
|
||||
// Code1=9A52, Code2=0000, path_len=0x00 (0 hops, hash_size=1)
|
||||
payload := []byte("hello")
|
||||
raw := []byte{0x00, 0x9A, 0x52, 0x00, 0x00, 0x00}
|
||||
raw = append(raw, payload...)
|
||||
hexStr := strings.ToUpper(hex.EncodeToString(raw))
|
||||
|
||||
decoded, err := DecodePacket(hexStr, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
if decoded.TransportCodes == nil {
|
||||
t.Fatal("expected TransportCodes, got nil")
|
||||
}
|
||||
if string(decoded.payloadRaw) != string(payload) {
|
||||
t.Errorf("payloadRaw = %v, want %v", decoded.payloadRaw, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGrpTxtShort(t *testing.T) {
|
||||
p := decodeGrpTxt([]byte{0x01, 0x02}, nil)
|
||||
if p.Error != "too short" {
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestHandleMessageAdvertForeign_FlagModeStoresWithFlag(t *testing.T) {
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
// Default mode (no ForeignAdverts.Mode set) MUST be "flag", per #730 design.
|
||||
handleMessage(store, "test", source, msg, nil, &Config{GeoFilter: gf})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{GeoFilter: gf})
|
||||
|
||||
var nodeCount int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&nodeCount); err != nil {
|
||||
@@ -70,7 +70,7 @@ func TestHandleMessageAdvertForeign_DropModeStillDrops(t *testing.T) {
|
||||
GeoFilter: gf,
|
||||
ForeignAdverts: &ForeignAdvertConfig{Mode: "drop"},
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var nodeCount int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&nodeCount); err != nil {
|
||||
@@ -99,7 +99,7 @@ func TestHandleMessageAdvertInRegion_NotFlaggedForeign(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{GeoFilter: gf})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{GeoFilter: gf})
|
||||
|
||||
var foreign int
|
||||
err := store.db.QueryRow("SELECT foreign_advert FROM nodes").Scan(&foreign)
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestBuildPacketData_PopulatesFromPubkey(t *testing.T) {
|
||||
Header: Header{PayloadType: PayloadADVERT},
|
||||
Payload: Payload{Type: "ADVERT", PubKey: pk},
|
||||
}
|
||||
pd := BuildPacketData(msg, decoded, "obs", "")
|
||||
pd := BuildPacketData(msg, decoded, "obs", "", nil)
|
||||
if pd.FromPubkey != pk {
|
||||
t.Fatalf("BuildPacketData FromPubkey = %q, want %q", pd.FromPubkey, pk)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func TestBuildPacketData_PopulatesFromPubkey(t *testing.T) {
|
||||
Header: Header{PayloadType: 2},
|
||||
Payload: Payload{Type: "TXT_MSG"},
|
||||
}
|
||||
pd2 := BuildPacketData(msg, decoded2, "obs", "")
|
||||
pd2 := BuildPacketData(msg, decoded2, "obs", "", nil)
|
||||
if pd2.FromPubkey != "" {
|
||||
t.Fatalf("BuildPacketData FromPubkey for non-ADVERT = %q, want empty", pd2.FromPubkey)
|
||||
}
|
||||
|
||||
+63
-4
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
@@ -210,6 +211,9 @@ func main() {
|
||||
log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted")
|
||||
}
|
||||
|
||||
regionKeys := loadRegionKeys(cfg)
|
||||
store.BackfillDefaultScopeAsync(regionKeys)
|
||||
|
||||
// Connect to each MQTT source
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
@@ -264,7 +268,7 @@ func main() {
|
||||
// Capture source for closure
|
||||
src := source
|
||||
opts.SetDefaultPublishHandler(func(c mqtt.Client, m mqtt.Message) {
|
||||
handleMessage(store, tag, src, m, channelKeys, cfg)
|
||||
handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg)
|
||||
})
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
@@ -398,7 +402,7 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions {
|
||||
return opts
|
||||
}
|
||||
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, cfg *Config) {
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionKeys map[string][]byte, cfg *Config) {
|
||||
// Liveness watchdog (#1212): record receipt before any processing so a
|
||||
// slow handler still counts as "source is alive". Cheap atomic store.
|
||||
markLivenessForTag(tag, time.Now())
|
||||
@@ -615,7 +619,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s",
|
||||
tag, truncPK, decoded.Payload.Name, lat, lon, firstNonEmpty(mqttMsg.Origin, observerID))
|
||||
}
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys)
|
||||
pktData.Foreign = foreign
|
||||
isNew, err := store.InsertTransmission(pktData)
|
||||
if err != nil {
|
||||
@@ -641,10 +645,16 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] node telemetry update error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
// Update default_scope when advert carries a matched transport scope (#899)
|
||||
if pktData.IsTransportScoped {
|
||||
if err := store.UpdateNodeDefaultScope(decoded.Payload.PubKey, pktData.ScopeName); err != nil {
|
||||
log.Printf("MQTT [%s] node default_scope update error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-ADVERT packets: store normally (routing/channel messages from
|
||||
// in-area observers are relevant regardless of relay hop origin).
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys)
|
||||
if _, err := store.InsertTransmission(pktData); err != nil {
|
||||
log.Printf("MQTT [%s] db insert error: %v", tag, err)
|
||||
}
|
||||
@@ -1085,6 +1095,55 @@ func loadChannelKeys(cfg *Config, configPath string) map[string]string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func loadRegionKeys(cfg *Config) map[string][]byte {
|
||||
keys := make(map[string][]byte)
|
||||
for _, raw := range cfg.HashRegions {
|
||||
name := strings.TrimSpace(raw)
|
||||
if name == "" {
|
||||
log.Printf("[regions] skipping empty hashRegions entry")
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(name, "#") {
|
||||
name = "#" + name
|
||||
}
|
||||
if _, exists := keys[name]; exists {
|
||||
log.Printf("[regions] duplicate region %q ignored", name)
|
||||
continue
|
||||
}
|
||||
h := sha256.Sum256([]byte(name))
|
||||
keys[name] = h[:16]
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
log.Printf("[regions] %d region key(s) loaded", len(keys))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// matchScope performs one HMAC-SHA256 per configured region. Expected
|
||||
// len(regionKeys) ≤ 50; beyond that, consider a pre-indexed lookup table.
|
||||
func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string {
|
||||
if code1 == "0000" || len(regionKeys) == 0 || len(payloadRaw) == 0 {
|
||||
return ""
|
||||
}
|
||||
for name, key := range regionKeys {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte{payloadType})
|
||||
mac.Write(payloadRaw)
|
||||
hmacBytes := mac.Sum(nil)
|
||||
code := uint16(hmacBytes[0]) | uint16(hmacBytes[1])<<8
|
||||
if code == 0 {
|
||||
code = 1
|
||||
} else if code == 0xFFFF {
|
||||
code = 0xFFFE
|
||||
}
|
||||
codeBytes := [2]byte{byte(code & 0xFF), byte(code >> 8)}
|
||||
if strings.ToUpper(hex.EncodeToString(codeBytes[:])) == code1 {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Version info (set via ldflags)
|
||||
var version = "dev"
|
||||
|
||||
|
||||
+109
-26
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
@@ -133,7 +135,7 @@ func TestHandleMessageRawPacket(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"myobs"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -150,7 +152,7 @@ func TestHandleMessageRawPacketAdvert(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
// Should create a node from the ADVERT
|
||||
var count int
|
||||
@@ -172,7 +174,7 @@ func TestHandleMessageInvalidJSON(t *testing.T) {
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: []byte(`not json`)}
|
||||
|
||||
// Should not panic
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -189,7 +191,7 @@ func TestHandleMessageStatusTopic(t *testing.T) {
|
||||
payload: []byte(`{"origin":"MyObserver"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var name, iata string
|
||||
err := store.db.QueryRow("SELECT name, iata FROM observers WHERE id = 'obs1'").Scan(&name, &iata)
|
||||
@@ -210,11 +212,11 @@ func TestHandleMessageSkipStatusTopics(t *testing.T) {
|
||||
|
||||
// meshcore/status should be skipped
|
||||
msg1 := &mockMessage{topic: "meshcore/status", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg1, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg1, nil, nil, &Config{})
|
||||
|
||||
// meshcore/events/connection should be skipped
|
||||
msg2 := &mockMessage{topic: "meshcore/events/connection", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg2, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg2, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -233,7 +235,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -246,7 +248,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs2/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg2, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg2, nil, nil, &Config{})
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 1 {
|
||||
@@ -264,7 +266,7 @@ func TestHandleMessageIATAFilterNoRegion(t *testing.T) {
|
||||
topic: "meshcore",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
// No region part → filter doesn't apply, message goes through
|
||||
// Actually the code checks len(parts) > 1 for IATA filter
|
||||
@@ -280,7 +282,7 @@ func TestHandleMessageNoRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"type":"companion","data":"something"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -298,7 +300,7 @@ func TestHandleMessageBadRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"ZZZZ"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -315,7 +317,7 @@ func TestHandleMessageWithSNRRSSIAsNumbers(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"RSSI":-95}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -334,7 +336,7 @@ func TestHandleMessageMinimalTopic(t *testing.T) {
|
||||
topic: "meshcore/SJC",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -355,7 +357,7 @@ func TestHandleMessageCorruptedAdvert(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
// Transmission should be inserted (even if advert is invalid)
|
||||
var count int
|
||||
@@ -381,7 +383,7 @@ func TestHandleMessageNoObserverID(t *testing.T) {
|
||||
topic: "packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `","origin":"obs1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -403,7 +405,7 @@ func TestHandleMessageSNRNotFloat(t *testing.T) {
|
||||
// SNR as a string value — should not parse as float
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":"bad","RSSI":"bad"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -419,7 +421,7 @@ func TestHandleMessageOriginExtraction(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","origin":"MyOrigin"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
// Verify origin was extracted to observer name
|
||||
var name string
|
||||
@@ -442,7 +444,7 @@ func TestHandleMessagePanicRecovery(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not panic — the defer/recover should catch it
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
}
|
||||
|
||||
func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
@@ -454,7 +456,7 @@ func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/status",
|
||||
payload: []byte(`{"type":"status"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var name string
|
||||
err := store.db.QueryRow("SELECT name FROM observers WHERE id = 'obs1'").Scan(&name)
|
||||
@@ -645,7 +647,7 @@ func TestHandleMessageWithLowercaseSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","snr":5.5,"rssi":-102}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -666,7 +668,7 @@ func TestHandleMessageSNRRSSIUppercaseWins(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"snr":1.0,"RSSI":-95,"rssi":-50}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -686,7 +688,7 @@ func TestHandleMessageNoSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -757,7 +759,7 @@ func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
topic: "meshcore/BFL/bfl-obs1/status",
|
||||
payload: []byte(`{"origin":"BFLObserver","stats":{"noise_floor":-105.0}}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var name string
|
||||
var noiseFloor *float64
|
||||
@@ -778,7 +780,7 @@ func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
topic: "meshcore/BFL/bfl-obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, pktMsg, nil, &Config{})
|
||||
handleMessage(store, "test", source, pktMsg, nil, nil, &Config{})
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 0 {
|
||||
@@ -786,6 +788,87 @@ func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRegionKeys(t *testing.T) {
|
||||
cfg := &Config{HashRegions: []string{"#belgium", "eu", " #Test ", "", "#belgium"}}
|
||||
keys := loadRegionKeys(cfg)
|
||||
|
||||
// Deduplication + normalization
|
||||
if len(keys) != 3 {
|
||||
t.Fatalf("len(keys) = %d, want 3", len(keys))
|
||||
}
|
||||
// Pre-computed: SHA256("#belgium")[:16]. Hardcoded so a change to the key
|
||||
// derivation algorithm (hash function, truncation length) breaks this test
|
||||
// even if both sides were updated together.
|
||||
wantBelgium, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27")
|
||||
if got := keys["#belgium"]; !bytes.Equal(got, wantBelgium) {
|
||||
t.Errorf("#belgium key mismatch: got %x, want %x", got, wantBelgium)
|
||||
}
|
||||
// "eu" should be normalized to "#eu"
|
||||
if _, ok := keys["#eu"]; !ok {
|
||||
t.Error("expected #eu key")
|
||||
}
|
||||
// " #Test " should be normalized to "#Test"
|
||||
if _, ok := keys["#Test"]; !ok {
|
||||
t.Error("expected #Test key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchScope(t *testing.T) {
|
||||
// Fixed known-answer vectors only — no in-test HMAC computation.
|
||||
// Keys and Code1 values are pre-computed externally so a wrong algorithm
|
||||
// that produces consistent wrong results on both sides would still fail.
|
||||
|
||||
// Vector 1: "#test"/payloadType=5/"hello" → Code1=2AB5
|
||||
// Key = SHA256("#test")[:16] = 9cd8fcf22a47333b591d96a2b848b73f
|
||||
testKey, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f")
|
||||
testKeys := map[string][]byte{"#test": testKey}
|
||||
if got := matchScope(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" {
|
||||
t.Errorf("#test vector: matchScope = %q, want #test", got)
|
||||
}
|
||||
|
||||
// Vector 2: "#belgium"/payloadType=5/"hello" → Code1=4A75
|
||||
// Key = SHA256("#belgium")[:16] = 7085b78ed010599094f8c8e7d1aa0e27
|
||||
belgiumKey, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27")
|
||||
belgiumKeys := map[string][]byte{"#belgium": belgiumKey}
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" {
|
||||
t.Errorf("#belgium vector: matchScope = %q, want #belgium", got)
|
||||
}
|
||||
|
||||
// Code1=0000 (unscoped transport) → no region matched
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "0000"); got != "" {
|
||||
t.Errorf("unscoped: matchScope = %q, want empty", got)
|
||||
}
|
||||
|
||||
// Code1 present but matches no configured region → empty string
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" {
|
||||
t.Errorf("no match: matchScope = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPacketDataScopeMatching(t *testing.T) {
|
||||
// Fixed known-answer packet: TRANSPORT_FLOOD, payloadType=5, payload="hello",
|
||||
// Code1=2AB5 (pre-computed for region "#test").
|
||||
// header=0x14 (route_type=0 FLOOD, payloadType=5 → 5<<2), Code1=[0x2A,0xB5],
|
||||
// Code2=[0,0], path_len=0, payload="hello" (68 65 6C 6C 6F).
|
||||
const rawHex = "142AB500000068656C6C6F"
|
||||
key, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f") // SHA256("#test")[:16]
|
||||
regionKeys := map[string][]byte{"#test": key}
|
||||
|
||||
decoded, err := DecodePacket(rawHex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys)
|
||||
if pktData.ScopeName != "#test" {
|
||||
t.Errorf("ScopeName = %q, want #test", pktData.ScopeName)
|
||||
}
|
||||
if !pktData.IsTransportScoped {
|
||||
t.Error("IsTransportScoped should be true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTConnectRetryTimeoutDoesNotBlock verifies that WaitTimeout returns within
|
||||
// the deadline for an unreachable broker when ConnectRetry=true (#910). Previously,
|
||||
// token.Wait() would block forever in this configuration.
|
||||
@@ -918,7 +1001,7 @@ func TestHandleMessageObserverIATAWhitelist(t *testing.T) {
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/GOT/obs1/status",
|
||||
payload: []byte(`{"origin":"node1","noise_floor":-110}`),
|
||||
}, nil, cfg)
|
||||
}, nil, nil, cfg)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs1'").Scan(&count)
|
||||
@@ -930,7 +1013,7 @@ func TestHandleMessageObserverIATAWhitelist(t *testing.T) {
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/ARN/obs2/status",
|
||||
payload: []byte(`{"origin":"node2","noise_floor":-105}`),
|
||||
}, nil, cfg)
|
||||
}, nil, nil, cfg)
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs2'").Scan(&count)
|
||||
if count != 1 {
|
||||
|
||||
@@ -61,7 +61,7 @@ func TestSigValidation_ValidAdvertStored(t *testing.T) {
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// Verify packet was stored
|
||||
var count int
|
||||
@@ -98,7 +98,7 @@ func TestSigValidation_TamperedSignatureDropped(t *testing.T) {
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// Verify packet was NOT stored in transmissions
|
||||
var txCount int
|
||||
@@ -157,7 +157,7 @@ func TestSigValidation_TruncatedAppdataDropped(t *testing.T) {
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+truncatedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
@@ -192,7 +192,7 @@ func TestSigValidation_DisabledByConfig(t *testing.T) {
|
||||
falseVal := false
|
||||
cfg := &Config{ValidateSignatures: &falseVal}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// With validation disabled, tampered packet should be stored
|
||||
var txCount int
|
||||
@@ -225,7 +225,7 @@ func TestSigValidation_DropCounterIncrements(t *testing.T) {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"Obs"}`)
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
}
|
||||
|
||||
if store.Stats.SignatureDrops.Load() != 3 {
|
||||
@@ -258,7 +258,7 @@ func TestSigValidation_LogContainsFields(t *testing.T) {
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"MyObserver"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var hash, reason, obsID, obsName, pubkey, nodeName string
|
||||
err = store.db.QueryRow("SELECT hash, reason, observer_id, observer_name, node_pubkey, node_name FROM dropped_packets LIMIT 1").
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/perfio"
|
||||
@@ -67,7 +66,7 @@ func writeStatsAtomic(path string, b []byte) error {
|
||||
// O_NOFOLLOW: if tmp is a pre-existing symlink, openat fails with ELOOP
|
||||
// instead of clobbering the symlink target. O_TRUNC zeroes existing
|
||||
// regular-file content. 0o600 — no need for world-readable.
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0o600)
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|oNoFollow, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import "syscall"
|
||||
|
||||
// oNoFollow is syscall.O_NOFOLLOW on platforms that define it (all non-Windows targets).
|
||||
// On Windows this constant does not exist; see stats_file_nofollow_windows.go.
|
||||
const oNoFollow = syscall.O_NOFOLLOW
|
||||
@@ -0,0 +1,8 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
// oNoFollow is 0 on Windows: O_NOFOLLOW is not defined in the Windows syscall
|
||||
// package. The ingestor is only deployed on Linux where the flag is enforced;
|
||||
// on Windows the flag is a no-op so the binary compiles and tests run.
|
||||
const oNoFollow = 0
|
||||
+187
-15
@@ -15,6 +15,10 @@ import (
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// routeTypeTransport covers FLOOD (0) and DIRECT (3) route types — packets
|
||||
// that carry transport-level scoping via Code1.
|
||||
const routeTypeTransportSQL = "route_type IN (0, 3)"
|
||||
|
||||
// DB wraps a read-only connection to the MeshCore SQLite database.
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
@@ -22,6 +26,8 @@ type DB struct {
|
||||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||||
hasResolvedPath bool // observations table has resolved_path column
|
||||
hasObsRawHex bool // observations table has raw_hex column (#881)
|
||||
hasScopeName bool // transmissions.scope_name column exists (#899)
|
||||
hasDefaultScope bool // nodes.default_scope column exists (#899)
|
||||
|
||||
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
|
||||
channelsCacheMu sync.Mutex
|
||||
@@ -83,6 +89,52 @@ func (db *DB) detectSchema() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
txRows, err := db.conn.Query("PRAGMA table_info(transmissions)")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer txRows.Close()
|
||||
for txRows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if txRows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||||
if colName == "scope_name" {
|
||||
db.hasScopeName = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodeRows, err := db.conn.Query("PRAGMA table_info(nodes)")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer nodeRows.Close()
|
||||
for nodeRows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if nodeRows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil {
|
||||
if colName == "default_scope" {
|
||||
db.hasDefaultScope = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nodeSelectCols returns the SELECT column list for nodes queries.
|
||||
// When hasDefaultScope is true, default_scope is appended as the last column.
|
||||
func (db *DB) nodeSelectCols() string {
|
||||
cols := "public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert"
|
||||
if db.hasDefaultScope {
|
||||
cols += ", default_scope"
|
||||
}
|
||||
return cols
|
||||
}
|
||||
|
||||
// transmissionBaseSQL returns the SELECT columns and JOIN clause for transmission-centric queries.
|
||||
@@ -108,6 +160,9 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) {
|
||||
)
|
||||
LEFT JOIN observers obs2 ON obs2.id = o.observer_id`
|
||||
}
|
||||
if db.hasScopeName {
|
||||
selectCols += `, t.scope_name`
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -118,13 +173,18 @@ func (db *DB) scanTransmissionRow(rows *sql.Rows) map[string]interface{} {
|
||||
var rawHex, hash, firstSeen, decodedJSON, observerID, observerName, observerIATA, pathJSON, direction sql.NullString
|
||||
var routeType, payloadType sql.NullInt64
|
||||
var snr, rssi sql.NullFloat64
|
||||
var scopeName sql.NullString
|
||||
|
||||
if err := rows.Scan(&id, &rawHex, &hash, &firstSeen, &routeType, &payloadType, &decodedJSON,
|
||||
&observationCount, &observerID, &observerName, &observerIATA, &snr, &rssi, &pathJSON, &direction); err != nil {
|
||||
scanArgs := []interface{}{&id, &rawHex, &hash, &firstSeen, &routeType, &payloadType, &decodedJSON,
|
||||
&observationCount, &observerID, &observerName, &observerIATA, &snr, &rssi, &pathJSON, &direction}
|
||||
if db.hasScopeName {
|
||||
scanArgs = append(scanArgs, &scopeName)
|
||||
}
|
||||
if err := rows.Scan(scanArgs...); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
m := map[string]interface{}{
|
||||
"id": id,
|
||||
"raw_hex": nullStr(rawHex),
|
||||
"hash": nullStr(hash),
|
||||
@@ -142,6 +202,10 @@ func (db *DB) scanTransmissionRow(rows *sql.Rows) map[string]interface{} {
|
||||
"path_json": nullStr(pathJSON),
|
||||
"direction": nullStr(direction),
|
||||
}
|
||||
if db.hasScopeName {
|
||||
m["scope_name"] = nullStr(scopeName)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Node represents a row from the nodes table.
|
||||
@@ -829,7 +893,7 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB
|
||||
var total int
|
||||
db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total)
|
||||
|
||||
querySQL := fmt.Sprintf("SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", w, order)
|
||||
querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", db.nodeSelectCols(), w, order)
|
||||
qArgs := append(args, limit, offset)
|
||||
|
||||
rows, err := db.conn.Query(querySQL, qArgs...)
|
||||
@@ -840,7 +904,7 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB
|
||||
|
||||
nodes := make([]map[string]interface{}, 0)
|
||||
for rows.Next() {
|
||||
n := scanNodeRow(rows)
|
||||
n := db.scanNodeRow(rows)
|
||||
if n != nil {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
@@ -855,8 +919,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
rows, err := db.conn.Query(`SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert
|
||||
FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?`,
|
||||
rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?", db.nodeSelectCols()),
|
||||
"%"+query+"%", query+"%", limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -865,7 +928,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er
|
||||
|
||||
nodes := make([]map[string]interface{}, 0)
|
||||
for rows.Next() {
|
||||
n := scanNodeRow(rows)
|
||||
n := db.scanNodeRow(rows)
|
||||
if n != nil {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
@@ -894,8 +957,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro
|
||||
}
|
||||
}
|
||||
rows, err := db.conn.Query(
|
||||
`SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert
|
||||
FROM nodes WHERE public_key LIKE ? LIMIT 2`,
|
||||
fmt.Sprintf("SELECT %s FROM nodes WHERE public_key LIKE ? LIMIT 2", db.nodeSelectCols()),
|
||||
prefix+"%",
|
||||
)
|
||||
if err != nil {
|
||||
@@ -905,7 +967,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro
|
||||
var first map[string]interface{}
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
n := scanNodeRow(rows)
|
||||
n := db.scanNodeRow(rows)
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
@@ -924,13 +986,13 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro
|
||||
|
||||
// GetNodeByPubkey returns a single node.
|
||||
func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) {
|
||||
rows, err := db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c, foreign_advert FROM nodes WHERE public_key = ?", pubkey)
|
||||
rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = ?", db.nodeSelectCols()), pubkey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
if rows.Next() {
|
||||
return scanNodeRow(rows), nil
|
||||
return db.scanNodeRow(rows), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1958,7 +2020,9 @@ func scanPacketRow(rows *sql.Rows) map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
func scanNodeRow(rows *sql.Rows) map[string]interface{} {
|
||||
// scanNodeRow scans a node row. When hasDefaultScope is true the SELECT must
|
||||
// include default_scope as the last column.
|
||||
func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} {
|
||||
var pk string
|
||||
var name, role, lastSeen, firstSeen sql.NullString
|
||||
var lat, lon sql.NullFloat64
|
||||
@@ -1966,8 +2030,13 @@ func scanNodeRow(rows *sql.Rows) map[string]interface{} {
|
||||
var batteryMv sql.NullInt64
|
||||
var temperatureC sql.NullFloat64
|
||||
var foreign sql.NullInt64
|
||||
var defaultScope sql.NullString
|
||||
|
||||
if err := rows.Scan(&pk, &name, &role, &lat, &lon, &lastSeen, &firstSeen, &advertCount, &batteryMv, &temperatureC, &foreign); err != nil {
|
||||
scanArgs := []interface{}{&pk, &name, &role, &lat, &lon, &lastSeen, &firstSeen, &advertCount, &batteryMv, &temperatureC, &foreign}
|
||||
if db.hasDefaultScope {
|
||||
scanArgs = append(scanArgs, &defaultScope)
|
||||
}
|
||||
if err := rows.Scan(scanArgs...); err != nil {
|
||||
return nil
|
||||
}
|
||||
m := map[string]interface{}{
|
||||
@@ -1994,6 +2063,9 @@ func scanNodeRow(rows *sql.Rows) map[string]interface{} {
|
||||
} else {
|
||||
m["temperature_c"] = nil
|
||||
}
|
||||
if db.hasDefaultScope {
|
||||
m["default_scope"] = nullStr(defaultScope)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -2435,6 +2507,106 @@ func (db *DB) GetSignatureDropCount() int64 {
|
||||
return count
|
||||
}
|
||||
|
||||
func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) {
|
||||
if !db.hasScopeName {
|
||||
return nil, fmt.Errorf("scope_name column not present — run ingestor to apply migrations")
|
||||
}
|
||||
|
||||
var since string
|
||||
var bucketExpr string
|
||||
switch window {
|
||||
case "1h":
|
||||
since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339)
|
||||
// 5-minute buckets
|
||||
bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`
|
||||
case "7d":
|
||||
since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
// 6-hour buckets
|
||||
bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`
|
||||
default: // "24h"
|
||||
window = "24h"
|
||||
since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
// 1-hour buckets
|
||||
bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)`
|
||||
}
|
||||
|
||||
resp := &ScopeStatsResponse{Window: window}
|
||||
|
||||
// Summary counts
|
||||
row := db.conn.QueryRow(`
|
||||
SELECT
|
||||
COUNT(*) AS transport_total,
|
||||
COUNT(scope_name) AS scoped,
|
||||
COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped,
|
||||
COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope
|
||||
FROM transmissions
|
||||
WHERE ` + routeTypeTransportSQL + ` AND first_seen >= ?
|
||||
`, since)
|
||||
if err := row.Scan(
|
||||
&resp.Summary.TransportTotal,
|
||||
&resp.Summary.Scoped,
|
||||
&resp.Summary.Unscoped,
|
||||
&resp.Summary.UnknownScope,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scope summary query: %w", err)
|
||||
}
|
||||
|
||||
// Per-region counts (named regions only)
|
||||
rows, err := db.conn.Query(`
|
||||
SELECT scope_name, COUNT(*) AS cnt
|
||||
FROM transmissions
|
||||
WHERE ` + routeTypeTransportSQL + ` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ?
|
||||
GROUP BY scope_name
|
||||
ORDER BY cnt DESC
|
||||
`, since)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope byRegion query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var rc ScopeRegionCount
|
||||
if rows.Scan(&rc.Name, &rc.Count) == nil {
|
||||
resp.ByRegion = append(resp.ByRegion, rc)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("scope byRegion iteration: %w", err)
|
||||
}
|
||||
if resp.ByRegion == nil {
|
||||
resp.ByRegion = []ScopeRegionCount{}
|
||||
}
|
||||
|
||||
// Time series
|
||||
tsQuery := fmt.Sprintf(`
|
||||
SELECT %s AS bucket,
|
||||
COUNT(scope_name) AS scoped,
|
||||
SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped
|
||||
FROM transmissions
|
||||
WHERE ` + routeTypeTransportSQL + ` AND first_seen >= ?
|
||||
GROUP BY bucket
|
||||
ORDER BY bucket
|
||||
`, bucketExpr)
|
||||
tsRows, err := db.conn.Query(tsQuery, since)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope timeseries query: %w", err)
|
||||
}
|
||||
defer tsRows.Close()
|
||||
for tsRows.Next() {
|
||||
var pt ScopeTimePoint
|
||||
if tsRows.Scan(&pt.T, &pt.Scoped, &pt.Unscoped) == nil {
|
||||
resp.TimeSeries = append(resp.TimeSeries, pt)
|
||||
}
|
||||
}
|
||||
if err := tsRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("scope timeseries iteration: %w", err)
|
||||
}
|
||||
if resp.TimeSeries == nil {
|
||||
resp.TimeSeries = []ScopeTimePoint{}
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// NodeForGeoPrune holds the minimal fields needed for geo-filter pruning.
|
||||
type NodeForGeoPrune struct {
|
||||
PubKey string
|
||||
|
||||
@@ -1469,6 +1469,73 @@ func TestOpenDBInvalidPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDetectSchemaScopeName verifies that OpenDB sets hasScopeName and
|
||||
// hasDefaultScope via the real detectSchema path when the columns are present.
|
||||
// The existing ScopeStats tests set these flags manually — this test ensures
|
||||
// the flag-setting code itself is covered.
|
||||
func TestDetectSchemaScopeName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "detect.db")
|
||||
|
||||
// Create file-based DB with the scope_name and default_scope columns.
|
||||
conn, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.SetMaxOpenConns(1)
|
||||
if _, err := conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil {
|
||||
conn.Close()
|
||||
t.Fatalf("create transmissions: %v", err)
|
||||
}
|
||||
if _, err := conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, default_scope TEXT)`); err != nil {
|
||||
conn.Close()
|
||||
t.Fatalf("create nodes: %v", err)
|
||||
}
|
||||
if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`); err != nil {
|
||||
conn.Close()
|
||||
t.Fatalf("create observations: %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDB: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if !db.hasScopeName {
|
||||
t.Error("hasScopeName should be true when scope_name column exists")
|
||||
}
|
||||
if !db.hasDefaultScope {
|
||||
t.Error("hasDefaultScope should be true when default_scope column exists")
|
||||
}
|
||||
|
||||
// Verify the flags stay false when the columns are absent.
|
||||
dbPath2 := filepath.Join(dir, "detect2.db")
|
||||
conn2, err := sql.Open("sqlite", dbPath2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn2.SetMaxOpenConns(1)
|
||||
conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`)
|
||||
conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`)
|
||||
conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`)
|
||||
conn2.Close()
|
||||
|
||||
db2, err := OpenDB(dbPath2)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDB2: %v", err)
|
||||
}
|
||||
defer db2.Close()
|
||||
|
||||
if db2.hasScopeName {
|
||||
t.Error("hasScopeName should be false when scope_name column is absent")
|
||||
}
|
||||
if db2.hasDefaultScope {
|
||||
t.Error("hasDefaultScope should be false when default_scope column is absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetChannelMessagesObserverFallback(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
@@ -2148,6 +2215,56 @@ func TestPerObservationRawHexEnrich(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetScopeStats(t *testing.T) {
|
||||
conn, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("sql.Open: %v", err)
|
||||
}
|
||||
conn.SetMaxOpenConns(1)
|
||||
db := &DB{conn: conn}
|
||||
defer db.conn.Close()
|
||||
|
||||
// Create minimal schema
|
||||
db.conn.Exec(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER,
|
||||
payload_type INTEGER, payload_version INTEGER, decoded_json TEXT,
|
||||
scope_name TEXT DEFAULT NULL
|
||||
)`)
|
||||
// Manually set hasScopeName since we bypassed the detector
|
||||
db.hasScopeName = true
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
// Transport scoped, known region
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', ?, 0, '#belgium')`, now)
|
||||
// Transport scoped, unknown
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', ?, 0, '')`, now)
|
||||
// Transport unscoped (NULL)
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', ?, 0, NULL)`, now)
|
||||
// Non-transport (should not count)
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', ?, 1, NULL)`, now)
|
||||
|
||||
stats, err := db.GetScopeStats("24h")
|
||||
if err != nil {
|
||||
t.Fatalf("GetScopeStats: %v", err)
|
||||
}
|
||||
if stats.Summary.TransportTotal != 3 {
|
||||
t.Errorf("TransportTotal = %d, want 3", stats.Summary.TransportTotal)
|
||||
}
|
||||
if stats.Summary.Scoped != 2 {
|
||||
t.Errorf("Scoped = %d, want 2", stats.Summary.Scoped)
|
||||
}
|
||||
if stats.Summary.Unscoped != 1 {
|
||||
t.Errorf("Unscoped = %d, want 1", stats.Summary.Unscoped)
|
||||
}
|
||||
if stats.Summary.UnknownScope != 1 {
|
||||
t.Errorf("UnknownScope = %d, want 1", stats.Summary.UnknownScope)
|
||||
}
|
||||
if len(stats.ByRegion) != 1 || stats.ByRegion[0].Name != "#belgium" || stats.ByRegion[0].Count != 1 {
|
||||
t.Errorf("ByRegion = %+v, want [{#belgium 1}]", stats.ByRegion)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadIndexesRelayHopsFromResolvedPath verifies that after Load(), relay
|
||||
// nodes that appear only in resolved_path (not in decoded_json) are indexed
|
||||
// in byNode. Regression for #692: indexByNode was called before observations
|
||||
|
||||
+69
-22
@@ -34,13 +34,6 @@ type Server struct {
|
||||
commit string
|
||||
buildTime string
|
||||
|
||||
// Guards s.cfg.GeoFilter — read by ingest/handler goroutines, written by PUT handler
|
||||
cfgMu sync.RWMutex
|
||||
|
||||
// Serializes concurrent PUT /api/config/geo-filter disk writes so requests
|
||||
// can't race on the .tmp file or interleave disk/memory updates.
|
||||
saveMu sync.Mutex
|
||||
|
||||
// Cached runtime.MemStats to avoid stop-the-world pauses on every health check
|
||||
memStatsMu sync.Mutex
|
||||
memStatsCache runtime.MemStats
|
||||
@@ -51,29 +44,26 @@ type Server struct {
|
||||
statsCache *StatsResponse
|
||||
statsCachedAt time.Time
|
||||
|
||||
// Guards s.cfg.GeoFilter — read by ingest/handler goroutines, written by PUT handler
|
||||
cfgMu sync.RWMutex
|
||||
|
||||
// Serializes concurrent PUT /api/config/geo-filter disk writes so requests
|
||||
// can't race on the .tmp file or interleave disk/memory updates.
|
||||
saveMu sync.Mutex
|
||||
|
||||
// Neighbor affinity graph (lazy-built, cached with TTL)
|
||||
neighborMu sync.Mutex
|
||||
neighborGraph *NeighborGraph
|
||||
|
||||
// Cached /api/scope-stats response — per-window, recomputed at most once every 30s
|
||||
scopeStatsMu sync.Mutex
|
||||
scopeStatsCache map[string]*ScopeStatsResponse
|
||||
scopeStatsCachedAt map[string]time.Time
|
||||
|
||||
// Router reference for OpenAPI spec generation
|
||||
router *mux.Router
|
||||
}
|
||||
|
||||
// getGeoFilter returns a pointer to the current geo_filter config under read lock.
|
||||
// Callers MUST NOT mutate the returned struct.
|
||||
func (s *Server) getGeoFilter() *GeoFilterConfig {
|
||||
s.cfgMu.RLock()
|
||||
defer s.cfgMu.RUnlock()
|
||||
return s.cfg.GeoFilter
|
||||
}
|
||||
|
||||
// setGeoFilter atomically swaps the geo_filter config; used by PUT /api/config/geo-filter.
|
||||
func (s *Server) setGeoFilter(gf *GeoFilterConfig) {
|
||||
s.cfgMu.Lock()
|
||||
defer s.cfgMu.Unlock()
|
||||
s.cfg.GeoFilter = gf
|
||||
}
|
||||
|
||||
// PerfStats tracks request performance.
|
||||
type PerfStats struct {
|
||||
mu sync.Mutex
|
||||
@@ -126,6 +116,21 @@ func (s *Server) getMemStats() runtime.MemStats {
|
||||
return s.memStatsCache
|
||||
}
|
||||
|
||||
// getGeoFilter returns a pointer to the current geo_filter config under read lock.
|
||||
// Callers MUST NOT mutate the returned struct.
|
||||
func (s *Server) getGeoFilter() *GeoFilterConfig {
|
||||
s.cfgMu.RLock()
|
||||
defer s.cfgMu.RUnlock()
|
||||
return s.cfg.GeoFilter
|
||||
}
|
||||
|
||||
// setGeoFilter atomically swaps the geo_filter config; used by PUT /api/config/geo-filter.
|
||||
func (s *Server) setGeoFilter(gf *GeoFilterConfig) {
|
||||
s.cfgMu.Lock()
|
||||
defer s.cfgMu.Unlock()
|
||||
s.cfg.GeoFilter = gf
|
||||
}
|
||||
|
||||
// RegisterRoutes sets up all HTTP routes on the given router.
|
||||
func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
s.router = r
|
||||
@@ -153,6 +158,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
// System endpoints
|
||||
r.HandleFunc("/api/health", s.handleHealth).Methods("GET")
|
||||
r.HandleFunc("/api/stats", s.handleStats).Methods("GET")
|
||||
r.HandleFunc("/api/scope-stats", s.handleScopeStats).Methods("GET")
|
||||
r.HandleFunc("/api/perf", s.handlePerf).Methods("GET")
|
||||
r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET")
|
||||
r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET")
|
||||
@@ -2944,6 +2950,47 @@ func (s *Server) handleDroppedPackets(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, results)
|
||||
}
|
||||
|
||||
func (s *Server) handleScopeStats(w http.ResponseWriter, r *http.Request) {
|
||||
const scopeStatsTTL = 30 * time.Second
|
||||
|
||||
window := r.URL.Query().Get("window")
|
||||
if window == "" {
|
||||
window = "24h"
|
||||
}
|
||||
if window != "1h" && window != "24h" && window != "7d" {
|
||||
writeError(w, 400, "window must be 1h, 24h, or 7d")
|
||||
return
|
||||
}
|
||||
|
||||
s.scopeStatsMu.Lock()
|
||||
if s.scopeStatsCache != nil {
|
||||
if cached, ok := s.scopeStatsCache[window]; ok && time.Since(s.scopeStatsCachedAt[window]) < scopeStatsTTL {
|
||||
s.scopeStatsMu.Unlock()
|
||||
writeJSON(w, cached)
|
||||
return
|
||||
}
|
||||
}
|
||||
s.scopeStatsMu.Unlock()
|
||||
|
||||
resp, err := s.db.GetScopeStats(window)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
s.scopeStatsMu.Lock()
|
||||
if s.scopeStatsCache == nil {
|
||||
s.scopeStatsCache = make(map[string]*ScopeStatsResponse)
|
||||
s.scopeStatsCachedAt = make(map[string]time.Time)
|
||||
}
|
||||
s.scopeStatsCache[window] = resp
|
||||
s.scopeStatsCachedAt[window] = time.Now()
|
||||
s.scopeStatsMu.Unlock()
|
||||
|
||||
writeJSON(w, resp)
|
||||
}
|
||||
|
||||
// handlePruneGeoFilter identifies (dry_run=true, default) or enqueues (confirm=true)
|
||||
// deletion of nodes whose GPS coordinates fall outside the currently configured
|
||||
// geo_filter. Nodes with no GPS fix are always kept. Requires geo_filter to be
|
||||
|
||||
@@ -4005,6 +4005,101 @@ func TestPacketDetailPrefersStoreOverDB(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleScopeStats(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil {
|
||||
t.Fatalf("add scope_name column: %v", err)
|
||||
}
|
||||
srv.db.hasScopeName = true
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
// 2 scoped (known region), 1 unknown-scoped (empty string), 1 unscoped (NULL)
|
||||
rows := []struct {
|
||||
hash string
|
||||
scope string
|
||||
route int
|
||||
}{
|
||||
{"h1", "#belgium", 0},
|
||||
{"h2", "#belgium", 3},
|
||||
{"h3", "", 0}, // transport-scoped, no region match
|
||||
{"h4_null", "", 0}, // will be inserted with NULL scope_name
|
||||
}
|
||||
for i, r := range rows {
|
||||
var scopeArg interface{} = r.scope
|
||||
if i == 3 {
|
||||
scopeArg = nil // unscoped (NULL)
|
||||
}
|
||||
if _, err := srv.db.conn.Exec(
|
||||
`INSERT INTO transmissions (raw_hex,hash,first_seen,route_type,payload_type,scope_name) VALUES (?,?,?,?,5,?)`,
|
||||
"aa", r.hash, now, r.route, scopeArg,
|
||||
); err != nil {
|
||||
t.Fatalf("seed row %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scope-stats?window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.handleScopeStats(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp ScopeStatsResponse
|
||||
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if resp.Window != "24h" {
|
||||
t.Errorf("window = %q, want 24h", resp.Window)
|
||||
}
|
||||
if resp.Summary.TransportTotal != 4 {
|
||||
t.Errorf("transportTotal = %d, want 4", resp.Summary.TransportTotal)
|
||||
}
|
||||
if resp.Summary.Scoped != 3 { // 2 named + 1 unknown-scoped (empty string, non-NULL)
|
||||
t.Errorf("scoped = %d, want 3", resp.Summary.Scoped)
|
||||
}
|
||||
if resp.Summary.Unscoped != 1 {
|
||||
t.Errorf("unscoped = %d, want 1", resp.Summary.Unscoped)
|
||||
}
|
||||
if resp.Summary.UnknownScope != 1 {
|
||||
t.Errorf("unknownScope = %d, want 1", resp.Summary.UnknownScope)
|
||||
}
|
||||
if len(resp.ByRegion) != 1 || resp.ByRegion[0].Name != "#belgium" || resp.ByRegion[0].Count != 2 {
|
||||
t.Errorf("byRegion = %v, want [{#belgium 2}]", resp.ByRegion)
|
||||
}
|
||||
if resp.TimeSeries == nil {
|
||||
t.Error("timeSeries is nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleScopeStatsInvalidWindow(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil {
|
||||
t.Fatalf("add scope_name column: %v", err)
|
||||
}
|
||||
srv.db.hasScopeName = true
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scope-stats?window=invalid", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.handleScopeStats(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("status = %d, want 400", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleScopeStatsNoColumn(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
// hasScopeName stays false (not set)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/scope-stats?window=24h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.handleScopeStats(w, req)
|
||||
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Errorf("status = %d, want 500", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// --- geo-filter write-back tests ---
|
||||
|
||||
func setupGeoFilterServer(t *testing.T, apiKey string) (*Server, *mux.Router, string) {
|
||||
@@ -4447,4 +4542,3 @@ func TestGetNodesForGeoPrune(t *testing.T) {
|
||||
// TestDeleteNodesByPubkeys was removed in PR #738 follow-up: the DELETE has
|
||||
// been relocated to the ingestor (cmd/ingestor/prune_geofilter.go). End-to-end
|
||||
// coverage of the prune flow now lives in cmd/ingestor/*_test.go.
|
||||
|
||||
|
||||
@@ -90,6 +90,33 @@ type StatsResponse struct {
|
||||
GoSysMB float64 `json:"goSysMB"` // runtime.MemStats.Sys (total Go-managed)
|
||||
}
|
||||
|
||||
// ─── Scope Stats ───────────────────────────────────────────────────────────────
|
||||
|
||||
type ScopeStatsSummary struct {
|
||||
TransportTotal int `json:"transportTotal"`
|
||||
Scoped int `json:"scoped"`
|
||||
Unscoped int `json:"unscoped"`
|
||||
UnknownScope int `json:"unknownScope"`
|
||||
}
|
||||
|
||||
type ScopeRegionCount struct {
|
||||
Name string `json:"name"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type ScopeTimePoint struct {
|
||||
T string `json:"t"`
|
||||
Scoped int `json:"scoped"`
|
||||
Unscoped int `json:"unscoped"`
|
||||
}
|
||||
|
||||
type ScopeStatsResponse struct {
|
||||
Window string `json:"window"`
|
||||
Summary ScopeStatsSummary `json:"summary"`
|
||||
ByRegion []ScopeRegionCount `json:"byRegion"`
|
||||
TimeSeries []ScopeTimePoint `json:"timeSeries"`
|
||||
}
|
||||
|
||||
// ─── Health ────────────────────────────────────────────────────────────────────
|
||||
|
||||
type MemoryStats struct {
|
||||
|
||||
@@ -263,6 +263,11 @@
|
||||
"_comment_mqttSources": "Each source connects to an MQTT broker. topics: what to subscribe to. iataFilter: only ingest packets from these regions (optional).",
|
||||
"_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.",
|
||||
"_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.",
|
||||
"hashRegions": [
|
||||
"#belgium",
|
||||
"#eu"
|
||||
],
|
||||
"_comment_hashRegions": "Region names for scope matching on transport-route packets. Key = SHA256('#name')[:16]. Add any region names used by nodes in your network.",
|
||||
"_comment_defaultRegion": "IATA code shown by default in region filters.",
|
||||
"_comment_mapDefaults": "Initial map center [lat, lon] and zoom level.",
|
||||
"_comment_regions": "IATA code → display name mapping for the region filter UI. Each key is a 3-letter IATA code that an observer is tagged with (resolved priority: MQTT payload `region` field > topic-derived region > mqttSources.region). Observers without an IATA tag will not appear under any region filter — only under 'All Regions'. The region filter dropdown shows one entry per code listed here PLUS any extra IATA codes the server discovers from observers at runtime (so you can omit codes here and they will still be selectable, just labelled with the bare IATA code instead of a friendly name). Selecting 'All Regions' (or no region) returns results from every observer including those with no IATA tag; selecting one or more codes restricts results to packets observed by observers tagged with those codes. The reserved value 'All' (case-insensitive) is treated as 'no filter' on the server, so the URL ?region=All behaves identically to omitting the param. Issue #770.",
|
||||
|
||||
+62
-1
@@ -40,6 +40,7 @@
|
||||
- [GET /api/analytics/hash-sizes](#get-apianalyticshash-sizes)
|
||||
- [GET /api/analytics/subpaths](#get-apianalyticssubpaths)
|
||||
- [GET /api/analytics/subpath-detail](#get-apianalyticssubpath-detail)
|
||||
- [GET /api/scope-stats](#get-apiscope-stats)
|
||||
- [GET /api/resolve-hops](#get-apiresolve-hops)
|
||||
- [GET /api/traces/:hash](#get-apitraceshash)
|
||||
- [GET /api/config/theme](#get-apiconfigtheme)
|
||||
@@ -308,7 +309,8 @@ Paginated node list with filtering.
|
||||
"hash_size": number | null, // latest hash size (1–3 bytes)
|
||||
"hash_size_inconsistent": boolean, // true if flip-flopping
|
||||
"hash_sizes_seen": [number] | undefined, // present only if >1 unique size seen
|
||||
"last_heard": string (ISO) | undefined // from in-memory packets or path relay
|
||||
"last_heard": string (ISO) | undefined, // from in-memory packets or path relay
|
||||
"default_scope": string | null | undefined // Most recently observed transport scope for this node. null = never observed transport-scoped, "" = observed scoped but no configured region matched, "#name" = matched region. Only present when ingestor has applied the nodes_default_scope_v1 migration.
|
||||
}
|
||||
],
|
||||
"total": number, // total matching count (before pagination)
|
||||
@@ -1456,6 +1458,65 @@ Detailed stats for a specific subpath.
|
||||
|
||||
---
|
||||
|
||||
## GET /api/scope-stats
|
||||
|
||||
Scope-based packet statistics over a time window. Requires ingestor `scope_name_v1` migration to have run.
|
||||
|
||||
### Query Parameters
|
||||
|
||||
| Param | Type | Default | Description |
|
||||
|----------|--------|---------|------------------------------------------------|
|
||||
| `window` | string | `24h` | Time window: `1h`, `24h`, `7d` |
|
||||
|
||||
### Response `200`
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"window": string, // echoed window ("1h", "24h", or "7d")
|
||||
"summary": {
|
||||
"transportTotal": number, // scoped + unscoped transport-route packets
|
||||
"scoped": number, // Code1 ≠ 0000 (named + unknown regions)
|
||||
"unscoped": number, // transport-route with Code1 = 0000
|
||||
"unknownScope": number // scoped but no configured region matched (subset of scoped)
|
||||
},
|
||||
"byRegion": [
|
||||
{ "name": string, "count": number } // region name and packet count
|
||||
],
|
||||
"timeSeries": [
|
||||
{ "t": string (ISO), "scoped": number, "unscoped": number } // bucket timestamps and counts
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- `transportTotal` = `scoped` + `unscoped` (only route_type 0 or 3 packets)
|
||||
- `scoped` = packets with Code1 ≠ 0000
|
||||
- `unscoped` = transport-route packets with Code1 = 0000
|
||||
- `unknownScope` = scoped packets that did not match any configured region name
|
||||
- Time-series bucket size depends on window:
|
||||
- `1h` window → 5-minute buckets
|
||||
- `24h` window → 1-hour buckets
|
||||
- `7d` window → 6-hour buckets
|
||||
- Cached 30 seconds
|
||||
|
||||
> **Note:** On deployments with pre-existing data, `unscoped` will be inflated until the async startup backfill completes, because transport-route rows inserted before the `scope_name_v1` migration ran have `scope_name = NULL` and are indistinguishable from Code1=0000 rows. The backfill goroutine populates them at startup but may take several minutes on large databases.
|
||||
|
||||
### Response `400`
|
||||
|
||||
```json
|
||||
{ "error": "window must be 1h, 24h, or 7d" }
|
||||
```
|
||||
|
||||
### Response `500` Internal Server Error
|
||||
|
||||
`scope_name` column does not exist (ingestor has not run migrations yet):
|
||||
|
||||
```json
|
||||
{ "error": "scope_name column not present — run ingestor to apply migrations" }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## GET /api/resolve-hops
|
||||
|
||||
Resolve path hop hex prefixes to node names with regional disambiguation.
|
||||
|
||||
@@ -1,674 +0,0 @@
|
||||
# Deep Linking P1 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make P1 UI states in nodes, packets, and channels URL-addressable so they survive refresh and can be shared.
|
||||
|
||||
**Architecture:** Each page reads URL params from `location.hash.split('?')[1]` on init (router strips query string before passing `routeParam`, so pages must read `location.hash` directly). State changes call `history.replaceState` to keep the URL in sync. localStorage remains the fallback default; URL params override when present.
|
||||
|
||||
**Tech Stack:** Vanilla JS (ES5/6), browser History API, URLSearchParams
|
||||
|
||||
---
|
||||
|
||||
## Files Changed
|
||||
|
||||
| File | Changes |
|
||||
|---|---|
|
||||
| `public/region-filter.js` | Add `setSelected(codesArray)`, track `_container` for re-render |
|
||||
| `public/nodes.js` | Read `?tab=`/`?search=` on init; `updateNodesUrl()` on tab/search change; expose `buildNodesQuery` on `window` |
|
||||
| `public/packets.js` | Read `?timeWindow=`/`?region=` on init; `updatePacketsUrl()` on timeWindow/region change; expose `buildPacketsUrl` on `window` |
|
||||
| `public/channels.js` | Read `?node=` on init; update URL in `showNodeDetail`/`closeNodeDetail` |
|
||||
| `test-frontend-helpers.js` | Add unit tests for `buildNodesQuery` and `buildPacketsUrl` |
|
||||
| `test-e2e-playwright.js` | Add Playwright tests: tab URL persistence, timeWindow URL persistence |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Add `setSelected` to RegionFilter
|
||||
|
||||
**Files:**
|
||||
- Modify: `public/region-filter.js`
|
||||
|
||||
- [ ] **Step 1: Write the failing unit test**
|
||||
|
||||
Add to `test-frontend-helpers.js` before the `// ===== SUMMARY =====` line:
|
||||
|
||||
```javascript
|
||||
// ===== REGION-FILTER.JS: setSelected =====
|
||||
console.log('\n=== region-filter.js: setSelected ===');
|
||||
{
|
||||
const ctx = makeSandbox();
|
||||
ctx.fetch = () => Promise.resolve({ json: () => Promise.resolve({ 'US-SFO': 'San Jose', 'US-LAX': 'Los Angeles' }) });
|
||||
loadInCtx(ctx, 'public/region-filter.js');
|
||||
|
||||
const RF = ctx.RegionFilter;
|
||||
RF.init(document.createElement('div'));
|
||||
|
||||
test('setSelected sets region codes', async () => {
|
||||
await RF.init(document.createElement('div'));
|
||||
RF.setSelected(['US-SFO', 'US-LAX']);
|
||||
assert.strictEqual(RF.getRegionParam(), 'US-SFO,US-LAX');
|
||||
});
|
||||
|
||||
test('setSelected with null clears selection', async () => {
|
||||
await RF.init(document.createElement('div'));
|
||||
RF.setSelected(['US-SFO']);
|
||||
RF.setSelected(null);
|
||||
assert.strictEqual(RF.getRegionParam(), '');
|
||||
});
|
||||
|
||||
test('setSelected with empty array clears selection', async () => {
|
||||
await RF.init(document.createElement('div'));
|
||||
RF.setSelected(['US-SFO']);
|
||||
RF.setSelected([]);
|
||||
assert.strictEqual(RF.getRegionParam(), '');
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -A2 "setSelected"
|
||||
```
|
||||
|
||||
Expected: `❌ setSelected sets region codes: RF.setSelected is not a function`
|
||||
|
||||
- [ ] **Step 3: Add `_container` tracking and `setSelected` to region-filter.js**
|
||||
|
||||
In `region-filter.js`, add `var _container = null;` after the existing module-level vars (after line 9 `var _listeners = [];`):
|
||||
|
||||
```javascript
|
||||
var _listeners = [];
|
||||
var _container = null; // ← add this line
|
||||
var _loaded = false;
|
||||
```
|
||||
|
||||
In `initFilter`, save the container:
|
||||
|
||||
```javascript
|
||||
async function initFilter(container, opts) {
|
||||
_container = container; // ← add this line
|
||||
if (opts && opts.dropdown) container._forceDropdown = true;
|
||||
await fetchRegions();
|
||||
render(container);
|
||||
}
|
||||
```
|
||||
|
||||
Add `setSelected` function before `// Expose globally`:
|
||||
|
||||
```javascript
|
||||
/** Override selected regions (e.g. from URL param). Persists to localStorage and re-renders. */
|
||||
function setSelected(codesArray) {
|
||||
_selected = (codesArray && codesArray.length > 0) ? new Set(codesArray) : null;
|
||||
saveToStorage();
|
||||
if (_container) render(_container);
|
||||
}
|
||||
```
|
||||
|
||||
Add `setSelected` to the public API object:
|
||||
|
||||
```javascript
|
||||
window.RegionFilter = {
|
||||
init: initFilter,
|
||||
render: render,
|
||||
getSelected: getSelected,
|
||||
getRegionParam: getRegionParam,
|
||||
regionQueryString: regionQueryString,
|
||||
onChange: onChange,
|
||||
offChange: offChange,
|
||||
fetchRegions: fetchRegions,
|
||||
setSelected: setSelected, // ← add this line
|
||||
};
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -E "(setSelected|FAIL|passed|failed)"
|
||||
```
|
||||
|
||||
Expected: 3 passing `setSelected` tests, overall pass.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add public/region-filter.js test-frontend-helpers.js
|
||||
git commit -m "feat: add RegionFilter.setSelected for URL param initialization (#536)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: nodes.js — tab and search deep linking
|
||||
|
||||
**Files:**
|
||||
- Modify: `public/nodes.js`
|
||||
- Test: `test-frontend-helpers.js`
|
||||
- Test: `test-e2e-playwright.js`
|
||||
|
||||
- [ ] **Step 1: Write the unit test (add to test-frontend-helpers.js)**
|
||||
|
||||
Add before the `// ===== SUMMARY =====` line:
|
||||
|
||||
```javascript
|
||||
// ===== NODES.JS: buildNodesQuery =====
|
||||
console.log('\n=== nodes.js: buildNodesQuery ===');
|
||||
{
|
||||
const ctx = makeSandbox();
|
||||
loadInCtx(ctx, 'public/roles.js');
|
||||
loadInCtx(ctx, 'public/app.js');
|
||||
|
||||
// Provide required globals for nodes.js IIFE to execute
|
||||
ctx.registerPage = () => {};
|
||||
ctx.RegionFilter = { init: () => Promise.resolve(), onChange: () => () => {}, offChange: () => {}, getSelected: () => null, getRegionParam: () => '' };
|
||||
ctx.onWS = () => {};
|
||||
ctx.offWS = () => {};
|
||||
ctx.debouncedOnWS = () => () => {};
|
||||
ctx.invalidateApiCache = () => {};
|
||||
ctx.favStar = () => '';
|
||||
ctx.bindFavStars = () => {};
|
||||
ctx.getFavorites = () => [];
|
||||
ctx.isFavorite = () => false;
|
||||
ctx.connectWS = () => {};
|
||||
ctx.HopResolver = { init: () => {}, resolve: () => ({}), ready: () => false };
|
||||
ctx.initTabBar = () => {};
|
||||
ctx.debounce = (fn) => fn;
|
||||
ctx.copyToClipboard = () => {};
|
||||
ctx.api = () => Promise.resolve({});
|
||||
ctx.escapeHtml = (s) => s;
|
||||
ctx.timeAgo = () => '';
|
||||
ctx.formatTimestampWithTooltip = () => '';
|
||||
ctx.getTimestampMode = () => 'ago';
|
||||
ctx.CLIENT_TTL = {};
|
||||
ctx.qrcode = null;
|
||||
|
||||
try {
|
||||
const src = fs.readFileSync('public/nodes.js', 'utf8');
|
||||
vm.runInContext(src, ctx);
|
||||
for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k];
|
||||
} catch (e) {
|
||||
console.log(' ⚠️ nodes.js sandbox load failed:', e.message.slice(0, 120));
|
||||
}
|
||||
|
||||
const buildNodesQuery = ctx.buildNodesQuery;
|
||||
|
||||
if (buildNodesQuery) {
|
||||
test('buildNodesQuery: all tab + no search = empty', () => {
|
||||
assert.strictEqual(buildNodesQuery('all', ''), '');
|
||||
});
|
||||
test('buildNodesQuery: repeater tab only', () => {
|
||||
assert.strictEqual(buildNodesQuery('repeater', ''), '?tab=repeater');
|
||||
});
|
||||
test('buildNodesQuery: search only (all tab)', () => {
|
||||
assert.strictEqual(buildNodesQuery('all', 'foo'), '?search=foo');
|
||||
});
|
||||
test('buildNodesQuery: tab + search combined', () => {
|
||||
assert.strictEqual(buildNodesQuery('companion', 'bar'), '?tab=companion&search=bar');
|
||||
});
|
||||
test('buildNodesQuery: null search treated as empty', () => {
|
||||
assert.strictEqual(buildNodesQuery('all', null), '');
|
||||
});
|
||||
test('buildNodesQuery: sensor tab', () => {
|
||||
assert.strictEqual(buildNodesQuery('sensor', ''), '?tab=sensor');
|
||||
});
|
||||
} else {
|
||||
console.log(' ⚠️ buildNodesQuery not exposed — skipping');
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails (or skips)**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -A3 "buildNodesQuery"
|
||||
```
|
||||
|
||||
Expected: `⚠️ buildNodesQuery not exposed — skipping`
|
||||
|
||||
- [ ] **Step 3: Add URL param reading and helpers to nodes.js**
|
||||
|
||||
**3a.** Add `buildNodesQuery` and `updateNodesUrl` functions inside the nodes.js IIFE, after the `TABS` definition (around line 86, before `function renderNodeTimestampHtml`):
|
||||
|
||||
```javascript
|
||||
function buildNodesQuery(tab, searchStr) {
|
||||
var parts = [];
|
||||
if (tab && tab !== 'all') parts.push('tab=' + encodeURIComponent(tab));
|
||||
if (searchStr) parts.push('search=' + encodeURIComponent(searchStr));
|
||||
return parts.length ? '?' + parts.join('&') : '';
|
||||
}
|
||||
window.buildNodesQuery = buildNodesQuery;
|
||||
|
||||
function updateNodesUrl() {
|
||||
history.replaceState(null, '', '#/nodes' + buildNodesQuery(activeTab, search));
|
||||
}
|
||||
```
|
||||
|
||||
**3b.** In the list-view branch of `init` (after the `return;` that ends the full-screen block at line 317), add URL param reading before `app.innerHTML`:
|
||||
|
||||
```javascript
|
||||
// Read URL params for list view (router strips query string from routeParam)
|
||||
const _listUrlParams = new URLSearchParams(location.hash.split('?')[1] || '');
|
||||
const _urlTab = _listUrlParams.get('tab');
|
||||
const _urlSearch = _listUrlParams.get('search');
|
||||
if (_urlTab && TABS.some(function(t) { return t.key === _urlTab; })) activeTab = _urlTab;
|
||||
if (_urlSearch) search = _urlSearch;
|
||||
|
||||
app.innerHTML = `<div class="nodes-page">
|
||||
```
|
||||
|
||||
**3c.** After `app.innerHTML = ...` (after the closing backtick at line ~330), populate the search input:
|
||||
|
||||
```javascript
|
||||
if (search) {
|
||||
var _si = document.getElementById('nodeSearch');
|
||||
if (_si) _si.value = search;
|
||||
}
|
||||
```
|
||||
|
||||
**3d.** In the search input event listener (around line 335), add `updateNodesUrl()`:
|
||||
|
||||
```javascript
|
||||
document.getElementById('nodeSearch').addEventListener('input', debounce(e => {
|
||||
search = e.target.value;
|
||||
updateNodesUrl();
|
||||
loadNodes();
|
||||
}, 250));
|
||||
```
|
||||
|
||||
**3e.** In the tab click handler inside `renderLeft` (around line 875), add `updateNodesUrl()`:
|
||||
|
||||
```javascript
|
||||
btn.addEventListener('click', () => { activeTab = btn.dataset.tab; updateNodesUrl(); loadNodes(); });
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run unit tests**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -E "(buildNodesQuery|✅|❌)" | grep -v "helpers"
|
||||
```
|
||||
|
||||
Expected: 6 passing `buildNodesQuery` tests.
|
||||
|
||||
- [ ] **Step 5: Write Playwright test (add to test-e2e-playwright.js)**
|
||||
|
||||
Add before the closing `await browser.close()` line:
|
||||
|
||||
```javascript
|
||||
// --- Group: Deep linking (#536) ---
|
||||
|
||||
// Test: nodes tab deep link
|
||||
await test('Nodes tab deep link restores active tab', async () => {
|
||||
await page.goto(BASE + '#/nodes?tab=repeater', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('.node-tab', { timeout: 8000 });
|
||||
const activeTab = await page.$('.node-tab.active');
|
||||
assert(activeTab, 'No active tab found');
|
||||
const tabText = await activeTab.textContent();
|
||||
assert(tabText.includes('Repeater'), `Expected Repeater tab active, got: ${tabText}`);
|
||||
const url = page.url();
|
||||
assert(url.includes('tab=repeater'), `URL should contain tab=repeater, got: ${url}`);
|
||||
});
|
||||
|
||||
// Test: nodes tab click updates URL
|
||||
await test('Nodes tab click updates URL', async () => {
|
||||
await page.goto(BASE + '#/nodes', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('.node-tab', { timeout: 8000 });
|
||||
const roomTab = await page.$('.node-tab[data-tab="room"]');
|
||||
if (roomTab) {
|
||||
await roomTab.click();
|
||||
await page.waitForTimeout(300);
|
||||
const url = page.url();
|
||||
assert(url.includes('tab=room'), `URL should contain tab=room after click, got: ${url}`);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run full test suite**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js
|
||||
```
|
||||
|
||||
Expected: all tests pass.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add public/nodes.js test-frontend-helpers.js test-e2e-playwright.js
|
||||
git commit -m "feat: deep link nodes tab and search query (#536)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: packets.js — timeWindow and region deep linking
|
||||
|
||||
**Files:**
|
||||
- Modify: `public/packets.js`
|
||||
- Test: `test-frontend-helpers.js`
|
||||
- Test: `test-e2e-playwright.js`
|
||||
|
||||
> Depends on Task 1 (RegionFilter.setSelected).
|
||||
|
||||
- [ ] **Step 1: Write the unit test**
|
||||
|
||||
Add to `test-frontend-helpers.js` before `// ===== SUMMARY =====`:
|
||||
|
||||
```javascript
|
||||
// ===== PACKETS.JS: buildPacketsUrl =====
|
||||
console.log('\n=== packets.js: buildPacketsUrl ===');
|
||||
{
|
||||
// Test the pure helper function
|
||||
// (loaded via packets.js after it exposes window.buildPacketsUrl)
|
||||
const ctx = makeSandbox();
|
||||
loadInCtx(ctx, 'public/roles.js');
|
||||
loadInCtx(ctx, 'public/app.js');
|
||||
|
||||
ctx.registerPage = () => {};
|
||||
ctx.RegionFilter = { init: () => Promise.resolve(), onChange: () => () => {}, offChange: () => {}, getSelected: () => null, getRegionParam: () => '', setSelected: () => {} };
|
||||
ctx.onWS = () => {};
|
||||
ctx.offWS = () => {};
|
||||
ctx.debouncedOnWS = () => () => {};
|
||||
ctx.invalidateApiCache = () => {};
|
||||
ctx.api = () => Promise.resolve({});
|
||||
ctx.observerMap = new Map();
|
||||
ctx.getParsedPath = () => [];
|
||||
ctx.getParsedDecoded = () => ({});
|
||||
ctx.clearParsedCache = () => {};
|
||||
ctx.escapeHtml = (s) => s;
|
||||
ctx.timeAgo = () => '';
|
||||
ctx.formatTimestampWithTooltip = () => '';
|
||||
ctx.getTimestampMode = () => 'ago';
|
||||
ctx.copyToClipboard = () => {};
|
||||
ctx.CLIENT_TTL = {};
|
||||
ctx.debounce = (fn) => fn;
|
||||
ctx.initTabBar = () => {};
|
||||
|
||||
try {
|
||||
const src = fs.readFileSync('public/packet-helpers.js', 'utf8');
|
||||
vm.runInContext(src, ctx);
|
||||
for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k];
|
||||
const src2 = fs.readFileSync('public/packets.js', 'utf8');
|
||||
vm.runInContext(src2, ctx);
|
||||
for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k];
|
||||
} catch (e) {
|
||||
console.log(' ⚠️ packets.js sandbox load failed:', e.message.slice(0, 120));
|
||||
}
|
||||
|
||||
const buildPacketsUrl = ctx.buildPacketsUrl;
|
||||
|
||||
if (buildPacketsUrl) {
|
||||
test('buildPacketsUrl: default (15min, no region) = bare #/packets', () => {
|
||||
assert.strictEqual(buildPacketsUrl(15, ''), '#/packets');
|
||||
});
|
||||
test('buildPacketsUrl: non-default timeWindow', () => {
|
||||
assert.strictEqual(buildPacketsUrl(60, ''), '#/packets?timeWindow=60');
|
||||
});
|
||||
test('buildPacketsUrl: region only', () => {
|
||||
assert.strictEqual(buildPacketsUrl(15, 'US-SFO'), '#/packets?region=US-SFO');
|
||||
});
|
||||
test('buildPacketsUrl: timeWindow + region', () => {
|
||||
assert.strictEqual(buildPacketsUrl(30, 'US-SFO,US-LAX'), '#/packets?timeWindow=30®ion=US-SFO%2CUS-LAX');
|
||||
});
|
||||
test('buildPacketsUrl: timeWindow=0 treated as default', () => {
|
||||
assert.strictEqual(buildPacketsUrl(0, ''), '#/packets');
|
||||
});
|
||||
} else {
|
||||
console.log(' ⚠️ buildPacketsUrl not exposed — skipping');
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it skips**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -A2 "buildPacketsUrl"
|
||||
```
|
||||
|
||||
Expected: `⚠️ buildPacketsUrl not exposed — skipping`
|
||||
|
||||
- [ ] **Step 3: Add helpers and URL param reading to packets.js**
|
||||
|
||||
**3a.** Add `buildPacketsUrl` and `updatePacketsUrl` inside the packets.js IIFE, after the existing constants at the top (around line 36, after `let showHexHashes`):
|
||||
|
||||
```javascript
|
||||
function buildPacketsUrl(timeWindowMin, regionParam) {
|
||||
var parts = [];
|
||||
if (timeWindowMin && timeWindowMin !== 15) parts.push('timeWindow=' + timeWindowMin);
|
||||
if (regionParam) parts.push('region=' + encodeURIComponent(regionParam));
|
||||
return '#/packets' + (parts.length ? '?' + parts.join('&') : '');
|
||||
}
|
||||
window.buildPacketsUrl = buildPacketsUrl;
|
||||
|
||||
function updatePacketsUrl() {
|
||||
history.replaceState(null, '', buildPacketsUrl(savedTimeWindowMin, RegionFilter.getRegionParam()));
|
||||
}
|
||||
```
|
||||
|
||||
**3b.** In the `init` function (around line 263), add URL param reading after the existing `routeParam`/`directObsId` parsing and before `app.innerHTML`:
|
||||
|
||||
```javascript
|
||||
// Read URL params for filter state (router strips query from routeParam; read from location.hash)
|
||||
var _initUrlParams = new URLSearchParams(location.hash.split('?')[1] || '');
|
||||
var _urlTimeWindow = Number(_initUrlParams.get('timeWindow'));
|
||||
if (Number.isFinite(_urlTimeWindow) && _urlTimeWindow > 0) {
|
||||
savedTimeWindowMin = _urlTimeWindow;
|
||||
localStorage.setItem('meshcore-time-window', String(_urlTimeWindow));
|
||||
}
|
||||
var _urlRegion = _initUrlParams.get('region');
|
||||
if (_urlRegion) {
|
||||
RegionFilter.setSelected(_urlRegion.split(',').filter(Boolean));
|
||||
}
|
||||
|
||||
app.innerHTML = `<div class="split-layout detail-collapsed">
|
||||
```
|
||||
|
||||
**3c.** In the time window change handler (around line 865), add `updatePacketsUrl()`:
|
||||
|
||||
```javascript
|
||||
fTimeWindow.addEventListener('change', () => {
|
||||
savedTimeWindowMin = Number(fTimeWindow.value);
|
||||
if (!Number.isFinite(savedTimeWindowMin) || savedTimeWindowMin <= 0) savedTimeWindowMin = 15;
|
||||
localStorage.setItem('meshcore-time-window', fTimeWindow.value);
|
||||
updatePacketsUrl();
|
||||
loadPackets();
|
||||
});
|
||||
```
|
||||
|
||||
**3d.** In the RegionFilter.onChange callback (around line 719), add `updatePacketsUrl()`:
|
||||
|
||||
```javascript
|
||||
RegionFilter.onChange(function() { updatePacketsUrl(); loadPackets(); });
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run unit tests**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js 2>&1 | grep -E "(buildPacketsUrl|✅|❌)" | grep -v "helpers"
|
||||
```
|
||||
|
||||
Expected: 5 passing `buildPacketsUrl` tests.
|
||||
|
||||
- [ ] **Step 5: Write Playwright test (add to test-e2e-playwright.js, inside the deep-linking group)**
|
||||
|
||||
```javascript
|
||||
// Test: packets timeWindow deep link
|
||||
await test('Packets timeWindow deep link restores dropdown', async () => {
|
||||
await page.goto(BASE + '#/packets?timeWindow=60', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#fTimeWindow', { timeout: 8000 });
|
||||
const val = await page.$eval('#fTimeWindow', el => el.value);
|
||||
assert(val === '60', `Expected timeWindow dropdown = 60, got: ${val}`);
|
||||
const url = page.url();
|
||||
assert(url.includes('timeWindow=60'), `URL should still contain timeWindow=60, got: ${url}`);
|
||||
});
|
||||
|
||||
// Test: timeWindow change updates URL
|
||||
await test('Packets timeWindow change updates URL', async () => {
|
||||
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#fTimeWindow', { timeout: 8000 });
|
||||
await page.selectOption('#fTimeWindow', '30');
|
||||
await page.waitForTimeout(300);
|
||||
const url = page.url();
|
||||
assert(url.includes('timeWindow=30'), `URL should contain timeWindow=30 after change, got: ${url}`);
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run full test suite**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js
|
||||
```
|
||||
|
||||
Expected: all tests pass.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add public/packets.js test-frontend-helpers.js test-e2e-playwright.js
|
||||
git commit -m "feat: deep link packets timeWindow and region filter (#536)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: channels.js — node panel deep linking
|
||||
|
||||
**Files:**
|
||||
- Modify: `public/channels.js`
|
||||
|
||||
No unit tests needed for this task — the URL manipulation is side-effectful (DOM + History API). Playwright tests cover it.
|
||||
|
||||
- [ ] **Step 1: Write the Playwright test (add to test-e2e-playwright.js, inside the deep-linking group)**
|
||||
|
||||
```javascript
|
||||
// Test: channels selected channel survives refresh (already implemented, verify it still works)
|
||||
await test('Channels channel selection is URL-addressable', async () => {
|
||||
await page.goto(BASE + '#/channels', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('.ch-item', { timeout: 8000 }).catch(() => null);
|
||||
const firstChannel = await page.$('.ch-item');
|
||||
if (firstChannel) {
|
||||
await firstChannel.click();
|
||||
await page.waitForTimeout(500);
|
||||
const url = page.url();
|
||||
assert(url.includes('#/channels/') || url.includes('#/channels'), `URL should reflect channel selection, got: ${url}`);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update `showNodeDetail` to write `?node=` to the URL**
|
||||
|
||||
In `channels.js`, in `showNodeDetail` (around line 171), add the URL update right after `selectedNode = name;`:
|
||||
|
||||
```javascript
|
||||
async function showNodeDetail(name) {
|
||||
_nodePanelTrigger = document.activeElement;
|
||||
if (_focusTrapCleanup) { _focusTrapCleanup(); _focusTrapCleanup = null; }
|
||||
const node = await lookupNode(name);
|
||||
selectedNode = name;
|
||||
var _chBase = selectedHash ? '#/channels/' + encodeURIComponent(selectedHash) : '#/channels';
|
||||
history.replaceState(null, '', _chBase + '?node=' + encodeURIComponent(name));
|
||||
|
||||
let panel = document.getElementById('chNodePanel');
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update `closeNodeDetail` to strip `?node=` from the URL**
|
||||
|
||||
In `closeNodeDetail` (around line 232), add URL restore right after `selectedNode = null;`:
|
||||
|
||||
```javascript
|
||||
function closeNodeDetail() {
|
||||
if (_focusTrapCleanup) { _focusTrapCleanup(); _focusTrapCleanup = null; }
|
||||
const panel = document.getElementById('chNodePanel');
|
||||
if (panel) panel.classList.remove('open');
|
||||
selectedNode = null;
|
||||
var _chRestoreUrl = selectedHash ? '#/channels/' + encodeURIComponent(selectedHash) : '#/channels';
|
||||
history.replaceState(null, '', _chRestoreUrl);
|
||||
if (_nodePanelTrigger && typeof _nodePanelTrigger.focus === 'function') {
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Read `?node=` on init and auto-open panel**
|
||||
|
||||
In `channels.js` `init` (line 316), add URL param reading at the very top of the function (before `app.innerHTML`):
|
||||
|
||||
```javascript
|
||||
function init(app, routeParam) {
|
||||
var _initUrlParams = new URLSearchParams(location.hash.split('?')[1] || '');
|
||||
var _pendingNode = _initUrlParams.get('node');
|
||||
|
||||
app.innerHTML = `<div class="ch-layout">
|
||||
```
|
||||
|
||||
Then update the `loadChannels().then(...)` call (around line 350) to auto-open the node panel:
|
||||
|
||||
```javascript
|
||||
loadChannels().then(async function () {
|
||||
if (routeParam) await selectChannel(routeParam);
|
||||
if (_pendingNode) showNodeDetail(_pendingNode);
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run full test suite**
|
||||
|
||||
```bash
|
||||
node test-frontend-helpers.js
|
||||
```
|
||||
|
||||
Expected: all tests pass (no channels unit tests, but regression tests still pass).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add public/channels.js
|
||||
git commit -m "feat: deep link channels node panel via ?node= (#536)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Run E2E Playwright tests
|
||||
|
||||
- [ ] **Step 1: Start the local server**
|
||||
|
||||
```bash
|
||||
cd cmd/server && go run . &
|
||||
```
|
||||
|
||||
Wait for it to be ready (check `http://localhost:3000`).
|
||||
|
||||
- [ ] **Step 2: Run Playwright tests**
|
||||
|
||||
```bash
|
||||
node test-e2e-playwright.js
|
||||
```
|
||||
|
||||
Expected: all tests pass including the new deep-linking group.
|
||||
|
||||
- [ ] **Step 3: If any deep-linking test fails, debug**
|
||||
|
||||
Common failures:
|
||||
- Selector `.node-tab.active` not found: check that nodes.js correctly reads `?tab=` from URL before rendering
|
||||
- `#fTimeWindow` value wrong: check that `savedTimeWindowMin` is overridden before the DOM is built
|
||||
- URL doesn't update: check `history.replaceState` calls in the change handlers
|
||||
|
||||
- [ ] **Step 4: Final commit (if any fixes needed)**
|
||||
|
||||
```bash
|
||||
git add public/nodes.js public/packets.js public/channels.js
|
||||
git commit -m "fix: deep linking E2E adjustments (#536)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage check:**
|
||||
- ✅ P1: Nodes role tab → Task 2
|
||||
- ✅ P1: Packets time window → Task 3
|
||||
- ✅ P1: Packets region filter → Task 3 (depends on Task 1)
|
||||
- ✅ P1: Channels selected channel → Already implemented via `#/channels/{hash}` (verified in channels.js init line 351)
|
||||
- ✅ P1: Channels node panel → Task 4
|
||||
- ✅ P2+ items → explicitly out of scope per issue
|
||||
|
||||
**Architecture note:** The router in `app.js` strips the query string at line 422 (`const route = hash.split('?')[0]`) before computing `basePage` and `routeParam`. Therefore `#/nodes?tab=repeater` gives `routeParam=null` (not `?tab=repeater`). All pages must read URL params from `location.hash` directly, not from `routeParam`. This is the established pattern in `analytics.js` and `nodes.js` (section scroll).
|
||||
|
||||
**Placeholder scan:** No TBDs, no "implement later", all code blocks complete. ✅
|
||||
|
||||
**Type consistency:**
|
||||
- `buildNodesQuery(tab, searchStr)` — used consistently in `updateNodesUrl()` and in tests ✅
|
||||
- `buildPacketsUrl(timeWindowMin, regionParam)` — used consistently in `updatePacketsUrl()` and in tests ✅
|
||||
- `RegionFilter.setSelected(codesArray)` — defined in Task 1, used in Task 3 ✅
|
||||
@@ -1,204 +0,0 @@
|
||||
# Scope Stats Page — Design Spec
|
||||
|
||||
**Issue**: Kpa-clawbot/CoreScope#899
|
||||
**Date**: 2026-04-23
|
||||
**Branch target**: `master`
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Add a dedicated **Scopes** page showing scope/region statistics for MeshCore transport-route packets. Scope filtering in MeshCore uses `TRANSPORT_FLOOD` (route_type 0) and `TRANSPORT_DIRECT` (route_type 3) packets that carry two 16-bit transport codes. Code1 ≠ `0000` means the packet is region-scoped.
|
||||
|
||||
Feature 3 from the issue (default scope per client via advert) is **not implemented** — the advert format has no scope field in the current firmware.
|
||||
|
||||
---
|
||||
|
||||
## How Scopes Work (Firmware)
|
||||
|
||||
Transport code derivation (authoritative source: `meshcore-dev/MeshCore`):
|
||||
|
||||
```
|
||||
key = SHA256("#regionname")[:16] // TransportKeyStore::getAutoKeyFor
|
||||
Code1 = HMAC-SHA256(key, type || payload) // TransportKey::calcTransportCode, 2-byte output
|
||||
```
|
||||
|
||||
Code1 is a **per-message** HMAC — the same region produces a different Code1 for every message. Identifying a region from Code1 requires knowing the region name in advance and recomputing the HMAC.
|
||||
|
||||
`Code1 = 0000` is the "no scope" sentinel (also `FFFF` is reserved). Packets with route_type 1 or 2 (plain FLOOD/DIRECT) carry no transport codes.
|
||||
|
||||
---
|
||||
|
||||
## Config
|
||||
|
||||
Add `hashRegions` to the ingestor `Config` struct in `cmd/ingestor/config.go`, mirroring `hashChannels`:
|
||||
|
||||
```json
|
||||
"hashRegions": ["#belgium", "#eu", "#brussels"]
|
||||
```
|
||||
|
||||
Normalization (same rules as `hashChannels`):
|
||||
- Trim whitespace
|
||||
- Prepend `#` if missing
|
||||
- Skip empty entries
|
||||
|
||||
---
|
||||
|
||||
## Ingestor Changes
|
||||
|
||||
### Key derivation (`loadRegionKeys`)
|
||||
|
||||
```go
|
||||
func loadRegionKeys(cfg *Config) map[string][]byte {
|
||||
// key = first 16 bytes of SHA256("#regionname")
|
||||
}
|
||||
```
|
||||
|
||||
Returns `map[string][]byte` (region name → 16-byte HMAC key). Called once at startup, stored on the `Store`.
|
||||
|
||||
### Decoder: expose raw payload bytes
|
||||
|
||||
Add `PayloadRaw []byte` to `DecodedPacket` in `cmd/ingestor/decoder.go`. Populated from the raw `buf` slice at the payload offset — zero-copy slice, no allocation. This is the **encrypted** payload bytes, matching what the firmware feeds into `calcTransportCode`.
|
||||
|
||||
### At-ingest region matching
|
||||
|
||||
In `BuildPacketData`:
|
||||
- Skip if `route_type` not in `{0, 3}` → `scope_name` stays `nil`
|
||||
- If `Code1 == "0000"` → `scope_name = nil` (unscoped transport, no scope involvement)
|
||||
- If `Code1 != "0000"` → try each region key:
|
||||
```
|
||||
HMAC-SHA256(key, payloadType_byte || PayloadRaw) → first 2 bytes as uint16
|
||||
```
|
||||
First match → `scope_name = "#regionname"`. No match → `scope_name = ""` (unknown scope).
|
||||
|
||||
Add `ScopeName *string` to `PacketData`.
|
||||
|
||||
### MQTT-sourced packets (DM / CHAN paths in main.go)
|
||||
|
||||
These are injected directly without going through `BuildPacketData`. They use `route_type = 1` (FLOOD), so they are never transport-route packets. No scope matching needed for these paths.
|
||||
|
||||
---
|
||||
|
||||
## Database
|
||||
|
||||
### Migration
|
||||
|
||||
```sql
|
||||
ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL;
|
||||
CREATE INDEX idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL;
|
||||
```
|
||||
|
||||
### Column semantics
|
||||
|
||||
| Value | Meaning |
|
||||
|-------|---------|
|
||||
| `NULL` | Either: non-transport-route packet (route_type 1/2), or transport-route with Code1=0000 |
|
||||
| `""` (empty string) | Transport-route, Code1 ≠ 0000, but no configured region matched |
|
||||
| `"#belgium"` | Matched named region |
|
||||
|
||||
The API stats queries resolve the NULL ambiguity by always filtering `route_type IN (0, 3)` first:
|
||||
- `unscoped` count = `route_type IN (0,3) AND scope_name IS NULL`
|
||||
- `scoped` count = `route_type IN (0,3) AND scope_name IS NOT NULL`
|
||||
|
||||
### Backfill
|
||||
|
||||
On migration, re-decode `raw_hex` for all rows where `route_type IN (0, 3)` and `scope_name IS NULL`. Run the same HMAC matching logic. Rows with `Code1 = 0000` remain `NULL`.
|
||||
|
||||
The backfill runs in the existing migration framework in `cmd/ingestor/db.go`. If no regions are configured, backfill is skipped.
|
||||
|
||||
---
|
||||
|
||||
## API
|
||||
|
||||
### `GET /api/scope-stats`
|
||||
|
||||
**Query param**: `window` — one of `1h`, `24h` (default), `7d`
|
||||
|
||||
**Time-series bucket sizes**:
|
||||
| Window | Bucket |
|
||||
|--------|--------|
|
||||
| `1h` | 5 min |
|
||||
| `24h` | 1 hour |
|
||||
| `7d` | 6 hours|
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"window": "24h",
|
||||
"summary": {
|
||||
"transportTotal": 1240,
|
||||
"scoped": 890,
|
||||
"unscoped": 350,
|
||||
"unknownScope": 42
|
||||
},
|
||||
"byRegion": [
|
||||
{ "name": "#belgium", "count": 612 },
|
||||
{ "name": "#eu", "count": 236 }
|
||||
],
|
||||
"timeSeries": [
|
||||
{ "t": "2026-04-23T10:00:00Z", "scoped": 45, "unscoped": 18 },
|
||||
{ "t": "2026-04-23T11:00:00Z", "scoped": 51, "unscoped": 22 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `transportTotal` = `scoped + unscoped` (transport-route packets only)
|
||||
- `scoped` = Code1 ≠ 0000 (named + unknown)
|
||||
- `unscoped` = transport-route with Code1 = 0000
|
||||
- `unknownScope` = scoped but no region name matched (subset of `scoped`)
|
||||
- `byRegion` sorted by count descending, excludes unknown
|
||||
- `timeSeries` covers the full window at the bucket granularity
|
||||
|
||||
Route: `GET /api/scope-stats` registered in `cmd/server/routes.go`.
|
||||
No auth required (same as other read endpoints).
|
||||
TTL cache: 30 seconds (heavier query than `/api/stats`).
|
||||
|
||||
---
|
||||
|
||||
## Frontend
|
||||
|
||||
### Navigation
|
||||
|
||||
Add nav link between Channels and Nodes in `public/index.html`:
|
||||
```html
|
||||
<a href="#/scopes" class="nav-link" data-route="scopes">Scopes</a>
|
||||
```
|
||||
|
||||
### `public/scopes.js`
|
||||
|
||||
Three sections on the page:
|
||||
|
||||
**1. Summary cards** (reuse existing card CSS pattern from home/analytics pages)
|
||||
- Transport total, Scoped, Unscoped, Unknown scope
|
||||
- Each card shows count + percentage of transport total
|
||||
|
||||
**2. Per-region table**
|
||||
Columns: Region, Messages, % of Scoped
|
||||
Sorted by count descending. Last row: "Unknown scope" (italic) if unknownScope > 0.
|
||||
Shows "No regions configured" message if `byRegion` is empty and `unknownScope = 0`.
|
||||
|
||||
**3. Time-series chart**
|
||||
- Window selector: `1h / 24h / 7d` (default 24h)
|
||||
- Two lines: **Scoped** (blue) and **Unscoped** (grey)
|
||||
- Uses the same lightweight canvas chart pattern as other pages (no external chart lib)
|
||||
|
||||
### Cache buster
|
||||
|
||||
`scopes.js` added to the `__BUST__` entries in `index.html` in the same commit.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit tests for `loadRegionKeys`: normalization, key bytes match firmware SHA256 derivation
|
||||
- Unit tests for HMAC matching: known Code1 value computed from firmware logic, verified against Go implementation
|
||||
- Integration test: ingest a synthetic transport-route packet with a known region, assert `scope_name` column is set correctly
|
||||
- API test: `GET /api/scope-stats` returns correct summary counts against fixture DB
|
||||
|
||||
---
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Feature 3 (default scope per client via advert) — firmware has no advert scope field
|
||||
- Drill-down from region row to filtered packet list (deferred)
|
||||
- Private regions (`$`-prefixed) — use secret keys not publicly derivable
|
||||
+167
-1
@@ -27,6 +27,10 @@
|
||||
function _stopRolesRefresh() {
|
||||
if (_rolesRefreshTimer) { clearInterval(_rolesRefreshTimer); _rolesRefreshTimer = null; }
|
||||
}
|
||||
var _scopesRefreshTimer = null;
|
||||
function _stopScopesRefresh() {
|
||||
if (_scopesRefreshTimer) { clearInterval(_scopesRefreshTimer); _scopesRefreshTimer = null; }
|
||||
}
|
||||
|
||||
// --- Status color helpers (read from CSS variables for theme support) ---
|
||||
function cssVar(name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); }
|
||||
@@ -123,6 +127,7 @@
|
||||
Placed after Clock Health (clock-skew posture is shown per-role
|
||||
inside this tab) and before Prefix Tool (utility tabs trail). -->
|
||||
<button class="tab-btn" data-tab="roles">Roles</button>
|
||||
<button class="tab-btn" data-tab="scopes">Scopes</button>
|
||||
<button class="tab-btn" data-tab="prefix-tool">Prefix Tool</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -160,6 +165,7 @@
|
||||
_currentTab = btn.dataset.tab;
|
||||
// #1085 — Roles tab owns its own 60s auto-refresh; stop it on switch.
|
||||
if (_currentTab !== 'roles') _stopRolesRefresh();
|
||||
if (_currentTab !== 'scopes') _stopScopesRefresh();
|
||||
_updateAnalyticsUrl();
|
||||
renderTab(_currentTab);
|
||||
});
|
||||
@@ -264,6 +270,7 @@
|
||||
case 'clock-health': await renderClockHealthTab(el); break;
|
||||
case 'roles': await renderRolesTab(el); break;
|
||||
case 'prefix-tool': await renderPrefixTool(el); break;
|
||||
case 'scopes': await renderScopesTab(el); break;
|
||||
}
|
||||
// Auto-apply column resizing to all analytics tables
|
||||
requestAnimationFrame(() => {
|
||||
@@ -2231,7 +2238,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
function destroy() { _stopRolesRefresh(); _analyticsData = {}; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } }
|
||||
function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = {}; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } }
|
||||
|
||||
// Expose for testing
|
||||
if (typeof window !== 'undefined') {
|
||||
@@ -3916,6 +3923,165 @@ function destroy() { _stopRolesRefresh(); _analyticsData = {}; _channelData = nu
|
||||
}
|
||||
}
|
||||
|
||||
// ===================== SCOPES =====================
|
||||
async function renderScopesTab(el) {
|
||||
var winKey = 'scopes_window';
|
||||
var selectedWindow = (typeof sessionStorage !== 'undefined' && sessionStorage.getItem(winKey)) || '24h';
|
||||
|
||||
// Fix 5: write static frame only once
|
||||
if (!el.querySelector('#scopes-cards')) {
|
||||
el.innerHTML =
|
||||
'<h3 style="margin:0 0 12px">Scope Statistics</h3>' +
|
||||
'<div style="margin-bottom:12px">' +
|
||||
['1h', '24h', '7d'].map(function(v) {
|
||||
return '<button class="tab-btn' + (selectedWindow === v ? ' active' : '') + '" data-win="' + v + '">' + v + '</button>';
|
||||
}).join('') +
|
||||
'</div>' +
|
||||
'<div id="scopes-cards" class="stats-grid" style="margin-bottom:16px"></div>' +
|
||||
'<div class="text-center text-muted" id="scopes-loading" style="padding:20px">Loading scope stats…</div>' +
|
||||
'<table class="data-table analytics-table" style="margin-bottom:8px">' +
|
||||
'<thead><tr><th>Region</th><th>Messages</th><th>% of Scoped</th></tr></thead>' +
|
||||
'<tbody id="scopes-tbody"></tbody>' +
|
||||
'</table>' +
|
||||
'<div id="scopes-chart"></div>';
|
||||
|
||||
// Attach window-button click listeners (once)
|
||||
el.querySelectorAll('[data-win]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
selectedWindow = btn.dataset.win;
|
||||
if (typeof sessionStorage !== 'undefined') sessionStorage.setItem(winKey, selectedWindow);
|
||||
el.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === selectedWindow); });
|
||||
load(selectedWindow);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function pct(n, total) {
|
||||
if (!total) return '—';
|
||||
return (n / total * 100).toFixed(1) + '%';
|
||||
}
|
||||
|
||||
async function load(w) {
|
||||
var loadingEl = document.getElementById('scopes-loading');
|
||||
if (loadingEl) loadingEl.style.display = '';
|
||||
try {
|
||||
// Fix 4: use api() instead of raw fetch()
|
||||
var data = await api('/api/scope-stats?window=' + encodeURIComponent(w), { ttl: 30000 });
|
||||
if (loadingEl) loadingEl.style.display = 'none';
|
||||
if (data.error) {
|
||||
var cardsEl2 = document.getElementById('scopes-cards');
|
||||
if (cardsEl2) cardsEl2.innerHTML = '<div class="text-center text-muted" style="padding:20px">' + esc(data.error) + '</div>';
|
||||
return;
|
||||
}
|
||||
updateData(data, w);
|
||||
} catch (err) {
|
||||
if (loadingEl) loadingEl.style.display = 'none';
|
||||
var cardsEl3 = document.getElementById('scopes-cards');
|
||||
if (cardsEl3) cardsEl3.innerHTML = '<div class="text-center" style="color:var(--status-red);padding:20px">Failed to load scope stats: ' + esc(String(err)) + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
function updateData(d, w) {
|
||||
var s = d.summary;
|
||||
var total = s.transportTotal || 0;
|
||||
|
||||
// Summary cards
|
||||
var cardsEl = document.getElementById('scopes-cards');
|
||||
if (cardsEl) {
|
||||
cardsEl.innerHTML = [
|
||||
{ label: 'Transport Total', value: total.toLocaleString(), note: '' },
|
||||
{ label: 'Scoped', value: s.scoped.toLocaleString(), note: pct(s.scoped, total) },
|
||||
{ label: 'Unscoped', value: s.unscoped.toLocaleString(), note: pct(s.unscoped, total) },
|
||||
{ label: 'Unknown Scope', value: s.unknownScope.toLocaleString(), note: pct(s.unknownScope, s.scoped) + ' of scoped' },
|
||||
].map(function(c) {
|
||||
return '<div class="stat-card"><div class="stat-value">' + c.value + '</div>' +
|
||||
'<div class="stat-label">' + c.label + '</div>' +
|
||||
(c.note ? '<div class="stat-note text-muted" style="font-size:11px">' + c.note + '</div>' : '') +
|
||||
'</div>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
// Per-region table
|
||||
var tbodyEl = document.getElementById('scopes-tbody');
|
||||
if (tbodyEl) {
|
||||
var tableBody = '';
|
||||
if (d.byRegion && d.byRegion.length) {
|
||||
tableBody = d.byRegion.map(function(r) {
|
||||
return '<tr><td><code>' + esc(r.name) + '</code></td>' +
|
||||
'<td>' + r.count.toLocaleString() + '</td>' +
|
||||
'<td>' + pct(r.count, s.scoped) + '</td></tr>';
|
||||
}).join('');
|
||||
if (s.unknownScope > 0) {
|
||||
tableBody += '<tr><td><em class="text-muted">Unknown scope</em></td>' +
|
||||
'<td>' + s.unknownScope.toLocaleString() + '</td>' +
|
||||
'<td>' + pct(s.unknownScope, s.scoped) + '</td></tr>';
|
||||
}
|
||||
} else if (s.scoped === 0) {
|
||||
tableBody = '<tr><td colspan="3" class="text-muted" style="text-align:center">No scoped messages in this window</td></tr>';
|
||||
} else {
|
||||
tableBody = '<tr><td colspan="3" class="text-muted" style="text-align:center">No regions configured — add <code>hashRegions</code> to your config</td></tr>';
|
||||
}
|
||||
tbodyEl.innerHTML = tableBody;
|
||||
}
|
||||
|
||||
// Time-series chart (two-line SVG)
|
||||
var chartEl = document.getElementById('scopes-chart');
|
||||
if (chartEl) {
|
||||
var chartHtml = '';
|
||||
if (d.timeSeries && d.timeSeries.length > 1) {
|
||||
var scopedVals = d.timeSeries.map(function(p) { return p.scoped; });
|
||||
var unscopedVals = d.timeSeries.map(function(p) { return p.unscoped; });
|
||||
var maxVal = Math.max(1, Math.max.apply(null, scopedVals.concat(unscopedVals)));
|
||||
var W = 800, H = 180, padL = 44, padT = 10, padR = 10;
|
||||
var plotW = W - padL - padR, plotH = H - 24 - padT;
|
||||
var n = d.timeSeries.length;
|
||||
|
||||
function pts(vals) {
|
||||
return vals.map(function(v, i) {
|
||||
var x = padL + i * plotW / Math.max(n - 1, 1);
|
||||
var y = padT + plotH - (v / maxVal) * plotH;
|
||||
return x.toFixed(1) + ',' + y.toFixed(1);
|
||||
}).join(' ');
|
||||
}
|
||||
|
||||
var grid = '';
|
||||
for (var gi = 0; gi <= 4; gi++) {
|
||||
var gy = padT + plotH * gi / 4;
|
||||
var gv = Math.round(maxVal * (4 - gi) / 4);
|
||||
grid += '<line x1="' + padL + '" y1="' + gy.toFixed(1) + '" x2="' + (W - padR) + '" y2="' + gy.toFixed(1) + '" stroke="var(--border)" stroke-dasharray="2"/>';
|
||||
grid += '<text x="' + (padL - 4) + '" y="' + (gy + 4).toFixed(1) + '" text-anchor="end" font-size="9" fill="var(--text-muted)">' + gv + '</text>';
|
||||
}
|
||||
|
||||
var legendX = padL + plotW - 120;
|
||||
chartHtml = '<div style="margin-top:16px">' +
|
||||
'<svg viewBox="0 0 ' + W + ' ' + H + '" style="width:100%;max-height:' + H + 'px" role="img" aria-label="Scope time series">' +
|
||||
grid +
|
||||
'<polyline points="' + pts(scopedVals) + '" fill="none" stroke="var(--accent)" stroke-width="2"/>' +
|
||||
'<polyline points="' + pts(unscopedVals) + '" fill="none" stroke="var(--text-muted)" stroke-width="1.5" stroke-dasharray="4"/>' +
|
||||
'<rect x="' + legendX + '" y="' + padT + '" width="10" height="10" fill="var(--accent)"/>' +
|
||||
'<text x="' + (legendX + 14) + '" y="' + (padT + 9) + '" font-size="10" fill="var(--text)">Scoped</text>' +
|
||||
'<rect x="' + legendX + '" y="' + (padT + 16) + '" width="10" height="10" fill="var(--text-muted)"/>' +
|
||||
'<text x="' + (legendX + 14) + '" y="' + (padT + 25) + '" font-size="10" fill="var(--text)">Unscoped</text>' +
|
||||
'</svg></div>';
|
||||
} else {
|
||||
chartHtml = '<p class="text-muted" style="font-size:0.85em;margin:12px 0 0">Insufficient data points to render chart — wait for more observations in this window.</p>';
|
||||
}
|
||||
chartEl.innerHTML = chartHtml;
|
||||
}
|
||||
}
|
||||
|
||||
load(selectedWindow);
|
||||
|
||||
// Fix 6: auto-refresh every 60s
|
||||
_stopScopesRefresh();
|
||||
_scopesRefreshTimer = setInterval(function() {
|
||||
if (_currentTab !== 'scopes') { _stopScopesRefresh(); return; }
|
||||
var cur = document.getElementById('analyticsContent');
|
||||
if (!cur) { _stopScopesRefresh(); return; }
|
||||
load(selectedWindow);
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
// #1085 — Roles tab (folded in from former /#/roles page).
|
||||
// Renders distribution of node roles + per-role clock-skew posture.
|
||||
// Auto-refreshes every 60s while the Roles tab is active (matches the
|
||||
|
||||
+1
-1
@@ -1078,7 +1078,7 @@
|
||||
padding: 2px 4px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.vcr-lcd-canvas { width: 78px; height: 18px; }
|
||||
.vcr-lcd-canvas { width: 74px; height: 18px; }
|
||||
.vcr-lcd-mode { font-size: 0.55rem; letter-spacing: 1px; }
|
||||
.vcr-lcd-pkts { font-size: 0.5rem; letter-spacing: 1px; }
|
||||
.vcr-scope-btns { order: 2; flex-shrink: 0; gap: 1px; }
|
||||
|
||||
@@ -2036,6 +2036,11 @@
|
||||
<table style="font-size:12px;width:100%;border-collapse:collapse;">
|
||||
<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Last Seen</td><td>${lastSeen}</td></tr>
|
||||
<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Adverts</td><td>${n.advert_count || 0}</td></tr>
|
||||
${'default_scope' in n ? `<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Scope</td><td>${
|
||||
n.default_scope === null ? '<span style="color:var(--text-muted)">—</span>'
|
||||
: n.default_scope === '' ? '<span style="color:var(--text-muted)">unknown scope</span>'
|
||||
: `<code style="color:var(--accent)">${escapeHtml(n.default_scope)}</code>`
|
||||
}</td></tr>` : ''}
|
||||
${hasLoc ? `<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Location</td><td>${n.lat.toFixed(5)}, ${n.lon.toFixed(5)}</td></tr>` : ''}
|
||||
${stats.avgSnr != null ? `<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Avg SNR</td><td>${stats.avgSnr.toFixed(1)} dB</td></tr>` : ''}
|
||||
${stats.avgHops != null ? `<tr><td style="color:var(--text-muted);padding:4px 8px 4px 0;">Avg Hops</td><td>${stats.avgHops.toFixed(1)}</td></tr>` : ''}
|
||||
|
||||
@@ -2923,6 +2923,7 @@
|
||||
${locationHtml ? `<dt>Location</dt><dd>${locationHtml}</dd>` : ''}
|
||||
<dt>SNR / RSSI</dt><dd>${snr != null ? snr + ' dB' : '—'} / ${rssi != null ? rssi + ' dBm' : '—'}</dd>
|
||||
<dt>Route Type</dt><dd>${routeTypeName(pkt.route_type)}</dd>
|
||||
${pkt.scope_name != null ? `<dt>Scope</dt><dd>${pkt.scope_name !== '' ? escapeHtml(pkt.scope_name) : '<span style="color:var(--text-muted)">unknown scope</span>'}</dd>` : ''}
|
||||
<dt>Payload Type</dt><dd><span class="badge badge-${payloadTypeColor(pkt.payload_type)}">${typeName}</span></dd>
|
||||
${hashSize ? `<dt>Hash Size</dt><dd>${hashSize} byte${hashSize !== 1 ? 's' : ''}</dd>` : ''}
|
||||
<dt>Timestamp</dt><dd>${renderTimestampCell(effectivePkt.timestamp)}</dd>
|
||||
|
||||
Reference in New Issue
Block a user