mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-09 17:49:44 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1a714731f0 | ||
|
|
1d449eabc7 | ||
|
|
42ff5a291b | ||
|
|
99029e41aa | ||
|
|
c99aa1dadf | ||
|
|
20843979a7 | ||
|
|
ea78581eea | ||
|
|
b5372d6f73 | ||
|
|
5afed0951b | ||
|
|
3630a32310 | ||
|
|
ff05db7367 | ||
|
|
441409203e | ||
|
|
a371d35bfd | ||
|
|
7c01a97178 | ||
|
|
f1eea9ee3c | ||
|
|
f30e6bef28 | ||
|
|
20f456da58 | ||
|
|
e31e14cae9 | ||
|
|
bb0f816a6b | ||
|
|
3f26dc7190 | ||
|
|
886aabf0ae | ||
|
|
a0fddb50aa | ||
|
|
bb09123f34 | ||
|
|
31a0a944f9 | ||
|
|
cad1f11073 | ||
|
|
7f024b7aa7 | ||
|
|
ddd18cb12f | ||
|
|
997bf190ce | ||
|
|
5ff4b75a07 | ||
|
|
2460e33f94 | ||
|
|
f701121672 | ||
|
|
d7fe24e2db | ||
|
|
a9732e64ae | ||
|
|
60be48dc5e | ||
|
|
9e90548637 | ||
|
|
a8e1cea683 | ||
|
|
bf674ebfa2 | ||
|
|
d596becca3 | ||
|
|
b9ba447046 | ||
|
|
b8846c2db2 | ||
|
|
34b8dc8961 | ||
|
|
fa3f623bd6 | ||
|
|
dfe383cc51 | ||
|
|
fa348efe2a | ||
|
|
a9a18ff051 | ||
|
|
ceea136e97 | ||
|
|
99dc4f805a | ||
|
|
ba7cd0fba7 | ||
|
|
6a648dea11 | ||
|
|
29157742eb | ||
|
|
ed19a19473 | ||
|
|
d27a7a653e | ||
|
|
0e286d85fd | ||
|
|
bffcbdaa0b | ||
|
|
3bdf72b4cf | ||
|
|
401fd070f8 | ||
|
|
1b315bf6d0 | ||
|
|
a815e70975 | ||
|
|
aa84ce1e6a | ||
|
|
2aea01f10c | ||
|
|
b7c2cb070c | ||
|
|
1de80a9eaf | ||
|
|
e6ace95059 | ||
|
|
f605d4ce7e | ||
|
|
84f03f4f41 | ||
|
|
8158631d02 | ||
|
|
14367488e2 | ||
|
|
71be54f085 |
@@ -290,6 +290,10 @@ jobs:
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Set up QEMU (arm64 runtime stage)
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
@@ -317,7 +321,7 @@ jobs:
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.docker-meta.outputs.tags }}
|
||||
labels: ${{ steps.docker-meta.outputs.labels }}
|
||||
build-args: |
|
||||
@@ -387,7 +391,10 @@ jobs:
|
||||
|
||||
- name: Deploy staging
|
||||
run: |
|
||||
# Stop old container and release memory
|
||||
# Force-remove the staging container regardless of how it was created
|
||||
# (compose-managed OR manually created via docker run)
|
||||
docker stop corescope-staging-go 2>/dev/null || true
|
||||
docker rm -f corescope-staging-go 2>/dev/null || true
|
||||
docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --timeout 30 2>/dev/null || true
|
||||
|
||||
# Wait for container to be fully gone and OS to reclaim memory (3GB limit)
|
||||
@@ -429,10 +436,11 @@ jobs:
|
||||
|
||||
- name: Smoke test staging API
|
||||
run: |
|
||||
if curl -sf http://localhost:82/api/stats | grep -q engine; then
|
||||
PORT="${STAGING_GO_HTTP_PORT:-80}"
|
||||
if curl -sf "http://localhost:${PORT}/api/stats" | grep -q engine; then
|
||||
echo "Staging verified — engine field present ✅"
|
||||
else
|
||||
echo "Staging /api/stats did not return engine field"
|
||||
echo "Staging /api/stats did not return engine field (port ${PORT})"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
+13
-7
@@ -1,19 +1,23 @@
|
||||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache build-base
|
||||
# Build stage always runs natively on the builder's arch ($BUILDPLATFORM)
|
||||
# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
|
||||
|
||||
ARG APP_VERSION=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_TIME=unknown
|
||||
# Provided by buildx for multi-arch builds
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
# Build server
|
||||
# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly)
|
||||
WORKDIR /build/server
|
||||
COPY cmd/server/go.mod cmd/server/go.sum ./
|
||||
COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
RUN go mod download
|
||||
COPY cmd/server/ ./
|
||||
RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
|
||||
# Build ingestor
|
||||
WORKDIR /build/ingestor
|
||||
@@ -22,7 +26,8 @@ COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
RUN go mod download
|
||||
COPY cmd/ingestor/ ./
|
||||
RUN go build -o /corescope-ingestor .
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -o /corescope-ingestor .
|
||||
|
||||
# Build decrypt CLI
|
||||
WORKDIR /build/decrypt
|
||||
@@ -30,7 +35,8 @@ COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./
|
||||
COPY internal/channel/ ../../internal/channel/
|
||||
RUN go mod download
|
||||
COPY cmd/decrypt/ ./
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /corescope-decrypt .
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags="-s -w" -o /corescope-decrypt .
|
||||
|
||||
# Runtime image
|
||||
FROM alpine:3.20
|
||||
|
||||
+22
-3
@@ -39,7 +39,8 @@ type Config struct {
|
||||
HashChannels []string `json:"hashChannels,omitempty"`
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
@@ -47,8 +48,9 @@ type GeoFilterConfig = geofilter.Config
|
||||
|
||||
// RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
}
|
||||
|
||||
// MetricsConfig controls observer metrics collection.
|
||||
@@ -56,6 +58,14 @@ type MetricsConfig struct {
|
||||
SampleIntervalSec int `json:"sampleIntervalSec"`
|
||||
}
|
||||
|
||||
// ShouldValidateSignatures returns true (default) unless explicitly disabled.
|
||||
func (c *Config) ShouldValidateSignatures() bool {
|
||||
if c.ValidateSignatures != nil {
|
||||
return *c.ValidateSignatures
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MetricsSampleInterval returns the configured sample interval or 300s default.
|
||||
func (c *Config) MetricsSampleInterval() int {
|
||||
if c.Metrics != nil && c.Metrics.SampleIntervalSec > 0 {
|
||||
@@ -80,6 +90,15 @@ func (c *Config) NodeDaysOrDefault() int {
|
||||
return 7
|
||||
}
|
||||
|
||||
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
|
||||
// A value of -1 means observers are never removed.
|
||||
func (c *Config) ObserverDaysOrDefault() int {
|
||||
if c.Retention != nil && c.Retention.ObserverDays != 0 {
|
||||
return c.Retention.ObserverDays
|
||||
}
|
||||
return 14
|
||||
}
|
||||
|
||||
// LoadConfig reads configuration from a JSON file, with env var overrides.
|
||||
// If the config file does not exist, sensible defaults are used (zero-config startup).
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// hmacSHA256 computes HMAC-SHA256 for test use.
|
||||
@@ -157,7 +158,7 @@ func TestHandleMessageChannelMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Alice: Hello everyone","channel_idx":3,"SNR":5.0,"RSSI":-95,"score":10,"direction":"rx","sender_timestamp":1700000000}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/2", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -217,7 +218,7 @@ func TestHandleMessageChannelMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -232,7 +233,7 @@ func TestHandleMessageChannelNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":"no sender here"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -249,7 +250,7 @@ func TestHandleMessageDirectMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Bob: Hey there","sender_timestamp":1700000000,"SNR":3.0,"rssi":-100,"Score":8,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc123", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -293,7 +294,7 @@ func TestHandleMessageDirectMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -308,7 +309,7 @@ func TestHandleMessageDirectNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: []byte(`{"text":"message with no colon"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -327,7 +328,7 @@ func TestHandleMessageUppercaseScoreDirection(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","Score":9.0,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var score *float64
|
||||
var direction *string
|
||||
@@ -348,7 +349,7 @@ func TestHandleMessageChannelLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":3.0,"rssi":-90,"Score":5,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/0", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -364,7 +365,7 @@ func TestHandleMessageDirectLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":2.0,"rssi":-85,"score":7,"direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -387,7 +388,7 @@ func TestHandleMessageAdvertWithTelemetry(t *testing.T) {
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
// Should have created transmission, node, and observer
|
||||
var txCount, nodeCount, obsCount int
|
||||
@@ -427,7 +428,7 @@ func TestHandleMessageAdvertGeoFiltered(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, gf)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{GeoFilter: gf})
|
||||
|
||||
// Geo-filtered adverts should not create nodes
|
||||
var nodeCount int
|
||||
@@ -664,7 +665,7 @@ func TestHandleMessageCorruptedAdvertNoNode(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -686,7 +687,7 @@ func TestHandleMessageNonAdvertPacket(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -863,7 +864,7 @@ func TestHandleMessageChannelLongSender(t *testing.T) {
|
||||
longText := "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -882,7 +883,7 @@ func TestHandleMessageDirectLongSender(t *testing.T) {
|
||||
longText := "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -899,7 +900,7 @@ func TestHandleMessageDirectUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"X: hi","Score":6,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/d1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -929,7 +930,7 @@ func TestHandleMessageChannelUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Y: hi","Score":4,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/5", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -960,7 +961,7 @@ func TestHandleMessageRawLowercaseScore(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","score":3.5}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var score *float64
|
||||
if err := store.db.QueryRow("SELECT score FROM observations LIMIT 1").Scan(&score); err != nil {
|
||||
@@ -979,7 +980,7 @@ func TestHandleMessageStatusNoOrigin(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs5/status",
|
||||
payload: []byte(`{"model":"L1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id = 'obs5'").Scan(&count); err != nil {
|
||||
@@ -1138,3 +1139,182 @@ func TestDecodeTraceWithPath(t *testing.T) {
|
||||
t.Errorf("flags=%v, want 3", p.TraceFlags)
|
||||
}
|
||||
}
|
||||
|
||||
// --- db.go: RemoveStaleObservers (soft-delete) ---
|
||||
|
||||
func TestRemoveStaleObservers(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert an observer with last_seen 30 days ago
|
||||
err := store.UpsertObserver("obs-old", "OldObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Override last_seen to 30 days ago
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Insert a recent observer
|
||||
err = store.UpsertObserver("obs-new", "NewObserver", "NYC", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Errorf("removed=%d, want 1", removed)
|
||||
}
|
||||
|
||||
// Observer should still be in the table (soft-delete), but marked inactive
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("observers count=%d, want 2 (soft-delete preserves row)", count)
|
||||
}
|
||||
|
||||
// Check that the old observer is marked inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 1 {
|
||||
t.Errorf("obs-old inactive=%d, want 1", inactive)
|
||||
}
|
||||
|
||||
// Check that the recent observer is still active
|
||||
var newInactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newInactive != 0 {
|
||||
t.Errorf("obs-new inactive=%d, want 0", newInactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversNone(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Errorf("removed=%d, want 0", removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversKeepForever(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert an old observer
|
||||
err := store.UpsertObserver("obs-ancient", "AncientObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// observerDays = -1 means keep forever
|
||||
removed, err := store.RemoveStaleObservers(-1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Errorf("removed=%d, want 0 (keep forever)", removed)
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("observers count=%d, want 1 (keep forever)", count)
|
||||
}
|
||||
|
||||
// Observer should NOT be marked inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 0 {
|
||||
t.Errorf("obs-ancient inactive=%d, want 0 (keep forever)", inactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversReactivation(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert and stale-mark an observer
|
||||
err := store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Errorf("removed=%d, want 1", removed)
|
||||
}
|
||||
|
||||
// Verify it's inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 1 {
|
||||
t.Errorf("inactive=%d, want 1 after soft-delete", inactive)
|
||||
}
|
||||
|
||||
// Now UpsertObserver should reactivate it
|
||||
err = store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 0 {
|
||||
t.Errorf("inactive=%d, want 0 after reactivation", inactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverDaysOrDefault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *Config
|
||||
want int
|
||||
}{
|
||||
{"nil retention", &Config{}, 14},
|
||||
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
|
||||
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
|
||||
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.ObserverDaysOrDefault()
|
||||
if got != tt.want {
|
||||
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+167
-11
@@ -22,6 +22,7 @@ type DBStats struct {
|
||||
NodeUpserts atomic.Int64
|
||||
ObserverUpserts atomic.Int64
|
||||
WriteErrors atomic.Int64
|
||||
SignatureDrops atomic.Int64
|
||||
}
|
||||
|
||||
// Store wraps the SQLite database for packet ingestion.
|
||||
@@ -110,7 +111,8 @@ func applySchema(db *sql.DB) error {
|
||||
radio TEXT,
|
||||
battery_mv INTEGER,
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen);
|
||||
@@ -195,7 +197,7 @@ func applySchema(db *sql.DB) error {
|
||||
t.created_at
|
||||
FROM observations o
|
||||
JOIN transmissions t ON t.id = o.transmission_id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0)
|
||||
`)
|
||||
if vErr != nil {
|
||||
return fmt.Errorf("packets_v view: %w", vErr)
|
||||
@@ -335,6 +337,19 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] observer_metrics timestamp index created")
|
||||
}
|
||||
|
||||
// Migration: add inactive column to observers for soft-delete retention
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding inactive column to observers...")
|
||||
_, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`)
|
||||
if err != nil {
|
||||
// Column may already exist (e.g. fresh install with schema above)
|
||||
log.Printf("[migration] observers.inactive: %v (may already exist)", err)
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`)
|
||||
log.Println("[migration] observers.inactive column added")
|
||||
}
|
||||
|
||||
// Migration: add packets_sent and packets_recv columns to observer_metrics
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
@@ -345,6 +360,54 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] packets_sent/packets_recv columns added")
|
||||
}
|
||||
|
||||
// Migration: add channel_hash column for fast channel queries (#762)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding channel_hash column to transmissions...")
|
||||
db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`)
|
||||
db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`)
|
||||
// Backfill: extract channel name for decrypted (CHAN) packets
|
||||
res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`)
|
||||
if err == nil {
|
||||
n, _ := res.RowsAffected()
|
||||
log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n)
|
||||
}
|
||||
// Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_'
|
||||
res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`)
|
||||
if err == nil {
|
||||
n, _ := res.RowsAffected()
|
||||
log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n)
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`)
|
||||
log.Println("[migration] channel_hash column added and backfilled")
|
||||
}
|
||||
|
||||
// Migration: dropped_packets table for signature validation failures (#793)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Creating dropped_packets table...")
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS dropped_packets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT,
|
||||
raw_hex TEXT,
|
||||
reason TEXT NOT NULL,
|
||||
observer_id TEXT,
|
||||
observer_name TEXT,
|
||||
node_pubkey TEXT,
|
||||
node_name TEXT,
|
||||
dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dropped_packets schema: %w", err)
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`)
|
||||
log.Println("[migration] dropped_packets table created")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -357,8 +420,8 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtInsertTransmission, err = s.db.Prepare(`
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -370,8 +433,12 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtInsertObservation, err = s.db.Prepare(`
|
||||
INSERT OR IGNORE INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp)
|
||||
INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET
|
||||
snr = COALESCE(excluded.snr, snr),
|
||||
rssi = COALESCE(excluded.rssi, rssi),
|
||||
score = COALESCE(excluded.score, score)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -481,7 +548,7 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
result, err := s.stmtInsertTransmission.Exec(
|
||||
data.RawHex, hash, now,
|
||||
data.RouteType, data.PayloadType, data.PayloadVersion,
|
||||
data.DecodedJSON,
|
||||
data.DecodedJSON, nilIfEmpty(data.ChannelHash),
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
@@ -622,10 +689,13 @@ func (s *Store) UpsertObserver(id, name, iata string, meta *ObserverMeta) error
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
} else {
|
||||
s.Stats.ObserverUpserts.Add(1)
|
||||
return err
|
||||
}
|
||||
return err
|
||||
s.Stats.ObserverUpserts.Add(1)
|
||||
|
||||
// Reactivate if this observer was previously marked inactive
|
||||
s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = ? AND inactive = 1`, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close checkpoints the WAL and closes the database.
|
||||
@@ -719,13 +789,14 @@ func (s *Store) Checkpoint() {
|
||||
|
||||
// LogStats logs current operational metrics.
|
||||
func (s *Store) LogStats() {
|
||||
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d",
|
||||
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d sig_drops=%d",
|
||||
s.Stats.TransmissionsInserted.Load(),
|
||||
s.Stats.DuplicateTransmissions.Load(),
|
||||
s.Stats.ObservationsInserted.Load(),
|
||||
s.Stats.NodeUpserts.Load(),
|
||||
s.Stats.ObserverUpserts.Load(),
|
||||
s.Stats.WriteErrors.Load(),
|
||||
s.Stats.SignatureDrops.Load(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -757,6 +828,71 @@ func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) {
|
||||
return moved, nil
|
||||
}
|
||||
|
||||
// RemoveStaleObservers marks observers that have not actively sent data in observerDays
|
||||
// as inactive (soft-delete). This preserves JOIN integrity for observations.observer_idx
|
||||
// and observer_metrics.observer_id — historical data still references the correct observer.
|
||||
// An observer must actively send data to stay listed — being seen by another node does not count.
|
||||
// observerDays <= -1 means never remove (keep forever).
|
||||
func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) {
|
||||
if observerDays <= -1 {
|
||||
return 0, nil // keep forever
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
|
||||
result, err := s.db.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mark stale observers inactive: %w", err)
|
||||
}
|
||||
removed, _ := result.RowsAffected()
|
||||
if removed > 0 {
|
||||
// Clean up orphaned metrics for now-inactive observers
|
||||
s.db.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
|
||||
log.Printf("Marked %d observer(s) as inactive (not seen in %d days)", removed, observerDays)
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// DroppedPacket holds data for a packet rejected during ingest.
|
||||
type DroppedPacket struct {
|
||||
Hash string
|
||||
RawHex string
|
||||
Reason string
|
||||
ObserverID string
|
||||
ObserverName string
|
||||
NodePubKey string
|
||||
NodeName string
|
||||
}
|
||||
|
||||
// InsertDroppedPacket records a rejected packet in the dropped_packets table.
|
||||
func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error {
|
||||
_, err := s.db.Exec(
|
||||
`INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName,
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
return fmt.Errorf("insert dropped packet: %w", err)
|
||||
}
|
||||
s.Stats.SignatureDrops.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PruneDroppedPackets removes dropped_packets older than retentionDays.
|
||||
func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) {
|
||||
if retentionDays <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339)
|
||||
result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < ?`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune dropped packets: %w", err)
|
||||
}
|
||||
n, _ := result.RowsAffected()
|
||||
if n > 0 {
|
||||
log.Printf("Pruned %d dropped packet(s) older than %d days", n, retentionDays)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// PacketData holds the data needed to insert a packet into the DB.
|
||||
type PacketData struct {
|
||||
RawHex string
|
||||
@@ -773,6 +909,15 @@ type PacketData struct {
|
||||
PayloadVersion int
|
||||
PathJSON string
|
||||
DecodedJSON string
|
||||
ChannelHash string // grouping key for channel queries (#762)
|
||||
}
|
||||
|
||||
// nilIfEmpty returns nil for empty strings (for nullable DB columns).
|
||||
func nilIfEmpty(s string) interface{} {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// MQTTPacketMessage is the JSON payload from an MQTT raw packet message.
|
||||
@@ -794,7 +939,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
|
||||
pathJSON = string(b)
|
||||
}
|
||||
|
||||
return &PacketData{
|
||||
pd := &PacketData{
|
||||
RawHex: msg.Raw,
|
||||
Timestamp: now,
|
||||
ObserverID: observerID,
|
||||
@@ -810,4 +955,15 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
|
||||
PathJSON: pathJSON,
|
||||
DecodedJSON: PayloadJSON(&decoded.Payload),
|
||||
}
|
||||
|
||||
// Populate channel_hash for fast channel queries (#762)
|
||||
if decoded.Header.PayloadType == PayloadGRP_TXT {
|
||||
if decoded.Payload.Type == "CHAN" && decoded.Payload.Channel != "" {
|
||||
pd.ChannelHash = decoded.Payload.Channel
|
||||
} else if decoded.Payload.Type == "GRP_TXT" && decoded.Payload.ChannelHashHex != "" {
|
||||
pd.ChannelHash = "enc_" + decoded.Payload.ChannelHashHex
|
||||
}
|
||||
}
|
||||
|
||||
return pd
|
||||
}
|
||||
|
||||
@@ -1882,3 +1882,89 @@ func TestExtractObserverMetaNewFields(t *testing.T) {
|
||||
t.Errorf("RecvErrors = %v, want 3", meta.RecvErrors)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInsertObservationSNRFillIn verifies that when the same observation is
|
||||
// received twice — first without SNR, then with SNR — the SNR is filled in
|
||||
// rather than silently discarded. The unique dedup index is
|
||||
// (transmission_id, observer_idx, COALESCE(path_json, '')); observer_idx must
|
||||
// be non-NULL for the conflict to fire (SQLite treats NULL != NULL).
|
||||
func TestInsertObservationSNRFillIn(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Register the observer so observer_idx is non-NULL (required for dedup).
|
||||
if err := s.UpsertObserver("pymc-obs1", "PyMC Observer", "SJC", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// First arrival: same observer, no SNR/RSSI (e.g. broker replay without RF fields).
|
||||
data1 := &PacketData{
|
||||
RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976",
|
||||
Timestamp: "2026-04-20T00:00:00Z",
|
||||
Hash: "snrfillin0001hash",
|
||||
RouteType: 1,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: nil,
|
||||
RSSI: nil,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr1, rssi1 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr1, &rssi1)
|
||||
if snr1 != nil || rssi1 != nil {
|
||||
t.Fatalf("precondition: first insert should have nil SNR/RSSI, got snr=%v rssi=%v", snr1, rssi1)
|
||||
}
|
||||
|
||||
// Second arrival: same packet, same observer, now WITH SNR/RSSI.
|
||||
snr := 10.5
|
||||
rssi := -88.0
|
||||
data2 := &PacketData{
|
||||
RawHex: data1.RawHex,
|
||||
Timestamp: data1.Timestamp,
|
||||
Hash: data1.Hash,
|
||||
RouteType: data1.RouteType,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: &snr,
|
||||
RSSI: &rssi,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr2, rssi2 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr2, &rssi2)
|
||||
if snr2 == nil || *snr2 != snr {
|
||||
t.Errorf("SNR not filled in by second arrival: got %v, want %v", snr2, snr)
|
||||
}
|
||||
if rssi2 == nil || *rssi2 != rssi {
|
||||
t.Errorf("RSSI not filled in by second arrival: got %v, want %v", rssi2, rssi)
|
||||
}
|
||||
|
||||
// Third arrival: same packet again, SNR absent — must NOT overwrite existing SNR.
|
||||
data3 := &PacketData{
|
||||
RawHex: data1.RawHex,
|
||||
Timestamp: data1.Timestamp,
|
||||
Hash: data1.Hash,
|
||||
RouteType: data1.RouteType,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: nil,
|
||||
RSSI: nil,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr3, rssi3 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr3, &rssi3)
|
||||
if snr3 == nil || *snr3 != snr {
|
||||
t.Errorf("SNR overwritten by null arrival: got %v, want %v", snr3, snr)
|
||||
}
|
||||
if rssi3 == nil || *rssi3 != rssi {
|
||||
t.Errorf("RSSI overwritten by null arrival: got %v, want %v", rssi3, rssi)
|
||||
}
|
||||
}
|
||||
|
||||
+45
-12
@@ -80,9 +80,10 @@ type TransportCodes struct {
|
||||
|
||||
// Path holds decoded path/hop information.
|
||||
type Path struct {
|
||||
HashSize int `json:"hashSize"`
|
||||
HashCount int `json:"hashCount"`
|
||||
Hops []string `json:"hops"`
|
||||
HashSize int `json:"hashSize"`
|
||||
HashCount int `json:"hashCount"`
|
||||
Hops []string `json:"hops"`
|
||||
HopsCompleted *int `json:"hopsCompleted,omitempty"`
|
||||
}
|
||||
|
||||
// AdvertFlags holds decoded advert flag bits.
|
||||
@@ -143,6 +144,7 @@ type DecodedPacket struct {
|
||||
Path Path `json:"path"`
|
||||
Payload Payload `json:"payload"`
|
||||
Raw string `json:"raw"`
|
||||
Anomaly string `json:"anomaly,omitempty"`
|
||||
}
|
||||
|
||||
func decodeHeader(b byte) Header {
|
||||
@@ -586,17 +588,35 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
payload := decodePayload(header.PayloadType, payloadBuf, channelKeys, validateSignatures)
|
||||
|
||||
// TRACE packets store hop IDs in the payload (buf[9:]) rather than the header
|
||||
// path field. The header path byte still encodes hashSize in bits 6-7, which
|
||||
// we use to split the payload path data into individual hop prefixes.
|
||||
// path field. Firmware always sends TRACE as DIRECT (route_type 2 or 3);
|
||||
// FLOOD-routed TRACEs are anomalous but handled gracefully (parsed, but
|
||||
// flagged). The TRACE flags byte (payload offset 8) encodes path_sz in
|
||||
// bits 0-1 as a power-of-two exponent: hash_bytes = 1 << path_sz.
|
||||
// NOT the header path byte's hash_size bits. The header path contains SNR
|
||||
// bytes — one per hop that actually forwarded.
|
||||
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
|
||||
// how far the trace got vs the full intended route.
|
||||
var anomaly string
|
||||
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
|
||||
// Flag anomalous routing — firmware only sends TRACE as DIRECT
|
||||
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
|
||||
anomaly = "TRACE packet with non-DIRECT routing (expected DIRECT or TRANSPORT_DIRECT)"
|
||||
}
|
||||
// The header path hops count represents SNR entries = completed hops
|
||||
hopsCompleted := path.HashCount
|
||||
pathBytes, err := hex.DecodeString(payload.PathData)
|
||||
if err == nil && path.HashSize > 0 {
|
||||
hops := make([]string, 0, len(pathBytes)/path.HashSize)
|
||||
for i := 0; i+path.HashSize <= len(pathBytes); i += path.HashSize {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+path.HashSize])))
|
||||
if err == nil && payload.TraceFlags != nil {
|
||||
// path_sz from flags byte is a power-of-two exponent per firmware:
|
||||
// hash_bytes = 1 << (flags & 0x03)
|
||||
pathSz := 1 << (*payload.TraceFlags & 0x03)
|
||||
hops := make([]string, 0, len(pathBytes)/pathSz)
|
||||
for i := 0; i+pathSz <= len(pathBytes); i += pathSz {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+pathSz])))
|
||||
}
|
||||
path.Hops = hops
|
||||
path.HashCount = len(hops)
|
||||
path.HashSize = pathSz
|
||||
path.HopsCompleted = &hopsCompleted
|
||||
}
|
||||
}
|
||||
|
||||
@@ -616,12 +636,14 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
Path: path,
|
||||
Payload: payload,
|
||||
Raw: strings.ToUpper(hexString),
|
||||
Anomaly: anomaly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
|
||||
// It hashes the header byte + payload (skipping path bytes) to produce a
|
||||
// path-independent identifier for the same transmission.
|
||||
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
|
||||
// route-independent identifier for the same logical packet. For TRACE packets,
|
||||
// path_len is included in the hash to match firmware behavior.
|
||||
func ComputeContentHash(rawHex string) string {
|
||||
buf, err := hex.DecodeString(rawHex)
|
||||
if err != nil || len(buf) < 2 {
|
||||
@@ -657,7 +679,18 @@ func ComputeContentHash(rawHex string) string {
|
||||
}
|
||||
|
||||
payload := buf[payloadStart:]
|
||||
toHash := append([]byte{headerByte}, payload...)
|
||||
|
||||
// Hash payload-type byte only (bits 2-5 of header), not the full header.
|
||||
// Firmware: SHA256(payload_type + [path_len for TRACE] + payload)
|
||||
// Using the full header caused different hashes for the same logical packet
|
||||
// when route type or version bits differed. See issue #786.
|
||||
payloadType := (headerByte >> 2) & 0x0F
|
||||
toHash := []byte{payloadType}
|
||||
if int(payloadType) == PayloadTRACE {
|
||||
// Firmware uses uint16_t path_len (2 bytes, little-endian)
|
||||
toHash = append(toHash, pathByte, 0x00)
|
||||
}
|
||||
toHash = append(toHash, payload...)
|
||||
|
||||
h := sha256.Sum256(toHash)
|
||||
return hex.EncodeToString(h[:])[:16]
|
||||
|
||||
@@ -926,6 +926,93 @@ func TestComputeContentHashLongFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashRouteTypeIndependence verifies that the same logical
|
||||
// packet produces the same content hash regardless of route type (issue #786).
|
||||
func TestComputeContentHashRouteTypeIndependence(t *testing.T) {
|
||||
// Same payload type (TXT_MSG=2, bits 2-5) with different route types.
|
||||
// Header 0x08 = route_type 0 (TRANSPORT_FLOOD), payload_type 2
|
||||
// Header 0x0A = route_type 2 (DIRECT), payload_type 2
|
||||
// Header 0x09 = route_type 1 (FLOOD), payload_type 2
|
||||
// pathByte=0x00, payload=D69FD7A5A7
|
||||
payloadHex := "D69FD7A5A7"
|
||||
|
||||
// FLOOD: header=0x09 (route_type 1), pathByte=0x00
|
||||
floodHex := "09" + "00" + payloadHex
|
||||
// DIRECT: header=0x0A (route_type 2), pathByte=0x00
|
||||
directHex := "0A" + "00" + payloadHex
|
||||
|
||||
hashFlood := ComputeContentHash(floodHex)
|
||||
hashDirect := ComputeContentHash(directHex)
|
||||
if hashFlood != hashDirect {
|
||||
t.Errorf("same payload with different route types produced different hashes: flood=%s direct=%s", hashFlood, hashDirect)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashTraceIncludesPathLen verifies TRACE packets include
|
||||
// path_len in the hash (matching firmware behavior).
|
||||
func TestComputeContentHashTraceIncludesPathLen(t *testing.T) {
|
||||
// TRACE = payload_type 0x09, so header bits 2-5 = 0x09 → header = 0x09<<2 | route=2 = 0x26
|
||||
// pathByte=0x01 (1 hop, 1-byte hash) → 1 path byte
|
||||
traceHeader1 := "26" // route=2, payload_type=9
|
||||
pathByte1 := "01"
|
||||
pathData1 := "AA"
|
||||
payload := "DEADBEEF"
|
||||
hex1 := traceHeader1 + pathByte1 + pathData1 + payload
|
||||
|
||||
// Same but pathByte=0x02 (2 hops) → 2 path bytes
|
||||
pathByte2 := "02"
|
||||
pathData2 := "AABB"
|
||||
hex2 := traceHeader1 + pathByte2 + pathData2 + payload
|
||||
|
||||
hash1 := ComputeContentHash(hex1)
|
||||
hash2 := ComputeContentHash(hex2)
|
||||
if hash1 == hash2 {
|
||||
t.Error("TRACE packets with different path_len should produce different hashes (path_len is part of hash input)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashMatchesFirmware verifies hash output matches what the
|
||||
// firmware would compute: SHA256(payload_type_byte + payload)[:16hex].
|
||||
func TestComputeContentHashMatchesFirmware(t *testing.T) {
|
||||
// header=0x0A → payload_type = (0x0A >> 2) & 0x0F = 2
|
||||
// pathByte=0x00, payload = D69FD7A5A7475DB07337749AE61FA53A4788E976
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
hash := ComputeContentHash(rawHex)
|
||||
|
||||
// Manually compute expected: SHA256(0x02 + payload_bytes)
|
||||
payloadBytes, _ := hex.DecodeString("D69FD7A5A7475DB07337749AE61FA53A4788E976")
|
||||
toHash := append([]byte{0x02}, payloadBytes...)
|
||||
expected := sha256.Sum256(toHash)
|
||||
expectedHex := hex.EncodeToString(expected[:])[:16]
|
||||
if hash != expectedHex {
|
||||
t.Errorf("hash=%s, want %s (firmware-compatible)", hash, expectedHex)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashTraceGoldenValue is a golden-value test that locks down
|
||||
// the 2-byte path_len (uint16 LE) behavior for TRACE hashing. If anyone removes
|
||||
// the 0x00 byte from the hash input, this test breaks.
|
||||
//
|
||||
// Packet: header=0x25 (FLOOD route=1, payload_type=TRACE=0x09), pathByte=0x02
|
||||
// (2 hops, 1-byte hash), path=[AA,BB], payload=[DE,AD,BE,EF].
|
||||
// Hash input: [0x09, 0x02, 0x00, 0xDE, 0xAD, 0xBE, 0xEF]
|
||||
// → SHA256 = b1baaf3bf0d0726c2672b1ec9e2665dc...
|
||||
// → first 16 hex chars = "b1baaf3bf0d0726c"
|
||||
func TestComputeContentHashTraceGoldenValue(t *testing.T) {
|
||||
// TRACE packet: header byte 0x25 = payload_type 9 (TRACE), route_type 1 (FLOOD)
|
||||
// pathByte 0x02 = hash_size 1, hash_count 2
|
||||
// 2 path bytes (AA, BB), then payload DEADBEEF
|
||||
rawHex := "2502AABBDEADBEEF"
|
||||
hash := ComputeContentHash(rawHex)
|
||||
|
||||
// Pre-computed: SHA256(0x09 0x02 0x00 0xDE 0xAD 0xBE 0xEF)[:16hex]
|
||||
// The 0x00 is the high byte of uint16_t path_len (little-endian).
|
||||
const golden = "b1baaf3bf0d0726c"
|
||||
if hash != golden {
|
||||
t.Errorf("TRACE golden hash = %s, want %s (2-byte path_len encoding)", hash, golden)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacketWithWhitespace(t *testing.T) {
|
||||
raw := "0A 00 D6 9F D7 A5 A7 47 5D B0 73 37 74 9A E6 1F A5 3A 47 88 E9 76"
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
|
||||
+60
-19
@@ -61,9 +61,14 @@ func main() {
|
||||
nodeDays := cfg.NodeDaysOrDefault()
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
|
||||
// Observer retention: remove stale observers on startup
|
||||
observerDays := cfg.ObserverDaysOrDefault()
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
|
||||
// Metrics retention: prune old metrics on startup
|
||||
metricsDays := cfg.MetricsRetentionDays()
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
|
||||
// Daily ticker for node retention
|
||||
retentionTicker := time.NewTicker(1 * time.Hour)
|
||||
@@ -73,11 +78,22 @@ func main() {
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for observer retention (every 24h, staggered 90s after startup)
|
||||
observerRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(90 * time.Second) // stagger after metrics prune
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
for range observerRetentionTicker.C {
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for metrics retention (every 24h)
|
||||
metricsRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range metricsRetentionTicker.C {
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -146,7 +162,7 @@ func main() {
|
||||
// Capture source for closure
|
||||
src := source
|
||||
opts.SetDefaultPublishHandler(func(c mqtt.Client, m mqtt.Message) {
|
||||
handleMessage(store, tag, src, m, channelKeys, cfg.GeoFilter)
|
||||
handleMessage(store, tag, src, m, channelKeys, cfg)
|
||||
})
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
@@ -181,7 +197,7 @@ func main() {
|
||||
log.Println("Done.")
|
||||
}
|
||||
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, geoFilter *GeoFilterConfig) {
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, cfg *Config) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("MQTT [%s] panic in handler: %v", tag, r)
|
||||
@@ -191,21 +207,6 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
topic := m.Topic()
|
||||
parts := strings.Split(topic, "/")
|
||||
|
||||
// IATA filter
|
||||
if len(source.IATAFilter) > 0 && len(parts) > 1 {
|
||||
region := parts[1]
|
||||
matched := false
|
||||
for _, f := range source.IATAFilter {
|
||||
if f == region {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
|
||||
return
|
||||
@@ -217,6 +218,9 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
}
|
||||
|
||||
// Status topic: meshcore/<region>/<observer_id>/status
|
||||
// IATA filter does NOT apply here — observer metadata (noise_floor, battery, etc.)
|
||||
// is region-independent and should be accepted from all observers regardless of
|
||||
// which IATA regions are configured for packet ingestion.
|
||||
if len(parts) >= 4 && parts[3] == "status" {
|
||||
observerID := parts[2]
|
||||
name, _ := msg["origin"].(string)
|
||||
@@ -245,10 +249,26 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
return
|
||||
}
|
||||
|
||||
// IATA filter applies to packet messages only — not status messages above.
|
||||
if len(source.IATAFilter) > 0 && len(parts) > 1 {
|
||||
region := parts[1]
|
||||
matched := false
|
||||
for _, f := range source.IATAFilter {
|
||||
if f == region {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Format 1: Raw packet (meshcoretomqtt / Cisien format)
|
||||
rawHex, _ := msg["raw"].(string)
|
||||
if rawHex != "" {
|
||||
decoded, err := DecodePacket(rawHex, channelKeys, false)
|
||||
validateSigs := cfg.ShouldValidateSignatures()
|
||||
decoded, err := DecodePacket(rawHex, channelKeys, validateSigs)
|
||||
if err != nil {
|
||||
log.Printf("MQTT [%s] decode error: %v", tag, err)
|
||||
return
|
||||
@@ -308,7 +328,27 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] skipping corrupted ADVERT: %s", tag, reason)
|
||||
return
|
||||
}
|
||||
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, geoFilter) {
|
||||
// Signature validation: drop adverts with invalid ed25519 signatures
|
||||
if validateSigs && decoded.Payload.SignatureValid != nil && !*decoded.Payload.SignatureValid {
|
||||
hash := ComputeContentHash(rawHex)
|
||||
truncPK := decoded.Payload.PubKey
|
||||
if len(truncPK) > 16 {
|
||||
truncPK = truncPK[:16]
|
||||
}
|
||||
log.Printf("MQTT [%s] DROPPED invalid signature: hash=%s name=%s observer=%s pubkey=%s",
|
||||
tag, hash, decoded.Payload.Name, firstNonEmpty(mqttMsg.Origin, observerID), truncPK)
|
||||
store.InsertDroppedPacket(&DroppedPacket{
|
||||
Hash: hash,
|
||||
RawHex: rawHex,
|
||||
Reason: "invalid signature",
|
||||
ObserverID: observerID,
|
||||
ObserverName: mqttMsg.Origin,
|
||||
NodePubKey: decoded.Payload.PubKey,
|
||||
NodeName: decoded.Payload.Name,
|
||||
})
|
||||
return
|
||||
}
|
||||
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, cfg.GeoFilter) {
|
||||
return
|
||||
}
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
|
||||
@@ -440,6 +480,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
PayloadType: 5, // GRP_TXT
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: string(decodedJSON),
|
||||
ChannelHash: channelName, // fast channel queries (#762)
|
||||
}
|
||||
|
||||
if _, err := store.InsertTransmission(pktData); err != nil {
|
||||
|
||||
+63
-22
@@ -130,7 +130,7 @@ func TestHandleMessageRawPacket(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"myobs"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -147,7 +147,7 @@ func TestHandleMessageRawPacketAdvert(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
// Should create a node from the ADVERT
|
||||
var count int
|
||||
@@ -169,7 +169,7 @@ func TestHandleMessageInvalidJSON(t *testing.T) {
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: []byte(`not json`)}
|
||||
|
||||
// Should not panic
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -186,7 +186,7 @@ func TestHandleMessageStatusTopic(t *testing.T) {
|
||||
payload: []byte(`{"origin":"MyObserver"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var name, iata string
|
||||
err := store.db.QueryRow("SELECT name, iata FROM observers WHERE id = 'obs1'").Scan(&name, &iata)
|
||||
@@ -207,11 +207,11 @@ func TestHandleMessageSkipStatusTopics(t *testing.T) {
|
||||
|
||||
// meshcore/status should be skipped
|
||||
msg1 := &mockMessage{topic: "meshcore/status", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg1, nil, nil)
|
||||
handleMessage(store, "test", source, msg1, nil, &Config{})
|
||||
|
||||
// meshcore/events/connection should be skipped
|
||||
msg2 := &mockMessage{topic: "meshcore/events/connection", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg2, nil, nil)
|
||||
handleMessage(store, "test", source, msg2, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -230,7 +230,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -243,7 +243,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs2/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg2, nil, nil)
|
||||
handleMessage(store, "test", source, msg2, nil, &Config{})
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 1 {
|
||||
@@ -261,7 +261,7 @@ func TestHandleMessageIATAFilterNoRegion(t *testing.T) {
|
||||
topic: "meshcore",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
// No region part → filter doesn't apply, message goes through
|
||||
// Actually the code checks len(parts) > 1 for IATA filter
|
||||
@@ -277,7 +277,7 @@ func TestHandleMessageNoRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"type":"companion","data":"something"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -295,7 +295,7 @@ func TestHandleMessageBadRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"ZZZZ"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -312,7 +312,7 @@ func TestHandleMessageWithSNRRSSIAsNumbers(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"RSSI":-95}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -331,7 +331,7 @@ func TestHandleMessageMinimalTopic(t *testing.T) {
|
||||
topic: "meshcore/SJC",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -352,7 +352,7 @@ func TestHandleMessageCorruptedAdvert(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
// Transmission should be inserted (even if advert is invalid)
|
||||
var count int
|
||||
@@ -378,7 +378,7 @@ func TestHandleMessageNoObserverID(t *testing.T) {
|
||||
topic: "packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `","origin":"obs1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -400,7 +400,7 @@ func TestHandleMessageSNRNotFloat(t *testing.T) {
|
||||
// SNR as a string value — should not parse as float
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":"bad","RSSI":"bad"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -416,7 +416,7 @@ func TestHandleMessageOriginExtraction(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","origin":"MyOrigin"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
// Verify origin was extracted to observer name
|
||||
var name string
|
||||
@@ -439,7 +439,7 @@ func TestHandleMessagePanicRecovery(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not panic — the defer/recover should catch it
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
}
|
||||
|
||||
func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
@@ -451,7 +451,7 @@ func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/status",
|
||||
payload: []byte(`{"type":"status"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var name string
|
||||
err := store.db.QueryRow("SELECT name FROM observers WHERE id = 'obs1'").Scan(&name)
|
||||
@@ -640,7 +640,7 @@ func TestHandleMessageWithLowercaseSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","snr":5.5,"rssi":-102}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -661,7 +661,7 @@ func TestHandleMessageSNRRSSIUppercaseWins(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"snr":1.0,"RSSI":-95,"rssi":-50}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -681,7 +681,7 @@ func TestHandleMessageNoSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -739,3 +739,44 @@ func TestToFloat64WithUnits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIATAFilterDoesNotDropStatusMessages verifies that status messages from
|
||||
// out-of-region observers are still processed (noise_floor, battery, etc.)
|
||||
// even when an IATA filter is configured for packet data.
|
||||
func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test", IATAFilter: []string{"SJC"}}
|
||||
|
||||
// BFL observer sends a status message with noise_floor — outside the IATA filter.
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/status",
|
||||
payload: []byte(`{"origin":"BFLObserver","stats":{"noise_floor":-105.0}}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var name string
|
||||
var noiseFloor *float64
|
||||
err := store.db.QueryRow("SELECT name, noise_floor FROM observers WHERE id = 'bfl-obs1'").Scan(&name, &noiseFloor)
|
||||
if err != nil {
|
||||
t.Fatalf("observer not found after status from out-of-region observer: %v", err)
|
||||
}
|
||||
if name != "BFLObserver" {
|
||||
t.Errorf("name=%q, want BFLObserver", name)
|
||||
}
|
||||
if noiseFloor == nil || *noiseFloor != -105.0 {
|
||||
t.Errorf("noise_floor=%v, want -105.0 — status message was dropped by IATA filter when it should not be", noiseFloor)
|
||||
}
|
||||
|
||||
// Verify that a packet from BFL is still filtered.
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
pktMsg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, pktMsg, nil, &Config{})
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("packet from out-of-region BFL should still be filtered by IATA")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildAdvertHex constructs a full ADVERT packet hex string.
|
||||
// header(1) + pathByte(1) + pubkey(32) + timestamp(4) + signature(64) + appdata
|
||||
func buildAdvertHex(pubKey ed25519.PublicKey, privKey ed25519.PrivateKey, timestamp uint32, appdata []byte) string {
|
||||
// Build signed message: pubkey(32) + timestamp(4 LE) + appdata
|
||||
msg := make([]byte, 32+4+len(appdata))
|
||||
copy(msg[0:32], pubKey)
|
||||
binary.LittleEndian.PutUint32(msg[32:36], timestamp)
|
||||
copy(msg[36:], appdata)
|
||||
|
||||
sig := ed25519.Sign(privKey, msg)
|
||||
|
||||
// Payload: pubkey(32) + timestamp(4) + signature(64) + appdata
|
||||
payload := make([]byte, 0, 100+len(appdata))
|
||||
payload = append(payload, pubKey...)
|
||||
ts := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(ts, timestamp)
|
||||
payload = append(payload, ts...)
|
||||
payload = append(payload, sig...)
|
||||
payload = append(payload, appdata...)
|
||||
|
||||
// Header: ADVERT (0x04 << 2) | FLOOD (1) = 0x11, pathByte=0 (no hops)
|
||||
header := byte(0x11)
|
||||
pathByte := byte(0x00)
|
||||
|
||||
pkt := append([]byte{header, pathByte}, payload...)
|
||||
return hex.EncodeToString(pkt)
|
||||
}
|
||||
|
||||
// makeAppdata builds minimal appdata: flags(1) + name
|
||||
func makeAppdata(name string) []byte {
|
||||
flags := byte(0x81) // hasName=true, type=companion(1)
|
||||
data := []byte{flags}
|
||||
data = append(data, []byte(name)...)
|
||||
data = append(data, 0x00) // null terminator
|
||||
return data
|
||||
}
|
||||
|
||||
func TestSigValidation_ValidAdvertStored(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("TestNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
|
||||
// Verify packet was stored
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count == 0 {
|
||||
t.Fatal("valid advert should be stored, got 0 transmissions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_TamperedSignatureDropped(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("BadNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper with signature (flip a byte in the signature area)
|
||||
// Signature starts at offset 2 (header+path) + 32 (pubkey) + 4 (timestamp) = 38
|
||||
// That's byte 38 in the packet, hex chars 76-77
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
tamperedHex := string(rawBytes)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
|
||||
// Verify packet was NOT stored in transmissions
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount != 0 {
|
||||
t.Fatalf("tampered advert should be dropped, got %d transmissions", txCount)
|
||||
}
|
||||
|
||||
// Verify it was recorded in dropped_packets
|
||||
var dropCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&dropCount)
|
||||
if dropCount == 0 {
|
||||
t.Fatal("tampered advert should be recorded in dropped_packets")
|
||||
}
|
||||
|
||||
// Verify drop counter incremented
|
||||
if store.Stats.SignatureDrops.Load() != 1 {
|
||||
t.Fatalf("expected 1 signature drop, got %d", store.Stats.SignatureDrops.Load())
|
||||
}
|
||||
|
||||
// Verify dropped_packets has correct fields
|
||||
var reason, nodeKey, nodeName, obsID string
|
||||
store.db.QueryRow("SELECT reason, node_pubkey, node_name, observer_id FROM dropped_packets LIMIT 1").Scan(&reason, &nodeKey, &nodeName, &obsID)
|
||||
if reason != "invalid signature" {
|
||||
t.Fatalf("expected reason 'invalid signature', got %q", reason)
|
||||
}
|
||||
if nodeKey == "" {
|
||||
t.Fatal("dropped packet should have node_pubkey")
|
||||
}
|
||||
if !strings.Contains(nodeName, "BadNode") {
|
||||
t.Fatalf("expected node_name to contain 'BadNode', got %q", nodeName)
|
||||
}
|
||||
if obsID != "obs1" {
|
||||
t.Fatalf("expected observer_id 'obs1', got %q", obsID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_TruncatedAppdataDropped(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("TruncNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Sign was computed with full appdata. Now truncate the raw hex to remove
|
||||
// some appdata bytes, making the signature invalid.
|
||||
// Truncate last 4 hex chars (2 bytes of appdata)
|
||||
truncatedHex := rawHex[:len(rawHex)-4]
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+truncatedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount != 0 {
|
||||
t.Fatalf("truncated advert should be dropped, got %d transmissions", txCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_DisabledByConfig(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("NoValNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper with signature
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
tamperedHex := string(rawBytes)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
|
||||
falseVal := false
|
||||
cfg := &Config{ValidateSignatures: &falseVal}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
|
||||
// With validation disabled, tampered packet should be stored
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount == 0 {
|
||||
t.Fatal("with validateSignatures=false, tampered advert should be stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_DropCounterIncrements(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
appdata := makeAppdata("Node")
|
||||
rawHex := buildAdvertHex(pub, priv, uint32(1700000000+i), appdata)
|
||||
// Tamper
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"Obs"}`)
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
}
|
||||
|
||||
if store.Stats.SignatureDrops.Load() != 3 {
|
||||
t.Fatalf("expected 3 signature drops, got %d", store.Stats.SignatureDrops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_LogContainsFields(t *testing.T) {
|
||||
// This test verifies the dropped_packets row has all required fields
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("LogTestNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"MyObserver"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, cfg)
|
||||
|
||||
var hash, reason, obsID, obsName, pubkey, nodeName string
|
||||
err = store.db.QueryRow("SELECT hash, reason, observer_id, observer_name, node_pubkey, node_name FROM dropped_packets LIMIT 1").
|
||||
Scan(&hash, &reason, &obsID, &obsName, &pubkey, &nodeName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if hash == "" {
|
||||
t.Error("dropped packet should have hash")
|
||||
}
|
||||
if reason != "invalid signature" {
|
||||
t.Errorf("expected reason 'invalid signature', got %q", reason)
|
||||
}
|
||||
if obsID != "obs1" {
|
||||
t.Errorf("expected observer_id 'obs1', got %q", obsID)
|
||||
}
|
||||
if obsName != "MyObserver" {
|
||||
t.Errorf("expected observer_name 'MyObserver', got %q", obsName)
|
||||
}
|
||||
if pubkey == "" {
|
||||
t.Error("dropped packet should have node_pubkey")
|
||||
}
|
||||
if !strings.Contains(nodeName, "LogTestNode") {
|
||||
t.Errorf("expected node_name containing 'LogTestNode', got %q", nodeName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneDroppedPackets(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Insert an old dropped packet
|
||||
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`)
|
||||
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`)
|
||||
|
||||
n, err := store.PruneDroppedPackets(30)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("expected 1 pruned, got %d", n)
|
||||
}
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Fatalf("expected 1 remaining, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldValidateSignatures_Default(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if !cfg.ShouldValidateSignatures() {
|
||||
t.Fatal("default should be true")
|
||||
}
|
||||
|
||||
falseVal := false
|
||||
cfg2 := &Config{ValidateSignatures: &falseVal}
|
||||
if cfg2.ShouldValidateSignatures() {
|
||||
t.Fatal("explicit false should be false")
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
cfg3 := &Config{ValidateSignatures: &trueVal}
|
||||
if !cfg3.ShouldValidateSignatures() {
|
||||
t.Fatal("explicit true should be true")
|
||||
}
|
||||
}
|
||||
|
||||
// newMockMsg creates a minimal mqtt.Message for testing.
|
||||
func newMockMsg(topic, payload string) *mockMessage {
|
||||
return &mockMessage{topic: topic, payload: []byte(payload)}
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// createTestDB creates a temporary SQLite database with N transmissions (1 obs each).
|
||||
func createTestDB(t *testing.T, numTx int) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
createTestDBAt(t, dbPath, numTx)
|
||||
return dbPath
|
||||
}
|
||||
|
||||
// loadStore creates a PacketStore from a test DB with given maxMemoryMB.
|
||||
func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore {
|
||||
t.Helper()
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: maxMemMB}
|
||||
store := NewPacketStore(db, cfg)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func TestBoundedLoad_LimitedMemory(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// Use 1MB budget — should load far fewer than 5000 packets
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
loaded := len(store.packets)
|
||||
if loaded >= 5000 {
|
||||
t.Errorf("expected bounded load to limit packets, got %d/5000", loaded)
|
||||
}
|
||||
if loaded < 1000 {
|
||||
t.Errorf("expected at least 1000 packets (minimum), got %d", loaded)
|
||||
}
|
||||
t.Logf("Loaded %d/5000 packets with 1MB budget", loaded)
|
||||
}
|
||||
|
||||
func TestBoundedLoad_NewestFirst(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
loaded := len(store.packets)
|
||||
if loaded >= 5000 {
|
||||
t.Skip("all packets loaded, can't verify newest-first")
|
||||
}
|
||||
|
||||
// The newest packet in DB has first_seen based on minute 5000.
|
||||
// The loaded packets should be the newest ones.
|
||||
// Last packet in store (sorted ASC) should be the newest in DB.
|
||||
last := store.packets[loaded-1]
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
newestExpected := base.Add(5000 * time.Minute).Format(time.RFC3339)
|
||||
if last.FirstSeen != newestExpected {
|
||||
t.Errorf("expected last packet to be newest (%s), got %s", newestExpected, last.FirstSeen)
|
||||
}
|
||||
|
||||
// First packet should NOT be the oldest in the DB (minute 1)
|
||||
first := store.packets[0]
|
||||
oldestAll := base.Add(1 * time.Minute).Format(time.RFC3339)
|
||||
if first.FirstSeen == oldestAll {
|
||||
t.Errorf("first loaded packet should not be the absolute oldest when bounded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedLoad_OldestLoadedSet(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if store.oldestLoaded == "" {
|
||||
t.Fatal("oldestLoaded should be set after bounded load")
|
||||
}
|
||||
if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen {
|
||||
t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen)
|
||||
}
|
||||
t.Logf("oldestLoaded = %s", store.oldestLoaded)
|
||||
}
|
||||
|
||||
func TestBoundedLoad_UnlimitedWithZero(t *testing.T) {
|
||||
dbPath := createTestDB(t, 200)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 0)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 200 {
|
||||
t.Errorf("expected all 200 packets with maxMemoryMB=0, got %d", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedLoad_AscendingOrder(t *testing.T) {
|
||||
dbPath := createTestDB(t, 3000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
// Verify packets are in ascending first_seen order
|
||||
for i := 1; i < len(store.packets); i++ {
|
||||
if store.packets[i].FirstSeen < store.packets[i-1].FirstSeen {
|
||||
t.Fatalf("packets not in ascending order at index %d: %s < %s",
|
||||
i, store.packets[i].FirstSeen, store.packets[i-1].FirstSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEstimateStoreTxBytesTypical(t *testing.T) {
|
||||
est := estimateStoreTxBytesTypical(10)
|
||||
if est < 1000 {
|
||||
t.Errorf("typical estimate too low: %d", est)
|
||||
}
|
||||
// Should be roughly proportional to observation count
|
||||
est1 := estimateStoreTxBytesTypical(1)
|
||||
est20 := estimateStoreTxBytesTypical(20)
|
||||
if est20 <= est1 {
|
||||
t.Errorf("estimate should grow with observations: 1obs=%d, 20obs=%d", est1, est20)
|
||||
}
|
||||
t.Logf("Typical estimate: 1obs=%d, 10obs=%d, 20obs=%d bytes", est1, est, est20)
|
||||
}
|
||||
|
||||
func BenchmarkLoad_Bounded(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench.db")
|
||||
createTestDBAt(b, dbPath, 5000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 1}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLoad_Unlimited(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench.db")
|
||||
createTestDBAt(b, dbPath, 5000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions
|
||||
// and realistic observation counts (1–5 per transmission).
|
||||
func BenchmarkLoad_30K_Bounded(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench30k.db")
|
||||
createTestDBWithObs(b, dbPath, 30000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 50}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions
|
||||
// and realistic observation counts (1–5 per transmission).
|
||||
func BenchmarkLoad_30K_Unlimited(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench30k.db")
|
||||
createTestDBWithObs(b, dbPath, 30000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// createTestDBAt is like createTestDB but writes to a specific path.
|
||||
func createTestDBAt(tb testing.TB, dbPath string, numTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(sql string) {
|
||||
if _, err := conn.Exec(sql); err != nil {
|
||||
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER,
|
||||
payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare transmissions insert: %v", err)
|
||||
}
|
||||
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare observations insert: %v", err)
|
||||
}
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
|
||||
obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts)
|
||||
}
|
||||
}
|
||||
|
||||
// createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx).
|
||||
func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(sqlStr string) {
|
||||
if _, err := conn.Exec(sqlStr); err != nil {
|
||||
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare transmissions: %v", err)
|
||||
}
|
||||
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare observations: %v", err)
|
||||
}
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"}
|
||||
obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"}
|
||||
obsID := 1
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%06d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i))
|
||||
nObs := (i % 5) + 1 // 1–5 observations per transmission
|
||||
for j := 0; j < nObs; j++ {
|
||||
snr := -5.0 + float64(j)*2.5
|
||||
rssi := -90.0 + float64(j)*5.0
|
||||
obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts)
|
||||
obsID++
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPacketsChannelFilter verifies /api/packets?channel=... actually filters
|
||||
// (regression test for #812).
|
||||
func TestPacketsChannelFilter(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
get := func(url string) map[string]interface{} {
|
||||
req := httptest.NewRequest("GET", url, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s: expected 200, got %d", url, w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode %s: %v", url, err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
all := get("/api/packets?limit=50")
|
||||
allTotal := int(all["total"].(float64))
|
||||
if allTotal < 2 {
|
||||
t.Fatalf("expected baseline >= 2 packets, got %d", allTotal)
|
||||
}
|
||||
|
||||
test := get("/api/packets?limit=50&channel=%23test")
|
||||
testTotal := int(test["total"].(float64))
|
||||
if testTotal == 0 {
|
||||
t.Fatalf("channel=#test: expected >= 1 match, got 0 (filter ignored?)")
|
||||
}
|
||||
if testTotal >= allTotal {
|
||||
t.Fatalf("channel=#test: expected fewer packets than baseline (%d), got %d", allTotal, testTotal)
|
||||
}
|
||||
|
||||
// Every returned packet must be a CHAN/GRP_TXT (payload_type=5) on #test.
|
||||
pkts, _ := test["packets"].([]interface{})
|
||||
for _, p := range pkts {
|
||||
m := p.(map[string]interface{})
|
||||
if pt, _ := m["payload_type"].(float64); int(pt) != 5 {
|
||||
t.Errorf("channel=#test: returned non-GRP_TXT packet (payload_type=%v)", m["payload_type"])
|
||||
}
|
||||
}
|
||||
|
||||
none := get("/api/packets?limit=50&channel=nonexistentchannel")
|
||||
if int(none["total"].(float64)) != 0 {
|
||||
t.Fatalf("channel=nonexistentchannel: expected total=0, got %v", none["total"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,748 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── Clock Skew Severity ────────────────────────────────────────────────────────
|
||||
|
||||
type SkewSeverity string
|
||||
|
||||
const (
|
||||
SkewOK SkewSeverity = "ok" // < 5 min
|
||||
SkewWarning SkewSeverity = "warning" // 5 min – 1 hour
|
||||
SkewCritical SkewSeverity = "critical" // 1 hour – 30 days
|
||||
SkewAbsurd SkewSeverity = "absurd" // > 30 days
|
||||
SkewNoClock SkewSeverity = "no_clock" // > 365 days — uninitialized RTC
|
||||
SkewBimodalClock SkewSeverity = "bimodal_clock" // mixed good+bad recent samples (flaky RTC)
|
||||
)
|
||||
|
||||
// Default thresholds in seconds.
|
||||
const (
|
||||
skewThresholdWarnSec = 5 * 60 // 5 minutes
|
||||
skewThresholdCriticalSec = 60 * 60 // 1 hour
|
||||
skewThresholdAbsurdSec = 30 * 24 * 3600 // 30 days
|
||||
skewThresholdNoClockSec = 365 * 24 * 3600 // 365 days — uninitialized RTC
|
||||
|
||||
// minDriftSamples is the minimum number of advert transmissions needed
|
||||
// to compute a meaningful linear drift rate.
|
||||
minDriftSamples = 5
|
||||
|
||||
// maxReasonableDriftPerDay caps drift display. Physically impossible
|
||||
// drift rates (> 1 day/day) indicate insufficient or outlier samples.
|
||||
maxReasonableDriftPerDay = 86400.0
|
||||
|
||||
// recentSkewWindowCount is the number of most-recent advert samples
|
||||
// used to derive the "current" skew for severity classification (see
|
||||
// issue #789). The all-time median is poisoned by historical bad
|
||||
// samples (e.g. a node that was off and then GPS-corrected); severity
|
||||
// must reflect current health, not lifetime statistics.
|
||||
recentSkewWindowCount = 5
|
||||
|
||||
// recentSkewWindowSec bounds the recent-window in time as well: only
|
||||
// samples from the last N seconds count as "recent" for severity.
|
||||
// The effective window is min(recentSkewWindowCount, samples in 1h).
|
||||
recentSkewWindowSec = 3600
|
||||
|
||||
// bimodalSkewThresholdSec is the absolute skew threshold (1 hour)
|
||||
// above which a sample is considered "bad" — likely firmware emitting
|
||||
// a nonsense timestamp from an uninitialized RTC, not real drift.
|
||||
// Chosen to match the warning/critical severity boundary: real clock
|
||||
// drift rarely exceeds 1 hour, while epoch-0 RTCs produce ~1.7B sec.
|
||||
bimodalSkewThresholdSec = 3600.0
|
||||
|
||||
// maxPlausibleSkewJumpSec is the largest skew change between
|
||||
// consecutive samples that we treat as physical drift. Anything larger
|
||||
// (e.g. a GPS sync that jumps the clock by minutes/days) is rejected
|
||||
// as an outlier when computing drift. Real microcontroller drift is
|
||||
// fractions of a second per advert; 60s is a generous safety factor.
|
||||
maxPlausibleSkewJumpSec = 60.0
|
||||
|
||||
// theilSenMaxPoints caps the number of points fed to Theil-Sen
|
||||
// regression (O(n²) in pairs). For nodes with thousands of samples we
|
||||
// keep the most-recent points, which are also the most relevant for
|
||||
// current drift.
|
||||
theilSenMaxPoints = 200
|
||||
)
|
||||
|
||||
// classifySkew maps absolute skew (seconds) to a severity level.
|
||||
// Float64 comparison is safe: inputs are rounded to 1 decimal via round(),
|
||||
// and thresholds are integer multiples of 60 — no rounding artifacts.
|
||||
func classifySkew(absSkewSec float64) SkewSeverity {
|
||||
switch {
|
||||
case absSkewSec >= skewThresholdNoClockSec:
|
||||
return SkewNoClock
|
||||
case absSkewSec >= skewThresholdAbsurdSec:
|
||||
return SkewAbsurd
|
||||
case absSkewSec >= skewThresholdCriticalSec:
|
||||
return SkewCritical
|
||||
case absSkewSec >= skewThresholdWarnSec:
|
||||
return SkewWarning
|
||||
default:
|
||||
return SkewOK
|
||||
}
|
||||
}
|
||||
|
||||
// ── Data Types ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// skewSample is a single raw skew measurement from one advert observation.
|
||||
type skewSample struct {
|
||||
advertTS int64 // node's advert Unix timestamp
|
||||
observedTS int64 // observation Unix timestamp
|
||||
observerID string // which observer saw this
|
||||
hash string // transmission hash (for multi-observer grouping)
|
||||
}
|
||||
|
||||
// ObserverCalibration holds the computed clock offset for an observer.
|
||||
type ObserverCalibration struct {
|
||||
ObserverID string `json:"observerID"`
|
||||
OffsetSec float64 `json:"offsetSec"` // positive = observer clock ahead
|
||||
Samples int `json:"samples"` // number of multi-observer packets used
|
||||
}
|
||||
|
||||
// NodeClockSkew is the API response for a single node's clock skew data.
|
||||
type NodeClockSkew struct {
|
||||
Pubkey string `json:"pubkey"`
|
||||
MeanSkewSec float64 `json:"meanSkewSec"` // corrected mean skew (positive = node ahead)
|
||||
MedianSkewSec float64 `json:"medianSkewSec"` // corrected median skew
|
||||
LastSkewSec float64 `json:"lastSkewSec"` // most recent corrected skew
|
||||
RecentMedianSkewSec float64 `json:"recentMedianSkewSec"` // median across most-recent samples (drives severity, see #789)
|
||||
DriftPerDaySec float64 `json:"driftPerDaySec"` // linear drift rate (sec/day)
|
||||
Severity SkewSeverity `json:"severity"`
|
||||
SampleCount int `json:"sampleCount"`
|
||||
Calibrated bool `json:"calibrated"` // true if observer calibration was applied
|
||||
LastAdvertTS int64 `json:"lastAdvertTS"` // most recent advert timestamp
|
||||
LastObservedTS int64 `json:"lastObservedTS"` // most recent observation timestamp
|
||||
Samples []SkewSample `json:"samples,omitempty"` // time-series for sparklines
|
||||
GoodFraction float64 `json:"goodFraction"` // fraction of recent samples with |skew| <= 1h
|
||||
RecentBadSampleCount int `json:"recentBadSampleCount"` // count of recent samples with |skew| > 1h
|
||||
RecentSampleCount int `json:"recentSampleCount"` // total recent samples in window
|
||||
NodeName string `json:"nodeName,omitempty"` // populated in fleet responses
|
||||
NodeRole string `json:"nodeRole,omitempty"` // populated in fleet responses
|
||||
}
|
||||
|
||||
// SkewSample is a single (timestamp, skew) point for sparkline rendering.
|
||||
type SkewSample struct {
|
||||
Timestamp int64 `json:"ts"` // Unix epoch of observation
|
||||
SkewSec float64 `json:"skew"` // corrected skew in seconds
|
||||
}
|
||||
|
||||
// txSkewResult maps tx hash → per-transmission skew stats. This is an
|
||||
// intermediate result keyed by hash (not pubkey); the store maps hash → pubkey
|
||||
// when building the final per-node view.
|
||||
type txSkewResult = map[string]*NodeClockSkew
|
||||
|
||||
// ── Clock Skew Engine ──────────────────────────────────────────────────────────
|
||||
|
||||
// ClockSkewEngine computes and caches clock skew data for nodes and observers.
|
||||
type ClockSkewEngine struct {
|
||||
mu sync.RWMutex
|
||||
observerOffsets map[string]float64 // observerID → calibrated offset (seconds)
|
||||
observerSamples map[string]int // observerID → number of multi-observer packets used
|
||||
nodeSkew txSkewResult
|
||||
lastComputed time.Time
|
||||
computeInterval time.Duration
|
||||
}
|
||||
|
||||
func NewClockSkewEngine() *ClockSkewEngine {
|
||||
return &ClockSkewEngine{
|
||||
observerOffsets: make(map[string]float64),
|
||||
observerSamples: make(map[string]int),
|
||||
nodeSkew: make(txSkewResult),
|
||||
computeInterval: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Recompute recalculates all clock skew data from the packet store.
|
||||
// Called periodically or on demand. Holds store RLock externally.
|
||||
// Uses read-copy-update: heavy computation runs outside the write lock,
|
||||
// then results are swapped in under a brief lock.
|
||||
func (e *ClockSkewEngine) Recompute(store *PacketStore) {
|
||||
// Fast path: check under read lock if recompute is needed.
|
||||
e.mu.RLock()
|
||||
fresh := time.Since(e.lastComputed) < e.computeInterval
|
||||
e.mu.RUnlock()
|
||||
if fresh {
|
||||
return
|
||||
}
|
||||
|
||||
// Phase 1: Collect skew samples from ADVERT packets (store RLock held by caller).
|
||||
samples := collectSamples(store)
|
||||
|
||||
// Phase 2–3: Compute outside the write lock.
|
||||
var newOffsets map[string]float64
|
||||
var newSamples map[string]int
|
||||
var newNodeSkew txSkewResult
|
||||
|
||||
if len(samples) > 0 {
|
||||
newOffsets, newSamples = calibrateObservers(samples)
|
||||
newNodeSkew = computeNodeSkew(samples, newOffsets)
|
||||
} else {
|
||||
newOffsets = make(map[string]float64)
|
||||
newSamples = make(map[string]int)
|
||||
newNodeSkew = make(txSkewResult)
|
||||
}
|
||||
|
||||
// Swap results under brief write lock.
|
||||
e.mu.Lock()
|
||||
// Re-check: another goroutine may have computed while we were working.
|
||||
if time.Since(e.lastComputed) < e.computeInterval {
|
||||
e.mu.Unlock()
|
||||
return
|
||||
}
|
||||
e.observerOffsets = newOffsets
|
||||
e.observerSamples = newSamples
|
||||
e.nodeSkew = newNodeSkew
|
||||
e.lastComputed = time.Now()
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// collectSamples extracts skew samples from ADVERT packets in the store.
|
||||
// Must be called with store.mu held (at least RLock).
|
||||
func collectSamples(store *PacketStore) []skewSample {
|
||||
adverts := store.byPayloadType[PayloadADVERT]
|
||||
if len(adverts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
samples := make([]skewSample, 0, len(adverts)*2)
|
||||
for _, tx := range adverts {
|
||||
decoded := tx.ParsedDecoded()
|
||||
if decoded == nil {
|
||||
continue
|
||||
}
|
||||
// Extract advert timestamp from decoded JSON.
|
||||
advertTS := extractTimestamp(decoded)
|
||||
if advertTS <= 0 {
|
||||
continue
|
||||
}
|
||||
// Sanity: skip timestamps before year 2020 or after year 2100.
|
||||
if advertTS < 1577836800 || advertTS > 4102444800 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, obs := range tx.Observations {
|
||||
obsTS := parseISO(obs.Timestamp)
|
||||
if obsTS <= 0 {
|
||||
continue
|
||||
}
|
||||
samples = append(samples, skewSample{
|
||||
advertTS: advertTS,
|
||||
observedTS: obsTS,
|
||||
observerID: obs.ObserverID,
|
||||
hash: tx.Hash,
|
||||
})
|
||||
}
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
// extractTimestamp gets the Unix timestamp from a decoded ADVERT payload.
|
||||
func extractTimestamp(decoded map[string]interface{}) int64 {
|
||||
// Try payload.timestamp first (nested in "payload" key).
|
||||
if payload, ok := decoded["payload"]; ok {
|
||||
if pm, ok := payload.(map[string]interface{}); ok {
|
||||
if ts := jsonNumber(pm, "timestamp"); ts > 0 {
|
||||
return ts
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: top-level timestamp.
|
||||
if ts := jsonNumber(decoded, "timestamp"); ts > 0 {
|
||||
return ts
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// jsonNumber extracts an int64 from a JSON-parsed map (handles float64 and json.Number).
|
||||
func jsonNumber(m map[string]interface{}, key string) int64 {
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int64(n)
|
||||
case int64:
|
||||
return n
|
||||
case int:
|
||||
return int64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseISO parses an ISO 8601 timestamp string to Unix seconds.
|
||||
func parseISO(s string) int64 {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
// Try with fractional seconds.
|
||||
t, err = time.Parse("2006-01-02T15:04:05.999999999Z07:00", s)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ── Phase 2: Observer Calibration ──────────────────────────────────────────────
|
||||
|
||||
// calibrateObservers computes each observer's clock offset using multi-observer
|
||||
// packets. Returns offset map and sample count map.
|
||||
func calibrateObservers(samples []skewSample) (map[string]float64, map[string]int) {
|
||||
// Group observations by packet hash.
|
||||
byHash := make(map[string][]skewSample)
|
||||
for _, s := range samples {
|
||||
byHash[s.hash] = append(byHash[s.hash], s)
|
||||
}
|
||||
|
||||
// For each multi-observer packet, compute per-observer deviation from median.
|
||||
deviations := make(map[string][]float64) // observerID → list of deviations
|
||||
for _, group := range byHash {
|
||||
if len(group) < 2 {
|
||||
continue // single-observer packet, can't calibrate
|
||||
}
|
||||
// Compute median observation timestamp for this packet.
|
||||
obsTimes := make([]float64, len(group))
|
||||
for i, s := range group {
|
||||
obsTimes[i] = float64(s.observedTS)
|
||||
}
|
||||
medianObs := median(obsTimes)
|
||||
for _, s := range group {
|
||||
dev := float64(s.observedTS) - medianObs
|
||||
deviations[s.observerID] = append(deviations[s.observerID], dev)
|
||||
}
|
||||
}
|
||||
|
||||
// Each observer's offset = median of its deviations.
|
||||
offsets := make(map[string]float64, len(deviations))
|
||||
counts := make(map[string]int, len(deviations))
|
||||
for obsID, devs := range deviations {
|
||||
offsets[obsID] = median(devs)
|
||||
counts[obsID] = len(devs)
|
||||
}
|
||||
return offsets, counts
|
||||
}
|
||||
|
||||
// ── Phase 3: Per-Node Skew ─────────────────────────────────────────────────────
|
||||
|
||||
// computeNodeSkew calculates corrected skew statistics for each node.
|
||||
func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) txSkewResult {
|
||||
// Compute corrected skew per sample, grouped by hash (each hash = one
|
||||
// node's advert transmission). The caller maps hash → pubkey via byNode.
|
||||
type correctedSample struct {
|
||||
skew float64
|
||||
observedTS int64
|
||||
calibrated bool
|
||||
}
|
||||
|
||||
byHash := make(map[string][]correctedSample)
|
||||
hashAdvertTS := make(map[string]int64)
|
||||
|
||||
for _, s := range samples {
|
||||
obsOffset, hasCal := obsOffsets[s.observerID]
|
||||
rawSkew := float64(s.advertTS - s.observedTS)
|
||||
corrected := rawSkew
|
||||
if hasCal {
|
||||
// Observer offset = obs_ts - median(all_obs_ts). If observer is ahead,
|
||||
// its obs_ts is inflated, making raw_skew too low. Add offset to correct.
|
||||
corrected = rawSkew + obsOffset
|
||||
}
|
||||
byHash[s.hash] = append(byHash[s.hash], correctedSample{
|
||||
skew: corrected,
|
||||
observedTS: s.observedTS,
|
||||
calibrated: hasCal,
|
||||
})
|
||||
hashAdvertTS[s.hash] = s.advertTS
|
||||
}
|
||||
|
||||
// Each hash represents one advert from one node. Compute median corrected
|
||||
// skew per hash (across multiple observers).
|
||||
|
||||
result := make(map[string]*NodeClockSkew) // keyed by hash for now
|
||||
for hash, cs := range byHash {
|
||||
skews := make([]float64, len(cs))
|
||||
for i, c := range cs {
|
||||
skews[i] = c.skew
|
||||
}
|
||||
medSkew := median(skews)
|
||||
meanSkew := mean(skews)
|
||||
|
||||
// Find latest observation.
|
||||
var latestObsTS int64
|
||||
var anyCal bool
|
||||
for _, c := range cs {
|
||||
if c.observedTS > latestObsTS {
|
||||
latestObsTS = c.observedTS
|
||||
}
|
||||
if c.calibrated {
|
||||
anyCal = true
|
||||
}
|
||||
}
|
||||
|
||||
absMedian := math.Abs(medSkew)
|
||||
result[hash] = &NodeClockSkew{
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(cs[len(cs)-1].skew, 1),
|
||||
Severity: classifySkew(absMedian),
|
||||
SampleCount: len(cs),
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: hashAdvertTS[hash],
|
||||
LastObservedTS: latestObsTS,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Integration with PacketStore ───────────────────────────────────────────────
|
||||
|
||||
// GetNodeClockSkew returns the clock skew data for a specific node (acquires RLock).
|
||||
func (s *PacketStore) GetNodeClockSkew(pubkey string) *NodeClockSkew {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.getNodeClockSkewLocked(pubkey)
|
||||
}
|
||||
|
||||
// getNodeClockSkewLocked returns clock skew for a node.
|
||||
// Must be called with s.mu held (at least RLock).
|
||||
func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
s.clockSkew.Recompute(s)
|
||||
|
||||
txs := s.byNode[pubkey]
|
||||
if len(txs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.clockSkew.mu.RLock()
|
||||
defer s.clockSkew.mu.RUnlock()
|
||||
|
||||
var allSkews []float64
|
||||
var lastSkew float64
|
||||
var lastObsTS, lastAdvTS int64
|
||||
var totalSamples int
|
||||
var anyCal bool
|
||||
var tsSkews []tsSkewPair
|
||||
|
||||
for _, tx := range txs {
|
||||
if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
|
||||
continue
|
||||
}
|
||||
cs, ok := s.clockSkew.nodeSkew[tx.Hash]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
allSkews = append(allSkews, cs.MedianSkewSec)
|
||||
totalSamples += cs.SampleCount
|
||||
if cs.Calibrated {
|
||||
anyCal = true
|
||||
}
|
||||
if cs.LastObservedTS > lastObsTS {
|
||||
lastObsTS = cs.LastObservedTS
|
||||
lastSkew = cs.LastSkewSec
|
||||
lastAdvTS = cs.LastAdvertTS
|
||||
}
|
||||
tsSkews = append(tsSkews, tsSkewPair{ts: cs.LastObservedTS, skew: cs.MedianSkewSec})
|
||||
}
|
||||
|
||||
if len(allSkews) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
medSkew := median(allSkews)
|
||||
meanSkew := mean(allSkews)
|
||||
|
||||
// Severity is derived from RECENT samples only (issue #789). The
|
||||
// all-time median is poisoned by historical bad data — a node that
|
||||
// was off for hours and then GPS-corrected can have median = -59M sec
|
||||
// while its current skew is -0.8s. Operators need severity to reflect
|
||||
// current health, so they trust the dashboard.
|
||||
//
|
||||
// Sort tsSkews by time and take the last recentSkewWindowCount samples
|
||||
// (or all samples within recentSkewWindowSec of the latest, whichever
|
||||
// gives FEWER samples — we want the more-current view; a chatty node
|
||||
// can fit dozens of samples in 1h, in which case the count cap wins).
|
||||
sort.Slice(tsSkews, func(i, j int) bool { return tsSkews[i].ts < tsSkews[j].ts })
|
||||
|
||||
recentSkew := lastSkew
|
||||
var recentVals []float64
|
||||
if n := len(tsSkews); n > 0 {
|
||||
latestTS := tsSkews[n-1].ts
|
||||
// Index-based window: last K samples.
|
||||
startByCount := n - recentSkewWindowCount
|
||||
if startByCount < 0 {
|
||||
startByCount = 0
|
||||
}
|
||||
// Time-based window: samples newer than latestTS - windowSec.
|
||||
startByTime := n - 1
|
||||
for i := n - 1; i >= 0; i-- {
|
||||
if latestTS-tsSkews[i].ts <= recentSkewWindowSec {
|
||||
startByTime = i
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Pick the narrower (larger-index) of the two windows — the most
|
||||
// current view of the node's clock health.
|
||||
start := startByCount
|
||||
if startByTime > start {
|
||||
start = startByTime
|
||||
}
|
||||
recentVals = make([]float64, 0, n-start)
|
||||
for i := start; i < n; i++ {
|
||||
recentVals = append(recentVals, tsSkews[i].skew)
|
||||
}
|
||||
if len(recentVals) > 0 {
|
||||
recentSkew = median(recentVals)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Bimodal detection (#845) ─────────────────────────────────────────
|
||||
// Split recent samples into "good" (|skew| <= 1h, real clock) and
|
||||
// "bad" (|skew| > 1h, firmware nonsense from uninitialized RTC).
|
||||
// Classification order (first match wins):
|
||||
// no_clock — goodFraction < 0.10 (essentially no real clock)
|
||||
// bimodal_clock — 0.10 <= goodFraction < 0.80 AND badCount > 0
|
||||
// ok/warn/etc. — goodFraction >= 0.80 (normal, outliers filtered)
|
||||
var goodSamples []float64
|
||||
for _, v := range recentVals {
|
||||
if math.Abs(v) <= bimodalSkewThresholdSec {
|
||||
goodSamples = append(goodSamples, v)
|
||||
}
|
||||
}
|
||||
recentSampleCount := len(recentVals)
|
||||
recentBadCount := recentSampleCount - len(goodSamples)
|
||||
var goodFraction float64
|
||||
if recentSampleCount > 0 {
|
||||
goodFraction = float64(len(goodSamples)) / float64(recentSampleCount)
|
||||
}
|
||||
|
||||
var severity SkewSeverity
|
||||
if goodFraction < 0.10 {
|
||||
// Essentially no real clock — classify as no_clock regardless
|
||||
// of the raw skew magnitude.
|
||||
severity = SkewNoClock
|
||||
} else if goodFraction < 0.80 && recentBadCount > 0 {
|
||||
// Bimodal: use median of GOOD samples as the "real" skew.
|
||||
severity = SkewBimodalClock
|
||||
if len(goodSamples) > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
} else {
|
||||
// Normal path: if there are good samples, use their median
|
||||
// (filters out rare outliers in ≥80% good case).
|
||||
if len(goodSamples) > 0 && recentBadCount > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
severity = classifySkew(math.Abs(recentSkew))
|
||||
}
|
||||
|
||||
// For no_clock / bimodal_clock nodes, skip drift when data is unreliable.
|
||||
var drift float64
|
||||
if severity != SkewNoClock && severity != SkewBimodalClock && len(tsSkews) >= minDriftSamples {
|
||||
drift = computeDrift(tsSkews)
|
||||
// Cap physically impossible drift rates.
|
||||
if math.Abs(drift) > maxReasonableDriftPerDay {
|
||||
drift = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Build sparkline samples from tsSkews (already sorted by time above).
|
||||
samples := make([]SkewSample, len(tsSkews))
|
||||
for i, p := range tsSkews {
|
||||
samples[i] = SkewSample{Timestamp: p.ts, SkewSec: round(p.skew, 1)}
|
||||
}
|
||||
|
||||
return &NodeClockSkew{
|
||||
Pubkey: pubkey,
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(lastSkew, 1),
|
||||
RecentMedianSkewSec: round(recentSkew, 1),
|
||||
DriftPerDaySec: round(drift, 2),
|
||||
Severity: severity,
|
||||
SampleCount: totalSamples,
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: lastAdvTS,
|
||||
LastObservedTS: lastObsTS,
|
||||
Samples: samples,
|
||||
GoodFraction: round(goodFraction, 2),
|
||||
RecentBadSampleCount: recentBadCount,
|
||||
RecentSampleCount: recentSampleCount,
|
||||
}
|
||||
}
|
||||
|
||||
// GetFleetClockSkew returns clock skew data for all nodes that have skew data.
|
||||
// Must NOT be called with s.mu held.
|
||||
func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
// Build name/role lookup from DB cache (requires s.mu held).
|
||||
allNodes, _ := s.getCachedNodesAndPM()
|
||||
nameMap := make(map[string]nodeInfo, len(allNodes))
|
||||
for _, ni := range allNodes {
|
||||
nameMap[ni.PublicKey] = ni
|
||||
}
|
||||
|
||||
var results []*NodeClockSkew
|
||||
for pubkey := range s.byNode {
|
||||
cs := s.getNodeClockSkewLocked(pubkey)
|
||||
if cs == nil {
|
||||
continue
|
||||
}
|
||||
// Enrich with node name/role.
|
||||
if ni, ok := nameMap[pubkey]; ok {
|
||||
cs.NodeName = ni.Name
|
||||
cs.NodeRole = ni.Role
|
||||
}
|
||||
// Omit samples in fleet response (too much data).
|
||||
cs.Samples = nil
|
||||
results = append(results, cs)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// GetObserverCalibrations returns the current observer clock offsets.
|
||||
func (s *PacketStore) GetObserverCalibrations() []ObserverCalibration {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.clockSkew.Recompute(s)
|
||||
|
||||
s.clockSkew.mu.RLock()
|
||||
defer s.clockSkew.mu.RUnlock()
|
||||
|
||||
result := make([]ObserverCalibration, 0, len(s.clockSkew.observerOffsets))
|
||||
for obsID, offset := range s.clockSkew.observerOffsets {
|
||||
result = append(result, ObserverCalibration{
|
||||
ObserverID: obsID,
|
||||
OffsetSec: round(offset, 1),
|
||||
Samples: s.clockSkew.observerSamples[obsID],
|
||||
})
|
||||
}
|
||||
// Sort by absolute offset descending.
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return math.Abs(result[i].OffsetSec) > math.Abs(result[j].OffsetSec)
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Math Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
func median(vals []float64) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
sorted := make([]float64, len(vals))
|
||||
copy(sorted, vals)
|
||||
sort.Float64s(sorted)
|
||||
n := len(sorted)
|
||||
if n%2 == 0 {
|
||||
return (sorted[n/2-1] + sorted[n/2]) / 2
|
||||
}
|
||||
return sorted[n/2]
|
||||
}
|
||||
|
||||
func mean(vals []float64) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, v := range vals {
|
||||
sum += v
|
||||
}
|
||||
return sum / float64(len(vals))
|
||||
}
|
||||
|
||||
// tsSkewPair is a (timestamp, skew) pair for drift estimation.
|
||||
type tsSkewPair struct {
|
||||
ts int64
|
||||
skew float64
|
||||
}
|
||||
|
||||
// computeDrift estimates linear drift in seconds per day from time-ordered
|
||||
// (timestamp, skew) pairs. Issue #789: a single GPS-correction event (huge
|
||||
// skew jump in seconds) used to dominate ordinary least squares and produce
|
||||
// absurd drift like 1.7M sec/day. We now:
|
||||
//
|
||||
// 1. Drop pairs whose consecutive skew jump exceeds maxPlausibleSkewJumpSec
|
||||
// (clock corrections, not physical drift). This protects both OLS-style
|
||||
// consumers and Theil-Sen.
|
||||
// 2. Use Theil-Sen regression — the slope is the median of all pairwise
|
||||
// slopes, naturally robust to remaining outliers (breakdown point ~29%).
|
||||
//
|
||||
// For very small samples after filtering we fall back to a simple slope
|
||||
// between first and last calibrated samples.
|
||||
func computeDrift(pairs []tsSkewPair) float64 {
|
||||
if len(pairs) < 2 {
|
||||
return 0
|
||||
}
|
||||
// Sort by timestamp.
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
return pairs[i].ts < pairs[j].ts
|
||||
})
|
||||
|
||||
// Time span too short? Skip.
|
||||
spanSec := float64(pairs[len(pairs)-1].ts - pairs[0].ts)
|
||||
if spanSec < 3600 { // need at least 1 hour of data
|
||||
return 0
|
||||
}
|
||||
|
||||
// Outlier filter: drop samples where the skew jumps more than
|
||||
// maxPlausibleSkewJumpSec from the running "stable" baseline.
|
||||
// We anchor on the first sample, then accept each subsequent point
|
||||
// that's within the threshold of the most recent accepted point —
|
||||
// this preserves a slow drift while rejecting correction events.
|
||||
filtered := make([]tsSkewPair, 0, len(pairs))
|
||||
filtered = append(filtered, pairs[0])
|
||||
for i := 1; i < len(pairs); i++ {
|
||||
prev := filtered[len(filtered)-1]
|
||||
if math.Abs(pairs[i].skew-prev.skew) <= maxPlausibleSkewJumpSec {
|
||||
filtered = append(filtered, pairs[i])
|
||||
}
|
||||
}
|
||||
// If the filter killed too much (e.g. unstable node), fall back to the
|
||||
// raw series so we at least produce *something* — it'll be capped by
|
||||
// maxReasonableDriftPerDay downstream.
|
||||
if len(filtered) < 2 || float64(filtered[len(filtered)-1].ts-filtered[0].ts) < 3600 {
|
||||
filtered = pairs
|
||||
}
|
||||
|
||||
// Cap point count for Theil-Sen (O(n²) on pairs). Keep most-recent.
|
||||
if len(filtered) > theilSenMaxPoints {
|
||||
filtered = filtered[len(filtered)-theilSenMaxPoints:]
|
||||
}
|
||||
|
||||
return theilSenSlope(filtered) * 86400 // sec/sec → sec/day
|
||||
}
|
||||
|
||||
// theilSenSlope returns the Theil-Sen estimator: median of all pairwise
|
||||
// slopes (yj - yi) / (tj - ti) for i < j. Naturally robust to outliers.
|
||||
// Pairs must be sorted by timestamp ascending.
|
||||
func theilSenSlope(pairs []tsSkewPair) float64 {
|
||||
n := len(pairs)
|
||||
if n < 2 {
|
||||
return 0
|
||||
}
|
||||
// Pre-allocate: n*(n-1)/2 pairs.
|
||||
slopes := make([]float64, 0, n*(n-1)/2)
|
||||
for i := 0; i < n; i++ {
|
||||
for j := i + 1; j < n; j++ {
|
||||
dt := float64(pairs[j].ts - pairs[i].ts)
|
||||
if dt <= 0 {
|
||||
continue
|
||||
}
|
||||
slopes = append(slopes, (pairs[j].skew-pairs[i].skew)/dt)
|
||||
}
|
||||
}
|
||||
if len(slopes) == 0 {
|
||||
return 0
|
||||
}
|
||||
return median(slopes)
|
||||
}
|
||||
@@ -0,0 +1,956 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── classifySkew ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestClassifySkew(t *testing.T) {
|
||||
tests := []struct {
|
||||
absSkew float64
|
||||
expected SkewSeverity
|
||||
}{
|
||||
{0, SkewOK},
|
||||
{60, SkewOK}, // 1 min
|
||||
{299, SkewOK}, // just under 5 min
|
||||
{300, SkewWarning}, // exactly 5 min
|
||||
{1800, SkewWarning}, // 30 min
|
||||
{3599, SkewWarning}, // just under 1 hour
|
||||
{3600, SkewCritical}, // exactly 1 hour
|
||||
{86400, SkewCritical}, // 1 day
|
||||
{2592000 - 1, SkewCritical}, // just under 30 days
|
||||
{2592000, SkewAbsurd}, // exactly 30 days
|
||||
{86400 * 365 - 1, SkewAbsurd}, // just under 365 days
|
||||
{86400 * 365, SkewNoClock}, // exactly 365 days
|
||||
{86400 * 365 * 10, SkewNoClock}, // 10 years (epoch-0 style)
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := classifySkew(tc.absSkew)
|
||||
if got != tc.expected {
|
||||
t.Errorf("classifySkew(%v) = %v, want %v", tc.absSkew, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── median ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestMedian(t *testing.T) {
|
||||
tests := []struct {
|
||||
vals []float64
|
||||
expected float64
|
||||
}{
|
||||
{nil, 0},
|
||||
{[]float64{}, 0},
|
||||
{[]float64{5}, 5},
|
||||
{[]float64{1, 3}, 2},
|
||||
{[]float64{3, 1, 2}, 2},
|
||||
{[]float64{4, 1, 3, 2}, 2.5},
|
||||
{[]float64{-10, 0, 10}, 0},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := median(tc.vals)
|
||||
if got != tc.expected {
|
||||
t.Errorf("median(%v) = %v, want %v", tc.vals, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMean(t *testing.T) {
|
||||
tests := []struct {
|
||||
vals []float64
|
||||
expected float64
|
||||
}{
|
||||
{nil, 0},
|
||||
{[]float64{10}, 10},
|
||||
{[]float64{2, 4, 6}, 4},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := mean(tc.vals)
|
||||
if got != tc.expected {
|
||||
t.Errorf("mean(%v) = %v, want %v", tc.vals, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── parseISO ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestParseISO(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected int64
|
||||
}{
|
||||
{"", 0},
|
||||
{"garbage", 0},
|
||||
{"2026-04-15T12:00:00Z", 1776254400},
|
||||
{"2026-04-15T12:00:00+00:00", 1776254400},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := parseISO(tc.input)
|
||||
if got != tc.expected {
|
||||
t.Errorf("parseISO(%q) = %v, want %v", tc.input, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── extractTimestamp ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestExtractTimestamp(t *testing.T) {
|
||||
// Nested payload.timestamp
|
||||
decoded := map[string]interface{}{
|
||||
"payload": map[string]interface{}{
|
||||
"timestamp": float64(1776340800),
|
||||
},
|
||||
}
|
||||
got := extractTimestamp(decoded)
|
||||
if got != 1776340800 {
|
||||
t.Errorf("extractTimestamp (nested) = %v, want 1776340800", got)
|
||||
}
|
||||
|
||||
// Top-level timestamp
|
||||
decoded2 := map[string]interface{}{
|
||||
"timestamp": float64(1776340900),
|
||||
}
|
||||
got2 := extractTimestamp(decoded2)
|
||||
if got2 != 1776340900 {
|
||||
t.Errorf("extractTimestamp (top-level) = %v, want 1776340900", got2)
|
||||
}
|
||||
|
||||
// No timestamp
|
||||
decoded3 := map[string]interface{}{"foo": "bar"}
|
||||
got3 := extractTimestamp(decoded3)
|
||||
if got3 != 0 {
|
||||
t.Errorf("extractTimestamp (missing) = %v, want 0", got3)
|
||||
}
|
||||
}
|
||||
|
||||
// ── calibrateObservers ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestCalibrateObservers_SingleObserver(t *testing.T) {
|
||||
// Single-observer packets can't calibrate — should return empty.
|
||||
samples := []skewSample{
|
||||
{advertTS: 1000, observedTS: 1000, observerID: "obs1", hash: "h1"},
|
||||
{advertTS: 2000, observedTS: 2000, observerID: "obs1", hash: "h2"},
|
||||
}
|
||||
offsets, _ := calibrateObservers(samples)
|
||||
if len(offsets) != 0 {
|
||||
t.Errorf("expected no offsets for single-observer, got %v", offsets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalibrateObservers_MultiObserver(t *testing.T) {
|
||||
// Packet h1 seen by 3 observers: obs1 at t=100, obs2 at t=110, obs3 at t=100.
|
||||
// Median observation = 100. obs1=0, obs2=+10, obs3=0
|
||||
// Packet h2 seen by 3 observers: obs1 at t=200, obs2 at t=210, obs3 at t=200.
|
||||
// Median observation = 200. obs1=0, obs2=+10, obs3=0
|
||||
samples := []skewSample{
|
||||
{advertTS: 100, observedTS: 100, observerID: "obs1", hash: "h1"},
|
||||
{advertTS: 100, observedTS: 110, observerID: "obs2", hash: "h1"},
|
||||
{advertTS: 100, observedTS: 100, observerID: "obs3", hash: "h1"},
|
||||
{advertTS: 200, observedTS: 200, observerID: "obs1", hash: "h2"},
|
||||
{advertTS: 200, observedTS: 210, observerID: "obs2", hash: "h2"},
|
||||
{advertTS: 200, observedTS: 200, observerID: "obs3", hash: "h2"},
|
||||
}
|
||||
offsets, _ := calibrateObservers(samples)
|
||||
if offsets["obs1"] != 0 {
|
||||
t.Errorf("obs1 offset = %v, want 0", offsets["obs1"])
|
||||
}
|
||||
if offsets["obs2"] != 10 {
|
||||
t.Errorf("obs2 offset = %v, want 10", offsets["obs2"])
|
||||
}
|
||||
if offsets["obs3"] != 0 {
|
||||
t.Errorf("obs3 offset = %v, want 0", offsets["obs3"])
|
||||
}
|
||||
}
|
||||
|
||||
// ── computeNodeSkew ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestComputeNodeSkew_BasicCorrection(t *testing.T) {
|
||||
// Validates observer offset correction direction.
|
||||
//
|
||||
// Setup: node is 60s ahead, obs1 accurate, obs2 is 10s ahead.
|
||||
// With 2 observers, median obs_ts = 1005.
|
||||
// obs1 offset = 1000 - 1005 = -5
|
||||
// obs2 offset = 1010 - 1005 = +5
|
||||
// Correction: corrected = raw_skew + obsOffset
|
||||
// obs1: raw=60, corrected = 60 + (-5) = 55
|
||||
// obs2: raw=50, corrected = 50 + 5 = 55
|
||||
// Both converge to 55 (not exact 60 because with only 2 observers,
|
||||
// the median can't fully distinguish which observer is drifted).
|
||||
|
||||
samples := []skewSample{
|
||||
// Same packet seen by accurate obs1 and obs2 (+10s ahead)
|
||||
{advertTS: 1060, observedTS: 1000, observerID: "obs1", hash: "h1"},
|
||||
{advertTS: 1060, observedTS: 1010, observerID: "obs2", hash: "h1"},
|
||||
}
|
||||
offsets, _ := calibrateObservers(samples)
|
||||
// median obs = 1005, obs1 offset = -5, obs2 offset = +5
|
||||
// So the median approach finds obs2 is +5 ahead (relative to median)
|
||||
|
||||
// Now compute node skew with those offsets:
|
||||
nodeSkew := computeNodeSkew(samples, offsets)
|
||||
cs, ok := nodeSkew["h1"]
|
||||
if !ok {
|
||||
t.Fatal("expected skew data for hash h1")
|
||||
}
|
||||
// With only 2 observers, median obs_ts = 1005.
|
||||
// obs1 offset = 1000-1005 = -5, obs2 offset = 1010-1005 = +5
|
||||
// raw from obs1 = 60, corrected = 60 + (-5) = 55
|
||||
// raw from obs2 = 50, corrected = 50 + 5 = 55
|
||||
// median = 55
|
||||
if cs.MedianSkewSec != 55 {
|
||||
t.Errorf("median skew = %v, want 55", cs.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeNodeSkew_ThreeObservers(t *testing.T) {
|
||||
// Node is exactly 60s ahead. obs1 accurate, obs2 accurate, obs3 +30s ahead.
|
||||
// advertTS = 1060, real time = 1000
|
||||
samples := []skewSample{
|
||||
{advertTS: 1060, observedTS: 1000, observerID: "obs1", hash: "h1"},
|
||||
{advertTS: 1060, observedTS: 1000, observerID: "obs2", hash: "h1"},
|
||||
{advertTS: 1060, observedTS: 1030, observerID: "obs3", hash: "h1"},
|
||||
}
|
||||
offsets, _ := calibrateObservers(samples)
|
||||
// median obs_ts = 1000. obs1=0, obs2=0, obs3=+30
|
||||
if offsets["obs3"] != 30 {
|
||||
t.Errorf("obs3 offset = %v, want 30", offsets["obs3"])
|
||||
}
|
||||
|
||||
nodeSkew := computeNodeSkew(samples, offsets)
|
||||
cs := nodeSkew["h1"]
|
||||
if cs == nil {
|
||||
t.Fatal("expected skew data for h1")
|
||||
}
|
||||
// raw from obs1 = 60, corrected = 60 + 0 = 60
|
||||
// raw from obs2 = 60, corrected = 60 + 0 = 60
|
||||
// raw from obs3 = 30, corrected = 30 + 30 = 60
|
||||
// All three converge to 60.
|
||||
if cs.MedianSkewSec != 60 {
|
||||
t.Errorf("median skew = %v, want 60 (node is 60s ahead)", cs.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// ── computeDrift ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestComputeDrift_Stable(t *testing.T) {
|
||||
// Constant skew = no drift.
|
||||
pairs := []tsSkewPair{
|
||||
{ts: 0, skew: 60},
|
||||
{ts: 7200, skew: 60},
|
||||
{ts: 14400, skew: 60},
|
||||
}
|
||||
drift := computeDrift(pairs)
|
||||
if drift != 0 {
|
||||
t.Errorf("drift = %v, want 0 for stable skew", drift)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_LinearDrift(t *testing.T) {
|
||||
// 1 second drift per hour = 24 sec/day.
|
||||
pairs := []tsSkewPair{
|
||||
{ts: 0, skew: 0},
|
||||
{ts: 3600, skew: 1},
|
||||
{ts: 7200, skew: 2},
|
||||
}
|
||||
drift := computeDrift(pairs)
|
||||
expected := 24.0
|
||||
if math.Abs(drift-expected) > 0.1 {
|
||||
t.Errorf("drift = %v, want ~%v", drift, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_TooFewSamples(t *testing.T) {
|
||||
pairs := []tsSkewPair{{ts: 0, skew: 10}}
|
||||
if computeDrift(pairs) != 0 {
|
||||
t.Error("expected 0 drift for single sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDrift_TooShortSpan(t *testing.T) {
|
||||
// Less than 1 hour apart.
|
||||
pairs := []tsSkewPair{
|
||||
{ts: 0, skew: 0},
|
||||
{ts: 1800, skew: 10},
|
||||
}
|
||||
if computeDrift(pairs) != 0 {
|
||||
t.Error("expected 0 drift for short time span")
|
||||
}
|
||||
}
|
||||
|
||||
// ── jsonNumber ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestJsonNumber(t *testing.T) {
|
||||
m := map[string]interface{}{
|
||||
"a": float64(42),
|
||||
"b": int64(99),
|
||||
"c": "not a number",
|
||||
"d": nil,
|
||||
}
|
||||
if jsonNumber(m, "a") != 42 {
|
||||
t.Error("float64 case failed")
|
||||
}
|
||||
if jsonNumber(m, "b") != 99 {
|
||||
t.Error("int64 case failed")
|
||||
}
|
||||
if jsonNumber(m, "c") != 0 {
|
||||
t.Error("string case should return 0")
|
||||
}
|
||||
if jsonNumber(m, "d") != 0 {
|
||||
t.Error("nil case should return 0")
|
||||
}
|
||||
if jsonNumber(m, "missing") != 0 {
|
||||
t.Error("missing key should return 0")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Integration: GetNodeClockSkew via PacketStore ──────────────────────────────
|
||||
|
||||
func TestGetNodeClockSkew_Integration(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
|
||||
// Simulate two ADVERT transmissions for the same node, seen by 2 observers each.
|
||||
// Node "AABB" has clock 120s ahead.
|
||||
pt := 4 // ADVERT
|
||||
tx1 := &StoreTx{
|
||||
Hash: "hash1",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":1700002320}}`, // obs=1700002200, node ahead by 120s
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: "2023-11-14T22:50:00Z"}, // 1700002200
|
||||
{ObserverID: "obs2", Timestamp: "2023-11-14T22:50:00Z"}, // 1700002200
|
||||
},
|
||||
}
|
||||
tx2 := &StoreTx{
|
||||
Hash: "hash2",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":1700005920}}`, // obs=1700005800, node ahead by 120s
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: "2023-11-14T23:50:00Z"}, // 1700005800
|
||||
{ObserverID: "obs2", Timestamp: "2023-11-14T23:50:00Z"}, // 1700005800
|
||||
},
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["AABB"] = []*StoreTx{tx1, tx2}
|
||||
ps.byPayloadType[4] = []*StoreTx{tx1, tx2}
|
||||
// Force recompute by setting interval to 0.
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
result := ps.GetNodeClockSkew("AABB")
|
||||
if result == nil {
|
||||
t.Fatal("expected clock skew result for node AABB")
|
||||
}
|
||||
if result.Pubkey != "AABB" {
|
||||
t.Errorf("pubkey = %q, want AABB", result.Pubkey)
|
||||
}
|
||||
// Both transmissions show 120s skew, so median should be 120.
|
||||
if result.MedianSkewSec != 120 {
|
||||
t.Errorf("median skew = %v, want 120", result.MedianSkewSec)
|
||||
}
|
||||
if result.SampleCount < 2 {
|
||||
t.Errorf("sample count = %v, want >= 2", result.SampleCount)
|
||||
}
|
||||
if result.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok (120s < 5min)", result.Severity)
|
||||
}
|
||||
// Drift should be ~0 since skew is constant.
|
||||
if math.Abs(result.DriftPerDaySec) > 1 {
|
||||
t.Errorf("drift = %v, want ~0 for constant skew", result.DriftPerDaySec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodeClockSkew_NoData(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
result := ps.GetNodeClockSkew("nonexistent")
|
||||
if result != nil {
|
||||
t.Error("expected nil for nonexistent node")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Sanity check tests (#XXX — clock skew crazy stats) ────────────────────────
|
||||
|
||||
func TestGetNodeClockSkew_NoClock_EpochZero(t *testing.T) {
|
||||
// Node with epoch-0 timestamp produces huge skew → no_clock severity, drift=0.
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4 // ADVERT
|
||||
|
||||
// Epoch-ish advert: advertTS near start of 2020, observed in 2023 → |skew| > 365 days
|
||||
var txs []*StoreTx
|
||||
baseObs := int64(1700000000) // ~Nov 2023
|
||||
for i := 0; i < 6; i++ {
|
||||
obsTS := baseObs + int64(i)*7200
|
||||
tx := &StoreTx{
|
||||
Hash: "epoch-h" + string(rune('0'+i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":1577836800}}`, // Jan 1 2020 — valid but way off
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["EPOCH"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
result := ps.GetNodeClockSkew("EPOCH")
|
||||
if result == nil {
|
||||
t.Fatal("expected clock skew result for epoch-0 node")
|
||||
}
|
||||
if result.Severity != SkewNoClock {
|
||||
t.Errorf("severity = %v, want no_clock", result.Severity)
|
||||
}
|
||||
if result.DriftPerDaySec != 0 {
|
||||
t.Errorf("drift = %v, want 0 for no_clock node", result.DriftPerDaySec)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodeClockSkew_TooFewSamplesForDrift(t *testing.T) {
|
||||
// Node with only 2 advert samples → drift should not be computed.
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 2; i++ {
|
||||
obsTS := baseObs + int64(i)*7200
|
||||
advTS := obsTS + 120 // 120s ahead
|
||||
tx := &StoreTx{
|
||||
Hash: "few-h" + string(rune('0'+i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["FEWSAMP"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
result := ps.GetNodeClockSkew("FEWSAMP")
|
||||
if result == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
if result.DriftPerDaySec != 0 {
|
||||
t.Errorf("drift = %v, want 0 for 2-sample node (minimum is %d)", result.DriftPerDaySec, minDriftSamples)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodeClockSkew_AbsurdDriftCapped(t *testing.T) {
|
||||
// Node with wildly varying skew producing |drift| > 86400 s/day → drift capped to 0.
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
// Create 6 samples with extreme skew variation to produce absurd drift.
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 6; i++ {
|
||||
obsTS := baseObs + int64(i)*3600
|
||||
// Alternate between huge positive and negative skew offsets
|
||||
skewOffset := int64(50000 * (1 - 2*(i%2))) // +50000 or -50000
|
||||
advTS := obsTS + skewOffset
|
||||
tx := &StoreTx{
|
||||
Hash: "wild-h" + string(rune('0'+i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["WILD"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
result := ps.GetNodeClockSkew("WILD")
|
||||
if result == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
if math.Abs(result.DriftPerDaySec) > maxReasonableDriftPerDay {
|
||||
t.Errorf("drift = %v, should be capped (|drift| > %v)", result.DriftPerDaySec, maxReasonableDriftPerDay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodeClockSkew_NormalNodeWithDrift(t *testing.T) {
|
||||
// Normal node with 6 samples and consistent linear drift → drift computed correctly.
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 6; i++ {
|
||||
obsTS := baseObs + int64(i)*7200 // every 2 hours
|
||||
// Drift: 1 sec/hour = 24 sec/day
|
||||
advTS := obsTS + 120 + int64(i) // skew grows by 1s per sample (2h apart)
|
||||
tx := &StoreTx{
|
||||
Hash: "norm-h" + string(rune('0'+i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["NORMAL"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
result := ps.GetNodeClockSkew("NORMAL")
|
||||
if result == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
if result.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", result.Severity)
|
||||
}
|
||||
// 1s per 7200s = 12 s/day
|
||||
if result.DriftPerDaySec == 0 {
|
||||
t.Error("expected non-zero drift for linearly drifting node")
|
||||
}
|
||||
if math.Abs(result.DriftPerDaySec) > maxReasonableDriftPerDay {
|
||||
t.Errorf("drift = %v, should be reasonable", result.DriftPerDaySec)
|
||||
}
|
||||
}
|
||||
|
||||
// formatInt64 is a test helper to format int64 as string for JSON embedding.
|
||||
func formatInt64(n int64) string {
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
|
||||
// ── #789: Recent-window severity & robust drift ───────────────────────────────
|
||||
|
||||
// TestSeverityUsesRecentNotMedian: 100 historical bad samples (skew=-60s,
|
||||
// each ~5min apart) followed by 5 fresh good samples (skew=-1s). All-time
|
||||
// median is still huge-ish but recent-window severity must reflect the
|
||||
// current healthy state.
|
||||
func TestSeverityUsesRecentNotMedian(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 105; i++ {
|
||||
obsTS := baseObs + int64(i)*300 // 5 min apart
|
||||
var skew int64 = -60
|
||||
if i >= 100 {
|
||||
skew = -1 // good samples at the tail
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("recent-h%03d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["RECENT"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("RECENT")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok (recent samples are healthy)", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec) > 5 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ~-1", r.RecentMedianSkewSec)
|
||||
}
|
||||
// Historical median should still be retained for context.
|
||||
if math.Abs(r.MedianSkewSec) < 30 {
|
||||
t.Errorf("medianSkewSec = %v, expected historical median to remain large", r.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriftRejectsCorrectionJump: 30 minutes of clean linear drift, then a
|
||||
// single 60-second skew jump. The pre-jump slope should win — drift must
|
||||
// not be catastrophically inflated by the correction event.
|
||||
func TestDriftRejectsCorrectionJump(t *testing.T) {
|
||||
pairs := []tsSkewPair{}
|
||||
// 30 min of stable, ~12 sec/day drift: 1s per 7200s.
|
||||
for i := 0; i < 12; i++ {
|
||||
ts := int64(i) * 300
|
||||
skew := float64(i) * (1.0 / 24.0) // ~0.04s per 5min step → 12 s/day
|
||||
pairs = append(pairs, tsSkewPair{ts: ts, skew: skew})
|
||||
}
|
||||
// Wait an hour, then a single 1000-sec correction jump (clearly outlier).
|
||||
pairs = append(pairs, tsSkewPair{ts: 3600 + 12*300, skew: 1000})
|
||||
|
||||
drift := computeDrift(pairs)
|
||||
// Without rejection this would be ~ (1000-0)/(end-0) * 86400 = enormous.
|
||||
if math.Abs(drift) > 100 {
|
||||
t.Errorf("drift = %v, expected small (~12 s/day), correction jump should be filtered", drift)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheilSenMatchesOLSWhenClean: on clean linear data Theil-Sen should
|
||||
// produce essentially the OLS answer.
|
||||
func TestTheilSenMatchesOLSWhenClean(t *testing.T) {
|
||||
// 1 sec drift per hour = 24 sec/day, 20 evenly-spaced samples.
|
||||
pairs := []tsSkewPair{}
|
||||
for i := 0; i < 20; i++ {
|
||||
pairs = append(pairs, tsSkewPair{
|
||||
ts: int64(i) * 600,
|
||||
skew: float64(i) * (600.0 / 3600.0),
|
||||
})
|
||||
}
|
||||
drift := computeDrift(pairs)
|
||||
if math.Abs(drift-24.0) > 0.25 { // ~1%
|
||||
t.Errorf("drift = %v, want ~24", drift)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReporterScenario_789: reproduce the exact scenario from issue #789.
|
||||
// Reporter saw mean=-52565156, median=-59063561, last=-0.8, sample count
|
||||
// 1662, drift +1793549.9 s/day, severity=absurd. After the fix, severity
|
||||
// must be ok (recent samples are healthy) and drift must be sane.
|
||||
func TestReporterScenario_789(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 1657 samples with the bad ~-683-day skew (the historical poison),
|
||||
// then 5 freshly corrected samples at -0.8s — totals 1662.
|
||||
for i := 0; i < 1662; i++ {
|
||||
obsTS := baseObs + int64(i)*60 // 1 min apart
|
||||
var skew int64
|
||||
if i < 1657 {
|
||||
skew = -59063561 // ~ -683 days
|
||||
} else {
|
||||
skew = -1 // corrected (rounded; reporter saw -0.8)
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("rep-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["REPNODE"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("REPNODE")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
// Severity must reflect current health, not the all-time median.
|
||||
if r.Severity != SkewOK && r.Severity != SkewWarning {
|
||||
t.Errorf("severity = %v, want ok/warning (recent samples are healthy)", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec) > 5 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want near 0", r.RecentMedianSkewSec)
|
||||
}
|
||||
// Drift must not be absurd. The historical jump is one event between
|
||||
// the 1657th and 1658th sample; outlier rejection must contain it.
|
||||
if math.Abs(r.DriftPerDaySec) > maxReasonableDriftPerDay {
|
||||
t.Errorf("drift = %v, must be <= cap %v", r.DriftPerDaySec, maxReasonableDriftPerDay)
|
||||
}
|
||||
// And it should be close to zero (stable historical + stable corrected).
|
||||
if math.Abs(r.DriftPerDaySec) > 1000 {
|
||||
t.Errorf("drift = %v, expected near zero after outlier rejection", r.DriftPerDaySec)
|
||||
}
|
||||
// Historical median is preserved as context.
|
||||
if math.Abs(r.MedianSkewSec) < 1e6 {
|
||||
t.Errorf("medianSkewSec = %v, expected historical poison preserved as context", r.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBimodalClock_845: 60% good samples → bimodal_clock severity.
|
||||
func TestBimodalClock_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 6 good samples (-5s each), 4 bad samples (-50000000s each) = 60% good
|
||||
// Interleave so the recent window (last 5) captures both good and bad.
|
||||
skews := []int64{-5, -5, -50000000, -5, -50000000, -5, -50000000, -5, -50000000, -5}
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS + skews[i]
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("bimodal-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["BIMODAL"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("BIMODAL")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewBimodalClock {
|
||||
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ≈ -5 (median of good samples)", r.RecentMedianSkewSec)
|
||||
}
|
||||
if r.GoodFraction < 0.5 || r.GoodFraction > 0.7 {
|
||||
t.Errorf("goodFraction = %v, want ~0.6", r.GoodFraction)
|
||||
}
|
||||
if r.RecentBadSampleCount < 1 {
|
||||
t.Errorf("recentBadSampleCount = %v, want > 0", r.RecentBadSampleCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllBad_NoClock_845: all samples bad → no_clock.
|
||||
func TestAllBad_NoClock_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS - 50000000
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("allbad-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["ALLBAD"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("ALLBAD")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewNoClock {
|
||||
t.Errorf("severity = %v, want no_clock", r.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMostlyGood_OK_845: 90% good 10% bad → ok (outlier filtered).
|
||||
func TestMostlyGood_OK_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 9 good at -5s, 1 bad at -50000000s
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
var skew int64
|
||||
if i < 9 {
|
||||
skew = -5
|
||||
} else {
|
||||
skew = -50000000
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("mostly-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["MOSTLY"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("MOSTLY")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
// 90% good → normal classification path, median of good samples = -5s → ok
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ≈ -5", r.RecentMedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleSample_845: one good sample → ok.
|
||||
func TestSingleSample_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
obsTS := int64(1700000000)
|
||||
advTS := obsTS - 30 // 30s skew
|
||||
tx := &StoreTx{
|
||||
Hash: "single-0001",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["SINGLE"] = []*StoreTx{tx}
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("SINGLE")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if r.RecentSampleCount != 1 {
|
||||
t.Errorf("recentSampleCount = %d, want 1", r.RecentSampleCount)
|
||||
}
|
||||
if r.GoodFraction != 1.0 {
|
||||
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFiftyFifty_Bimodal_845: 50% good / 50% bad → bimodal_clock.
|
||||
func TestFiftyFifty_Bimodal_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
var skew int64
|
||||
if i%2 == 0 {
|
||||
skew = -10
|
||||
} else {
|
||||
skew = -50000000
|
||||
}
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("fifty-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS+skew) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["FIFTY"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("FIFTY")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewBimodalClock {
|
||||
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
|
||||
}
|
||||
if r.GoodFraction < 0.4 || r.GoodFraction > 0.6 {
|
||||
t.Errorf("goodFraction = %v, want ~0.5", r.GoodFraction)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllGood_OK_845: all samples good → ok, no bimodal.
|
||||
func TestAllGood_OK_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("allgood-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS-3) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["ALLGOOD"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("ALLGOOD")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if r.GoodFraction != 1.0 {
|
||||
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
|
||||
}
|
||||
if r.RecentBadSampleCount != 0 {
|
||||
t.Errorf("recentBadSampleCount = %v, want 0", r.RecentBadSampleCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCollisionDetailsIncludeNodePairs verifies that collision details contain
|
||||
// the correct prefix and matching node pairs (#757).
|
||||
func TestCollisionDetailsIncludeNodePairs(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Insert two repeater nodes with the same 3-byte prefix "AABB11"
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Node Alpha', 'repeater')`)
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11eeff334455', 'Node Beta', 'repeater')`)
|
||||
|
||||
// Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
result := store.GetAnalyticsHashCollisions("")
|
||||
bySize, ok := result["by_size"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size map")
|
||||
}
|
||||
|
||||
size3, ok := bySize["3"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size[3] map")
|
||||
}
|
||||
|
||||
collisions, ok := size3["collisions"].([]collisionEntry)
|
||||
if !ok {
|
||||
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
|
||||
}
|
||||
|
||||
// Find our collision
|
||||
var found *collisionEntry
|
||||
for i := range collisions {
|
||||
if collisions[i].Prefix == "AABB11" {
|
||||
found = &collisions[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatal("expected collision with prefix AABB11")
|
||||
}
|
||||
if found.Appearances != 2 {
|
||||
t.Errorf("expected 2 appearances, got %d", found.Appearances)
|
||||
}
|
||||
if len(found.Nodes) != 2 {
|
||||
t.Fatalf("expected 2 nodes in collision, got %d", len(found.Nodes))
|
||||
}
|
||||
|
||||
// Verify node pairs
|
||||
pubkeys := map[string]bool{}
|
||||
names := map[string]bool{}
|
||||
for _, n := range found.Nodes {
|
||||
pubkeys[n.PublicKey] = true
|
||||
names[n.Name] = true
|
||||
}
|
||||
if !pubkeys["aabb11ccdd001122"] {
|
||||
t.Error("expected node aabb11ccdd001122 in collision")
|
||||
}
|
||||
if !pubkeys["aabb11eeff334455"] {
|
||||
t.Error("expected node aabb11eeff334455 in collision")
|
||||
}
|
||||
if !names["Node Alpha"] {
|
||||
t.Error("expected Node Alpha in collision")
|
||||
}
|
||||
if !names["Node Beta"] {
|
||||
t.Error("expected Node Beta in collision")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollisionDetailsEmptyWhenNoCollisions verifies that collision details are
|
||||
// empty when there are no collisions (#757).
|
||||
func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Insert one repeater node with 3-byte hash
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`)
|
||||
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
result := store.GetAnalyticsHashCollisions("")
|
||||
bySize, ok := result["by_size"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size map")
|
||||
}
|
||||
|
||||
size3, ok := bySize["3"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size[3] map")
|
||||
}
|
||||
|
||||
collisions, ok := size3["collisions"].([]collisionEntry)
|
||||
if !ok {
|
||||
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
|
||||
}
|
||||
|
||||
if len(collisions) != 0 {
|
||||
t.Errorf("expected 0 collisions, got %d", len(collisions))
|
||||
}
|
||||
}
|
||||
+54
-4
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
)
|
||||
@@ -16,6 +17,17 @@ type Config struct {
|
||||
APIKey string `json:"apiKey"`
|
||||
DBPath string `json:"dbPath"`
|
||||
|
||||
// NodeBlacklist is a list of public keys to exclude from all API responses.
|
||||
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
|
||||
// Use this to filter out trolls, nodes with offensive names, or nodes
|
||||
// reporting deliberately false data (e.g. wrong GPS position) that the
|
||||
// operator refuses to fix.
|
||||
NodeBlacklist []string `json:"nodeBlacklist"`
|
||||
|
||||
// blacklistSetCached is the lazily-built set version of NodeBlacklist.
|
||||
blacklistSetCached map[string]bool
|
||||
blacklistOnce sync.Once
|
||||
|
||||
Branding map[string]interface{} `json:"branding"`
|
||||
Theme map[string]interface{} `json:"theme"`
|
||||
ThemeDark map[string]interface{} `json:"themeDark"`
|
||||
@@ -103,16 +115,18 @@ type NeighborGraphConfig struct {
|
||||
// PacketStoreConfig controls in-memory packet store limits.
|
||||
type PacketStoreConfig struct {
|
||||
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
MaxResolvedPubkeyIndexEntries int `json:"maxResolvedPubkeyIndexEntries"` // warning threshold for index size (0 = 5M default)
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
type GeoFilterConfig = geofilter.Config
|
||||
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
}
|
||||
|
||||
// MetricsRetentionDays returns configured metrics retention or 30 days default.
|
||||
@@ -165,6 +179,15 @@ func (c *Config) NodeDaysOrDefault() int {
|
||||
return 7
|
||||
}
|
||||
|
||||
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
|
||||
// A value of -1 means observers are never removed.
|
||||
func (c *Config) ObserverDaysOrDefault() int {
|
||||
if c.Retention != nil && c.Retention.ObserverDays != 0 {
|
||||
return c.Retention.ObserverDays
|
||||
}
|
||||
return 14
|
||||
}
|
||||
|
||||
type HealthThresholds struct {
|
||||
InfraDegradedHours float64 `json:"infraDegradedHours"`
|
||||
InfraSilentHours float64 `json:"infraSilentHours"`
|
||||
@@ -338,3 +361,30 @@ func (c *Config) PropagationBufferMs() int {
|
||||
}
|
||||
return 5000
|
||||
}
|
||||
|
||||
// blacklistSet lazily builds and caches the nodeBlacklist as a set for O(1) lookups.
|
||||
// Uses sync.Once to eliminate the data race on first concurrent access.
|
||||
func (c *Config) blacklistSet() map[string]bool {
|
||||
c.blacklistOnce.Do(func() {
|
||||
if len(c.NodeBlacklist) == 0 {
|
||||
return
|
||||
}
|
||||
m := make(map[string]bool, len(c.NodeBlacklist))
|
||||
for _, pk := range c.NodeBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.blacklistSetCached = m
|
||||
})
|
||||
return c.blacklistSetCached
|
||||
}
|
||||
|
||||
// IsBlacklisted returns true if the given public key is in the nodeBlacklist.
|
||||
func (c *Config) IsBlacklisted(pubkey string) bool {
|
||||
if c == nil || len(c.NodeBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
|
||||
}
|
||||
|
||||
@@ -365,3 +365,25 @@ func TestPropagationBufferMs(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestObserverDaysOrDefault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *Config
|
||||
want int
|
||||
}{
|
||||
{"nil retention", &Config{}, 14},
|
||||
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
|
||||
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
|
||||
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.ObserverDaysOrDefault()
|
||||
if got != tt.want {
|
||||
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+205
-59
@@ -41,7 +41,7 @@ func setupTestDBv2(t *testing.T) *DB {
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
|
||||
decoded_json TEXT, created_at TEXT DEFAULT (datetime('now'))
|
||||
decoded_json TEXT, channel_hash TEXT DEFAULT NULL, created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
@@ -585,12 +585,15 @@ func TestHandlePacketsMultiNodeWithStore(t *testing.T) {
|
||||
func TestHandlePacketDetailNoStore(t *testing.T) {
|
||||
_, router := setupNoStoreServer(t)
|
||||
|
||||
// With no in-memory store, handlePacketDetail now falls back to the DB
|
||||
// (#827). The seeded transmissions are present in the DB, so by-hash and
|
||||
// by-ID lookups succeed; only truly absent IDs return 404.
|
||||
t.Run("by hash", func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/api/packets/abc123def4567890", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -598,8 +601,8 @@ func TestHandlePacketDetailNoStore(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/api/packets/1", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3217,6 +3220,189 @@ func TestGetNodeHashSizeInfoEdgeCases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashSizeTransportRoutePathByteOffset verifies that transport routes (0, 3)
|
||||
// read the path byte from offset 5 (after 4 transport code bytes), not offset 1.
|
||||
// Regression test for #744 / #722.
|
||||
func TestHashSizeTransportRoutePathByteOffset(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
|
||||
|
||||
// Route type 0 (TRANSPORT_FLOOD): header=0x04 (payload_type=1, route_type=0)
|
||||
// 4 transport bytes + path byte at offset 5.
|
||||
// Path byte 0x80 → hash_size bits = 10 → size 3
|
||||
// If bug is present, code reads byte 1 (0xAA) → hash_size bits = 10 → size 3 (coincidence)
|
||||
// Use path byte 0x40 (hash_size=2) and transport byte 0x01 at offset 1 (hash_size=1 if misread)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('100102030440aabb', 'tf_offset', ?, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
// Route type 3 (TRANSPORT_DIRECT): header=0x13 (payload_type=4, route_type=3)
|
||||
// 4 transport bytes + path byte at offset 5.
|
||||
// Path byte 0xC1 → hash_size bits = 11 → size 4, hop_count = 1 (not zero-hop)
|
||||
// Byte 1 = 0x05 → hash_size bits = 00 → size 1 if misread
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1305060708C1bbcc', 'td_offset', ?, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
// Route type 1 (FLOOD): header=0x11 (payload_type=4, route_type=1)
|
||||
// Path byte at offset 1. Path byte 0x80 → hash_size bits = 10 → size 3
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1180aabbccdd', 'flood_offset', ?, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
info := store.GetNodeHashSizeInfo()
|
||||
|
||||
// Transport flood node: path byte 0x40 → hash_size = 2
|
||||
if ni, ok := info["aaaa000000000001"]; !ok {
|
||||
t.Error("transport flood node missing from hash size info")
|
||||
} else if ni.HashSize != 2 {
|
||||
t.Errorf("transport flood node: want HashSize=2 (from path byte at offset 5), got %d", ni.HashSize)
|
||||
}
|
||||
|
||||
// Transport direct node: path byte 0xC1 → hash_size = 4
|
||||
if ni, ok := info["aaaa000000000002"]; !ok {
|
||||
t.Error("transport direct node missing from hash size info")
|
||||
} else if ni.HashSize != 4 {
|
||||
t.Errorf("transport direct node: want HashSize=4 (from path byte at offset 5), got %d", ni.HashSize)
|
||||
}
|
||||
|
||||
// Regular flood node: path byte 0x80 → hash_size = 3
|
||||
if ni, ok := info["aaaa000000000003"]; !ok {
|
||||
t.Error("regular flood node missing from hash size info")
|
||||
} else if ni.HashSize != 3 {
|
||||
t.Errorf("regular flood node: want HashSize=3 (from path byte at offset 1), got %d", ni.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashSizeTransportDirectZeroHopSkipped verifies that RouteTransportDirect
|
||||
// zero-hop adverts are skipped (same as RouteDirect). Regression test for #744.
|
||||
func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
|
||||
|
||||
// RouteDirect (2) zero-hop: path byte 0x40 → hop_count=0, hash_size bits=01
|
||||
// Should be skipped (existing behavior)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1240aabbccdd', 'direct_zh', ?, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
// RouteTransportDirect (3) zero-hop: 4 transport bytes + path byte 0x40 → hop_count=0
|
||||
// Should ALSO be skipped (this was the missing case)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('130102030440aabb', 'tdirect_zh', ?, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
// RouteDirect (2) non-zero-hop: path byte 0x41 → hop_count=1
|
||||
// Should NOT be skipped
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1241aabbccdd', 'direct_1h', ?, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
info := store.GetNodeHashSizeInfo()
|
||||
|
||||
// RouteDirect zero-hop should be absent
|
||||
if _, ok := info["bbbb000000000001"]; ok {
|
||||
t.Error("RouteDirect zero-hop advert should be skipped")
|
||||
}
|
||||
|
||||
// RouteTransportDirect zero-hop should also be absent
|
||||
if _, ok := info["bbbb000000000002"]; ok {
|
||||
t.Error("RouteTransportDirect zero-hop advert should be skipped")
|
||||
}
|
||||
|
||||
// RouteDirect non-zero-hop should be present with hash_size=2
|
||||
if ni, ok := info["bbbb000000000003"]; !ok {
|
||||
t.Error("RouteDirect non-zero-hop should be in hash size info")
|
||||
} else if ni.HashSize != 2 {
|
||||
t.Errorf("RouteDirect non-zero-hop: want HashSize=2, got %d", ni.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyticsHashSizesZeroHopSkip verifies that computeAnalyticsHashSizes
|
||||
// does not overwrite a node's hash_size with a zero-hop advert's unreliable value.
|
||||
// Regression test for #744.
|
||||
func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
|
||||
|
||||
pk := "cccc000000000001"
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'ZH-Analytics', 'repeater')`, pk)
|
||||
|
||||
decoded := `{"pubKey":"` + pk + `","name":"ZH-Analytics","type":"ADVERT"}`
|
||||
|
||||
// First: a flood advert with hashSize=2 (reliable, multi-hop)
|
||||
// header 0x11 = route_type 1 (flood), payload_type 4
|
||||
// pathByte 0x41 = hashSize bits 01 → size 2, hop_count 1
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1141aabbccdd', 'az_flood', ?, 1, 4, ?)`, recent, decoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '["aabb"]', ?)`, recentEpoch)
|
||||
|
||||
// Second: a direct zero-hop advert with pathByte=0x00 → would give hashSize=1
|
||||
// header 0x12 = route_type 2 (direct), payload_type 4
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1200aabbccdd', 'az_direct', ?, 2, 4, ?)`, recent, decoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
result := store.GetAnalyticsHashSizes("")
|
||||
|
||||
// The node should appear in multiByteNodes (hashSize=2 from the flood advert)
|
||||
// If the zero-hop bug is present, hashSize would be 1 and the node would NOT
|
||||
// appear in multiByteNodes.
|
||||
multiByteNodes, ok := result["multiByteNodes"].([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected multiByteNodes slice in analytics hash sizes")
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, n := range multiByteNodes {
|
||||
if n["pubkey"] == pk {
|
||||
found = true
|
||||
if hs, ok := n["hashSize"].(int); ok && hs != 2 {
|
||||
t.Errorf("expected hashSize=2 from flood advert, got %d", hs)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("node should appear in multiByteNodes with hashSize=2; zero-hop advert should not overwrite to 1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleResolveHopsEdgeCases(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
@@ -4133,88 +4319,48 @@ func TestIndexByNodePreCheck(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestIndexByNodeResolvedPath tests that resolved_path entries are indexed in byNode.
|
||||
// TestIndexByNodeResolvedPath tests that indexByNode only indexes decoded JSON pubkeys.
|
||||
// After #800, resolved_path entries are handled via the decode-window, not indexByNode.
|
||||
func TestIndexByNodeResolvedPath(t *testing.T) {
|
||||
store := &PacketStore{
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
}
|
||||
|
||||
t.Run("indexes resolved path pubkeys from observations", func(t *testing.T) {
|
||||
relayPK := "aabb1122334455ff"
|
||||
t.Run("decoded JSON pubkeys still indexed", func(t *testing.T) {
|
||||
pk := "aabb1122334455ff"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp1",
|
||||
DecodedJSON: `{"type":"CHAN","text":"hello"}`, // no pubKey fields
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&relayPK}},
|
||||
},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[relayPK]) != 1 {
|
||||
t.Errorf("expected relay pubkey indexed, got %d", len(store.byNode[relayPK]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips null entries in resolved path", func(t *testing.T) {
|
||||
pk := "cc11dd22ee33ff44"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp2",
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{nil, &pk, nil}},
|
||||
},
|
||||
DecodedJSON: `{"pubKey":"` + pk + `"}`,
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[pk]) != 1 {
|
||||
t.Errorf("expected resolved pubkey indexed, got %d", len(store.byNode[pk]))
|
||||
}
|
||||
// Verify nil entries didn't create empty-string keys
|
||||
if _, exists := store.byNode[""]; exists {
|
||||
t.Error("nil/empty resolved path entries should not create byNode entries")
|
||||
t.Errorf("expected decoded pubkey indexed, got %d", len(store.byNode[pk]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("relay-only node appears in byNode", func(t *testing.T) {
|
||||
// A packet with no decoded pubkey fields, only a relay in resolved path
|
||||
relayOnly := "relay0only0pubkey"
|
||||
t.Run("resolved path pubkeys NOT indexed by indexByNode", func(t *testing.T) {
|
||||
// After #800, indexByNode only handles decoded JSON fields.
|
||||
// Resolved path pubkeys are handled by the decode-window.
|
||||
tx := &StoreTx{
|
||||
Hash: "rp3",
|
||||
// No DecodedJSON at all — pure relay
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&relayOnly}},
|
||||
},
|
||||
Hash: "rp2",
|
||||
DecodedJSON: `{"type":"CHAN","text":"hello"}`, // no pubKey fields
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[relayOnly]) != 1 {
|
||||
t.Errorf("expected relay-only node indexed, got %d", len(store.byNode[relayOnly]))
|
||||
}
|
||||
// No new entries expected since there are no decoded pubkeys
|
||||
})
|
||||
|
||||
t.Run("dedup between decoded JSON and resolved path", func(t *testing.T) {
|
||||
t.Run("dedup within decoded JSON", func(t *testing.T) {
|
||||
pk := "dedup0test0pk1234"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp4",
|
||||
DecodedJSON: `{"pubKey":"` + pk + `"}`,
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&pk}},
|
||||
},
|
||||
DecodedJSON: `{"pubKey":"` + pk + `","destPubKey":"` + pk + `"}`,
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[pk]) != 1 {
|
||||
t.Errorf("expected dedup to keep 1 entry, got %d", len(store.byNode[pk]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("indexes tx.ResolvedPath when observations empty", func(t *testing.T) {
|
||||
rpPK := "txlevel0resolved1"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp5",
|
||||
ResolvedPath: []*string{&rpPK},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[rpPK]) != 1 {
|
||||
t.Errorf("expected tx-level resolved path indexed, got %d", len(store.byNode[rpPK]))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkIndexByNode measures indexByNode performance with and without pubkey
|
||||
|
||||
+335
-74
@@ -8,6 +8,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
@@ -19,6 +20,12 @@ type DB struct {
|
||||
path string // filesystem path to the database file
|
||||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||||
hasResolvedPath bool // observations table has resolved_path column
|
||||
|
||||
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
|
||||
channelsCacheMu sync.Mutex
|
||||
channelsCacheKey string
|
||||
channelsCacheRes []map[string]interface{}
|
||||
channelsCacheExp time.Time
|
||||
}
|
||||
|
||||
// OpenDB opens a read-only SQLite connection with WAL mode.
|
||||
@@ -377,6 +384,7 @@ type PacketQuery struct {
|
||||
Until string
|
||||
Region string
|
||||
Node string
|
||||
Channel string // channel_hash filter (#812). Plain names like "#test"/"public" or "enc_<HEX>" for encrypted
|
||||
Order string // ASC or DESC
|
||||
ExpandObservations bool // when true, include observation sub-maps in txToMap output
|
||||
}
|
||||
@@ -613,6 +621,11 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
|
||||
where = append(where, "t.decoded_json LIKE ?")
|
||||
args = append(args, "%"+pk+"%")
|
||||
}
|
||||
if q.Channel != "" {
|
||||
// channel_hash column is indexed for payload_type = 5; filter is exact match.
|
||||
where = append(where, "t.channel_hash = ?")
|
||||
args = append(args, q.Channel)
|
||||
}
|
||||
if q.Observer != "" {
|
||||
ids := strings.Split(q.Observer, ",")
|
||||
placeholders := strings.Repeat("?,", len(ids))
|
||||
@@ -679,6 +692,20 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// GetObservationsForHash returns all observations for the transmission with
|
||||
// the given content hash. Used as a fallback by the packet-detail handler
|
||||
// when the in-memory PacketStore has pruned the entry but the DB still has it.
|
||||
func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} {
|
||||
var txID int
|
||||
err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?",
|
||||
strings.ToLower(hash)).Scan(&txID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
obsByTx := db.getObservationsForTransmissions([]int{txID})
|
||||
return obsByTx[txID]
|
||||
}
|
||||
|
||||
|
||||
// GetNodes returns filtered, paginated node list.
|
||||
func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortBy, region string) ([]map[string]interface{}, int, map[string]int, error) {
|
||||
@@ -1153,69 +1180,219 @@ func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) {
|
||||
// Queries transmissions directly (not a VIEW) to avoid observation-level
|
||||
// duplicates that could cause stale lastMessage when an older message has
|
||||
// a later re-observation timestamp.
|
||||
func (db *DB) GetChannels() ([]map[string]interface{}, error) {
|
||||
rows, err := db.conn.Query(`SELECT decoded_json, first_seen FROM transmissions WHERE payload_type = 5 ORDER BY first_seen ASC`)
|
||||
func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) {
|
||||
regionParam := ""
|
||||
if len(region) > 0 {
|
||||
regionParam = region[0]
|
||||
}
|
||||
|
||||
// Check cache (60s TTL)
|
||||
db.channelsCacheMu.Lock()
|
||||
if db.channelsCacheRes != nil && db.channelsCacheKey == regionParam && time.Now().Before(db.channelsCacheExp) {
|
||||
res := db.channelsCacheRes
|
||||
db.channelsCacheMu.Unlock()
|
||||
return res, nil
|
||||
}
|
||||
db.channelsCacheMu.Unlock()
|
||||
|
||||
regionCodes := normalizeRegionCodes(regionParam)
|
||||
|
||||
var querySQL string
|
||||
args := make([]interface{}, 0, len(regionCodes))
|
||||
|
||||
if len(regionCodes) > 0 {
|
||||
placeholders := make([]string, len(regionCodes))
|
||||
for i, code := range regionCodes {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, code)
|
||||
}
|
||||
regionPlaceholder := strings.Join(placeholders, ",")
|
||||
if db.isV3 {
|
||||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(t.first_seen) AS last_activity,
|
||||
(SELECT t2.decoded_json FROM transmissions t2
|
||||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||||
FROM transmissions t
|
||||
JOIN observations o ON o.transmission_id = t.id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
WHERE t.payload_type = 5
|
||||
AND t.channel_hash IS NOT NULL
|
||||
AND t.channel_hash NOT LIKE 'enc_%%'
|
||||
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
|
||||
GROUP BY t.channel_hash
|
||||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||||
} else {
|
||||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(t.first_seen) AS last_activity,
|
||||
(SELECT t2.decoded_json FROM transmissions t2
|
||||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||||
FROM transmissions t
|
||||
JOIN observations o ON o.transmission_id = t.id
|
||||
WHERE t.payload_type = 5
|
||||
AND t.channel_hash IS NOT NULL
|
||||
AND t.channel_hash NOT LIKE 'enc_%%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM observers obs
|
||||
WHERE obs.id = o.observer_id
|
||||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||||
)
|
||||
GROUP BY t.channel_hash
|
||||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||||
}
|
||||
} else {
|
||||
querySQL = `SELECT channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(first_seen) AS last_activity,
|
||||
(SELECT t2.decoded_json FROM transmissions t2
|
||||
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
|
||||
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
|
||||
FROM transmissions t
|
||||
WHERE payload_type = 5
|
||||
AND channel_hash IS NOT NULL
|
||||
AND channel_hash NOT LIKE 'enc_%%'
|
||||
GROUP BY channel_hash
|
||||
ORDER BY last_activity DESC`
|
||||
}
|
||||
|
||||
rows, err := db.conn.Query(querySQL, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
channelMap := map[string]map[string]interface{}{}
|
||||
channels := make([]map[string]interface{}, 0)
|
||||
for rows.Next() {
|
||||
var dj, fs sql.NullString
|
||||
rows.Scan(&dj, &fs)
|
||||
if !dj.Valid {
|
||||
var chHash, lastActivity, sampleJSON sql.NullString
|
||||
var msgCount int
|
||||
if err := rows.Scan(&chHash, &msgCount, &lastActivity, &sampleJSON); err != nil {
|
||||
continue
|
||||
}
|
||||
var decoded map[string]interface{}
|
||||
if json.Unmarshal([]byte(dj.String), &decoded) != nil {
|
||||
continue
|
||||
}
|
||||
dtype, _ := decoded["type"].(string)
|
||||
if dtype != "CHAN" {
|
||||
continue
|
||||
}
|
||||
// Filter out garbage-decrypted channel names/messages (pre-#197 data still in DB)
|
||||
chanStr, _ := decoded["channel"].(string)
|
||||
textStr, _ := decoded["text"].(string)
|
||||
if hasGarbageChars(chanStr) || hasGarbageChars(textStr) {
|
||||
continue
|
||||
}
|
||||
channelName, _ := decoded["channel"].(string)
|
||||
channelName := nullStr(chHash)
|
||||
if channelName == "" {
|
||||
channelName = "unknown"
|
||||
continue
|
||||
}
|
||||
key := channelName
|
||||
|
||||
ch, exists := channelMap[key]
|
||||
if !exists {
|
||||
ch = map[string]interface{}{
|
||||
"hash": key, "name": channelName,
|
||||
"lastMessage": nil, "lastSender": nil,
|
||||
"messageCount": 0, "lastActivity": nullStr(fs),
|
||||
}
|
||||
channelMap[key] = ch
|
||||
}
|
||||
ch["messageCount"] = ch["messageCount"].(int) + 1
|
||||
if fs.Valid {
|
||||
ch["lastActivity"] = fs.String
|
||||
}
|
||||
if text, ok := decoded["text"].(string); ok && text != "" {
|
||||
idx := strings.Index(text, ": ")
|
||||
if idx > 0 {
|
||||
ch["lastMessage"] = text[idx+2:]
|
||||
} else {
|
||||
ch["lastMessage"] = text
|
||||
}
|
||||
if sender, ok := decoded["sender"].(string); ok {
|
||||
ch["lastSender"] = sender
|
||||
var lastMessage, lastSender interface{}
|
||||
if sampleJSON.Valid {
|
||||
var decoded map[string]interface{}
|
||||
if json.Unmarshal([]byte(sampleJSON.String), &decoded) == nil {
|
||||
if text, ok := decoded["text"].(string); ok && text != "" {
|
||||
idx := strings.Index(text, ": ")
|
||||
if idx > 0 {
|
||||
lastMessage = text[idx+2:]
|
||||
} else {
|
||||
lastMessage = text
|
||||
}
|
||||
if sender, ok := decoded["sender"].(string); ok {
|
||||
lastSender = sender
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
channels = append(channels, map[string]interface{}{
|
||||
"hash": channelName, "name": channelName,
|
||||
"lastMessage": lastMessage, "lastSender": lastSender,
|
||||
"messageCount": msgCount, "lastActivity": nullStr(lastActivity),
|
||||
})
|
||||
}
|
||||
|
||||
channels := make([]map[string]interface{}, 0, len(channelMap))
|
||||
for _, ch := range channelMap {
|
||||
channels = append(channels, ch)
|
||||
// Store in cache (60s TTL)
|
||||
db.channelsCacheMu.Lock()
|
||||
db.channelsCacheRes = channels
|
||||
db.channelsCacheKey = regionParam
|
||||
db.channelsCacheExp = time.Now().Add(60 * time.Second)
|
||||
db.channelsCacheMu.Unlock()
|
||||
|
||||
return channels, nil
|
||||
}
|
||||
|
||||
// GetEncryptedChannels returns channels where all messages are undecryptable (no key).
|
||||
// Uses channel_hash column (prefixed with 'enc_') for fast grouped queries.
|
||||
func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, error) {
|
||||
regionParam := ""
|
||||
if len(region) > 0 {
|
||||
regionParam = region[0]
|
||||
}
|
||||
regionCodes := normalizeRegionCodes(regionParam)
|
||||
|
||||
var querySQL string
|
||||
args := make([]interface{}, 0, len(regionCodes))
|
||||
|
||||
if len(regionCodes) > 0 {
|
||||
placeholders := make([]string, len(regionCodes))
|
||||
for i, code := range regionCodes {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, code)
|
||||
}
|
||||
regionPlaceholder := strings.Join(placeholders, ",")
|
||||
if db.isV3 {
|
||||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(t.first_seen) AS last_activity
|
||||
FROM transmissions t
|
||||
JOIN observations o ON o.transmission_id = t.id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
WHERE t.payload_type = 5
|
||||
AND t.channel_hash LIKE 'enc_%%'
|
||||
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
|
||||
GROUP BY t.channel_hash
|
||||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||||
} else {
|
||||
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(t.first_seen) AS last_activity
|
||||
FROM transmissions t
|
||||
JOIN observations o ON o.transmission_id = t.id
|
||||
WHERE t.payload_type = 5
|
||||
AND t.channel_hash LIKE 'enc_%%'
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM observers obs
|
||||
WHERE obs.id = o.observer_id
|
||||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||||
)
|
||||
GROUP BY t.channel_hash
|
||||
ORDER BY last_activity DESC`, regionPlaceholder)
|
||||
}
|
||||
} else {
|
||||
querySQL = `SELECT channel_hash,
|
||||
COUNT(*) AS msg_count,
|
||||
MAX(first_seen) AS last_activity
|
||||
FROM transmissions
|
||||
WHERE payload_type = 5
|
||||
AND channel_hash LIKE 'enc_%%'
|
||||
GROUP BY channel_hash
|
||||
ORDER BY last_activity DESC`
|
||||
}
|
||||
|
||||
rows, err := db.conn.Query(querySQL, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
channels := make([]map[string]interface{}, 0)
|
||||
for rows.Next() {
|
||||
var chHash, lastActivity sql.NullString
|
||||
var msgCount int
|
||||
if err := rows.Scan(&chHash, &msgCount, &lastActivity); err != nil {
|
||||
continue
|
||||
}
|
||||
fullHash := nullStrVal(chHash) // e.g. "enc_3A"
|
||||
hexPart := strings.TrimPrefix(fullHash, "enc_")
|
||||
channels = append(channels, map[string]interface{}{
|
||||
"hash": fullHash,
|
||||
"name": "Encrypted (0x" + hexPart + ")",
|
||||
"lastMessage": nil,
|
||||
"lastSender": nil,
|
||||
"messageCount": msgCount,
|
||||
"lastActivity": nullStr(lastActivity),
|
||||
"encrypted": true,
|
||||
})
|
||||
}
|
||||
return channels, nil
|
||||
}
|
||||
@@ -1244,15 +1421,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
|
||||
regionPlaceholders = strings.Join(placeholders, ",")
|
||||
}
|
||||
|
||||
// Fetch messages with channel_hash filter (pagination applied in Go after dedup)
|
||||
var querySQL string
|
||||
args := make([]interface{}, 0, len(regionArgs))
|
||||
args := []interface{}{channelHash}
|
||||
if db.isV3 {
|
||||
querySQL = `SELECT o.id, t.hash, t.decoded_json, t.first_seen,
|
||||
obs.id, obs.name, o.snr, o.path_json
|
||||
FROM observations o
|
||||
JOIN transmissions t ON t.id = o.transmission_id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
WHERE t.payload_type = 5`
|
||||
WHERE t.channel_hash = ? AND t.payload_type = 5`
|
||||
if len(regionCodes) > 0 {
|
||||
querySQL += fmt.Sprintf(" AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)", regionPlaceholders)
|
||||
args = append(args, regionArgs...)
|
||||
@@ -1264,14 +1442,11 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
|
||||
o.observer_id, o.observer_name, o.snr, o.path_json
|
||||
FROM observations o
|
||||
JOIN transmissions t ON t.id = o.transmission_id
|
||||
WHERE t.payload_type = 5`
|
||||
WHERE t.channel_hash = ? AND t.payload_type = 5`
|
||||
if len(regionCodes) > 0 {
|
||||
querySQL += fmt.Sprintf(` AND EXISTS (
|
||||
SELECT 1
|
||||
FROM observers obs
|
||||
WHERE obs.id = o.observer_id
|
||||
AND UPPER(TRIM(obs.iata)) IN (%s)
|
||||
)`, regionPlaceholders)
|
||||
SELECT 1 FROM observers obs WHERE obs.id = o.observer_id
|
||||
AND UPPER(TRIM(obs.iata)) IN (%s))`, regionPlaceholders)
|
||||
args = append(args, regionArgs...)
|
||||
}
|
||||
querySQL += `
|
||||
@@ -1303,17 +1478,6 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
|
||||
if json.Unmarshal([]byte(dj.String), &decoded) != nil {
|
||||
continue
|
||||
}
|
||||
dtype, _ := decoded["type"].(string)
|
||||
if dtype != "CHAN" {
|
||||
continue
|
||||
}
|
||||
ch, _ := decoded["channel"].(string)
|
||||
if ch == "" {
|
||||
ch = "unknown"
|
||||
}
|
||||
if ch != channelHash {
|
||||
continue
|
||||
}
|
||||
|
||||
text, _ := decoded["text"].(string)
|
||||
sender, _ := decoded["sender"].(string)
|
||||
@@ -1373,18 +1537,18 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
|
||||
}
|
||||
}
|
||||
|
||||
total := len(msgOrder)
|
||||
// Return latest messages (tail)
|
||||
start := total - limit - offset
|
||||
// Return latest messages (tail) with pagination
|
||||
msgTotal := len(msgOrder)
|
||||
start := msgTotal - limit - offset
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
end := total - offset
|
||||
end := msgTotal - offset
|
||||
if end < 0 {
|
||||
end = 0
|
||||
}
|
||||
if end > total {
|
||||
end = total
|
||||
if end > msgTotal {
|
||||
end = msgTotal
|
||||
}
|
||||
|
||||
messages := make([]map[string]interface{}, 0)
|
||||
@@ -1395,7 +1559,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
|
||||
messages = append(messages, m.Data)
|
||||
}
|
||||
|
||||
return messages, total, nil
|
||||
return messages, msgTotal, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2069,6 +2233,35 @@ func (db *DB) PruneOldMetrics(retentionDays int) (int64, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// RemoveStaleObservers marks observers that have not actively sent data in observerDays
|
||||
// as inactive (soft-delete). This preserves JOIN integrity for observations.observer_idx
|
||||
// and observer_metrics.observer_id — historical data still references the correct observer.
|
||||
// An observer must actively send data to stay listed — being seen by another node does not count.
|
||||
// observerDays <= -1 means never remove (keep forever).
|
||||
func (db *DB) RemoveStaleObservers(observerDays int) (int64, error) {
|
||||
if observerDays <= -1 {
|
||||
return 0, nil // keep forever
|
||||
}
|
||||
rw, err := openRW(db.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
|
||||
res, err := rw.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n > 0 {
|
||||
// Clean up orphaned metrics for now-inactive observers
|
||||
rw.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
|
||||
log.Printf("[observers] Marked %d observer(s) as inactive (not seen in %d days)", n, observerDays)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// TouchNodeLastSeen updates last_seen for a node identified by full public key.
|
||||
// Only updates if the new timestamp is newer than the existing value (or NULL).
|
||||
// Returns nil even if no rows are affected (node doesn't exist).
|
||||
@@ -2079,3 +2272,71 @@ func (db *DB) TouchNodeLastSeen(pubkey string, timestamp string) error {
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetDroppedPackets returns recently dropped packets, newest first.
|
||||
func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map[string]interface{}, error) {
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
query := `SELECT id, hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name, dropped_at FROM dropped_packets`
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
if observerID != "" {
|
||||
conditions = append(conditions, "observer_id = ?")
|
||||
args = append(args, observerID)
|
||||
}
|
||||
if nodePubkey != "" {
|
||||
conditions = append(conditions, "node_pubkey = ?")
|
||||
args = append(args, nodePubkey)
|
||||
}
|
||||
if len(conditions) > 0 {
|
||||
query += " WHERE " + strings.Join(conditions, " AND ")
|
||||
}
|
||||
query += " ORDER BY dropped_at DESC LIMIT ?"
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []map[string]interface{}
|
||||
for rows.Next() {
|
||||
var id int
|
||||
var hash, rawHex, reason, obsID, obsName, pubkey, name, droppedAt sql.NullString
|
||||
if err := rows.Scan(&id, &hash, &rawHex, &reason, &obsID, &obsName, &pubkey, &name, &droppedAt); err != nil {
|
||||
continue
|
||||
}
|
||||
row := map[string]interface{}{
|
||||
"id": id,
|
||||
"hash": nullStr(hash),
|
||||
"reason": nullStr(reason),
|
||||
"observer_id": nullStr(obsID),
|
||||
"observer_name": nullStr(obsName),
|
||||
"node_pubkey": nullStr(pubkey),
|
||||
"node_name": nullStr(name),
|
||||
"dropped_at": nullStr(droppedAt),
|
||||
}
|
||||
// Only include raw_hex if explicitly requested (it's large)
|
||||
if rawHex.Valid {
|
||||
row["raw_hex"] = rawHex.String
|
||||
}
|
||||
results = append(results, row)
|
||||
}
|
||||
if results == nil {
|
||||
results = []map[string]interface{}{}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// GetSignatureDropCount returns the total number of dropped packets.
|
||||
func (db *DB) GetSignatureDropCount() int64 {
|
||||
var count int64
|
||||
// Table may not exist yet if ingestor hasn't run the migration
|
||||
err := db.conn.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
+81
-24
@@ -60,6 +60,7 @@ func setupTestDB(t *testing.T) *DB {
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
channel_hash TEXT DEFAULT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
@@ -124,10 +125,10 @@ func seedTestData(t *testing.T, db *DB) {
|
||||
VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, ?, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo)
|
||||
|
||||
// Seed transmissions
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, yesterday)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test')`, recent)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday)
|
||||
// Second ADVERT for same node with different hash_size (raw_hex byte 0x1F → hs=1 vs 0xBB → hs=3)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('AA1F', 'def456abc1230099', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}')`, yesterday)
|
||||
@@ -735,12 +736,12 @@ func TestGetChannelMessagesRegionFiltering(t *testing.T) {
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', ' sfo ')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'chanregion0001', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}')`, ts1)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
'{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}', '#region')`, ts1)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB', 'chanregion0002', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}')`, ts2)
|
||||
'{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}', '#region')`, ts2)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '[]', ?)`, epoch1)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
@@ -1119,6 +1120,7 @@ func setupTestDBV2(t *testing.T) *DB {
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
channel_hash TEXT DEFAULT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
@@ -1202,12 +1204,12 @@ func TestGetChannelMessagesDedup(t *testing.T) {
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', 'SFO')`)
|
||||
|
||||
// Insert two transmissions with same hash to test dedup
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'chanmsg00000001', '2026-01-15T10:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#general","text":"User1: Hello","sender":"User1"}')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
'{"type":"CHAN","channel":"#general","text":"User1: Hello","sender":"User1"}', '#general')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB', 'chanmsg00000002', '2026-01-15T10:01:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#general","text":"User2: World","sender":"User2"}')`)
|
||||
'{"type":"CHAN","channel":"#general","text":"User2: World","sender":"User2"}', '#general')`)
|
||||
|
||||
// Observations: first msg seen by two observers (dedup), second by one
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
@@ -1251,9 +1253,9 @@ func TestGetChannelMessagesNoSender(t *testing.T) {
|
||||
defer db.Close()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('CC', 'chanmsg00000003', '2026-01-15T10:02:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#noname","text":"plain text no colon"}')`)
|
||||
'{"type":"CHAN","channel":"#noname","text":"plain text no colon"}', '#noname')`)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 12.0, -90, null, 1736935300)`)
|
||||
|
||||
@@ -1356,9 +1358,9 @@ func TestGetChannelMessagesObserverFallback(t *testing.T) {
|
||||
defer db.Close()
|
||||
|
||||
// Observer with ID but no name entry (observer_idx won't match)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'chanmsg00000004', '2026-01-15T10:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#obs","text":"Sender: Test","sender":"Sender"}')`)
|
||||
'{"type":"CHAN","channel":"#obs","text":"Sender: Test","sender":"Sender"}', '#obs')`)
|
||||
// Observation without observer (observer_idx = NULL)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, NULL, 12.0, -90, null, 1736935200)`)
|
||||
@@ -1380,12 +1382,12 @@ func TestGetChannelsMultiple(t *testing.T) {
|
||||
defer db.Close()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer', 'SJC')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'chan1hash', '2026-01-15T10:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#alpha","text":"Alice: Hello","sender":"Alice"}')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
'{"type":"CHAN","channel":"#alpha","text":"Alice: Hello","sender":"Alice"}', '#alpha')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB', 'chan2hash', '2026-01-15T10:01:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#beta","text":"Bob: World","sender":"Bob"}')`)
|
||||
'{"type":"CHAN","channel":"#beta","text":"Bob: World","sender":"Bob"}', '#beta')`)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('CC', 'chan3hash', '2026-01-15T10:02:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"","text":"No channel"}')`)
|
||||
@@ -1468,13 +1470,13 @@ func TestGetChannelsStaleMessage(t *testing.T) {
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer2', 'SFO')`)
|
||||
|
||||
// Older message (first_seen T1)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'oldhash1', '2026-01-15T10:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#test","text":"Alice: Old message","sender":"Alice"}')`)
|
||||
'{"type":"CHAN","channel":"#test","text":"Alice: Old message","sender":"Alice"}', '#test')`)
|
||||
// Newer message (first_seen T2 > T1)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB', 'newhash2', '2026-01-15T10:05:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#test","text":"Bob: New message","sender":"Bob"}')`)
|
||||
'{"type":"CHAN","channel":"#test","text":"Bob: New message","sender":"Bob"}', '#test')`)
|
||||
|
||||
// Observations: older message re-observed AFTER newer message (stale scenario)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
|
||||
@@ -1504,6 +1506,61 @@ func TestGetChannelsStaleMessage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetChannelsRegionFiltering(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer1', 'SJC')`)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer2', 'SFO')`)
|
||||
|
||||
// Channel message seen only in SJC
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA', 'hash1', '2026-01-15T10:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#sjc-only","text":"Alice: Hello SJC","sender":"Alice"}', '#sjc-only')`)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
|
||||
VALUES (1, 1, 12.0, -90, 1736935200)`)
|
||||
|
||||
// Channel message seen only in SFO
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB', 'hash2', '2026-01-15T10:05:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#sfo-only","text":"Bob: Hello SFO","sender":"Bob"}', '#sfo-only')`)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
|
||||
VALUES (2, 2, 14.0, -88, 1736935500)`)
|
||||
|
||||
// No region filter — both channels
|
||||
all, err := db.GetChannels()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 channels without region filter, got %d", len(all))
|
||||
}
|
||||
|
||||
// Filter SJC — only #sjc-only
|
||||
sjc, err := db.GetChannels("SJC")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sjc) != 1 {
|
||||
t.Fatalf("expected 1 channel for SJC, got %d", len(sjc))
|
||||
}
|
||||
if sjc[0]["name"] != "#sjc-only" {
|
||||
t.Errorf("expected channel '#sjc-only', got %q", sjc[0]["name"])
|
||||
}
|
||||
|
||||
// Filter SFO — only #sfo-only
|
||||
sfo, err := db.GetChannels("SFO")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sfo) != 1 {
|
||||
t.Fatalf("expected 1 channel for SFO, got %d", len(sfo))
|
||||
}
|
||||
if sfo[0]["name"] != "#sfo-only" {
|
||||
t.Errorf("expected channel '#sfo-only', got %q", sfo[0]["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeTelemetryFields(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
+36
-8
@@ -116,6 +116,7 @@ type DecodedPacket struct {
|
||||
Path Path `json:"path"`
|
||||
Payload Payload `json:"payload"`
|
||||
Raw string `json:"raw"`
|
||||
Anomaly string `json:"anomaly,omitempty"`
|
||||
}
|
||||
|
||||
func decodeHeader(b byte) Header {
|
||||
@@ -388,22 +389,34 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
|
||||
payload := decodePayload(header.PayloadType, payloadBuf, validateSignatures)
|
||||
|
||||
// TRACE packets store hop IDs in the payload (buf[9:]) rather than the header
|
||||
// path field. The header path byte still encodes hashSize in bits 6-7, which
|
||||
// we use to split the payload path data into individual hop prefixes.
|
||||
// The header path contains SNR bytes — one per hop that actually forwarded.
|
||||
// path field. Firmware always sends TRACE as DIRECT (route_type 2 or 3);
|
||||
// FLOOD-routed TRACEs are anomalous but handled gracefully (parsed, but
|
||||
// flagged). The TRACE flags byte (payload offset 8) encodes path_sz in
|
||||
// bits 0-1 as a power-of-two exponent: hash_bytes = 1 << path_sz.
|
||||
// NOT the header path byte's hash_size bits. The header path contains SNR
|
||||
// bytes — one per hop that actually forwarded.
|
||||
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
|
||||
// how far the trace got vs the full intended route.
|
||||
var anomaly string
|
||||
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
|
||||
// Flag anomalous routing — firmware only sends TRACE as DIRECT
|
||||
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
|
||||
anomaly = "TRACE packet with non-DIRECT routing (expected DIRECT or TRANSPORT_DIRECT)"
|
||||
}
|
||||
// The header path hops count represents SNR entries = completed hops
|
||||
hopsCompleted := path.HashCount
|
||||
pathBytes, err := hex.DecodeString(payload.PathData)
|
||||
if err == nil && path.HashSize > 0 {
|
||||
hops := make([]string, 0, len(pathBytes)/path.HashSize)
|
||||
for i := 0; i+path.HashSize <= len(pathBytes); i += path.HashSize {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+path.HashSize])))
|
||||
if err == nil && payload.TraceFlags != nil {
|
||||
// path_sz from flags byte is a power-of-two exponent per firmware:
|
||||
// hash_bytes = 1 << (flags & 0x03)
|
||||
pathSz := 1 << (*payload.TraceFlags & 0x03)
|
||||
hops := make([]string, 0, len(pathBytes)/pathSz)
|
||||
for i := 0; i+pathSz <= len(pathBytes); i += pathSz {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+pathSz])))
|
||||
}
|
||||
path.Hops = hops
|
||||
path.HashCount = len(hops)
|
||||
path.HashSize = pathSz
|
||||
path.HopsCompleted = &hopsCompleted
|
||||
}
|
||||
}
|
||||
@@ -424,6 +437,7 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
|
||||
Path: path,
|
||||
Payload: payload,
|
||||
Raw: strings.ToUpper(hexString),
|
||||
Anomaly: anomaly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -528,6 +542,9 @@ func BuildBreakdown(hexString string) *Breakdown {
|
||||
}
|
||||
|
||||
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
|
||||
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
|
||||
// route-independent identifier for the same logical packet. For TRACE packets,
|
||||
// path_len is included in the hash to match firmware behavior.
|
||||
func ComputeContentHash(rawHex string) string {
|
||||
buf, err := hex.DecodeString(rawHex)
|
||||
if err != nil || len(buf) < 2 {
|
||||
@@ -563,7 +580,18 @@ func ComputeContentHash(rawHex string) string {
|
||||
}
|
||||
|
||||
payload := buf[payloadStart:]
|
||||
toHash := append([]byte{headerByte}, payload...)
|
||||
|
||||
// Hash payload-type byte only (bits 2-5 of header), not the full header.
|
||||
// Firmware: SHA256(payload_type + [path_len for TRACE] + payload)
|
||||
// Using the full header caused different hashes for the same logical packet
|
||||
// when route type or version bits differed. See issue #786.
|
||||
payloadType := (headerByte >> 2) & 0x0F
|
||||
toHash := []byte{payloadType}
|
||||
if int(payloadType) == PayloadTRACE {
|
||||
// Firmware uses uint16_t path_len (2 bytes, little-endian)
|
||||
toHash = append(toHash, pathByte, 0x00)
|
||||
}
|
||||
toHash = append(toHash, payload...)
|
||||
|
||||
h := sha256.Sum256(toHash)
|
||||
return hex.EncodeToString(h[:])[:16]
|
||||
|
||||
@@ -357,6 +357,10 @@ func TestDecodePacket_TraceHopsCompleted(t *testing.T) {
|
||||
if *pkt.Path.HopsCompleted != 2 {
|
||||
t.Errorf("expected HopsCompleted=2, got %d", *pkt.Path.HopsCompleted)
|
||||
}
|
||||
// FLOOD routing for TRACE is anomalous
|
||||
if pkt.Anomaly == "" {
|
||||
t.Error("expected anomaly flag for FLOOD-routed TRACE")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceNoSNR(t *testing.T) {
|
||||
@@ -407,6 +411,124 @@ func TestDecodePacket_TraceFullyCompleted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceFlags1_TwoBytePathSz(t *testing.T) {
|
||||
// TRACE with flags=1 → path_sz = 1 << (1 & 0x03) = 2-byte hashes
|
||||
// Firmware always sends TRACE as DIRECT (route_type=2), so header byte =
|
||||
// (0<<6)|(9<<2)|2 = 0x26. path_length 0x00 = 0 SNR bytes.
|
||||
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"01" + // flags = 1 → path_sz = 2
|
||||
"AABBCCDD" // 4 bytes = 2 hops of 2-byte each
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if len(pkt.Path.Hops) != 2 {
|
||||
t.Errorf("expected 2 hops (2-byte path_sz), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
|
||||
}
|
||||
if pkt.Path.HashSize != 2 {
|
||||
t.Errorf("expected HashSize=2, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
if pkt.Anomaly != "" {
|
||||
t.Errorf("expected no anomaly for DIRECT TRACE, got %q", pkt.Anomaly)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceFlags2_FourBytePathSz(t *testing.T) {
|
||||
// TRACE with flags=2 → path_sz = 1 << (2 & 0x03) = 4-byte hashes
|
||||
// DIRECT route_type (0x26)
|
||||
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"02" + // flags = 2 → path_sz = 4
|
||||
"AABBCCDD11223344" // 8 bytes = 2 hops of 4-byte each
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if len(pkt.Path.Hops) != 2 {
|
||||
t.Errorf("expected 2 hops (4-byte path_sz), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
|
||||
}
|
||||
if pkt.Path.HashSize != 4 {
|
||||
t.Errorf("expected HashSize=4, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TracePathSzUnevenPayload(t *testing.T) {
|
||||
// TRACE with flags=1 → path_sz=2, but 5 bytes of path data (not evenly divisible)
|
||||
// Should produce 2 hops (4 bytes) and ignore the trailing byte
|
||||
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"01" + // flags = 1 → path_sz = 2
|
||||
"AABBCCDDEE" // 5 bytes → 2 hops, 1 byte remainder ignored
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if len(pkt.Path.Hops) != 2 {
|
||||
t.Errorf("expected 2 hops (trailing byte ignored), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceTransportDirect(t *testing.T) {
|
||||
// TRACE via TRANSPORT_DIRECT (route_type=3) — includes 4 transport code bytes
|
||||
// header: (0<<6)|(9<<2)|3 = 0x27
|
||||
hex := "27" + // header (TRANSPORT_DIRECT+TRACE)
|
||||
"AABB" + "CCDD" + // transport codes (2+2 bytes)
|
||||
"02" + // path_length: hash_count=2 SNR bytes
|
||||
"EEFF" + // 2 SNR bytes
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"00" + // flags = 0 → path_sz = 1
|
||||
"112233" // 3 hops (1-byte each)
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.TransportCodes == nil {
|
||||
t.Fatal("expected transport codes for TRANSPORT_DIRECT")
|
||||
}
|
||||
if pkt.TransportCodes.Code1 != "AABB" {
|
||||
t.Errorf("expected Code1=AABB, got %s", pkt.TransportCodes.Code1)
|
||||
}
|
||||
if len(pkt.Path.Hops) != 3 {
|
||||
t.Errorf("expected 3 hops, got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
|
||||
}
|
||||
if pkt.Path.HopsCompleted == nil || *pkt.Path.HopsCompleted != 2 {
|
||||
t.Errorf("expected HopsCompleted=2, got %v", pkt.Path.HopsCompleted)
|
||||
}
|
||||
if pkt.Anomaly != "" {
|
||||
t.Errorf("expected no anomaly for TRANSPORT_DIRECT TRACE, got %q", pkt.Anomaly)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceFloodRouteAnomaly(t *testing.T) {
|
||||
// TRACE via FLOOD (route_type=1) — anomalous per firmware (firmware only
|
||||
// sends TRACE as DIRECT). Should still parse but flag the anomaly.
|
||||
hex := "2500" + // header (FLOOD+TRACE) + path_length (0 SNR)
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"01" + // flags = 1 → path_sz = 2
|
||||
"AABBCCDD" // 4 bytes = 2 hops of 2-byte each
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("should not crash on anomalous FLOOD+TRACE: %v", err)
|
||||
}
|
||||
if len(pkt.Path.Hops) != 2 {
|
||||
t.Errorf("expected 2 hops even for anomalous FLOOD route, got %d", len(pkt.Path.Hops))
|
||||
}
|
||||
if pkt.Anomaly == "" {
|
||||
t.Error("expected anomaly flag for FLOOD-routed TRACE, got empty string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeAdvertSignatureValidation(t *testing.T) {
|
||||
pub, priv, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// seedEncryptedChannelData adds undecryptable GRP_TXT packets to the test DB.
|
||||
func seedEncryptedChannelData(t *testing.T, db *DB) {
|
||||
t.Helper()
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Two encrypted GRP_TXT packets on channel hash "A1B2"
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('EE01', 'enc_hash_001', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('EE02', 'enc_hash_002', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent)
|
||||
|
||||
// Observations for both
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
}
|
||||
|
||||
func TestGetEncryptedChannels(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
seedEncryptedChannelData(t, db)
|
||||
|
||||
channels, err := db.GetEncryptedChannels()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected 1 encrypted channel, got %d", len(channels))
|
||||
}
|
||||
ch := channels[0]
|
||||
if ch["hash"] != "enc_A1B2" {
|
||||
t.Errorf("expected hash enc_A1B2, got %v", ch["hash"])
|
||||
}
|
||||
if ch["encrypted"] != true {
|
||||
t.Errorf("expected encrypted=true, got %v", ch["encrypted"])
|
||||
}
|
||||
if ch["messageCount"] != 2 {
|
||||
t.Errorf("expected messageCount=2, got %v", ch["messageCount"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelsAPIExcludesEncrypted(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
// Seed encrypted data into the server's DB
|
||||
// setupTestServer uses seedTestData which has no encrypted packets,
|
||||
// so default /api/channels should NOT include encrypted channels.
|
||||
req := httptest.NewRequest("GET", "/api/channels", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
channels := body["channels"].([]interface{})
|
||||
|
||||
for _, ch := range channels {
|
||||
m := ch.(map[string]interface{})
|
||||
if enc, ok := m["encrypted"]; ok && enc == true {
|
||||
t.Errorf("default /api/channels should not include encrypted channels, found: %v", m["hash"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelsAPIIncludesEncryptedWithParam(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
// Add encrypted data to the server's DB
|
||||
seedEncryptedChannelData(t, srv.db)
|
||||
// Reload store so in-memory also has the data
|
||||
store := NewPacketStore(srv.db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/channels?includeEncrypted=true", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
channels := body["channels"].([]interface{})
|
||||
|
||||
foundEncrypted := false
|
||||
for _, ch := range channels {
|
||||
m := ch.(map[string]interface{})
|
||||
if enc, ok := m["encrypted"]; ok && enc == true {
|
||||
foundEncrypted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundEncrypted {
|
||||
t.Error("expected encrypted channels with includeEncrypted=true, found none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelMessagesExcludesEncrypted(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
seedEncryptedChannelData(t, srv.db)
|
||||
store := NewPacketStore(srv.db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
|
||||
// Request messages for the encrypted channel — should return empty
|
||||
req := httptest.NewRequest("GET", "/api/channels/enc_A1B2/messages", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
messages, ok := body["messages"].([]interface{})
|
||||
if !ok {
|
||||
// messages might be null/missing — that's fine, means no messages
|
||||
return
|
||||
}
|
||||
// Encrypted messages should not be returned as readable messages
|
||||
for _, msg := range messages {
|
||||
m := msg.(map[string]interface{})
|
||||
if text, ok := m["text"].(string); ok && text != "" {
|
||||
t.Errorf("encrypted channel should not return readable messages, got text: %s", text)
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
-13
@@ -247,6 +247,11 @@ func TestEvictStale_CleansNodeIndexes(t *testing.T) {
|
||||
|
||||
func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Create a temp DB for on-demand SQL fetch during eviction
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
store := &PacketStore{
|
||||
packets: make([]*StoreTx, 0),
|
||||
byHash: make(map[string]*StoreTx),
|
||||
@@ -267,25 +272,33 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
rfCacheTTL: 15 * time.Second,
|
||||
retentionHours: 24,
|
||||
db: db,
|
||||
useResolvedPathIndex: true,
|
||||
}
|
||||
store.initResolvedPathIndex()
|
||||
|
||||
// Create a packet indexed only via resolved_path (no decoded JSON pubkeys)
|
||||
// Create a packet indexed via resolved_path pubkeys
|
||||
relayPK := "relay0001abcdef"
|
||||
txID := 1
|
||||
obsID := 100
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
ID: txID,
|
||||
Hash: "hash_rp_001",
|
||||
FirstSeen: now.Add(-48 * time.Hour).UTC().Format(time.RFC3339),
|
||||
}
|
||||
rpPtr := &relayPK
|
||||
obs := &StoreObs{
|
||||
ID: 100,
|
||||
TransmissionID: 1,
|
||||
ID: obsID,
|
||||
TransmissionID: txID,
|
||||
ObserverID: "obs0",
|
||||
Timestamp: tx.FirstSeen,
|
||||
ResolvedPath: []*string{rpPtr},
|
||||
}
|
||||
tx.Observations = append(tx.Observations, obs)
|
||||
tx.ResolvedPath = []*string{rpPtr}
|
||||
|
||||
// Insert into DB so on-demand SQL fetch works during eviction
|
||||
db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)",
|
||||
txID, tx.Hash, tx.FirstSeen)
|
||||
db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)",
|
||||
obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`)
|
||||
|
||||
store.packets = append(store.packets, tx)
|
||||
store.byHash[tx.Hash] = tx
|
||||
@@ -293,8 +306,9 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
store.byObsID[obs.ID] = obs
|
||||
store.byObserver["obs0"] = append(store.byObserver["obs0"], obs)
|
||||
|
||||
// Index via resolved_path
|
||||
store.indexByNode(tx)
|
||||
// Index relay via decode-window simulation
|
||||
store.addToByNode(tx, relayPK)
|
||||
store.addToResolvedPubkeyIndex(txID, []string{relayPK})
|
||||
|
||||
// Verify indexed
|
||||
if len(store.byNode[relayPK]) != 1 {
|
||||
@@ -304,7 +318,7 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
t.Fatalf("expected nodeHashes[%s] to contain %s", relayPK, tx.Hash)
|
||||
}
|
||||
|
||||
evicted := store.EvictStale()
|
||||
evicted := store.RunEviction()
|
||||
if evicted != 1 {
|
||||
t.Fatalf("expected 1 evicted, got %d", evicted)
|
||||
}
|
||||
@@ -316,6 +330,14 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
if _, exists := store.nodeHashes[relayPK]; exists {
|
||||
t.Fatalf("expected nodeHashes[%s] to be deleted after eviction", relayPK)
|
||||
}
|
||||
// Verify resolved pubkey index is cleaned up
|
||||
h := resolvedPubkeyHash(relayPK)
|
||||
if len(store.resolvedPubkeyIndex[h]) != 0 {
|
||||
t.Fatalf("expected resolvedPubkeyIndex to be empty after eviction")
|
||||
}
|
||||
if _, exists := store.resolvedPubkeyReverse[txID]; exists {
|
||||
t.Fatalf("expected resolvedPubkeyReverse to be empty after eviction")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvictStale_RunEvictionThreadSafe(t *testing.T) {
|
||||
@@ -541,13 +563,19 @@ func TestEstimateStoreTxBytes(t *testing.T) {
|
||||
PathJSON: `["aa","bb"]`,
|
||||
}
|
||||
est := estimateStoreTxBytes(tx)
|
||||
// Verify the function returns a reasonable value matching our manual calculation
|
||||
// Manual calculation: base + string lengths + index entries + perTxMaps + path hops + subpaths
|
||||
hops := int64(len(txGetParsedPath(tx)))
|
||||
manualCalc := int64(storeTxBaseBytes) + int64(len(tx.RawHex)+len(tx.Hash)+len(tx.DecodedJSON)+len(tx.PathJSON)) + int64(numIndexesPerTx*indexEntryBytes)
|
||||
manualCalc += perTxMapsBytes
|
||||
manualCalc += hops * perPathHopBytes
|
||||
if hops > 1 {
|
||||
manualCalc += (hops * (hops - 1) / 2) * perSubpathEntryBytes
|
||||
}
|
||||
if est != manualCalc {
|
||||
t.Fatalf("estimateStoreTxBytes = %d, want %d (manual calc)", est, manualCalc)
|
||||
}
|
||||
if est < 600 || est > 800 {
|
||||
t.Fatalf("estimateStoreTxBytes = %d, expected in range [600, 800]", est)
|
||||
if est < 600 || est > 1200 {
|
||||
t.Fatalf("estimateStoreTxBytes = %d, expected in range [600, 1200]", est)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// migrateContentHashesAsync recomputes content hashes in batches after the
|
||||
// server is already serving HTTP. Packets whose hash changes are updated in
|
||||
// both the DB and the in-memory byHash index. The migration is idempotent:
|
||||
// once all hashes match the current formula it completes instantly.
|
||||
func migrateContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[hash-migrate] panic recovered: %v", r)
|
||||
}
|
||||
store.hashMigrationComplete.Store(true)
|
||||
}()
|
||||
|
||||
// Snapshot the packet slice length under lock (packets only grow).
|
||||
store.mu.RLock()
|
||||
total := len(store.packets)
|
||||
store.mu.RUnlock()
|
||||
|
||||
migrated := 0
|
||||
for offset := 0; offset < total; offset += batchSize {
|
||||
end := offset + batchSize
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
|
||||
// Collect stale hashes in this batch under RLock.
|
||||
type hashUpdate struct {
|
||||
tx *StoreTx
|
||||
oldHash string
|
||||
newHash string
|
||||
}
|
||||
var updates []hashUpdate
|
||||
|
||||
store.mu.RLock()
|
||||
for _, tx := range store.packets[offset:end] {
|
||||
if tx.RawHex == "" {
|
||||
continue
|
||||
}
|
||||
newHash := ComputeContentHash(tx.RawHex)
|
||||
if newHash != tx.Hash {
|
||||
updates = append(updates, hashUpdate{tx: tx, oldHash: tx.Hash, newHash: newHash})
|
||||
}
|
||||
}
|
||||
store.mu.RUnlock()
|
||||
|
||||
if len(updates) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Write batch to DB in a single transaction.
|
||||
dbTx, err := store.db.conn.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[hash-migrate] begin tx: %v", err)
|
||||
continue
|
||||
}
|
||||
stmt, err := dbTx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?")
|
||||
if err != nil {
|
||||
log.Printf("[hash-migrate] prepare: %v", err)
|
||||
dbTx.Rollback()
|
||||
continue
|
||||
}
|
||||
|
||||
for _, u := range updates {
|
||||
if _, err := stmt.Exec(u.newHash, u.tx.ID); err != nil {
|
||||
// UNIQUE constraint = two old hashes map to the same new hash (duplicate).
|
||||
// Merge observations to the surviving tx, delete the duplicate.
|
||||
log.Printf("[hash-migrate] tx %d collides — merging duplicate", u.tx.ID)
|
||||
var survID int
|
||||
if err2 := dbTx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil {
|
||||
dbTx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.tx.ID)
|
||||
dbTx.Exec("DELETE FROM transmissions WHERE id = ?", u.tx.ID)
|
||||
u.newHash = "" // mark for in-memory removal only
|
||||
}
|
||||
}
|
||||
}
|
||||
stmt.Close()
|
||||
|
||||
if err := dbTx.Commit(); err != nil {
|
||||
log.Printf("[hash-migrate] commit: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update in-memory index under write lock.
|
||||
store.mu.Lock()
|
||||
for _, u := range updates {
|
||||
delete(store.byHash, u.oldHash)
|
||||
if u.newHash == "" {
|
||||
// Merged duplicate — remove from packets slice and indexes.
|
||||
delete(store.byTxID, u.tx.ID)
|
||||
// Move observations to survivor if present.
|
||||
if surv := store.byHash[ComputeContentHash(u.tx.RawHex)]; surv != nil {
|
||||
for _, obs := range u.tx.Observations {
|
||||
surv.Observations = append(surv.Observations, obs)
|
||||
surv.ObservationCount++
|
||||
}
|
||||
}
|
||||
} else {
|
||||
u.tx.Hash = u.newHash
|
||||
store.byHash[u.newHash] = u.tx
|
||||
}
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
migrated += len(updates)
|
||||
|
||||
// Yield to let HTTP handlers run.
|
||||
time.Sleep(yieldDuration)
|
||||
}
|
||||
|
||||
if migrated > 0 {
|
||||
log.Printf("[hash-migrate] Migrated %d content hashes to new formula", migrated)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMigrateContentHashesAsync(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
// Insert a packet with a manually wrong hash (simulating old formula).
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
correctHash := ComputeContentHash(rawHex)
|
||||
wrongHash := "deadbeef12345678"
|
||||
|
||||
_, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type)
|
||||
VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if store.byHash[wrongHash] == nil {
|
||||
t.Fatal("expected packet under wrong hash before migration")
|
||||
}
|
||||
|
||||
migrateContentHashesAsync(store, 100, time.Millisecond)
|
||||
|
||||
if !store.hashMigrationComplete.Load() {
|
||||
t.Error("expected hashMigrationComplete to be true")
|
||||
}
|
||||
if store.byHash[wrongHash] != nil {
|
||||
t.Error("old hash should be removed from index")
|
||||
}
|
||||
if store.byHash[correctHash] == nil {
|
||||
t.Error("new hash should be in index")
|
||||
}
|
||||
|
||||
var dbHash string
|
||||
err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = ?", rawHex).Scan(&dbHash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dbHash != correctHash {
|
||||
t.Errorf("DB hash = %s, want %s", dbHash, correctHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateContentHashesAsync_NoOp(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
correctHash := ComputeContentHash(rawHex)
|
||||
|
||||
_, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type)
|
||||
VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, correctHash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
migrateContentHashesAsync(store, 100, time.Millisecond)
|
||||
|
||||
if !store.hashMigrationComplete.Load() {
|
||||
t.Error("expected hashMigrationComplete to be true")
|
||||
}
|
||||
if store.byHash[correctHash] == nil {
|
||||
t.Error("hash should remain in index")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9"
|
||||
|
||||
// setupIssue673Store builds an in-memory store with one repeater node having:
|
||||
// - one ADVERT packet (legitimately indexed in byNode)
|
||||
// - one GRP_TXT packet whose decoded text contains the node's pubkey (false-positive candidate)
|
||||
func setupIssue673Store(t *testing.T) (*PacketStore, *DB) {
|
||||
t.Helper()
|
||||
db := setupTestDB(t)
|
||||
|
||||
_, err := db.conn.Exec(
|
||||
"INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)",
|
||||
issue673NodePK, "Quail Hollow Park", "repeater",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ps := NewPacketStore(db, nil)
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
pt4 := 4 // ADVERT
|
||||
pt5 := 5 // GRP_TXT
|
||||
|
||||
advertDecoded, _ := json.Marshal(map[string]interface{}{"pubKey": issue673NodePK})
|
||||
advert := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "advert_hash_673",
|
||||
PayloadType: &pt4,
|
||||
DecodedJSON: string(advertDecoded),
|
||||
FirstSeen: now,
|
||||
}
|
||||
|
||||
otherPK := "aabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccdd"
|
||||
chatDecoded, _ := json.Marshal(map[string]interface{}{
|
||||
"srcPubKey": otherPK,
|
||||
"text": "Check out node " + issue673NodePK + " on the analyzer",
|
||||
})
|
||||
chat := &StoreTx{
|
||||
ID: 2,
|
||||
Hash: "chat_hash_673",
|
||||
PayloadType: &pt5,
|
||||
DecodedJSON: string(chatDecoded),
|
||||
FirstSeen: now,
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.packets = append(ps.packets, advert, chat)
|
||||
ps.byHash[advert.Hash] = advert
|
||||
ps.byHash[chat.Hash] = chat
|
||||
ps.byTxID[advert.ID] = advert
|
||||
ps.byTxID[chat.ID] = chat
|
||||
ps.byNode[issue673NodePK] = []*StoreTx{advert}
|
||||
ps.mu.Unlock()
|
||||
|
||||
return ps, db
|
||||
}
|
||||
|
||||
// TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText verifies that a GRP_TXT packet
|
||||
// whose message text contains a node's pubkey is not counted in that node's analytics.
|
||||
func TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText(t *testing.T) {
|
||||
ps, db := setupIssue673Store(t)
|
||||
defer db.Close()
|
||||
|
||||
analytics, err := ps.GetNodeAnalytics(issue673NodePK, 30)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if analytics == nil {
|
||||
t.Fatal("expected analytics, got nil")
|
||||
}
|
||||
|
||||
for _, ptc := range analytics.PacketTypeBreakdown {
|
||||
if ptc.PayloadType == 5 {
|
||||
t.Errorf("GRP_TXT (type 5) should not appear in analytics for repeater node, got count=%d", ptc.Count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterPackets_NodeQueryDoesNotMatchChatText verifies that the slow path of
|
||||
// filterPackets (node filter combined with Since) does not return a GRP_TXT packet
|
||||
// whose pubkey appears only in message text, not in a structured pubkey field.
|
||||
func TestFilterPackets_NodeQueryDoesNotMatchChatText(t *testing.T) {
|
||||
ps, db := setupIssue673Store(t)
|
||||
defer db.Close()
|
||||
|
||||
yesterday := time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
result := ps.QueryPackets(PacketQuery{Node: issue673NodePK, Since: yesterday, Limit: 50})
|
||||
|
||||
if result.Total != 1 {
|
||||
t.Errorf("expected 1 packet for node (ADVERT only), got %d", result.Total)
|
||||
}
|
||||
for _, pkt := range result.Packets {
|
||||
if pkt["hash"] == "chat_hash_673" {
|
||||
t.Errorf("GRP_TXT with pubkey in message text was incorrectly returned for node query")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestRepro810 reproduces #810: when the longest-path observation has NULL
|
||||
// resolved_path but a shorter-path observation has one, fetchResolvedPathForTxBest
|
||||
// returns nil → /api/nodes/{pk}/health.recentPackets[].resolved_path is missing
|
||||
// while /api/packets shows it.
|
||||
func TestRepro810(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent)
|
||||
// Longest-path obs WITHOUT resolved_path
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch)
|
||||
// Shorter-path obs WITH resolved_path
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
// Sanity: /api/packets should show resolved_path for this tx.
|
||||
reqP := httptest.NewRequest("GET", "/api/packets?limit=10", nil)
|
||||
wP := httptest.NewRecorder()
|
||||
router.ServeHTTP(wP, reqP)
|
||||
var pktsBody map[string]interface{}
|
||||
json.Unmarshal(wP.Body.Bytes(), &pktsBody)
|
||||
pkts, _ := pktsBody["packets"].([]interface{})
|
||||
hasOnPackets := false
|
||||
for _, p := range pkts {
|
||||
pm := p.(map[string]interface{})
|
||||
if pm["hash"] == "testhash00000001" && pm["resolved_path"] != nil {
|
||||
hasOnPackets = true
|
||||
}
|
||||
}
|
||||
if !hasOnPackets {
|
||||
t.Fatal("precondition: /api/packets must report resolved_path for tx")
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
rp, _ := body["recentPackets"].([]interface{})
|
||||
if len(rp) == 0 {
|
||||
t.Fatal("no recentPackets")
|
||||
}
|
||||
for _, p := range rp {
|
||||
pm := p.(map[string]interface{})
|
||||
if pm["hash"] == "testhash00000001" {
|
||||
if pm["resolved_path"] == nil {
|
||||
t.Fatal("BUG #810: /health.recentPackets resolved_path is nil despite /api/packets reporting it")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("tx not found in recentPackets")
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
"path/filepath"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// setupTestDB871 creates a test DB with schema and returns a read-only *DB handle.
|
||||
func setupTestDB871(t *testing.T) (*DB, *sql.DB) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test871.db")
|
||||
|
||||
// Open writable connection for setup
|
||||
rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = rw.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
public_key TEXT PRIMARY KEY,
|
||||
name TEXT, role TEXT,
|
||||
lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT,
|
||||
advert_count INTEGER DEFAULT 0,
|
||||
battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
route_type INTEGER,
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS observers (
|
||||
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
id TEXT NOT NULL UNIQUE,
|
||||
name TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL,
|
||||
observer_id TEXT,
|
||||
observer_name TEXT,
|
||||
direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Open read-only handle for the store
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
db.Close()
|
||||
rw.Close()
|
||||
})
|
||||
|
||||
return db, rw
|
||||
}
|
||||
|
||||
// TestEnrichObsFallbackToDB verifies that enrichObs falls back to the DB when
|
||||
// the parent transmission has been evicted from memory (#871 root cause).
|
||||
func TestEnrichObsFallbackToDB(t *testing.T) {
|
||||
db, rw := setupTestDB871(t)
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
_, err := rw.Exec(
|
||||
`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type, decoded_json) VALUES (?, ?, ?, ?, ?)`,
|
||||
"aabbcc", "abc123", now, 4, `{"pubKey":"pk1"}`,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewPacketStore(db, &PacketStoreConfig{})
|
||||
|
||||
// Observation references tx_id=1, but tx is NOT in byTxID (simulates eviction)
|
||||
obs := &StoreObs{
|
||||
ID: 1,
|
||||
TransmissionID: 1,
|
||||
ObserverID: "obs1",
|
||||
ObserverName: "Observer1",
|
||||
Timestamp: now,
|
||||
}
|
||||
|
||||
result := store.enrichObs(obs)
|
||||
|
||||
// hash must be present from DB fallback
|
||||
if result["hash"] == nil {
|
||||
t.Errorf("enrichObs: hash is nil — DB fallback failed")
|
||||
}
|
||||
if h, ok := result["hash"].(string); !ok || h != "abc123" {
|
||||
t.Errorf("enrichObs: expected hash 'abc123', got %v", result["hash"])
|
||||
}
|
||||
if result["payload_type"] == nil {
|
||||
t.Errorf("enrichObs: payload_type is nil — DB fallback failed")
|
||||
}
|
||||
|
||||
// When tx IS in memory, it should use the in-memory path
|
||||
pt := 4
|
||||
store.byTxID[1] = &StoreTx{
|
||||
ID: 1, Hash: "abc123", FirstSeen: now,
|
||||
PayloadType: &pt, RawHex: "aabbcc",
|
||||
}
|
||||
|
||||
result2 := store.enrichObs(obs)
|
||||
if result2["hash"] == nil {
|
||||
t.Errorf("enrichObs with in-memory tx: hash is nil")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetNodeHealthRecentPacketsNoNilFields verifies that GetNodeHealth's
|
||||
// recentPackets never contains entries with nil hash or timestamp.
|
||||
func TestGetNodeHealthRecentPacketsNoNilFields(t *testing.T) {
|
||||
db, rw := setupTestDB871(t)
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
_, err := rw.Exec(
|
||||
`INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)`,
|
||||
"pk1", "TestNode", "repeater", now,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewPacketStore(db, &PacketStoreConfig{})
|
||||
|
||||
pt := 4
|
||||
tx := &StoreTx{
|
||||
ID: 1, Hash: "hash1", FirstSeen: now,
|
||||
PayloadType: &pt, DecodedJSON: `{"pubKey":"pk1"}`,
|
||||
obsKeys: make(map[string]bool), observerSet: make(map[string]bool),
|
||||
}
|
||||
store.byTxID[1] = tx
|
||||
store.byHash["hash1"] = tx
|
||||
store.byNode["pk1"] = []*StoreTx{tx}
|
||||
store.nodeHashes["pk1"] = map[string]bool{"hash1": true}
|
||||
|
||||
result, err := store.GetNodeHealth("pk1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("GetNodeHealth returned nil")
|
||||
}
|
||||
|
||||
packets, ok := result["recentPackets"].([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("recentPackets is not []map[string]interface{}")
|
||||
}
|
||||
|
||||
for i, p := range packets {
|
||||
if p["hash"] == nil {
|
||||
t.Errorf("recentPackets[%d] has nil hash", i)
|
||||
}
|
||||
if p["timestamp"] == nil {
|
||||
t.Errorf("recentPackets[%d] has nil timestamp", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnrichObsNilDB verifies enrichObs doesn't panic when db is nil.
|
||||
func TestEnrichObsNilDB(t *testing.T) {
|
||||
store := &PacketStore{
|
||||
byTxID: make(map[int]*StoreTx),
|
||||
byObsID: make(map[int]*StoreObs),
|
||||
}
|
||||
|
||||
obs := &StoreObs{
|
||||
ID: 1, TransmissionID: 999,
|
||||
Timestamp: "2026-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
// Should not panic
|
||||
result := store.enrichObs(obs)
|
||||
if result["hash"] != nil {
|
||||
t.Errorf("expected nil hash when no DB and no in-memory tx, got %v", result["hash"])
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,14 @@ func main() {
|
||||
// Resolve DB path
|
||||
resolvedDB := cfg.ResolveDBPath(configDir)
|
||||
log.Printf("[config] port=%d db=%s public=%s", cfg.Port, resolvedDB, publicDir)
|
||||
if len(cfg.NodeBlacklist) > 0 {
|
||||
log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist))
|
||||
for _, pk := range cfg.NodeBlacklist {
|
||||
if trimmed := strings.ToLower(strings.TrimSpace(pk)); trimmed != "" {
|
||||
log.Printf("[config] blacklisted: %s", trimmed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Open database
|
||||
database, err := OpenDB(resolvedDB)
|
||||
@@ -325,6 +333,40 @@ func main() {
|
||||
log.Printf("[metrics-prune] auto-prune enabled: metrics older than %d days", metricsDays)
|
||||
}
|
||||
|
||||
// Auto-prune stale observers
|
||||
var stopObserverPrune func()
|
||||
{
|
||||
observerDays := cfg.ObserverDaysOrDefault()
|
||||
if observerDays <= -1 {
|
||||
// -1 means keep forever, skip
|
||||
} else {
|
||||
observerPruneTicker := time.NewTicker(24 * time.Hour)
|
||||
observerPruneDone := make(chan struct{})
|
||||
stopObserverPrune = func() {
|
||||
observerPruneTicker.Stop()
|
||||
close(observerPruneDone)
|
||||
}
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[observer-prune] panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
time.Sleep(3 * time.Minute) // stagger after metrics prune
|
||||
database.RemoveStaleObservers(observerDays)
|
||||
for {
|
||||
select {
|
||||
case <-observerPruneTicker.C:
|
||||
database.RemoveStaleObservers(observerDays)
|
||||
case <-observerPruneDone:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[observer-prune] auto-prune enabled: observers not seen in %d days will be removed", observerDays)
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-prune old neighbor edges
|
||||
var stopEdgePrune func()
|
||||
{
|
||||
@@ -386,6 +428,9 @@ func main() {
|
||||
if stopMetricsPrune != nil {
|
||||
stopMetricsPrune()
|
||||
}
|
||||
if stopObserverPrune != nil {
|
||||
stopObserverPrune()
|
||||
}
|
||||
if stopEdgePrune != nil {
|
||||
stopEdgePrune()
|
||||
}
|
||||
@@ -412,6 +457,9 @@ func main() {
|
||||
// Start async backfill in background — HTTP is now available.
|
||||
go backfillResolvedPathsAsync(store, dbPath, 5000, 100*time.Millisecond, cfg.BackfillHours())
|
||||
|
||||
// Migrate old content hashes in background (one-time, idempotent).
|
||||
go migrateContentHashesAsync(store, 5000, 100*time.Millisecond)
|
||||
|
||||
if err := httpServer.ListenAndServe(); err != http.ErrServerClosed {
|
||||
log.Fatalf("[server] %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemorySnapshot is a point-in-time view of process memory across several
|
||||
// vantage points. Values are in MB (1024*1024 bytes), rounded to one decimal.
|
||||
//
|
||||
// Field invariants (typical, not guaranteed under exotic conditions):
|
||||
//
|
||||
// processRSSMB >= goSysMB >= goHeapInuseMB >= storeDataMB
|
||||
//
|
||||
// - processRSSMB is what the kernel charges the process (resident set).
|
||||
// Read from /proc/self/status `VmRSS:` on Linux; falls back to goSysMB
|
||||
// on other platforms or when /proc is unavailable.
|
||||
// - goSysMB is the total memory obtained from the OS by the Go runtime
|
||||
// (heap, stacks, GC metadata, mspans, mcache, etc.). Includes
|
||||
// fragmentation and unused-but-mapped span overhead.
|
||||
// - goHeapInuseMB is the live, in-use Go heap (HeapInuse). Excludes
|
||||
// idle spans and runtime overhead.
|
||||
// - storeDataMB is the in-store packet byte estimate (transmissions +
|
||||
// observations). Subset of HeapInuse. Does not include index maps,
|
||||
// analytics caches, broadcast queues, or runtime overhead. Used as
|
||||
// the input to the eviction watermark.
|
||||
//
|
||||
// processRSSMB and storeDataMB are monotonic only relative to ingest +
|
||||
// eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB
|
||||
// fluctuate with GC.
|
||||
//
|
||||
// cgoBytesMB intentionally absent: this build uses the pure-Go
|
||||
// modernc.org/sqlite driver, so there is no cgo allocator to measure.
|
||||
// Reintroduce only if we ever switch back to mattn/go-sqlite3.
|
||||
type MemorySnapshot struct {
|
||||
ProcessRSSMB float64 `json:"processRSSMB"`
|
||||
GoHeapInuseMB float64 `json:"goHeapInuseMB"`
|
||||
GoSysMB float64 `json:"goSysMB"`
|
||||
StoreDataMB float64 `json:"storeDataMB"`
|
||||
}
|
||||
|
||||
// rssCache rate-limits the /proc/self/status read. Go memory stats are
|
||||
// already cached by Server.getMemStats (5s TTL). We use a tighter 1s TTL
|
||||
// here so processRSSMB stays reasonably fresh during ops debugging
|
||||
// without paying the syscall cost on every /api/stats hit.
|
||||
var (
|
||||
rssCacheMu sync.Mutex
|
||||
rssCacheValueMB float64
|
||||
rssCacheCachedAt time.Time
|
||||
)
|
||||
|
||||
const rssCacheTTL = 1 * time.Second
|
||||
|
||||
// getMemorySnapshot composes a MemorySnapshot using the Server's existing
|
||||
// runtime.MemStats cache (5s TTL, used by /api/health and /api/perf too)
|
||||
// plus a rate-limited /proc RSS read. storeDataMB is supplied by the
|
||||
// caller because the packet store is the source of truth.
|
||||
func (s *Server) getMemorySnapshot(storeDataMB float64) MemorySnapshot {
|
||||
ms := s.getMemStats()
|
||||
|
||||
rssCacheMu.Lock()
|
||||
if time.Since(rssCacheCachedAt) > rssCacheTTL {
|
||||
rssCacheValueMB = readProcRSSMB()
|
||||
rssCacheCachedAt = time.Now()
|
||||
}
|
||||
rssMB := rssCacheValueMB
|
||||
rssCacheMu.Unlock()
|
||||
|
||||
if rssMB <= 0 {
|
||||
// Fallback when /proc is unavailable (non-Linux, sandboxes, etc.).
|
||||
// runtime.Sys is an upper bound on Go-attributable memory and a
|
||||
// reasonable proxy for pure-Go builds.
|
||||
rssMB = float64(ms.Sys) / 1048576.0
|
||||
}
|
||||
|
||||
return MemorySnapshot{
|
||||
ProcessRSSMB: roundMB(rssMB),
|
||||
GoHeapInuseMB: roundMB(float64(ms.HeapInuse) / 1048576.0),
|
||||
GoSysMB: roundMB(float64(ms.Sys) / 1048576.0),
|
||||
StoreDataMB: roundMB(storeDataMB),
|
||||
}
|
||||
}
|
||||
|
||||
// readProcRSSMB parses /proc/self/status for the VmRSS line. Returns 0 on
|
||||
// any failure (file missing, malformed line, parse error) — the caller
|
||||
// then uses a runtime fallback. Linux only; macOS/Windows return 0.
|
||||
//
|
||||
// Safety notes (djb): the file path is hard-coded, no untrusted input is
|
||||
// concatenated. We bound the read at 8 KiB (the whole status file is
|
||||
// well under 4 KiB on modern kernels) so a corrupt /proc can't OOM us.
|
||||
// We only parse digits with strconv; no shell, no exec, no format strings.
|
||||
func readProcRSSMB() float64 {
|
||||
const maxStatusBytes = 8 * 1024
|
||||
f, err := os.Open("/proc/self/status")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
buf := make([]byte, maxStatusBytes)
|
||||
n, err := f.Read(buf)
|
||||
if err != nil && n == 0 {
|
||||
return 0
|
||||
}
|
||||
for _, line := range strings.Split(string(buf[:n]), "\n") {
|
||||
if !strings.HasPrefix(line, "VmRSS:") {
|
||||
continue
|
||||
}
|
||||
// Format: "VmRSS:\t 123456 kB"
|
||||
fields := strings.Fields(line[len("VmRSS:"):])
|
||||
if len(fields) < 2 {
|
||||
return 0
|
||||
}
|
||||
kb, err := strconv.ParseFloat(fields[0], 64)
|
||||
if err != nil || kb < 0 {
|
||||
return 0
|
||||
}
|
||||
// Unit is kB per kernel convention; convert to MB.
|
||||
return kb / 1024.0
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func roundMB(v float64) float64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(int64(v*10+0.5)) / 10.0
|
||||
}
|
||||
@@ -6,10 +6,17 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// recentTS returns a timestamp string N hours ago, ensuring test data
|
||||
// stays within the 7-day advert window used by computeNodeHashSizeInfo.
|
||||
func recentTS(hoursAgo int) string {
|
||||
return time.Now().UTC().Add(-time.Duration(hoursAgo) * time.Hour).Format("2006-01-02T15:04:05.000Z")
|
||||
}
|
||||
|
||||
// setupCapabilityTestDB creates a minimal in-memory DB with nodes table.
|
||||
func setupCapabilityTestDB(t *testing.T) *DB {
|
||||
t.Helper()
|
||||
@@ -69,7 +76,7 @@ func makeTestAdvert(pubkey string, hashSize int) *StoreTx {
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: string(decoded),
|
||||
PathJSON: `["` + prefix + `"]`,
|
||||
FirstSeen: "2026-04-11T00:00:00.000Z",
|
||||
FirstSeen: recentTS(24),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,12 +87,12 @@ func TestMultiByteCapability_Confirmed(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepA", "repeater", "2026-04-11T00:00:00Z")
|
||||
"aabbccdd11223344", "RepA", "repeater", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 2))
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -107,7 +114,7 @@ func TestMultiByteCapability_Suspected(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepB", "repeater", "2026-04-10T00:00:00Z")
|
||||
"aabbccdd11223344", "RepB", "repeater", recentTS(48))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
@@ -119,11 +126,11 @@ func TestMultiByteCapability_Suspected(t *testing.T) {
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aabb"]`,
|
||||
FirstSeen: "2026-04-10T00:00:00.000Z",
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -145,14 +152,14 @@ func TestMultiByteCapability_Unknown(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepC", "repeater", "2026-04-08T00:00:00Z")
|
||||
"aabbccdd11223344", "RepC", "repeater", recentTS(72))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
// Advert with 1-byte hash only
|
||||
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 1))
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -173,9 +180,9 @@ func TestMultiByteCapability_PrefixCollision(t *testing.T) {
|
||||
|
||||
// Two repeaters sharing 1-byte prefix "aa"
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabb000000000001", "RepConfirmed", "repeater", "2026-04-11T00:00:00Z")
|
||||
"aabb000000000001", "RepConfirmed", "repeater", recentTS(24))
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aacc000000000002", "RepOther", "repeater", "2026-04-11T00:00:00Z")
|
||||
"aacc000000000002", "RepOther", "repeater", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
@@ -190,11 +197,11 @@ func TestMultiByteCapability_PrefixCollision(t *testing.T) {
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aa"]`,
|
||||
FirstSeen: "2026-04-10T00:00:00.000Z",
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(caps))
|
||||
}
|
||||
@@ -221,7 +228,7 @@ func TestMultiByteCapability_TraceExcluded(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepTrace", "repeater", "2026-04-10T00:00:00Z")
|
||||
"aabbccdd11223344", "RepTrace", "repeater", recentTS(48))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
@@ -233,11 +240,11 @@ func TestMultiByteCapability_TraceExcluded(t *testing.T) {
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aabb"]`,
|
||||
FirstSeen: "2026-04-10T00:00:00.000Z",
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -253,7 +260,7 @@ func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepNonTrace", "repeater", "2026-04-10T00:00:00Z")
|
||||
"aabbccdd11223344", "RepNonTrace", "repeater", recentTS(48))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
@@ -265,11 +272,11 @@ func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) {
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aabb"]`,
|
||||
FirstSeen: "2026-04-10T00:00:00.000Z",
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -285,7 +292,7 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) {
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepConfirmedTrace", "repeater", "2026-04-11T00:00:00Z")
|
||||
"aabbccdd11223344", "RepConfirmedTrace", "repeater", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
@@ -300,11 +307,11 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) {
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aabb"]`,
|
||||
FirstSeen: "2026-04-10T00:00:00.000Z",
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
caps := store.computeMultiByteCapability()
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
@@ -312,3 +319,117 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) {
|
||||
t.Errorf("expected confirmed (unaffected by TRACE), got %s", caps[0].Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiByteCapability_CompanionConfirmed tests that a companion with
|
||||
// multi-byte advert is classified as "confirmed", not "unknown" (Bug 1, #754).
|
||||
func TestMultiByteCapability_CompanionConfirmed(t *testing.T) {
|
||||
db := setupCapabilityTestDB(t)
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "CompA", "companion", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 2))
|
||||
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(caps))
|
||||
}
|
||||
if caps[0].Status != "confirmed" {
|
||||
t.Errorf("expected confirmed for companion, got %s", caps[0].Status)
|
||||
}
|
||||
if caps[0].Role != "companion" {
|
||||
t.Errorf("expected role companion, got %s", caps[0].Role)
|
||||
}
|
||||
if caps[0].Evidence != "advert" {
|
||||
t.Errorf("expected advert evidence, got %s", caps[0].Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiByteCapability_RoleColumnPopulated tests that the Role field is
|
||||
// populated for all node types (Bug 2, #754).
|
||||
func TestMultiByteCapability_RoleColumnPopulated(t *testing.T) {
|
||||
db := setupCapabilityTestDB(t)
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabb000000000001", "Rep1", "repeater", recentTS(24))
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"ccdd000000000002", "Comp1", "companion", recentTS(24))
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"eeff000000000003", "Room1", "room_server", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
addTestPacket(store, makeTestAdvert("aabb000000000001", 2))
|
||||
addTestPacket(store, makeTestAdvert("ccdd000000000002", 2))
|
||||
addTestPacket(store, makeTestAdvert("eeff000000000003", 1))
|
||||
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
if len(caps) != 3 {
|
||||
t.Fatalf("expected 3 entries, got %d", len(caps))
|
||||
}
|
||||
|
||||
roleByName := map[string]string{}
|
||||
for _, c := range caps {
|
||||
roleByName[c.Name] = c.Role
|
||||
}
|
||||
if roleByName["Rep1"] != "repeater" {
|
||||
t.Errorf("Rep1 role: expected repeater, got %s", roleByName["Rep1"])
|
||||
}
|
||||
if roleByName["Comp1"] != "companion" {
|
||||
t.Errorf("Comp1 role: expected companion, got %s", roleByName["Comp1"])
|
||||
}
|
||||
if roleByName["Room1"] != "room_server" {
|
||||
t.Errorf("Room1 role: expected room_server, got %s", roleByName["Room1"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultiByteCapability_AdopterEvidenceTakesPrecedence tests that when
|
||||
// adopter data shows hashSize >= 2 but path evidence says "suspected",
|
||||
// the node is upgraded to "confirmed" (Bug 3, #754).
|
||||
func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) {
|
||||
db := setupCapabilityTestDB(t)
|
||||
defer db.conn.Close()
|
||||
|
||||
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
|
||||
"aabbccdd11223344", "RepAdopter", "repeater", recentTS(24))
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
// Only a path-based packet (no advert) — would normally be "suspected"
|
||||
pathByte := buildPathByte(2, 1)
|
||||
rawHex := "01" + pathByte + "aabb"
|
||||
pt := 1
|
||||
pkt := &StoreTx{
|
||||
RawHex: rawHex,
|
||||
PayloadType: &pt,
|
||||
PathJSON: `["aabb"]`,
|
||||
FirstSeen: recentTS(48),
|
||||
}
|
||||
addTestPacket(store, pkt)
|
||||
|
||||
// Without adopter data: should be suspected
|
||||
caps := store.computeMultiByteCapability(nil)
|
||||
capByName := map[string]MultiByteCapEntry{}
|
||||
for _, c := range caps {
|
||||
capByName[c.Name] = c
|
||||
}
|
||||
if capByName["RepAdopter"].Status != "suspected" {
|
||||
t.Errorf("without adopter data: expected suspected, got %s", capByName["RepAdopter"].Status)
|
||||
}
|
||||
|
||||
// With adopter data showing hashSize 2: should be confirmed
|
||||
adopterHS := map[string]int{"aabbccdd11223344": 2}
|
||||
caps = store.computeMultiByteCapability(adopterHS)
|
||||
capByName = map[string]MultiByteCapEntry{}
|
||||
for _, c := range caps {
|
||||
capByName[c.Name] = c
|
||||
}
|
||||
if capByName["RepAdopter"].Status != "confirmed" {
|
||||
t.Errorf("with adopter data: expected confirmed, got %s", capByName["RepAdopter"].Status)
|
||||
}
|
||||
if capByName["RepAdopter"].Evidence != "advert" {
|
||||
t.Errorf("with adopter data: expected advert evidence, got %s", capByName["RepAdopter"].Evidence)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,6 +94,10 @@ func (s *Server) getNeighborGraph() *NeighborGraph {
|
||||
|
||||
func (s *Server) handleNodeNeighbors(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := strings.ToLower(mux.Vars(r)["pubkey"])
|
||||
if s.cfg.IsBlacklisted(pubkey) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
|
||||
minCount := 1
|
||||
if v := r.URL.Query().Get("min_count"); v != "" {
|
||||
@@ -272,6 +276,11 @@ func (s *Server) handleNeighborGraph(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Filter blacklisted nodes from graph.
|
||||
if s.cfg != nil && (s.cfg.IsBlacklisted(e.NodeA) || s.cfg.IsBlacklisted(e.NodeB)) {
|
||||
continue
|
||||
}
|
||||
|
||||
ge := GraphEdge{
|
||||
Source: e.NodeA,
|
||||
Target: e.NodeB,
|
||||
@@ -373,6 +382,26 @@ func (s *Server) buildNodeInfoMap() map[string]nodeInfo {
|
||||
for _, n := range nodes {
|
||||
m[strings.ToLower(n.PublicKey)] = n
|
||||
}
|
||||
|
||||
// Enrich observer-only nodes: if an observer pubkey isn't already in the
|
||||
// map (i.e. it's not also a repeater/companion), add it with role "observer".
|
||||
if s.db != nil {
|
||||
rows, err := s.db.conn.Query("SELECT id, name FROM observers")
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, name string
|
||||
if rows.Scan(&id, &name) != nil {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(id)
|
||||
if _, exists := m[key]; !exists {
|
||||
m[key] = nodeInfo{PublicKey: id, Name: name, Role: "observer"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -8,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────────
|
||||
@@ -457,3 +459,69 @@ func TestNeighborGraphAPI_ResponseShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Tests: buildNodeInfoMap observer enrichment (#753) ────────────────────────
|
||||
|
||||
func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) {
|
||||
// Create a temp SQLite DB with nodes and observers tables.
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := tmpDir + "/test.db"
|
||||
|
||||
conn, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// Create tables
|
||||
for _, stmt := range []string{
|
||||
"CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)",
|
||||
"CREATE TABLE observers (id TEXT, name TEXT)",
|
||||
"INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)",
|
||||
"INSERT INTO observers VALUES ('BBBB2222', 'Observer-Alpha')",
|
||||
"INSERT INTO observers VALUES ('AAAA1111', 'Obs-also-repeater')",
|
||||
} {
|
||||
if _, err := conn.Exec(stmt); err != nil {
|
||||
t.Fatalf("exec %q: %v", stmt, err)
|
||||
}
|
||||
}
|
||||
conn.Close()
|
||||
|
||||
// Open via our DB wrapper
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.conn.Close()
|
||||
|
||||
// Build a PacketStore with this DB (minimal — just need getCachedNodesAndPM)
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
srv := &Server{
|
||||
db: db,
|
||||
store: store,
|
||||
perfStats: NewPerfStats(),
|
||||
}
|
||||
|
||||
m := srv.buildNodeInfoMap()
|
||||
|
||||
// AAAA1111 should be from nodes table (repeater), NOT overwritten by observer
|
||||
if info, ok := m["aaaa1111"]; !ok {
|
||||
t.Error("expected aaaa1111 in map")
|
||||
} else if info.Role != "repeater" {
|
||||
t.Errorf("expected role=repeater for aaaa1111, got %q", info.Role)
|
||||
}
|
||||
|
||||
// BBBB2222 should be enriched from observers table
|
||||
if info, ok := m["bbbb2222"]; !ok {
|
||||
t.Error("expected bbbb2222 in map (observer-only node)")
|
||||
} else {
|
||||
if info.Role != "observer" {
|
||||
t.Errorf("expected role=observer for bbbb2222, got %q", info.Role)
|
||||
}
|
||||
if info.Name != "Observer-Alpha" {
|
||||
t.Errorf("expected name=Observer-Alpha for bbbb2222, got %q", info.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,7 +166,7 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
|
||||
|
||||
// Phase 1: Extract edges from every transmission + observation.
|
||||
for _, tx := range packets {
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT
|
||||
fromNode := extractFromNode(tx)
|
||||
// Pre-compute lowered originator once per tx (not per observation).
|
||||
fromLower := ""
|
||||
|
||||
@@ -381,7 +381,13 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
|
||||
}
|
||||
}
|
||||
for _, obs := range tx.Observations {
|
||||
if obs.ResolvedPath == nil && obs.PathJSON != "" && obs.PathJSON != "[]" {
|
||||
// Check if this observation has been resolved: look up in the index.
|
||||
// If the tx has no reverse-map entries AND path is non-empty, it needs backfill.
|
||||
hasRP := false
|
||||
if _, ok := store.resolvedPubkeyReverse[tx.ID]; ok {
|
||||
hasRP = true
|
||||
}
|
||||
if !hasRP && obs.PathJSON != "" && obs.PathJSON != "[]" {
|
||||
allPending = append(allPending, obsRef{
|
||||
obsID: obs.ID,
|
||||
pathJSON: obs.PathJSON,
|
||||
@@ -482,24 +488,61 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
|
||||
}
|
||||
}
|
||||
|
||||
// Update in-memory state and re-pick best observation under a single
|
||||
// write lock. The per-tx pickBestObservation is O(observations) which is
|
||||
// typically <10 per tx — negligible cost vs. the race risk of splitting
|
||||
// the lock (pollAndMerge can append to tx.Observations concurrently).
|
||||
// Update in-memory state: update resolved pubkey index, re-pick best observation,
|
||||
// and invalidate LRU cache entries for backfilled observations (#800).
|
||||
//
|
||||
// Lock ordering: always take s.mu BEFORE lruMu. The read path
|
||||
// (fetchResolvedPathForObs) takes lruMu independently of s.mu,
|
||||
// so we must NOT hold s.mu while taking lruMu. Instead, collect
|
||||
// obsIDs to invalidate under s.mu, release it, then take lruMu.
|
||||
store.mu.Lock()
|
||||
affectedSet := make(map[string]bool)
|
||||
lruInvalidate := make([]int, 0, len(results))
|
||||
for _, r := range results {
|
||||
if obs, ok := store.byObsID[r.obsID]; ok {
|
||||
obs.ResolvedPath = r.rp
|
||||
}
|
||||
// Remove old index entries for this tx, then re-add with new pubkeys
|
||||
if !affectedSet[r.txHash] {
|
||||
affectedSet[r.txHash] = true
|
||||
if tx, ok := store.byHash[r.txHash]; ok {
|
||||
pickBestObservation(tx)
|
||||
store.removeFromResolvedPubkeyIndex(tx.ID)
|
||||
}
|
||||
}
|
||||
// Add new resolved pubkeys to index
|
||||
if tx, ok := store.byHash[r.txHash]; ok {
|
||||
pks := extractResolvedPubkeys(r.rp)
|
||||
store.addToResolvedPubkeyIndex(tx.ID, pks)
|
||||
// Update byNode for relay nodes
|
||||
for _, pk := range pks {
|
||||
store.addToByNode(tx, pk)
|
||||
}
|
||||
// Update byPathHop resolved-key entries
|
||||
hopsSeen := make(map[string]bool)
|
||||
for _, hop := range txGetParsedPath(tx) {
|
||||
hopsSeen[strings.ToLower(hop)] = true
|
||||
}
|
||||
for _, pk := range pks {
|
||||
if !hopsSeen[pk] {
|
||||
hopsSeen[pk] = true
|
||||
store.byPathHop[pk] = append(store.byPathHop[pk], tx)
|
||||
}
|
||||
}
|
||||
}
|
||||
lruInvalidate = append(lruInvalidate, r.obsID)
|
||||
}
|
||||
// Re-pick best observation for affected transmissions
|
||||
for txHash := range affectedSet {
|
||||
if tx, ok := store.byHash[txHash]; ok {
|
||||
pickBestObservation(tx)
|
||||
}
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
// Invalidate LRU entries AFTER releasing s.mu to maintain lock
|
||||
// ordering (lruMu must never be taken while s.mu is held).
|
||||
store.lruMu.Lock()
|
||||
for _, obsID := range lruInvalidate {
|
||||
store.lruDelete(obsID)
|
||||
}
|
||||
store.lruMu.Unlock()
|
||||
}
|
||||
|
||||
totalProcessed += len(chunk)
|
||||
@@ -525,7 +568,7 @@ type edgeCandidate struct {
|
||||
// For ADVERTs: originator↔path[0] (if unambiguous). For ALL types: observer↔path[last] (if unambiguous).
|
||||
// Also handles zero-hop ADVERTs (originator↔observer direct link).
|
||||
func extractEdgesFromObs(obs *StoreObs, tx *StoreTx, pm *prefixMap) []edgeCandidate {
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
|
||||
isAdvert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT
|
||||
fromNode := extractFromNode(tx)
|
||||
path := parsePathJSON(obs.PathJSON)
|
||||
observerPK := strings.ToLower(obs.ObserverID)
|
||||
|
||||
@@ -27,7 +27,7 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) {
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT, hash TEXT UNIQUE, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
|
||||
decoded_json TEXT
|
||||
decoded_json TEXT, channel_hash TEXT DEFAULT NULL
|
||||
)`)
|
||||
conn.Exec(`CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY, name TEXT, iata TEXT
|
||||
@@ -203,14 +203,14 @@ func TestLoadNeighborEdgesFromDB(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
|
||||
// Verify resolved_path appears in broadcast maps
|
||||
pk := "aabbccdd"
|
||||
// After #800 refactor, resolved_path is no longer stored on StoreTx/StoreObs structs.
|
||||
// Broadcast maps carry resolved_path from the decode-window, not from struct fields.
|
||||
// This test verifies pickBestObservation no longer sets ResolvedPath on tx.
|
||||
obs := &StoreObs{
|
||||
ID: 1,
|
||||
ObserverID: "obs1",
|
||||
ObserverName: "Observer 1",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
@@ -221,32 +221,26 @@ func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
|
||||
}
|
||||
pickBestObservation(tx)
|
||||
|
||||
if tx.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be set on tx after pickBestObservation")
|
||||
}
|
||||
if *tx.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("expected resolved path to be aabbccdd, got %s", *tx.ResolvedPath[0])
|
||||
// tx should NOT have a ResolvedPath field anymore (compile-time guard)
|
||||
// Verify the best observation's fields are propagated correctly
|
||||
if tx.ObserverID != "obs1" {
|
||||
t.Errorf("expected ObserverID=obs1, got %s", tx.ObserverID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInTxToMap(t *testing.T) {
|
||||
pk := "aabbccdd"
|
||||
// After #800, txToMap no longer includes resolved_path from the struct.
|
||||
// resolved_path is only available via on-demand SQL fetch (txToMapWithRP).
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
obsKeys: make(map[string]bool),
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
PathJSON: `["aa"]`,
|
||||
obsKeys: make(map[string]bool),
|
||||
}
|
||||
|
||||
m := txToMap(tx)
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path not in txToMap output")
|
||||
}
|
||||
rpSlice, ok := rp.([]*string)
|
||||
if !ok || len(rpSlice) != 1 || *rpSlice[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected resolved_path: %v", rp)
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should not be in txToMap output (removed in #800)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,27 +359,21 @@ func TestLoadWithResolvedPath(t *testing.T) {
|
||||
t.Fatalf("expected 1 observation, got %d", len(tx.Observations))
|
||||
}
|
||||
|
||||
obs := tx.Observations[0]
|
||||
if obs.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be loaded")
|
||||
}
|
||||
if len(obs.ResolvedPath) != 1 || *obs.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected ResolvedPath: %v", obs.ResolvedPath)
|
||||
}
|
||||
|
||||
// Check that pickBestObservation propagated resolved_path to tx
|
||||
if tx.ResolvedPath == nil || len(tx.ResolvedPath) != 1 {
|
||||
t.Error("expected ResolvedPath to be propagated to tx")
|
||||
// After #800, ResolvedPath is not stored on StoreObs struct.
|
||||
// Instead, resolved pubkeys are in the membership index.
|
||||
_ = tx.Observations[0] // obs exists
|
||||
h := resolvedPubkeyHash("aabbccdd")
|
||||
if len(store.resolvedPubkeyIndex[h]) != 1 {
|
||||
t.Fatal("expected resolved pubkey to be indexed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInAPIResponse(t *testing.T) {
|
||||
// Test that TransmissionResp properly marshals resolved_path
|
||||
pk := "aabbccddee"
|
||||
// After #800, TransmissionResp no longer has ResolvedPath field.
|
||||
// resolved_path is included dynamically in map-based API responses.
|
||||
resp := TransmissionResp{
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
ResolvedPath: []*string{&pk, nil},
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
}
|
||||
|
||||
data, err := json.Marshal(resp)
|
||||
@@ -396,19 +384,9 @@ func TestResolvedPathInAPIResponse(t *testing.T) {
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path missing from JSON")
|
||||
}
|
||||
rpArr, ok := rp.([]interface{})
|
||||
if !ok || len(rpArr) != 2 {
|
||||
t.Fatalf("unexpected resolved_path shape: %v", rp)
|
||||
}
|
||||
if rpArr[0] != "aabbccddee" {
|
||||
t.Errorf("first element wrong: %v", rpArr[0])
|
||||
}
|
||||
if rpArr[1] != nil {
|
||||
t.Errorf("second element should be null: %v", rpArr[1])
|
||||
// resolved_path should NOT be in the marshaled JSON
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should not be in TransmissionResp JSON (#800)")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func TestConfigIsBlacklisted(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"AA", "BB", "cc"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
pubkey string
|
||||
want bool
|
||||
}{
|
||||
{"AA", true},
|
||||
{"aa", true}, // case-insensitive
|
||||
{"BB", true},
|
||||
{"CC", true}, // lowercase "cc" matches uppercase
|
||||
{"DD", false},
|
||||
{"", false},
|
||||
{"AAB", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := cfg.IsBlacklisted(tt.pubkey)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsBlacklisted(%q) = %v, want %v", tt.pubkey, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigIsBlacklistedEmpty(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if cfg.IsBlacklisted("anything") {
|
||||
t.Error("empty blacklist should not match anything")
|
||||
}
|
||||
if cfg.IsBlacklisted("") {
|
||||
t.Error("empty blacklist should not match empty string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigBlacklistWhitespace(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{" AA ", "BB"},
|
||||
}
|
||||
if !cfg.IsBlacklisted("AA") {
|
||||
t.Error("trimmed key should match")
|
||||
}
|
||||
if !cfg.IsBlacklisted(" AA ") {
|
||||
t.Error("whitespace-padded key should match after trimming")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigBlacklistEmptyEntries(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"", " ", "AA"},
|
||||
}
|
||||
if !cfg.IsBlacklisted("AA") {
|
||||
t.Error("non-empty entry should match")
|
||||
}
|
||||
if cfg.IsBlacklisted("") {
|
||||
t.Error("empty blacklist entry should not match empty pubkey")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistFiltersHandleNodes(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))")
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes?limit=50", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp NodeListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
for _, node := range resp.Nodes {
|
||||
if pk, _ := node["public_key"].(string); pk == "badnode" {
|
||||
t.Error("blacklisted node should not appear in nodes list")
|
||||
}
|
||||
}
|
||||
if resp.Total == 0 {
|
||||
t.Error("expected at least one non-blacklisted node")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistFiltersNodeDetail(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/badnode", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("expected 404 for blacklisted node, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistFiltersNodeSearch(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', datetime('now'))")
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/search?q=Troll", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp NodeSearchResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
for _, node := range resp.Nodes {
|
||||
if pk, _ := node["public_key"].(string); pk == "badnode" {
|
||||
t.Error("blacklisted node should not appear in search results")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoBlacklistPassesAll(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', datetime('now'))")
|
||||
|
||||
cfg := &Config{}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes?limit=50", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp NodeListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
if resp.Total == 0 {
|
||||
t.Error("without blacklist, node should appear")
|
||||
}
|
||||
}
|
||||
|
||||
// setupTestRouter creates a mux.Router and registers server routes.
|
||||
func setupTestRouter(srv *Server) *mux.Router {
|
||||
r := mux.NewRouter()
|
||||
srv.RegisterRoutes(r)
|
||||
srv.router = r
|
||||
return r
|
||||
}
|
||||
func TestBlacklistFiltersNeighborGraph(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
db := setupTestDB(t)
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/neighbor-graph", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
// Check edges don't contain blacklisted node
|
||||
if edges, ok := resp["edges"].([]interface{}); ok {
|
||||
for _, e := range edges {
|
||||
if edge, ok := e.(map[string]interface{}); ok {
|
||||
if src, _ := edge["source"].(string); src == "badnode" {
|
||||
t.Error("blacklisted node should not appear as edge source in neighbor graph")
|
||||
}
|
||||
if tgt, _ := edge["target"].(string); tgt == "badnode" {
|
||||
t.Error("blacklisted node should not appear as edge target in neighbor graph")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check nodes list doesn't contain blacklisted node
|
||||
if nodes, ok := resp["nodes"].([]interface{}); ok {
|
||||
for _, n := range nodes {
|
||||
if node, ok := n.(map[string]interface{}); ok {
|
||||
if pk, _ := node["pubkey"].(string); pk == "badnode" {
|
||||
t.Error("blacklisted node should not appear in neighbor graph nodes")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistFiltersResolveHops(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=badnode", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp ResolveHopsResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
if hr, ok := resp.Resolved["badnode"]; ok {
|
||||
for _, c := range hr.Candidates {
|
||||
if c.Pubkey == "badnode" {
|
||||
t.Error("blacklisted node should not appear as resolve-hops candidate")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistFiltersSubpathDetail(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"badnode"},
|
||||
}
|
||||
db := setupTestDB(t)
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/subpath-detail?hops=badnode,othernode", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("expected 404 for subpath-detail with blacklisted hop, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistConcurrentIsBlacklisted(t *testing.T) {
|
||||
cfg := &Config{
|
||||
NodeBlacklist: []string{"AA", "BB", "CC"},
|
||||
}
|
||||
|
||||
errc := make(chan error, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
go func() {
|
||||
for j := 0; j < 100; j++ {
|
||||
cfg.IsBlacklisted("AA")
|
||||
cfg.IsBlacklisted("BB")
|
||||
cfg.IsBlacklisted("DD")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// If sync.Once is wrong, this would panic or race.
|
||||
// We can't run the race detector on ARM, but at least verify no panics.
|
||||
done := false
|
||||
for !done {
|
||||
select {
|
||||
case <-errc:
|
||||
t.Error("concurrent IsBlacklisted panicked")
|
||||
default:
|
||||
done = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,475 @@
|
||||
package main
|
||||
|
||||
// Lock ordering contract (MUST be followed everywhere):
|
||||
//
|
||||
// s.mu → s.lruMu (s.mu is the outer lock, lruMu is the inner lock)
|
||||
//
|
||||
// • Never acquire s.lruMu while holding s.mu.
|
||||
// • fetchResolvedPathForObs takes lruMu independently — callers under s.mu
|
||||
// must NOT call it directly; instead collect IDs under s.mu, release, then
|
||||
// do LRU ops under lruMu separately.
|
||||
// • The backfill path (backfillResolvedPathsAsync) follows this by collecting
|
||||
// obsIDs to invalidate under s.mu, releasing it, then taking lruMu.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"hash/fnv"
|
||||
"log"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// resolvedPubkeyHash computes a fast 64-bit hash for membership index keying.
|
||||
// Uses FNV-1a from stdlib — good distribution, no external dependency.
|
||||
func resolvedPubkeyHash(pk string) uint64 {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(strings.ToLower(pk)))
|
||||
return h.Sum64()
|
||||
}
|
||||
|
||||
// addToResolvedPubkeyIndex adds a txID under each resolved pubkey hash.
|
||||
// Deduplicates both within a single call AND across calls — won't add the
|
||||
// same (hash, txID) pair twice even when called multiple times for the same tx.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) addToResolvedPubkeyIndex(txID int, resolvedPubkeys []string) {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
seen := make(map[uint64]bool, len(resolvedPubkeys))
|
||||
for _, pk := range resolvedPubkeys {
|
||||
if pk == "" {
|
||||
continue
|
||||
}
|
||||
h := resolvedPubkeyHash(pk)
|
||||
if seen[h] {
|
||||
continue
|
||||
}
|
||||
seen[h] = true
|
||||
|
||||
// Cross-call dedup: check if (h, txID) already exists in forward index.
|
||||
existing := s.resolvedPubkeyIndex[h]
|
||||
alreadyPresent := false
|
||||
for _, id := range existing {
|
||||
if id == txID {
|
||||
alreadyPresent = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if alreadyPresent {
|
||||
continue
|
||||
}
|
||||
|
||||
s.resolvedPubkeyIndex[h] = append(existing, txID)
|
||||
s.resolvedPubkeyReverse[txID] = append(s.resolvedPubkeyReverse[txID], h)
|
||||
}
|
||||
}
|
||||
|
||||
// removeFromResolvedPubkeyIndex removes all index entries for a txID using the reverse map.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) removeFromResolvedPubkeyIndex(txID int) {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
hashes := s.resolvedPubkeyReverse[txID]
|
||||
for _, h := range hashes {
|
||||
list := s.resolvedPubkeyIndex[h]
|
||||
// Remove ALL occurrences of txID (not just the first) to prevent orphans.
|
||||
filtered := list[:0]
|
||||
for _, id := range list {
|
||||
if id != txID {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
delete(s.resolvedPubkeyIndex, h)
|
||||
} else {
|
||||
s.resolvedPubkeyIndex[h] = filtered
|
||||
}
|
||||
}
|
||||
delete(s.resolvedPubkeyReverse, txID)
|
||||
}
|
||||
|
||||
// extractResolvedPubkeys extracts all non-nil, non-empty pubkeys from a resolved path.
|
||||
func extractResolvedPubkeys(rp []*string) []string {
|
||||
if len(rp) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, 0, len(rp))
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" {
|
||||
result = append(result, *p)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// mergeResolvedPubkeys collects unique non-empty pubkeys from multiple resolved paths.
|
||||
func mergeResolvedPubkeys(paths ...[]*string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
for _, rp := range paths {
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" && !seen[*p] {
|
||||
seen[*p] = true
|
||||
result = append(result, *p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// nodeInResolvedPathViaIndex checks whether a transmission is associated with
|
||||
// a target pubkey using the membership index + collision-safety SQL check.
|
||||
// Must be called under s.mu RLock at minimum.
|
||||
func (s *PacketStore) nodeInResolvedPathViaIndex(tx *StoreTx, targetPK string) bool {
|
||||
if !s.useResolvedPathIndex {
|
||||
// Flag off: can't disambiguate, keep candidate (conservative)
|
||||
return true
|
||||
}
|
||||
|
||||
// If this tx has no indexed pubkeys at all, we can't disambiguate —
|
||||
// keep the candidate (same as old behavior for NULL resolved_path).
|
||||
if _, hasReverse := s.resolvedPubkeyReverse[tx.ID]; !hasReverse {
|
||||
return true
|
||||
}
|
||||
|
||||
h := resolvedPubkeyHash(targetPK)
|
||||
txIDs := s.resolvedPubkeyIndex[h]
|
||||
|
||||
// Check if this tx's ID is in the candidate list
|
||||
for _, id := range txIDs {
|
||||
if id == tx.ID {
|
||||
// Found in index. Collision-safety: verify with SQL.
|
||||
if s.db != nil && s.db.conn != nil {
|
||||
return s.confirmResolvedPathContains(tx.ID, targetPK)
|
||||
}
|
||||
return true // no DB, trust the index
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// confirmResolvedPathContains verifies an exact pubkey match in resolved_path
|
||||
// via SQL. This is the collision-safety fallback for the membership index.
|
||||
func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return true
|
||||
}
|
||||
// Use INSTR with surrounding quotes for exact match — avoids LIKE escape issues.
|
||||
// resolved_path format: ["pubkey1","pubkey2",...]
|
||||
needle := `"` + strings.ToLower(pubkey) + `"`
|
||||
var count int
|
||||
err := s.db.conn.QueryRow(
|
||||
`SELECT COUNT(*) FROM observations WHERE transmission_id = ? AND INSTR(LOWER(resolved_path), ?) > 0`,
|
||||
txID, needle,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
return true // on error, keep the candidate
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// fetchResolvedPathsForTx fetches resolved_path from SQLite for all observations
|
||||
// of a transmission. Used for on-demand API responses and eviction cleanup.
|
||||
func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.db.conn.Query(
|
||||
`SELECT id, resolved_path FROM observations WHERE transmission_id = ? AND resolved_path IS NOT NULL`,
|
||||
txID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := make(map[int][]*string)
|
||||
for rows.Next() {
|
||||
var obsID int
|
||||
var rpJSON sql.NullString
|
||||
if err := rows.Scan(&obsID, &rpJSON); err != nil {
|
||||
continue
|
||||
}
|
||||
if rpJSON.Valid && rpJSON.String != "" {
|
||||
result[obsID] = unmarshalResolvedPath(rpJSON.String)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fetchResolvedPathForObs fetches resolved_path for a single observation,
|
||||
// using the LRU cache.
|
||||
func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check LRU cache first
|
||||
s.lruMu.RLock()
|
||||
if s.apiResolvedPathLRU != nil {
|
||||
if entry, ok := s.apiResolvedPathLRU[obsID]; ok {
|
||||
s.lruMu.RUnlock()
|
||||
return entry
|
||||
}
|
||||
}
|
||||
s.lruMu.RUnlock()
|
||||
|
||||
var rpJSON sql.NullString
|
||||
err := s.db.conn.QueryRow(
|
||||
`SELECT resolved_path FROM observations WHERE id = ?`, obsID,
|
||||
).Scan(&rpJSON)
|
||||
if err != nil || !rpJSON.Valid {
|
||||
return nil
|
||||
}
|
||||
rp := unmarshalResolvedPath(rpJSON.String)
|
||||
|
||||
// Store in LRU
|
||||
s.lruMu.Lock()
|
||||
s.lruPut(obsID, rp)
|
||||
s.lruMu.Unlock()
|
||||
|
||||
return rp
|
||||
}
|
||||
|
||||
// fetchResolvedPathForTxBest returns the best observation's resolved_path for a tx.
|
||||
//
|
||||
// "Best" = the longest path_json among observations that actually have a stored
|
||||
// resolved_path. Earlier versions picked the longest-path obs unconditionally
|
||||
// and queried SQL for that single ID — if the longest-path obs had NULL
|
||||
// resolved_path while a shorter sibling had one, the call returned nil and
|
||||
// callers (e.g. /api/nodes/{pk}/health.recentPackets) lost the field. Fixes
|
||||
// #810 by checking all observations and falling back to the longest sibling
|
||||
// that has a stored path.
|
||||
func (s *PacketStore) fetchResolvedPathForTxBest(tx *StoreTx) []*string {
|
||||
if tx == nil || len(tx.Observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Fast path: try the longest-path obs first via the LRU/SQL helper.
|
||||
longest := tx.Observations[0]
|
||||
longestLen := pathLen(longest.PathJSON)
|
||||
for _, obs := range tx.Observations[1:] {
|
||||
if l := pathLen(obs.PathJSON); l > longestLen {
|
||||
longest = obs
|
||||
longestLen = l
|
||||
}
|
||||
}
|
||||
if rp := s.fetchResolvedPathForObs(longest.ID); rp != nil {
|
||||
return rp
|
||||
}
|
||||
// Fallback: longest-path obs has no stored resolved_path. Query all
|
||||
// observations for this tx and pick the one with the longest path_json
|
||||
// that actually has a stored resolved_path.
|
||||
rpMap := s.fetchResolvedPathsForTx(tx.ID)
|
||||
if len(rpMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
var bestRP []*string
|
||||
bestObsID := 0
|
||||
bestLen := -1
|
||||
for _, obs := range tx.Observations {
|
||||
rp, ok := rpMap[obs.ID]
|
||||
if !ok || rp == nil {
|
||||
continue
|
||||
}
|
||||
if l := pathLen(obs.PathJSON); l > bestLen {
|
||||
bestLen = l
|
||||
bestRP = rp
|
||||
bestObsID = obs.ID
|
||||
}
|
||||
}
|
||||
// Populate LRU so repeat lookups for this tx don't re-issue the multi-row
|
||||
// SQL fallback (e.g. dashboard polling /api/nodes/{pk}/health).
|
||||
if bestRP != nil && bestObsID != 0 {
|
||||
s.lruMu.Lock()
|
||||
s.lruPut(bestObsID, bestRP)
|
||||
s.lruMu.Unlock()
|
||||
}
|
||||
return bestRP
|
||||
}
|
||||
|
||||
// --- Simple LRU cache for resolved paths ---
|
||||
|
||||
const lruMaxSize = 10000
|
||||
|
||||
// lruPut adds an entry. Must be called under s.lruMu write lock.
|
||||
func (s *PacketStore) lruPut(obsID int, rp []*string) {
|
||||
if s.apiResolvedPathLRU == nil {
|
||||
return
|
||||
}
|
||||
if _, exists := s.apiResolvedPathLRU[obsID]; exists {
|
||||
return
|
||||
}
|
||||
// Compact lruOrder if stale entries exceed 50% of capacity.
|
||||
// This prevents effective capacity degradation after bulk deletions.
|
||||
if len(s.lruOrder) >= lruMaxSize && len(s.apiResolvedPathLRU) < lruMaxSize/2 {
|
||||
compacted := make([]int, 0, len(s.apiResolvedPathLRU))
|
||||
for _, id := range s.lruOrder {
|
||||
if _, ok := s.apiResolvedPathLRU[id]; ok {
|
||||
compacted = append(compacted, id)
|
||||
}
|
||||
}
|
||||
s.lruOrder = compacted
|
||||
}
|
||||
if len(s.lruOrder) >= lruMaxSize {
|
||||
// Evict oldest, skipping stale entries
|
||||
for len(s.lruOrder) > 0 {
|
||||
evictID := s.lruOrder[0]
|
||||
s.lruOrder = s.lruOrder[1:]
|
||||
if _, ok := s.apiResolvedPathLRU[evictID]; ok {
|
||||
delete(s.apiResolvedPathLRU, evictID)
|
||||
break
|
||||
}
|
||||
// stale entry — skip and continue
|
||||
}
|
||||
}
|
||||
s.apiResolvedPathLRU[obsID] = rp
|
||||
s.lruOrder = append(s.lruOrder, obsID)
|
||||
}
|
||||
|
||||
// lruDelete removes an entry. Must be called under s.lruMu write lock.
|
||||
func (s *PacketStore) lruDelete(obsID int) {
|
||||
if s.apiResolvedPathLRU == nil {
|
||||
return
|
||||
}
|
||||
delete(s.apiResolvedPathLRU, obsID)
|
||||
// Don't scan lruOrder — eviction handles stale entries naturally.
|
||||
}
|
||||
|
||||
// resolvedPubkeysForEvictionBatch fetches resolved pubkeys for multiple txIDs
|
||||
// from SQL in a single batched query. Returns a map from txID to unique pubkeys.
|
||||
// MUST be called WITHOUT holding s.mu — this is the whole point of the batch approach.
|
||||
// Chunks queries to stay under SQLite's 500-parameter limit.
|
||||
func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]string {
|
||||
result := make(map[int][]string, len(txIDs))
|
||||
if len(txIDs) == 0 || s.db == nil || s.db.conn == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
const chunkSize = 499 // SQLite SQLITE_MAX_VARIABLE_NUMBER default is 999; stay well under
|
||||
for start := 0; start < len(txIDs); start += chunkSize {
|
||||
end := start + chunkSize
|
||||
if end > len(txIDs) {
|
||||
end = len(txIDs)
|
||||
}
|
||||
chunk := txIDs[start:end]
|
||||
|
||||
// Build query with placeholders
|
||||
placeholders := make([]byte, 0, len(chunk)*2)
|
||||
args := make([]interface{}, len(chunk))
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
placeholders = append(placeholders, ',')
|
||||
}
|
||||
placeholders = append(placeholders, '?')
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" +
|
||||
string(placeholders) + ") AND resolved_path IS NOT NULL"
|
||||
|
||||
rows, err := s.db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var txID int
|
||||
var rpJSON sql.NullString
|
||||
if err := rows.Scan(&txID, &rpJSON); err != nil {
|
||||
continue
|
||||
}
|
||||
if !rpJSON.Valid || rpJSON.String == "" {
|
||||
continue
|
||||
}
|
||||
rp := unmarshalResolvedPath(rpJSON.String)
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" {
|
||||
result[txID] = append(result[txID], *p)
|
||||
}
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
|
||||
// Deduplicate per-txID
|
||||
for txID, pks := range result {
|
||||
seen := make(map[string]bool, len(pks))
|
||||
deduped := pks[:0]
|
||||
for _, pk := range pks {
|
||||
if !seen[pk] {
|
||||
seen[pk] = true
|
||||
deduped = append(deduped, pk)
|
||||
}
|
||||
}
|
||||
result[txID] = deduped
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// initResolvedPathIndex initializes the resolved path index data structures.
|
||||
func (s *PacketStore) initResolvedPathIndex() {
|
||||
s.resolvedPubkeyIndex = make(map[uint64][]int, 4096)
|
||||
s.resolvedPubkeyReverse = make(map[int][]uint64, 4096)
|
||||
s.apiResolvedPathLRU = make(map[int][]*string, lruMaxSize)
|
||||
s.lruOrder = make([]int, 0, lruMaxSize)
|
||||
}
|
||||
|
||||
// CompactResolvedPubkeyIndex reclaims memory from the resolved pubkey index maps
|
||||
// after eviction. It removes empty forward-index entries (shouldn't exist if
|
||||
// removeFromResolvedPubkeyIndex is correct, but defense in depth) and clips
|
||||
// oversized slice backing arrays where cap > 2*len.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) CompactResolvedPubkeyIndex() {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
for h, ids := range s.resolvedPubkeyIndex {
|
||||
if len(ids) == 0 {
|
||||
delete(s.resolvedPubkeyIndex, h)
|
||||
continue
|
||||
}
|
||||
// Clip oversized backing arrays: if cap > 2*len, reallocate.
|
||||
if cap(ids) > 2*len(ids)+8 {
|
||||
clipped := make([]int, len(ids))
|
||||
copy(clipped, ids)
|
||||
s.resolvedPubkeyIndex[h] = clipped
|
||||
}
|
||||
}
|
||||
for txID, hashes := range s.resolvedPubkeyReverse {
|
||||
if len(hashes) == 0 {
|
||||
delete(s.resolvedPubkeyReverse, txID)
|
||||
continue
|
||||
}
|
||||
if cap(hashes) > 2*len(hashes)+8 {
|
||||
clipped := make([]uint64, len(hashes))
|
||||
copy(clipped, hashes)
|
||||
s.resolvedPubkeyReverse[txID] = clipped
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// defaultMaxResolvedPubkeyIndexEntries is the default hard cap for the forward
|
||||
// index. When exceeded, a warning is logged. No auto-eviction — that's the
|
||||
// eviction ticker's job.
|
||||
const defaultMaxResolvedPubkeyIndexEntries = 5_000_000
|
||||
|
||||
// CheckResolvedPubkeyIndexSize logs a warning if the resolved pubkey forward
|
||||
// index exceeds the configured maximum entries. Must be called under s.mu
|
||||
// read lock at minimum.
|
||||
func (s *PacketStore) CheckResolvedPubkeyIndexSize() {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
maxEntries := s.maxResolvedPubkeyIndexEntries
|
||||
if maxEntries <= 0 {
|
||||
maxEntries = defaultMaxResolvedPubkeyIndexEntries
|
||||
}
|
||||
fwdLen := len(s.resolvedPubkeyIndex)
|
||||
revLen := len(s.resolvedPubkeyReverse)
|
||||
if fwdLen > maxEntries || revLen > maxEntries {
|
||||
log.Printf("[store] WARNING: resolvedPubkeyIndex size exceeds limit — forward=%d reverse=%d limit=%d",
|
||||
fwdLen, revLen, maxEntries)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+393
-32
@@ -124,6 +124,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.Handle("/api/perf/reset", s.requireAPIKey(http.HandlerFunc(s.handlePerfReset))).Methods("POST")
|
||||
r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST")
|
||||
r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET")
|
||||
r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET")
|
||||
|
||||
// Packet endpoints
|
||||
r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")
|
||||
@@ -142,6 +143,9 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/nodes/{pubkey}/health", s.handleNodeHealth).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/{pubkey}/paths", s.handleNodePaths).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/{pubkey}/analytics", s.handleNodeAnalytics).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/clock-skew", s.handleFleetClockSkew).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/{pubkey}/clock-skew", s.handleNodeClockSkew).Methods("GET")
|
||||
r.HandleFunc("/api/observers/clock-skew", s.handleObserverClockSkew).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/{pubkey}/neighbors", s.handleNodeNeighbors).Methods("GET")
|
||||
r.HandleFunc("/api/nodes/{pubkey}", s.handleNodeDetail).Methods("GET")
|
||||
r.HandleFunc("/api/nodes", s.handleNodes).Methods("GET")
|
||||
@@ -565,6 +569,16 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
backfillProgress = 1
|
||||
}
|
||||
|
||||
// Memory accounting (#832). storeDataMB is the in-store packet byte
|
||||
// estimate (the old "trackedMB"); processRSSMB / goHeapInuseMB / goSysMB
|
||||
// give ops the breakdown needed to reason about real RSS. All values
|
||||
// share a single 1s-cached snapshot to amortize ReadMemStats cost.
|
||||
var storeDataMB float64
|
||||
if s.store != nil {
|
||||
storeDataMB = s.store.trackedMemoryMB()
|
||||
}
|
||||
mem := s.getMemorySnapshot(storeDataMB)
|
||||
|
||||
resp := &StatsResponse{
|
||||
TotalPackets: stats.TotalPackets,
|
||||
TotalTransmissions: &stats.TotalTransmissions,
|
||||
@@ -584,8 +598,16 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
Companions: counts["companions"],
|
||||
Sensors: counts["sensors"],
|
||||
},
|
||||
Backfilling: backfilling,
|
||||
BackfillProgress: backfillProgress,
|
||||
Backfilling: backfilling,
|
||||
BackfillProgress: backfillProgress,
|
||||
SignatureDrops: s.db.GetSignatureDropCount(),
|
||||
HashMigrationComplete: s.store != nil && s.store.hashMigrationComplete.Load(),
|
||||
|
||||
TrackedMB: mem.StoreDataMB, // deprecated alias
|
||||
StoreDataMB: mem.StoreDataMB,
|
||||
ProcessRSSMB: mem.ProcessRSSMB,
|
||||
GoHeapInuseMB: mem.GoHeapInuseMB,
|
||||
GoSysMB: mem.GoSysMB,
|
||||
}
|
||||
|
||||
s.statsMu.Lock()
|
||||
@@ -768,6 +790,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
|
||||
Until: r.URL.Query().Get("until"),
|
||||
Region: r.URL.Query().Get("region"),
|
||||
Node: r.URL.Query().Get("node"),
|
||||
Channel: r.URL.Query().Get("channel"),
|
||||
Order: "DESC",
|
||||
ExpandObservations: r.URL.Query().Get("expand") == "observations",
|
||||
}
|
||||
@@ -870,9 +893,11 @@ func (s *Server) handleBatchObservations(w http.ResponseWriter, r *http.Request)
|
||||
func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
param := mux.Vars(r)["id"]
|
||||
var packet map[string]interface{}
|
||||
fromDB := false
|
||||
|
||||
isHash := hashPattern.MatchString(strings.ToLower(param))
|
||||
if s.store != nil {
|
||||
if hashPattern.MatchString(strings.ToLower(param)) {
|
||||
if isHash {
|
||||
packet = s.store.GetPacketByHash(param)
|
||||
}
|
||||
if packet == nil {
|
||||
@@ -885,6 +910,25 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// DB fallback: in-memory PacketStore prunes old entries, but the SQLite
|
||||
// DB retains them and is the source for /api/nodes recentAdverts. Without
|
||||
// this fallback, links from node-detail pages 404 once the packet ages out.
|
||||
if packet == nil && s.db != nil {
|
||||
if isHash {
|
||||
if dbPkt, err := s.db.GetPacketByHash(param); err == nil && dbPkt != nil {
|
||||
packet = dbPkt
|
||||
fromDB = true
|
||||
}
|
||||
}
|
||||
if packet == nil {
|
||||
if id, parseErr := strconv.Atoi(param); parseErr == nil {
|
||||
if dbPkt, err := s.db.GetTransmissionByID(id); err == nil && dbPkt != nil {
|
||||
packet = dbPkt
|
||||
fromDB = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if packet == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
@@ -895,6 +939,9 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store != nil {
|
||||
observations = s.store.GetObservationsForHash(hash)
|
||||
}
|
||||
if len(observations) == 0 && fromDB && s.db != nil && hash != "" {
|
||||
observations = s.db.GetObservationsForHash(hash)
|
||||
}
|
||||
observationCount := len(observations)
|
||||
if observationCount == 0 {
|
||||
observationCount = 1
|
||||
@@ -1048,6 +1095,17 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
|
||||
total = len(filtered)
|
||||
nodes = filtered
|
||||
}
|
||||
// Filter blacklisted nodes
|
||||
if len(s.cfg.NodeBlacklist) > 0 {
|
||||
filtered := nodes[:0]
|
||||
for _, node := range nodes {
|
||||
if pk, ok := node["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
|
||||
filtered = append(filtered, node)
|
||||
}
|
||||
}
|
||||
total = len(filtered)
|
||||
nodes = filtered
|
||||
}
|
||||
writeJSON(w, NodeListResponse{Nodes: nodes, Total: total, Counts: counts})
|
||||
}
|
||||
|
||||
@@ -1062,11 +1120,25 @@ func (s *Server) handleNodeSearch(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
// Filter blacklisted nodes from search results
|
||||
if len(s.cfg.NodeBlacklist) > 0 {
|
||||
filtered := make([]map[string]interface{}, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
if pk, ok := node["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
|
||||
filtered = append(filtered, node)
|
||||
}
|
||||
}
|
||||
nodes = filtered
|
||||
}
|
||||
writeJSON(w, NodeSearchResponse{Nodes: nodes})
|
||||
}
|
||||
|
||||
func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := mux.Vars(r)["pubkey"]
|
||||
if s.cfg.IsBlacklisted(pubkey) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
node, err := s.db.GetNodeByPubkey(pubkey)
|
||||
if err != nil || node == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
@@ -1092,6 +1164,10 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleNodeHealth(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := mux.Vars(r)["pubkey"]
|
||||
if s.cfg.IsBlacklisted(pubkey) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
if s.store != nil {
|
||||
result, err := s.store.GetNodeHealth(pubkey)
|
||||
if err != nil || result == nil {
|
||||
@@ -1112,7 +1188,19 @@ func (s *Server) handleBulkHealth(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if s.store != nil {
|
||||
region := r.URL.Query().Get("region")
|
||||
writeJSON(w, s.store.GetBulkHealth(limit, region))
|
||||
results := s.store.GetBulkHealth(limit, region)
|
||||
// Filter blacklisted nodes
|
||||
if len(s.cfg.NodeBlacklist) > 0 {
|
||||
filtered := make([]map[string]interface{}, 0, len(results))
|
||||
for _, entry := range results {
|
||||
if pk, ok := entry["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
|
||||
filtered = append(filtered, entry)
|
||||
}
|
||||
}
|
||||
writeJSON(w, filtered)
|
||||
return
|
||||
}
|
||||
writeJSON(w, results)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1131,6 +1219,10 @@ func (s *Server) handleNetworkStatus(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := mux.Vars(r)["pubkey"]
|
||||
if s.cfg.IsBlacklisted(pubkey) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
node, err := s.db.GetNodeByPubkey(pubkey)
|
||||
if err != nil || node == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
@@ -1182,14 +1274,52 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
// Post-filter: verify target node actually appears in each candidate's resolved_path.
|
||||
// The byPathHop index uses short prefixes which can collide (e.g. "c0" matches multiple nodes).
|
||||
// We lean on resolved_path (from neighbor affinity graph) to disambiguate.
|
||||
filtered := candidates[:0] // reuse backing array
|
||||
for _, tx := range candidates {
|
||||
if nodeInResolvedPath(tx, lowerPK) {
|
||||
filtered = append(filtered, tx)
|
||||
//
|
||||
// Collect candidate IDs and index membership under the read lock, then release
|
||||
// the lock before running SQL queries (confirmResolvedPathContains does disk I/O).
|
||||
type candidateCheck struct {
|
||||
tx *StoreTx
|
||||
hasReverse bool
|
||||
inIndex bool
|
||||
}
|
||||
checks := make([]candidateCheck, len(candidates))
|
||||
for i, tx := range candidates {
|
||||
cc := candidateCheck{tx: tx}
|
||||
if !s.store.useResolvedPathIndex {
|
||||
cc.inIndex = true // flag off — keep all
|
||||
} else if _, hasRev := s.store.resolvedPubkeyReverse[tx.ID]; !hasRev {
|
||||
cc.inIndex = true // no indexed pubkeys — keep (conservative)
|
||||
} else {
|
||||
h := resolvedPubkeyHash(lowerPK)
|
||||
for _, id := range s.store.resolvedPubkeyIndex[h] {
|
||||
if id == tx.ID {
|
||||
cc.hasReverse = true // needs SQL confirmation
|
||||
break
|
||||
}
|
||||
}
|
||||
// If not in index at all, it's a definite no
|
||||
}
|
||||
checks[i] = cc
|
||||
}
|
||||
s.store.mu.RUnlock()
|
||||
|
||||
// Now run SQL checks outside the lock for candidates that need confirmation.
|
||||
filtered := candidates[:0]
|
||||
for _, cc := range checks {
|
||||
if cc.inIndex {
|
||||
filtered = append(filtered, cc.tx)
|
||||
} else if cc.hasReverse {
|
||||
if s.store.confirmResolvedPathContains(cc.tx.ID, lowerPK) {
|
||||
filtered = append(filtered, cc.tx)
|
||||
}
|
||||
}
|
||||
// else: not in index → exclude
|
||||
}
|
||||
candidates = filtered
|
||||
|
||||
// Re-acquire read lock for the aggregation phase that reads store data.
|
||||
s.store.mu.RLock()
|
||||
|
||||
type pathAgg struct {
|
||||
Hops []PathHopResp
|
||||
Count int
|
||||
@@ -1294,6 +1424,10 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := mux.Vars(r)["pubkey"]
|
||||
if s.cfg.IsBlacklisted(pubkey) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
days := queryInt(r, "days", 7)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
@@ -1315,6 +1449,36 @@ func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, 404, "Not found")
|
||||
}
|
||||
|
||||
func (s *Server) handleNodeClockSkew(w http.ResponseWriter, r *http.Request) {
|
||||
pubkey := mux.Vars(r)["pubkey"]
|
||||
if s.store == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
result := s.store.GetNodeClockSkew(pubkey)
|
||||
if result == nil {
|
||||
writeError(w, 404, "No clock skew data for this node")
|
||||
return
|
||||
}
|
||||
writeJSON(w, result)
|
||||
}
|
||||
|
||||
func (s *Server) handleObserverClockSkew(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store == nil {
|
||||
writeJSON(w, []ObserverCalibration{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, s.store.GetObserverCalibrations())
|
||||
}
|
||||
|
||||
func (s *Server) handleFleetClockSkew(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store == nil {
|
||||
writeJSON(w, []*NodeClockSkew{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, s.store.GetFleetClockSkew())
|
||||
}
|
||||
|
||||
// --- Analytics Handlers ---
|
||||
|
||||
func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1340,7 +1504,11 @@ func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *Server) handleAnalyticsTopology(w http.ResponseWriter, r *http.Request) {
|
||||
region := r.URL.Query().Get("region")
|
||||
if s.store != nil {
|
||||
writeJSON(w, s.store.GetAnalyticsTopology(region))
|
||||
data := s.store.GetAnalyticsTopology(region)
|
||||
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
|
||||
data = s.filterBlacklistedFromTopology(data)
|
||||
}
|
||||
writeJSON(w, data)
|
||||
return
|
||||
}
|
||||
writeJSON(w, TopologyResponse{
|
||||
@@ -1428,7 +1596,11 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
maxLen := queryInt(r, "maxLen", 8)
|
||||
limit := queryInt(r, "limit", 100)
|
||||
writeJSON(w, s.store.GetAnalyticsSubpaths(region, minLen, maxLen, limit))
|
||||
data := s.store.GetAnalyticsSubpaths(region, minLen, maxLen, limit)
|
||||
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
|
||||
data = s.filterBlacklistedFromSubpaths(data)
|
||||
}
|
||||
writeJSON(w, data)
|
||||
return
|
||||
}
|
||||
writeJSON(w, SubpathsResponse{
|
||||
@@ -1480,6 +1652,11 @@ func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
results := s.store.GetAnalyticsSubpathsBulk(region, groups)
|
||||
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
|
||||
for i, r := range results {
|
||||
results[i] = s.filterBlacklistedFromSubpaths(r)
|
||||
}
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{"results": results})
|
||||
}
|
||||
|
||||
@@ -1499,6 +1676,15 @@ func (s *Server) handleAnalyticsSubpathDetail(w http.ResponseWriter, r *http.Req
|
||||
writeJSON(w, ErrorResp{Error: "Need at least 2 hops"})
|
||||
return
|
||||
}
|
||||
// Reject if any hop is a blacklisted node.
|
||||
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
|
||||
for _, hop := range rawHops {
|
||||
if s.cfg.IsBlacklisted(hop) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.store != nil {
|
||||
writeJSON(w, s.store.GetSubpathDetail(rawHops))
|
||||
return
|
||||
@@ -1564,6 +1750,10 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
|
||||
if pm != nil {
|
||||
if matched, ok := pm.m[hopLower]; ok {
|
||||
for _, ni := range matched {
|
||||
// Skip blacklisted nodes from resolution results.
|
||||
if s.cfg != nil && s.cfg.IsBlacklisted(ni.PublicKey) {
|
||||
continue
|
||||
}
|
||||
c := HopCandidate{Pubkey: ni.PublicKey}
|
||||
if ni.Name != "" {
|
||||
c.Name = ni.Name
|
||||
@@ -1632,7 +1822,8 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Use the resolved node as the default (best-effort pick).
|
||||
if best != nil {
|
||||
// Skip if the best pick is a blacklisted node.
|
||||
if best != nil && !(s.cfg != nil && s.cfg.IsBlacklisted(best.PublicKey)) {
|
||||
hr.Name = best.Name
|
||||
hr.Pubkey = best.PublicKey
|
||||
}
|
||||
@@ -1655,18 +1846,35 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store != nil {
|
||||
region := r.URL.Query().Get("region")
|
||||
channels := s.store.GetChannels(region)
|
||||
region := r.URL.Query().Get("region")
|
||||
includeEncrypted := r.URL.Query().Get("includeEncrypted") == "true"
|
||||
// Prefer DB for full history (in-memory store has limited retention)
|
||||
if s.db != nil {
|
||||
channels, err := s.db.GetChannels(region)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
if includeEncrypted {
|
||||
encrypted, err := s.db.GetEncryptedChannels(region)
|
||||
if err != nil {
|
||||
log.Printf("WARN GetEncryptedChannels: %v", err)
|
||||
} else {
|
||||
channels = append(channels, encrypted...)
|
||||
}
|
||||
}
|
||||
writeJSON(w, ChannelListResponse{Channels: channels})
|
||||
return
|
||||
}
|
||||
channels, err := s.db.GetChannels()
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
if s.store != nil {
|
||||
channels := s.store.GetChannels(region)
|
||||
if includeEncrypted {
|
||||
channels = append(channels, s.store.GetEncryptedChannels(region)...)
|
||||
}
|
||||
writeJSON(w, ChannelListResponse{Channels: channels})
|
||||
return
|
||||
}
|
||||
writeJSON(w, ChannelListResponse{Channels: channels})
|
||||
writeJSON(w, ChannelListResponse{Channels: []map[string]interface{}{}})
|
||||
}
|
||||
|
||||
func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1674,17 +1882,22 @@ func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) {
|
||||
limit := queryInt(r, "limit", 100)
|
||||
offset := queryInt(r, "offset", 0)
|
||||
region := r.URL.Query().Get("region")
|
||||
// Prefer DB for full history (in-memory store has limited retention)
|
||||
if s.db != nil {
|
||||
messages, total, err := s.db.GetChannelMessages(hash, limit, offset, region)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
|
||||
return
|
||||
}
|
||||
if s.store != nil {
|
||||
messages, total := s.store.GetChannelMessages(hash, limit, offset, region)
|
||||
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
|
||||
return
|
||||
}
|
||||
messages, total, err := s.db.GetChannelMessages(hash, limit, offset, region)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
|
||||
writeJSON(w, ChannelMessagesResponse{Messages: []map[string]interface{}{}, Total: 0})
|
||||
}
|
||||
|
||||
func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1870,7 +2083,7 @@ func (s *Server) handleObserverAnalytics(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
snrBuckets[bucket].Count++
|
||||
}
|
||||
if i < 20 {
|
||||
if i < 20 && enriched["hash"] != nil {
|
||||
recentPackets = append(recentPackets, enriched)
|
||||
}
|
||||
}
|
||||
@@ -2153,9 +2366,6 @@ func mapSliceToTransmissions(maps []map[string]interface{}) []TransmissionResp {
|
||||
tx.PathJSON = m["path_json"]
|
||||
tx.Direction = m["direction"]
|
||||
tx.Score = m["score"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
tx.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, tx)
|
||||
}
|
||||
return result
|
||||
@@ -2176,10 +2386,10 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
|
||||
obs.SNR = m["snr"]
|
||||
obs.RSSI = m["rssi"]
|
||||
obs.PathJSON = m["path_json"]
|
||||
obs.ResolvedPath = m["resolved_path"]
|
||||
obs.Direction = m["direction"]
|
||||
obs.RawHex = m["raw_hex"]
|
||||
obs.Timestamp = m["timestamp"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
obs.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, obs)
|
||||
}
|
||||
return result
|
||||
@@ -2330,16 +2540,167 @@ func (s *Server) handleAdminPrune(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, 400, "days parameter required (or set retention.packetDays in config)")
|
||||
return
|
||||
}
|
||||
|
||||
results := map[string]interface{}{}
|
||||
|
||||
// Prune old packets
|
||||
n, err := s.db.PruneOldPackets(days)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
writeJSON(w, map[string]interface{}{"deleted": n, "days": days})
|
||||
results["packets_deleted"] = n
|
||||
results["deleted"] = n // legacy alias
|
||||
|
||||
// Also mark stale observers as inactive if observerDays is configured
|
||||
observerDays := s.cfg.ObserverDaysOrDefault()
|
||||
if observerDays > 0 {
|
||||
obsN, obsErr := s.db.RemoveStaleObservers(observerDays)
|
||||
if obsErr != nil {
|
||||
log.Printf("[prune] observer prune error: %v", obsErr)
|
||||
} else {
|
||||
results["observers_inactive"] = obsN
|
||||
}
|
||||
}
|
||||
|
||||
results["days"] = days
|
||||
writeJSON(w, results)
|
||||
}
|
||||
|
||||
// constantTimeEqual compares two strings in constant time to prevent timing attacks.
|
||||
func constantTimeEqual(a, b string) bool {
|
||||
return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
|
||||
}
|
||||
|
||||
// filterBlacklistedFromTopology removes blacklisted node references from the
|
||||
// topology analytics response (TopRepeaters, TopPairs, BestPathList, MultiObsNodes, PerObserverReach).
|
||||
func (s *Server) filterBlacklistedFromTopology(data map[string]interface{}) map[string]interface{} {
|
||||
// Filter TopRepeaters
|
||||
if repeaters, ok := data["topRepeaters"]; ok {
|
||||
if arr, ok := repeaters.([]TopRepeater); ok {
|
||||
var filtered []TopRepeater
|
||||
for _, r := range arr {
|
||||
if pk, ok := r.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, r)
|
||||
}
|
||||
data["topRepeaters"] = filtered
|
||||
}
|
||||
}
|
||||
|
||||
// Filter TopPairs
|
||||
if pairs, ok := data["topPairs"]; ok {
|
||||
if arr, ok := pairs.([]TopPair); ok {
|
||||
var filtered []TopPair
|
||||
for _, p := range arr {
|
||||
if pkA, ok := p.PubkeyA.(string); ok && s.cfg.IsBlacklisted(pkA) {
|
||||
continue
|
||||
}
|
||||
if pkB, ok := p.PubkeyB.(string); ok && s.cfg.IsBlacklisted(pkB) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
data["topPairs"] = filtered
|
||||
}
|
||||
}
|
||||
|
||||
// Filter BestPathList
|
||||
if paths, ok := data["bestPathList"]; ok {
|
||||
if arr, ok := paths.([]BestPathEntry); ok {
|
||||
var filtered []BestPathEntry
|
||||
for _, p := range arr {
|
||||
if pk, ok := p.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
data["bestPathList"] = filtered
|
||||
}
|
||||
}
|
||||
|
||||
// Filter MultiObsNodes
|
||||
if nodes, ok := data["multiObsNodes"]; ok {
|
||||
if arr, ok := nodes.([]MultiObsNode); ok {
|
||||
var filtered []MultiObsNode
|
||||
for _, n := range arr {
|
||||
if pk, ok := n.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, n)
|
||||
}
|
||||
data["multiObsNodes"] = filtered
|
||||
}
|
||||
}
|
||||
|
||||
// Filter PerObserverReach
|
||||
if reach, ok := data["perObserverReach"]; ok {
|
||||
if m, ok := reach.(map[string]*ObserverReach); ok {
|
||||
for k, v := range m {
|
||||
for ri := range v.Rings {
|
||||
var filteredNodes []ReachNode
|
||||
for _, rn := range v.Rings[ri].Nodes {
|
||||
if pk, ok := rn.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
|
||||
continue
|
||||
}
|
||||
filteredNodes = append(filteredNodes, rn)
|
||||
}
|
||||
v.Rings[ri].Nodes = filteredNodes
|
||||
}
|
||||
m[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// filterBlacklistedFromSubpaths removes blacklisted node references from
|
||||
// the subpaths analytics response.
|
||||
func (s *Server) filterBlacklistedFromSubpaths(data map[string]interface{}) map[string]interface{} {
|
||||
if subpaths, ok := data["subpaths"]; ok {
|
||||
if arr, ok := subpaths.([]interface{}); ok {
|
||||
var filtered []interface{}
|
||||
for _, item := range arr {
|
||||
if m, ok := item.(map[string]interface{}); ok {
|
||||
if hops, ok := m["hops"].([]interface{}); ok {
|
||||
skip := false
|
||||
for _, h := range hops {
|
||||
if hp, ok := h.(string); ok && s.cfg.IsBlacklisted(hp) {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
data["subpaths"] = filtered
|
||||
}
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// handleDroppedPackets returns recently dropped packets for investigation.
|
||||
func (s *Server) handleDroppedPackets(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 100
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
limit = n
|
||||
}
|
||||
}
|
||||
observerID := r.URL.Query().Get("observer")
|
||||
nodePubkey := r.URL.Query().Get("pubkey")
|
||||
|
||||
results, err := s.db.GetDroppedPackets(limit, observerID, nodePubkey)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, results)
|
||||
}
|
||||
|
||||
+211
-47
@@ -2219,8 +2219,8 @@ pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk)
|
||||
|
||||
decoded := `{"name":"TestNode","pubKey":"` + pk + `"}`
|
||||
raw1 := "04" + "00" + "aabb"
|
||||
raw2 := "04" + "40" + "aabb"
|
||||
raw1 := "11" + "01" + "aabb"
|
||||
raw2 := "11" + "41" + "aabb"
|
||||
|
||||
payloadType := 4
|
||||
for i := 0; i < 3; i++ {
|
||||
@@ -2267,8 +2267,8 @@ pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk)
|
||||
|
||||
decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}`
|
||||
raw1byte := "04" + "00" + "aabb" // pathByte=0x00 → hashSize=1 (direct send, no hops)
|
||||
raw2byte := "04" + "40" + "aabb" // pathByte=0x40 → hashSize=2
|
||||
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
|
||||
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
|
||||
|
||||
payloadType := 4
|
||||
// 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte)
|
||||
@@ -2310,8 +2310,8 @@ func TestGetNodeHashSizeInfoLatestWins(t *testing.T) {
|
||||
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'LatestWins', 'repeater')", pk)
|
||||
|
||||
decoded := `{"name":"LatestWins","pubKey":"` + pk + `"}`
|
||||
raw1byte := "04" + "00" + "aabb" // pathByte=0x00 → hashSize=1
|
||||
raw2byte := "04" + "40" + "aabb" // pathByte=0x40 → hashSize=2
|
||||
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
|
||||
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
|
||||
|
||||
payloadType := 4
|
||||
// 4 historical 1-byte adverts, then 1 recent 2-byte advert (latest).
|
||||
@@ -3681,67 +3681,55 @@ func TestNodePathsPrefixCollisionFilter(t *testing.T) {
|
||||
func TestNodeInResolvedPath(t *testing.T) {
|
||||
target := "aabbccdd11223344"
|
||||
|
||||
// Case 1: tx.ResolvedPath contains target
|
||||
pk := "aabbccdd11223344"
|
||||
tx1 := &StoreTx{ResolvedPath: []*string{&pk}}
|
||||
if !nodeInResolvedPath(tx1, target) {
|
||||
t.Error("should match when ResolvedPath contains target")
|
||||
// After #800, nodeInResolvedPath is replaced by nodeInResolvedPathViaIndex
|
||||
// which uses the membership index. Test the index-based approach.
|
||||
store := &PacketStore{
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
useResolvedPathIndex: true,
|
||||
}
|
||||
store.initResolvedPathIndex()
|
||||
|
||||
// Case 1: tx indexed with target pubkey
|
||||
tx1 := &StoreTx{ID: 1}
|
||||
store.addToResolvedPubkeyIndex(1, []string{target})
|
||||
if !store.nodeInResolvedPathViaIndex(tx1, target) {
|
||||
t.Error("should match when index contains target")
|
||||
}
|
||||
|
||||
// Case 2: tx.ResolvedPath contains different node
|
||||
other := "aacafe0000000000"
|
||||
tx2 := &StoreTx{ResolvedPath: []*string{&other}}
|
||||
if nodeInResolvedPath(tx2, target) {
|
||||
t.Error("should not match when ResolvedPath contains different node")
|
||||
// Case 2: tx indexed with different pubkey
|
||||
tx2 := &StoreTx{ID: 2}
|
||||
store.addToResolvedPubkeyIndex(2, []string{"aacafe0000000000"})
|
||||
if store.nodeInResolvedPathViaIndex(tx2, target) {
|
||||
t.Error("should not match when index contains different node")
|
||||
}
|
||||
|
||||
// Case 3: nil ResolvedPath — should match (no data to disambiguate, keep it)
|
||||
tx3 := &StoreTx{}
|
||||
if !nodeInResolvedPath(tx3, target) {
|
||||
t.Error("should match when ResolvedPath is nil (no data to disambiguate)")
|
||||
}
|
||||
|
||||
// Case 4: ResolvedPath with nil elements only — has data but no match
|
||||
tx4 := &StoreTx{ResolvedPath: []*string{nil, nil}}
|
||||
if nodeInResolvedPath(tx4, target) {
|
||||
t.Error("should not match when all ResolvedPath elements are nil")
|
||||
}
|
||||
|
||||
// Case 5: target in observation but not in tx.ResolvedPath
|
||||
tx5 := &StoreTx{
|
||||
ResolvedPath: []*string{&other},
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&pk}},
|
||||
},
|
||||
}
|
||||
if !nodeInResolvedPath(tx5, target) {
|
||||
t.Error("should match when observation's ResolvedPath contains target")
|
||||
// Case 3: tx not in index at all — should match (no data to disambiguate)
|
||||
tx3 := &StoreTx{ID: 3}
|
||||
if !store.nodeInResolvedPathViaIndex(tx3, target) {
|
||||
t.Error("should match when tx has no index entries (no data to disambiguate)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathHopIndexIncrementalUpdate(t *testing.T) {
|
||||
// Test that addTxToPathHopIndex and removeTxFromPathHopIndex work correctly
|
||||
// After #800, addTxToPathHopIndex only indexes raw hops (not resolved pubkeys).
|
||||
// Resolved pubkeys are handled by the resolved pubkey membership index.
|
||||
idx := make(map[string][]*StoreTx)
|
||||
|
||||
pk1 := "fullpubkey1"
|
||||
tx1 := &StoreTx{
|
||||
ID: 1,
|
||||
PathJSON: `["ab","cd"]`,
|
||||
ResolvedPath: []*string{&pk1, nil},
|
||||
}
|
||||
|
||||
addTxToPathHopIndex(idx, tx1)
|
||||
|
||||
// Should be indexed under "ab", "cd", and "fullpubkey1"
|
||||
// Should be indexed under "ab" and "cd" only (no resolved pubkey)
|
||||
if len(idx["ab"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'ab', got %d", len(idx["ab"]))
|
||||
}
|
||||
if len(idx["cd"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'cd', got %d", len(idx["cd"]))
|
||||
}
|
||||
if len(idx["fullpubkey1"]) != 1 {
|
||||
t.Errorf("expected 1 entry for resolved pubkey, got %d", len(idx["fullpubkey1"]))
|
||||
}
|
||||
|
||||
// Add another tx with overlapping hop
|
||||
tx2 := &StoreTx{
|
||||
@@ -3766,9 +3754,6 @@ func TestPathHopIndexIncrementalUpdate(t *testing.T) {
|
||||
if _, ok := idx["cd"]; ok {
|
||||
t.Error("expected 'cd' key to be deleted after removal")
|
||||
}
|
||||
if _, ok := idx["fullpubkey1"]; ok {
|
||||
t.Error("expected resolved pubkey key to be deleted after removal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAPIEndpoints(t *testing.T) {
|
||||
@@ -3808,3 +3793,182 @@ func TestMetricsAPIEndpoints(t *testing.T) {
|
||||
t.Errorf("expected 1 observer in summary, got %v", resp2["observers"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeHealth_RecentPackets_ResolvedPath verifies that recentPackets in the
|
||||
// node health endpoint include resolved_path (regression for Codex review item #2).
|
||||
func TestNodeHealth_RecentPackets_ResolvedPath(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
rp, ok := body["recentPackets"].([]interface{})
|
||||
if !ok || len(rp) == 0 {
|
||||
t.Fatal("expected non-empty recentPackets")
|
||||
}
|
||||
// At least one packet should have resolved_path (tx 1 has observations with resolved_path)
|
||||
found := false
|
||||
for _, p := range rp {
|
||||
pm, ok := p.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if pm["resolved_path"] != nil {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("expected at least one recentPacket with resolved_path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketsExpand_ResolvedPath verifies that expandObservations=true includes
|
||||
// resolved_path on expanded observations (regression for Codex review item #3).
|
||||
func TestPacketsExpand_ResolvedPath(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/packets?expand=observations&limit=10", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
packets, ok := body["packets"].([]interface{})
|
||||
if !ok || len(packets) == 0 {
|
||||
t.Fatal("expected non-empty packets")
|
||||
}
|
||||
// Find a packet with observations that should have resolved_path
|
||||
found := false
|
||||
for _, p := range packets {
|
||||
pm, ok := p.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
obs, ok := pm["observations"].([]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, o := range obs {
|
||||
om, ok := o.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if om["resolved_path"] != nil {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("expected at least one expanded observation with resolved_path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetailFallsBackToDBWhenStoreMisses verifies that handlePacketDetail
|
||||
// serves transmissions present in the DB but absent from the in-memory store.
|
||||
// This is the recentAdverts → "Not found" bug (#827).
|
||||
func TestPacketDetailFallsBackToDBWhenStoreMisses(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
// Insert a transmission directly into the DB AFTER store.Load(), so the
|
||||
// in-memory PacketStore won't see it. Mirrors the production case where
|
||||
// the store has pruned an entry but the DB still has it.
|
||||
const dbOnlyHash = "deadbeef00112233"
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('FFEE', ?, ?, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
var txID int
|
||||
if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", dbOnlyHash).Scan(&txID); err != nil {
|
||||
t.Fatalf("lookup tx id: %v", err)
|
||||
}
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO observations
|
||||
(transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (?, 1, 7.5, -99, '[]', ?)`, txID, time.Now().Unix()); err != nil {
|
||||
t.Fatalf("insert obs: %v", err)
|
||||
}
|
||||
|
||||
// Confirm the store really doesn't have it (precondition for the fix).
|
||||
if got := srv.store.GetPacketByHash(dbOnlyHash); got != nil {
|
||||
t.Fatalf("test precondition failed: store unexpectedly has %s", dbOnlyHash)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/packets/"+dbOnlyHash, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkt, ok := body["packet"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected packet object")
|
||||
}
|
||||
if pkt["hash"] != dbOnlyHash {
|
||||
t.Errorf("expected hash %s, got %v", dbOnlyHash, pkt["hash"])
|
||||
}
|
||||
// Observations fallback should populate from DB too.
|
||||
obs, _ := body["observations"].([]interface{})
|
||||
if len(obs) == 0 {
|
||||
t.Errorf("expected DB observations to be returned, got 0")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetail404WhenAbsentFromBoth verifies that a hash present in
|
||||
// neither store nor DB still returns 404 (no false positives from the fallback).
|
||||
func TestPacketDetail404WhenAbsentFromBoth(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/packets/0011223344556677", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Errorf("expected 404, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetailPrefersStoreOverDB verifies the store result wins when the
|
||||
// hash exists in both — the DB fallback must not double-fetch / overwrite.
|
||||
func TestPacketDetailPrefersStoreOverDB(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
// abc123def4567890 is seeded in both DB and (after Load) the store.
|
||||
const hash = "abc123def4567890"
|
||||
if got := srv.store.GetPacketByHash(hash); got == nil {
|
||||
t.Fatalf("test precondition failed: store should have %s", hash)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/packets/"+hash, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
pkt, _ := body["packet"].(map[string]interface{})
|
||||
if pkt == nil || pkt["hash"] != hash {
|
||||
t.Fatalf("expected packet with hash %s, got %v", hash, pkt)
|
||||
}
|
||||
// observation_count comes from store observations (2 seeded for tx 1).
|
||||
if cnt, _ := body["observation_count"].(float64); cnt != 2 {
|
||||
t.Errorf("expected observation_count=2 (from store), got %v", body["observation_count"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestStatsMemoryFields verifies that /api/stats exposes the new memory
|
||||
// breakdown introduced for issue #832: storeDataMB, processRSSMB,
|
||||
// goHeapInuseMB, goSysMB, plus the deprecated trackedMB alias.
|
||||
//
|
||||
// We assert presence, type, sign, and ordering invariants — but NOT
|
||||
// "RSS within X% of true RSS" because that is flaky in CI under cgo,
|
||||
// containerization, and shared-runner load.
|
||||
func TestStatsMemoryFields(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/stats", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
|
||||
required := []string{"trackedMB", "storeDataMB", "processRSSMB", "goHeapInuseMB", "goSysMB"}
|
||||
values := make(map[string]float64, len(required))
|
||||
for _, k := range required {
|
||||
v, ok := body[k]
|
||||
if !ok {
|
||||
t.Fatalf("missing field %q in /api/stats response", k)
|
||||
}
|
||||
f, ok := v.(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %q is %T, expected float64", k, v)
|
||||
}
|
||||
if f < 0 {
|
||||
t.Errorf("field %q is negative: %v", k, f)
|
||||
}
|
||||
values[k] = f
|
||||
}
|
||||
|
||||
// trackedMB is a deprecated alias for storeDataMB; they must match.
|
||||
if values["trackedMB"] != values["storeDataMB"] {
|
||||
t.Errorf("trackedMB (%v) != storeDataMB (%v); they must remain aliased",
|
||||
values["trackedMB"], values["storeDataMB"])
|
||||
}
|
||||
|
||||
// Ordering invariants. goSys is the runtime's view of total OS memory;
|
||||
// HeapInuse is a subset of it. storeData is a subset of HeapInuse.
|
||||
// processRSS may be 0 in environments without /proc — treat 0 as
|
||||
// "unknown" rather than a failure.
|
||||
if values["goHeapInuseMB"] > values["goSysMB"]+0.5 {
|
||||
t.Errorf("invariant violated: goHeapInuseMB (%v) > goSysMB (%v)",
|
||||
values["goHeapInuseMB"], values["goSysMB"])
|
||||
}
|
||||
if values["storeDataMB"] > values["goHeapInuseMB"]+0.5 && values["storeDataMB"] > 0 {
|
||||
// In the test fixture storeDataMB is typically 0 (no packets in
|
||||
// store); only enforce the bound when both are nonzero.
|
||||
t.Errorf("invariant violated: storeDataMB (%v) > goHeapInuseMB (%v)",
|
||||
values["storeDataMB"], values["goHeapInuseMB"])
|
||||
}
|
||||
if values["processRSSMB"] > 0 && values["goSysMB"] > 0 {
|
||||
// goSys can briefly exceed RSS if pages are reserved-but-not-touched,
|
||||
// so allow some slack.
|
||||
if values["goSysMB"] > values["processRSSMB"]*4 {
|
||||
t.Errorf("suspicious: goSysMB (%v) >> processRSSMB (%v)",
|
||||
values["goSysMB"], values["processRSSMB"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsMemoryFieldsRawJSON spot-checks that the JSON wire format uses
|
||||
// the documented camelCase names (no accidental rename through struct tags).
|
||||
func TestStatsMemoryFieldsRawJSON(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/stats", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
body := w.Body.String()
|
||||
for _, key := range []string{
|
||||
`"trackedMB":`, `"storeDataMB":`,
|
||||
`"processRSSMB":`, `"goHeapInuseMB":`, `"goSysMB":`,
|
||||
} {
|
||||
if !strings.Contains(body, key) {
|
||||
t.Errorf("missing %s in raw response: %s", key, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
+698
-266
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func f64(v float64) *float64 { return &v }
|
||||
|
||||
func TestDedupeTopHopsByPair(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: f64(5.0), Hash: "h1", Timestamp: "t1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: f64(8.0), Hash: "h2", Timestamp: "t2"},
|
||||
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 80, Type: "R↔R", SNR: f64(3.0), Hash: "h3", Timestamp: "t3"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 70, Type: "R↔R", SNR: f64(6.0), Hash: "h4", Timestamp: "t4"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 60, Type: "R↔R", SNR: f64(4.0), Hash: "h5", Timestamp: "t5"},
|
||||
{FromPk: "CCC", ToPk: "DDD", FromName: "C", ToName: "D", Dist: 50, Type: "C↔R", SNR: f64(7.0), Hash: "h6", Timestamp: "t6"},
|
||||
}
|
||||
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
|
||||
if len(result) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(result))
|
||||
}
|
||||
|
||||
// First entry: A↔B pair, max distance = 100, obsCount = 5
|
||||
ab := result[0]
|
||||
if ab["dist"].(float64) != 100 {
|
||||
t.Errorf("expected dist 100, got %v", ab["dist"])
|
||||
}
|
||||
if ab["obsCount"].(int) != 5 {
|
||||
t.Errorf("expected obsCount 5, got %v", ab["obsCount"])
|
||||
}
|
||||
if ab["hash"].(string) != "h1" {
|
||||
t.Errorf("expected hash h1 (from max-dist record), got %v", ab["hash"])
|
||||
}
|
||||
if ab["bestSnr"].(float64) != 8.0 {
|
||||
t.Errorf("expected bestSnr 8.0, got %v", ab["bestSnr"])
|
||||
}
|
||||
// medianSnr of [3,4,5,6,8] = 5.0
|
||||
if ab["medianSnr"].(float64) != 5.0 {
|
||||
t.Errorf("expected medianSnr 5.0, got %v", ab["medianSnr"])
|
||||
}
|
||||
|
||||
// Second entry: C↔D pair
|
||||
cd := result[1]
|
||||
if cd["dist"].(float64) != 50 {
|
||||
t.Errorf("expected dist 50, got %v", cd["dist"])
|
||||
}
|
||||
if cd["obsCount"].(int) != 1 {
|
||||
t.Errorf("expected obsCount 1, got %v", cd["obsCount"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsReversePairMerges(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 50, Type: "R↔R", Hash: "h1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 80, Type: "R↔R", Hash: "h2"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(result))
|
||||
}
|
||||
if result[0]["obsCount"].(int) != 2 {
|
||||
t.Errorf("expected obsCount 2, got %v", result[0]["obsCount"])
|
||||
}
|
||||
if result[0]["dist"].(float64) != 80 {
|
||||
t.Errorf("expected dist 80, got %v", result[0]["dist"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsNilSNR(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: nil, Hash: "h1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: nil, Hash: "h2"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(result))
|
||||
}
|
||||
if result[0]["bestSnr"] != nil {
|
||||
t.Errorf("expected bestSnr nil, got %v", result[0]["bestSnr"])
|
||||
}
|
||||
if result[0]["medianSnr"] != nil {
|
||||
t.Errorf("expected medianSnr nil, got %v", result[0]["medianSnr"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsLimit(t *testing.T) {
|
||||
// Generate 25 unique pairs, verify limit=20 caps output
|
||||
hops := make([]distHopRecord, 25)
|
||||
for i := range hops {
|
||||
hops[i] = distHopRecord{
|
||||
FromPk: "A", ToPk: string(rune('a' + i)),
|
||||
Dist: float64(i), Type: "R↔R", Hash: "h",
|
||||
}
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 20 {
|
||||
t.Errorf("expected 20 entries, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsEvenMedian(t *testing.T) {
|
||||
// Even count: median = avg of two middle values
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "A", ToPk: "B", Dist: 10, Type: "R↔R", SNR: f64(2.0), Hash: "h1"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 20, Type: "R↔R", SNR: f64(4.0), Hash: "h2"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 30, Type: "R↔R", SNR: f64(6.0), Hash: "h3"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 40, Type: "R↔R", SNR: f64(8.0), Hash: "h4"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
// sorted SNR: [2,4,6,8], median = (4+6)/2 = 5.0
|
||||
if result[0]["medianSnr"].(float64) != 5.0 {
|
||||
t.Errorf("expected medianSnr 5.0, got %v", result[0]["medianSnr"])
|
||||
}
|
||||
}
|
||||
+10
-4
@@ -42,14 +42,20 @@
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"snr": {
|
||||
"type": "number"
|
||||
},
|
||||
"hash": {
|
||||
"type": "string"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string"
|
||||
},
|
||||
"bestSnr": {
|
||||
"type": "number"
|
||||
},
|
||||
"medianSnr": {
|
||||
"type": "number"
|
||||
},
|
||||
"obsCount": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1580,4 +1586,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,13 +69,11 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
pk := "relay1"
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{&pk},
|
||||
}
|
||||
// After #800, touchRelayLastSeen takes a []string of pubkeys (from decode-window)
|
||||
pks := []string{"relay1"}
|
||||
|
||||
now := time.Now()
|
||||
s.touchRelayLastSeen(tx, now)
|
||||
s.touchRelayLastSeen(pks, now)
|
||||
|
||||
// Verify it was written
|
||||
var lastSeen sql.NullString
|
||||
@@ -88,7 +86,7 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
db.conn.Exec("UPDATE nodes SET last_seen = NULL WHERE public_key = ?", "relay1")
|
||||
|
||||
// Call again within 5 minutes — should be debounced (no write)
|
||||
s.touchRelayLastSeen(tx, now.Add(2*time.Minute))
|
||||
s.touchRelayLastSeen(pks, now.Add(2*time.Minute))
|
||||
|
||||
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
|
||||
if lastSeen.Valid {
|
||||
@@ -96,14 +94,14 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
}
|
||||
|
||||
// Call after 5 minutes — should write again
|
||||
s.touchRelayLastSeen(tx, now.Add(6*time.Minute))
|
||||
s.touchRelayLastSeen(pks, now.Add(6*time.Minute))
|
||||
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
|
||||
if !lastSeen.Valid {
|
||||
t.Fatal("expected write after debounce interval expired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchRelayLastSeen_SkipsNilResolvedPath(t *testing.T) {
|
||||
func TestTouchRelayLastSeen_SkipsEmptyPubkeys(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
@@ -112,13 +110,9 @@ func TestTouchRelayLastSeen_SkipsNilResolvedPath(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// tx with nil entries and empty resolved_path
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{nil, nil},
|
||||
}
|
||||
|
||||
// Should not panic or error
|
||||
s.touchRelayLastSeen(tx, time.Now())
|
||||
// Empty pubkeys — should not panic or error
|
||||
s.touchRelayLastSeen([]string{}, time.Now())
|
||||
s.touchRelayLastSeen(nil, time.Now())
|
||||
}
|
||||
|
||||
func TestTouchRelayLastSeen_NilDB(t *testing.T) {
|
||||
@@ -127,11 +121,6 @@ func TestTouchRelayLastSeen_NilDB(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
pk := "abc"
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{&pk},
|
||||
}
|
||||
|
||||
// Should not panic with nil db
|
||||
s.touchRelayLastSeen(tx, time.Now())
|
||||
s.touchRelayLastSeen([]string{"abc"}, time.Now())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestEstimateStoreTxBytes_ReasonableValues verifies the estimate function
|
||||
// returns reasonable values for different packet sizes.
|
||||
func TestEstimateStoreTxBytes_ReasonableValues(t *testing.T) {
|
||||
tx := &StoreTx{
|
||||
Hash: "abcdef1234567890",
|
||||
RawHex: "deadbeef",
|
||||
DecodedJSON: `{"type":"GRP_TXT"}`,
|
||||
PathJSON: `["hop1","hop2","hop3"]`,
|
||||
parsedPath: []string{"hop1", "hop2", "hop3"},
|
||||
pathParsed: true,
|
||||
}
|
||||
got := estimateStoreTxBytes(tx)
|
||||
|
||||
// Should be at least base (384) + maps (200) + indexes + path/subpath costs
|
||||
if got < 700 {
|
||||
t.Errorf("estimate too low for 3-hop tx: %d", got)
|
||||
}
|
||||
if got > 5000 {
|
||||
t.Errorf("estimate unreasonably high for 3-hop tx: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEstimateStoreTxBytes_ManyHopsSubpaths verifies that packets with many
|
||||
// hops estimate significantly more due to O(path²) subpath index entries.
|
||||
func TestEstimateStoreTxBytes_ManyHopsSubpaths(t *testing.T) {
|
||||
tx2 := &StoreTx{
|
||||
Hash: "aabb",
|
||||
parsedPath: []string{"a", "b"},
|
||||
pathParsed: true,
|
||||
}
|
||||
tx10 := &StoreTx{
|
||||
Hash: "aabb",
|
||||
parsedPath: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"},
|
||||
pathParsed: true,
|
||||
}
|
||||
est2 := estimateStoreTxBytes(tx2)
|
||||
est10 := estimateStoreTxBytes(tx10)
|
||||
|
||||
// 10 hops → 45 subpath combos × 40 = 1800 bytes just for subpaths
|
||||
if est10 <= est2 {
|
||||
t.Errorf("10-hop (%d) should estimate more than 2-hop (%d)", est10, est2)
|
||||
}
|
||||
if est10 < est2+1500 {
|
||||
t.Errorf("10-hop (%d) should estimate at least 1500 more than 2-hop (%d)", est10, est2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEstimateStoreObsBytes_AfterRefactor verifies that after #800 refactor,
|
||||
// observations no longer have ResolvedPath overhead in their estimate.
|
||||
func TestEstimateStoreObsBytes_AfterRefactor(t *testing.T) {
|
||||
obs := &StoreObs{
|
||||
ObserverID: "obs1",
|
||||
PathJSON: `["a","b"]`,
|
||||
}
|
||||
|
||||
est := estimateStoreObsBytes(obs)
|
||||
if est <= 0 {
|
||||
t.Errorf("estimate should be positive, got %d", est)
|
||||
}
|
||||
// After #800, all obs estimates should be the same (no RP field variation)
|
||||
obs2 := &StoreObs{
|
||||
ObserverID: "obs1",
|
||||
PathJSON: `["a","b"]`,
|
||||
}
|
||||
est2 := estimateStoreObsBytes(obs2)
|
||||
if est != est2 {
|
||||
t.Errorf("estimates should be equal after #800 (no RP field), got %d vs %d", est, est2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEstimateStoreObsBytes_ManyObservations verifies that 15 observations
|
||||
// estimate significantly more than 1.
|
||||
func TestEstimateStoreObsBytes_ManyObservations(t *testing.T) {
|
||||
est1 := estimateStoreObsBytes(&StoreObs{ObserverID: "a", PathJSON: `["x"]`})
|
||||
est15 := int64(0)
|
||||
for i := 0; i < 15; i++ {
|
||||
est15 += estimateStoreObsBytes(&StoreObs{ObserverID: "a", PathJSON: `["x"]`})
|
||||
}
|
||||
if est15 <= est1*10 {
|
||||
t.Errorf("15 obs total (%d) should be >10x single obs (%d)", est15, est1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrackedBytesMatchesSumAfterInsert verifies that trackedBytes equals the
|
||||
// sum of individual estimates after inserting packets via makeTestStore.
|
||||
func TestTrackedBytesMatchesSumAfterInsert(t *testing.T) {
|
||||
store := makeTestStore(20, time.Now().Add(-2*time.Hour), 5)
|
||||
|
||||
// Manually compute trackedBytes as sum of estimates
|
||||
var expectedSum int64
|
||||
for _, tx := range store.packets {
|
||||
expectedSum += estimateStoreTxBytes(tx)
|
||||
for _, obs := range tx.Observations {
|
||||
expectedSum += estimateStoreObsBytes(obs)
|
||||
}
|
||||
}
|
||||
|
||||
if store.trackedBytes != expectedSum {
|
||||
t.Errorf("trackedBytes=%d, expected sum=%d", store.trackedBytes, expectedSum)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvictionTriggersWithImprovedEstimates verifies that eviction triggers
|
||||
// at the right point with the improved (higher) estimates.
|
||||
func TestEvictionTriggersWithImprovedEstimates(t *testing.T) {
|
||||
store := makeTestStore(100, time.Now().Add(-10*time.Hour), 5)
|
||||
|
||||
// trackedBytes for 100 packets is small — artificially set maxMemoryMB
|
||||
// so highWatermark is just below trackedBytes to trigger eviction.
|
||||
highWatermarkBytes := store.trackedBytes - 1000
|
||||
if highWatermarkBytes < 1 {
|
||||
highWatermarkBytes = 1
|
||||
}
|
||||
// maxMemoryMB * 1048576 = highWatermark, so maxMemoryMB = ceil(highWatermarkBytes / 1048576)
|
||||
// But that'll be 0 for small values. Instead, directly set trackedBytes high.
|
||||
store.trackedBytes = 6 * 1048576 // 6MB
|
||||
store.maxMemoryMB = 3 // 3MB limit
|
||||
|
||||
beforeCount := len(store.packets)
|
||||
store.RunEviction()
|
||||
afterCount := len(store.packets)
|
||||
|
||||
if afterCount >= beforeCount {
|
||||
t.Errorf("expected eviction to remove packets: before=%d, after=%d, trackedBytes=%d, maxMB=%d",
|
||||
beforeCount, afterCount, store.trackedBytes, store.maxMemoryMB)
|
||||
}
|
||||
// trackedBytes should have decreased
|
||||
if store.trackedBytes >= 6*1048576 {
|
||||
t.Errorf("trackedBytes should have decreased after eviction")
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEstimateStoreTxBytes verifies the estimate function is fast.
|
||||
func BenchmarkEstimateStoreTxBytes(b *testing.B) {
|
||||
tx := &StoreTx{
|
||||
Hash: "abcdef1234567890",
|
||||
RawHex: "deadbeefdeadbeef",
|
||||
DecodedJSON: `{"type":"GRP_TXT","payload":"hello"}`,
|
||||
PathJSON: `["hop1","hop2","hop3","hop4","hop5"]`,
|
||||
parsedPath: []string{"hop1", "hop2", "hop3", "hop4", "hop5"},
|
||||
pathParsed: true,
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
estimateStoreTxBytes(tx)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkEstimateStoreObsBytes verifies the obs estimate function is fast.
|
||||
func BenchmarkEstimateStoreObsBytes(b *testing.B) {
|
||||
obs := &StoreObs{
|
||||
ObserverID: "observer1234",
|
||||
PathJSON: `["a","b","c"]`,
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
estimateStoreObsBytes(obs)
|
||||
}
|
||||
}
|
||||
+28
-5
@@ -68,8 +68,26 @@ type StatsResponse struct {
|
||||
Commit string `json:"commit"`
|
||||
BuildTime string `json:"buildTime"`
|
||||
Counts RoleCounts `json:"counts"`
|
||||
Backfilling bool `json:"backfilling"`
|
||||
BackfillProgress float64 `json:"backfillProgress"`
|
||||
Backfilling bool `json:"backfilling"`
|
||||
BackfillProgress float64 `json:"backfillProgress"`
|
||||
SignatureDrops int64 `json:"signatureDrops,omitempty"`
|
||||
HashMigrationComplete bool `json:"hashMigrationComplete"`
|
||||
|
||||
// Memory accounting (issue #832). All values in MB.
|
||||
//
|
||||
// StoreDataMB ("trackedMB" historically) is the in-store packet byte
|
||||
// estimate — useful packet bytes only. Subset of HeapInuse. Used as
|
||||
// the eviction watermark input. NOT a proxy for RSS; ops dashboards
|
||||
// should prefer ProcessRSSMB for capacity decisions.
|
||||
//
|
||||
// Old field name TrackedMB is retained for backward compatibility
|
||||
// with pre-v3.6 consumers; it carries the same value as StoreDataMB
|
||||
// and is deprecated.
|
||||
TrackedMB float64 `json:"trackedMB"` // deprecated alias for storeDataMB
|
||||
StoreDataMB float64 `json:"storeDataMB"` // in-store packet bytes (subset of heap)
|
||||
ProcessRSSMB float64 `json:"processRSSMB"` // process RSS from /proc (Linux) or runtime.Sys fallback
|
||||
GoHeapInuseMB float64 `json:"goHeapInuseMB"` // runtime.MemStats.HeapInuse
|
||||
GoSysMB float64 `json:"goSysMB"` // runtime.MemStats.Sys (total Go-managed)
|
||||
}
|
||||
|
||||
// ─── Health ────────────────────────────────────────────────────────────────────
|
||||
@@ -176,6 +194,7 @@ type PerfPacketStoreStats struct {
|
||||
MaxPackets int `json:"maxPackets"`
|
||||
EstimatedMB float64 `json:"estimatedMB"`
|
||||
TrackedMB float64 `json:"trackedMB"`
|
||||
AvgBytesPerPacket int64 `json:"avgBytesPerPacket"`
|
||||
MaxMB int `json:"maxMB"`
|
||||
Indexes PacketStoreIndexes `json:"indexes"`
|
||||
}
|
||||
@@ -244,7 +263,6 @@ type TransmissionResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
Direction interface{} `json:"direction"`
|
||||
Score interface{} `json:"score,omitempty"`
|
||||
Observations []ObservationResp `json:"observations,omitempty"`
|
||||
@@ -259,7 +277,9 @@ type ObservationResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
ResolvedPath interface{} `json:"resolved_path,omitempty"`
|
||||
Direction interface{} `json:"direction,omitempty"`
|
||||
RawHex interface{} `json:"raw_hex,omitempty"`
|
||||
Timestamp interface{} `json:"timestamp"`
|
||||
}
|
||||
|
||||
@@ -468,6 +488,7 @@ type NodeAnalyticsResponse struct {
|
||||
PeerInteractions []PeerInteraction `json:"peerInteractions"`
|
||||
UptimeHeatmap []HeatmapCell `json:"uptimeHeatmap"`
|
||||
ComputedStats ComputedNodeStats `json:"computedStats"`
|
||||
ClockSkew *NodeClockSkew `json:"clockSkew,omitempty"`
|
||||
}
|
||||
|
||||
// ─── Analytics — RF ────────────────────────────────────────────────────────────
|
||||
@@ -660,7 +681,9 @@ type DistanceHop struct {
|
||||
ToPk string `json:"toPk"`
|
||||
Dist float64 `json:"dist"`
|
||||
Type string `json:"type"`
|
||||
SNR interface{} `json:"snr"`
|
||||
BestSnr interface{} `json:"bestSnr"`
|
||||
MedianSnr interface{} `json:"medianSnr"`
|
||||
ObsCount int `json:"obsCount"`
|
||||
Hash string `json:"hash"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
+15
-2
@@ -1,10 +1,13 @@
|
||||
{
|
||||
"port": 3000,
|
||||
"apiKey": "your-secret-api-key-here",
|
||||
"nodeBlacklist": [],
|
||||
"_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.",
|
||||
"retention": {
|
||||
"nodeDays": 7,
|
||||
"observerDays": 14,
|
||||
"packetDays": 30,
|
||||
"_comment": "nodeDays: nodes not seen in N days are moved to inactive_nodes (default 7). packetDays: transmissions+observations older than N days are deleted daily (0 = disabled)."
|
||||
"_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)."
|
||||
},
|
||||
"https": {
|
||||
"cert": "/path/to/cert.pem",
|
||||
@@ -125,7 +128,7 @@
|
||||
}
|
||||
],
|
||||
"channelKeys": {
|
||||
"public": "8b3387e9c5cdea6ac9e5edbaa115cd72"
|
||||
"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72"
|
||||
},
|
||||
"hashChannels": [
|
||||
"#LongFast",
|
||||
@@ -153,6 +156,16 @@
|
||||
],
|
||||
"zoom": 9
|
||||
},
|
||||
"geo_filter": {
|
||||
"polygon": [
|
||||
[37.80, -122.52],
|
||||
[37.80, -121.80],
|
||||
[37.20, -121.80],
|
||||
[37.20, -122.52]
|
||||
],
|
||||
"bufferKm": 20,
|
||||
"_comment": "Optional. Restricts ingestion and API responses to nodes within the polygon + bufferKm. Polygon is an array of [lat, lon] pairs (minimum 3). Use tools/geofilter-builder.html to draw a polygon visually. Remove this section to disable filtering. Nodes with no GPS fix are always allowed through."
|
||||
},
|
||||
"regions": {
|
||||
"SJC": "San Jose, US",
|
||||
"SFO": "San Francisco, US",
|
||||
|
||||
@@ -176,6 +176,19 @@ Lower values = fresher data but more server load.
|
||||
|
||||
Provide cert and key paths to enable HTTPS.
|
||||
|
||||
## Geographic filtering
|
||||
|
||||
```json
|
||||
"geo_filter": {
|
||||
"polygon": [[51.55, 3.80], [51.55, 5.90], [50.65, 5.90], [50.65, 3.80]],
|
||||
"bufferKm": 20
|
||||
}
|
||||
```
|
||||
|
||||
Restricts ingestion and API responses to nodes within the polygon plus a buffer margin. Remove the block to disable filtering. Nodes with no GPS fix always pass through.
|
||||
|
||||
See [Geographic Filtering](geofilter.md) for the full guide including the visual polygon builder and the prune script for cleaning up historical data.
|
||||
|
||||
## Home page
|
||||
|
||||
The `home` section customizes the onboarding experience. See `config.example.json` for the full structure including `steps`, `checklist`, and `footerLinks`.
|
||||
|
||||
@@ -66,6 +66,12 @@ Click **Import JSON** and paste a previously exported theme. The customizer load
|
||||
|
||||
Click **Reset to Defaults** to restore all settings to the built-in defaults.
|
||||
|
||||
## GeoFilter Builder
|
||||
|
||||
The Export tab includes a **GeoFilter Builder →** link. Click it to open a Leaflet map where you can draw a polygon boundary for your deployment area. The tool generates a `geo_filter` block you can paste directly into `config.json`.
|
||||
|
||||
See [Geographic Filtering](geofilter.md) for full details on what geo filtering does and how to configure it.
|
||||
|
||||
## How it works
|
||||
|
||||
The customizer writes CSS custom properties (variables) to override the defaults. Exported JSON maps directly to the `theme`, `nodeColors`, `branding`, and `home` sections of [config.json](configuration.md).
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# Geographic Filtering
|
||||
|
||||
CoreScope supports geographic filtering to restrict which nodes are ingested and returned in API responses. This is useful for public-facing deployments that should only show activity in a specific region.
|
||||
|
||||
## How it works
|
||||
|
||||
Geographic filtering operates at two levels:
|
||||
|
||||
- **Ingest time** — ADVERT packets carrying GPS coordinates are rejected by the ingestor if the node falls outside the configured area. The node never reaches the database.
|
||||
- **API responses** — Nodes already in the database are filtered from the `/api/nodes` response if they fall outside the area. This covers nodes ingested before the filter was configured.
|
||||
|
||||
Nodes with no GPS fix (`lat=0, lon=0` or missing coordinates) always pass the filter regardless of configuration.
|
||||
|
||||
## Configuration
|
||||
|
||||
Add a `geo_filter` block to `config.json`:
|
||||
|
||||
```json
|
||||
"geo_filter": {
|
||||
"polygon": [
|
||||
[51.55, 3.80],
|
||||
[51.55, 5.90],
|
||||
[50.65, 5.90],
|
||||
[50.65, 3.80]
|
||||
],
|
||||
"bufferKm": 20
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `polygon` | `[[lat, lon], ...]` | Array of at least 3 coordinate pairs defining the boundary |
|
||||
| `bufferKm` | number | Extra distance (km) around the polygon edge that is also accepted. `0` = exact boundary |
|
||||
|
||||
Both the server and the ingestor read `geo_filter` from `config.json`. Restart both after changing this section.
|
||||
|
||||
To disable filtering entirely, remove the `geo_filter` block.
|
||||
|
||||
### Legacy bounding box
|
||||
|
||||
An older bounding box format is also supported as a fallback when no `polygon` is present:
|
||||
|
||||
```json
|
||||
"geo_filter": {
|
||||
"latMin": 50.65,
|
||||
"latMax": 51.55,
|
||||
"lonMin": 3.80,
|
||||
"lonMax": 5.90
|
||||
}
|
||||
```
|
||||
|
||||
Prefer the polygon format — it supports irregular shapes and the `bufferKm` margin.
|
||||
|
||||
## API endpoint
|
||||
|
||||
The current geo filter configuration is exposed at:
|
||||
|
||||
```
|
||||
GET /api/config/geo-filter
|
||||
```
|
||||
|
||||
The frontend reads this endpoint to display the active filter. No authentication is required (the endpoint returns config, not private data).
|
||||
|
||||
## GeoFilter Builder
|
||||
|
||||
The simplest way to create a polygon is the included visual builder:
|
||||
|
||||
**File:** `tools/geofilter-builder.html`
|
||||
|
||||
Open it directly in a browser — it runs entirely client-side, no server required:
|
||||
|
||||
```bash
|
||||
# From the project root
|
||||
open tools/geofilter-builder.html # macOS
|
||||
xdg-open tools/geofilter-builder.html # Linux
|
||||
start tools/geofilter-builder.html # Windows
|
||||
```
|
||||
|
||||
**Workflow:**
|
||||
|
||||
1. The map opens centered on Belgium by default. Navigate to your region.
|
||||
2. Click on the map to add polygon vertices. Each click adds a numbered point.
|
||||
3. Add at least 3 points to form a closed polygon.
|
||||
4. Adjust **Buffer km** (default 20) to add a margin around the polygon edge.
|
||||
5. The generated JSON block appears at the bottom of the page — copy it directly into `config.json`.
|
||||
6. Use **↩ Undo** to remove the last point, **✕ Clear** to start over.
|
||||
|
||||
The output is a complete `{ "geo_filter": { ... } }` block ready to paste into `config.json`.
|
||||
|
||||
## Cleaning up historical nodes
|
||||
|
||||
The ingestor prevents new out-of-bounds nodes from being ingested, but it does not retroactively remove nodes that were stored before the filter was configured. For that, use the prune script.
|
||||
|
||||
**File:** `scripts/prune-nodes-outside-geo-filter.py`
|
||||
|
||||
```bash
|
||||
# Dry run — shows what would be deleted without making any changes
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py --dry-run
|
||||
|
||||
# Default paths: /app/data/meshcore.db and /app/config.json
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py
|
||||
|
||||
# Custom paths
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py /path/to/meshcore.db \
|
||||
--config /path/to/config.json
|
||||
|
||||
# In Docker — run inside the container
|
||||
docker exec -it meshcore-analyzer \
|
||||
python3 /app/scripts/prune-nodes-outside-geo-filter.py --dry-run
|
||||
```
|
||||
|
||||
The script reads `geo_filter.polygon` and `geo_filter.bufferKm` from config, lists the nodes that fall outside, then asks for `yes` confirmation before deleting. Nodes without coordinates are always kept.
|
||||
|
||||
This is a **one-time migration tool** — run it once after first configuring `geo_filter` to clean up pre-filter data. The ingestor handles all subsequent filtering automatically at ingest time.
|
||||
+189
-24
@@ -28,7 +28,7 @@
|
||||
|
||||
function barChart(data, labels, colors, w = 800, h = 220, pad = 40) {
|
||||
const max = Math.max(...data, 1);
|
||||
const barW = Math.min((w - pad * 2) / data.length - 2, 30);
|
||||
const barW = Math.max(1, Math.min((w - pad * 2) / data.length - 2, 30));
|
||||
let svg = `<svg viewBox="0 0 ${w} ${h}" style="width:100%;max-height:${h}px" role="img" aria-label="Bar chart showing data distribution"><title>Bar chart showing data distribution</title>`;
|
||||
// Grid
|
||||
for (let i = 0; i <= 4; i++) {
|
||||
@@ -87,6 +87,7 @@
|
||||
<button class="tab-btn" data-tab="distance">Distance</button>
|
||||
<button class="tab-btn" data-tab="neighbor-graph">Neighbor Graph</button>
|
||||
<button class="tab-btn" data-tab="rf-health">RF Health</button>
|
||||
<button class="tab-btn" data-tab="clock-health">Clock Health</button>
|
||||
<button class="tab-btn" data-tab="prefix-tool">Prefix Tool</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -181,6 +182,7 @@
|
||||
case 'distance': await renderDistanceTab(el); break;
|
||||
case 'neighbor-graph': await renderNeighborGraphTab(el); break;
|
||||
case 'rf-health': await renderRFHealthTab(el); break;
|
||||
case 'clock-health': await renderClockHealthTab(el); break;
|
||||
case 'prefix-tool': await renderPrefixTool(el); break;
|
||||
}
|
||||
// Auto-apply column resizing to all analytics tables
|
||||
@@ -261,7 +263,25 @@
|
||||
<div class="analytics-row">
|
||||
<div class="analytics-card flex-1">
|
||||
<h3>📈 Packets / Hour</h3>
|
||||
${barChart(rf.packetsPerHour.map(h=>h.count), rf.packetsPerHour.map(h=>h.hour.slice(11)+'h'), 'var(--accent)')}
|
||||
${(() => {
|
||||
const pph = rf.packetsPerHour;
|
||||
const counts = pph.map(h => h.count);
|
||||
// Decimate x-axis labels to avoid overlap
|
||||
const totalHours = pph.length;
|
||||
// Pick label interval: <=24h show every 6h, <=72h every 12h, else every 24h
|
||||
const labelInterval = totalHours <= 24 ? 6 : totalHours <= 72 ? 12 : 24;
|
||||
const labels = pph.map((h, i) => {
|
||||
const hh = h.hour.slice(11, 13); // "HH"
|
||||
const hourNum = parseInt(hh, 10);
|
||||
if (hourNum % labelInterval === 0) {
|
||||
// For multi-day ranges, show date on 00h boundaries
|
||||
if (totalHours > 48 && hourNum === 0) return h.hour.slice(5, 10);
|
||||
return hh + 'h';
|
||||
}
|
||||
return ''; // skip label
|
||||
});
|
||||
return barChart(counts, labels, 'var(--accent)');
|
||||
})()}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -622,14 +642,13 @@
|
||||
if (!data || !data.rings.length) return '<div class="text-muted">No path data for this observer</div>';
|
||||
let html = `<div class="reach-rings">`;
|
||||
data.rings.forEach(ring => {
|
||||
const opacity = Math.max(0.3, 1 - ring.hops * 0.06);
|
||||
const nodeLinks = ring.nodes.slice(0, 8).map(n => {
|
||||
const label = n.name ? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>` : `<span class="mono">${n.hop}</span>`;
|
||||
const detail = n.distRange ? ` <span class="text-muted">(${n.distRange})</span>` : '';
|
||||
return label + detail;
|
||||
}).join(', ');
|
||||
const extra = ring.nodes.length > 8 ? ` <span class="text-muted">+${ring.nodes.length - 8} more</span>` : '';
|
||||
html += `<div class="reach-ring" style="opacity:${opacity}">
|
||||
html += `<div class="reach-ring">
|
||||
<div class="reach-hop">${ring.hops} hop${ring.hops > 1 ? 's' : ''}</div>
|
||||
<div class="reach-nodes">${nodeLinks}${extra}</div>
|
||||
<div class="reach-count">${ring.nodes.length} node${ring.nodes.length > 1 ? 's' : ''}</div>
|
||||
@@ -673,7 +692,6 @@
|
||||
});
|
||||
let html = '<div class="reach-rings">';
|
||||
Object.entries(byDist).sort((a, b) => +a[0] - +b[0]).forEach(([dist, nodes]) => {
|
||||
const opacity = Math.max(0.3, 1 - (+dist) * 0.06);
|
||||
const nodeLinks = nodes.slice(0, 10).map(n => {
|
||||
const label = n.name
|
||||
? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>`
|
||||
@@ -681,7 +699,7 @@
|
||||
return label + ` <span class="text-muted">via ${esc(n.observer_name)}</span>`;
|
||||
}).join(', ');
|
||||
const extra = nodes.length > 10 ? ` <span class="text-muted">+${nodes.length - 10} more</span>` : '';
|
||||
html += `<div class="reach-ring" style="opacity:${opacity}">
|
||||
html += `<div class="reach-ring">
|
||||
<div class="reach-hop">${dist} hop${+dist > 1 ? 's' : ''}</div>
|
||||
<div class="reach-nodes">${nodeLinks}${extra}</div>
|
||||
<div class="reach-count">${nodes.length} node${nodes.length > 1 ? 's' : ''}</div>
|
||||
@@ -838,29 +856,44 @@
|
||||
}
|
||||
}
|
||||
|
||||
var CHANNEL_TIMELINE_MAX_SERIES = 8;
|
||||
|
||||
function renderChannelTimeline(data) {
|
||||
if (!data.length) return '<div class="text-muted">No data</div>';
|
||||
var hours = []; var hourSet = {};
|
||||
var channelList = []; var channelSet = {};
|
||||
var lookup = {};
|
||||
var maxCount = 1;
|
||||
var channelVolume = {};
|
||||
for (var i = 0; i < data.length; i++) {
|
||||
var d = data[i];
|
||||
if (!hourSet[d.hour]) { hourSet[d.hour] = 1; hours.push(d.hour); }
|
||||
if (!channelSet[d.channel]) { channelSet[d.channel] = 1; channelList.push(d.channel); }
|
||||
lookup[d.hour + '|' + d.channel] = d.count;
|
||||
if (d.count > maxCount) maxCount = d.count;
|
||||
channelVolume[d.channel] = (channelVolume[d.channel] || 0) + d.count;
|
||||
}
|
||||
hours.sort();
|
||||
// Sort channels by total volume descending, cap to top N
|
||||
channelList.sort(function(a, b) { return channelVolume[b] - channelVolume[a]; });
|
||||
var hiddenCount = Math.max(0, channelList.length - CHANNEL_TIMELINE_MAX_SERIES);
|
||||
var visibleChannels = channelList.slice(0, CHANNEL_TIMELINE_MAX_SERIES);
|
||||
|
||||
var maxCount = 1;
|
||||
for (var vi = 0; vi < visibleChannels.length; vi++) {
|
||||
for (var hi2 = 0; hi2 < hours.length; hi2++) {
|
||||
var c = lookup[hours[hi2] + '|' + visibleChannels[vi]] || 0;
|
||||
if (c > maxCount) maxCount = c;
|
||||
}
|
||||
}
|
||||
|
||||
var colors = ['#ef4444','#22c55e','#3b82f6','#f59e0b','#8b5cf6','#ec4899','#14b8a6','#64748b'];
|
||||
var w = 600, h = 180, pad = 35;
|
||||
var xScale = (w - pad * 2) / Math.max(hours.length - 1, 1);
|
||||
var yScale = (h - pad * 2) / maxCount;
|
||||
var svg = '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:100%;max-height:180px" role="img" aria-label="Channel message activity over time"><title>Channel message activity over time</title>';
|
||||
for (var ci = 0; ci < channelList.length; ci++) {
|
||||
for (var ci = 0; ci < visibleChannels.length; ci++) {
|
||||
var pts = [];
|
||||
for (var hi = 0; hi < hours.length; hi++) {
|
||||
var count = lookup[hours[hi] + '|' + channelList[ci]] || 0;
|
||||
var count = lookup[hours[hi] + '|' + visibleChannels[ci]] || 0;
|
||||
var x = pad + hi * xScale;
|
||||
var y = h - pad - count * yScale;
|
||||
pts.push(x + ',' + y);
|
||||
@@ -874,8 +907,11 @@
|
||||
}
|
||||
svg += '</svg>';
|
||||
var legendParts = [];
|
||||
for (var lci = 0; lci < channelList.length; lci++) {
|
||||
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(channelList[lci]) + '</span>');
|
||||
for (var lci = 0; lci < visibleChannels.length; lci++) {
|
||||
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(visibleChannels[lci]) + '</span>');
|
||||
}
|
||||
if (hiddenCount > 0) {
|
||||
legendParts.push('<span class="text-muted">+' + hiddenCount + ' more</span>');
|
||||
}
|
||||
svg += '<div class="timeline-legend">' + legendParts.join('') + '</div>';
|
||||
return svg;
|
||||
@@ -998,6 +1034,7 @@
|
||||
return (filtered.length ? '<table class="analytics-table" id="mbAdoptersTable" style="margin-top:12px">' +
|
||||
'<thead><tr>' +
|
||||
'<th scope="col" data-sort="name">Node</th>' +
|
||||
'<th scope="col" data-sort="role">Role</th>' +
|
||||
'<th scope="col" data-sort="status">Status</th>' +
|
||||
'<th scope="col" data-sort="hashSize">Hash Size</th>' +
|
||||
'<th scope="col" data-sort="packets">Adverts</th>' +
|
||||
@@ -1005,8 +1042,10 @@
|
||||
'</tr></thead>' +
|
||||
'<tbody>' +
|
||||
filtered.map(function(r) {
|
||||
var roleColor = (window.ROLE_COLORS || {})[r.role] || '#6b7280';
|
||||
return '<tr class="clickable-row" data-action="navigate" data-value="#/nodes/' + encodeURIComponent(r.pubkey) + '" tabindex="0" role="row">' +
|
||||
'<td><strong>' + esc(r.name) + '</strong></td>' +
|
||||
'<td><span class="badge" style="background:' + roleColor + '20;color:' + roleColor + '">' + esc(r.role || 'unknown') + '</span></td>' +
|
||||
'<td><span style="color:' + (statusColor[r.status] || statusColor.unknown) + '">' +
|
||||
(statusIcon[r.status] || '❓') + ' ' + (statusLabel[r.status] || 'Unknown') + '</span></td>' +
|
||||
'<td><span class="badge badge-hash-' + r.hashSize + '">' + r.hashSize + '-byte</span></td>' +
|
||||
@@ -1190,10 +1229,10 @@
|
||||
else matrixDesc.textContent = '3-byte prefix space is too large to visualize as a matrix — collision table is shown below.';
|
||||
}
|
||||
renderHashMatrixFromServer(cData.by_size[String(bytes)], bytes);
|
||||
// Hide collision risk card for 3-byte — stats are shown in the matrix panel
|
||||
// Show collision risk section for all byte sizes
|
||||
const riskCard = document.getElementById('collisionRiskSection');
|
||||
if (riskCard) riskCard.style.display = bytes === 3 ? 'none' : '';
|
||||
if (bytes !== 3) renderCollisionsFromServer(cData.by_size[String(bytes)], bytes);
|
||||
if (riskCard) riskCard.style.display = '';
|
||||
renderCollisionsFromServer(cData.by_size[String(bytes)], bytes);
|
||||
}
|
||||
|
||||
// Wire up selector
|
||||
@@ -1285,9 +1324,9 @@
|
||||
<div class="analytics-stat-value" style="font-size:16px">${pctStr}%</div>
|
||||
<div style="font-size:10px;color:var(--text-muted);margin-top:2px">${usedCount > 256 ? usedCount + ' of ' : 'of '}${spaceLabel} possible</div>
|
||||
</div>
|
||||
<div class="analytics-stat-card" style="flex:1;min-width:110px;border-color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--border)'}">
|
||||
<div class="analytics-stat-card" style="flex:1;min-width:110px;border-color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--border)'}${collisionCount > 0 ? ';cursor:pointer' : ''}" ${collisionCount > 0 ? 'onclick="document.getElementById(\'collisionRiskSection\')?.scrollIntoView({behavior:\'smooth\',block:\'start\'})"' : ''} ${collisionCount > 0 ? 'title="Click to see collision details"' : ''}>
|
||||
<div class="analytics-stat-label">Prefix collisions</div>
|
||||
<div class="analytics-stat-value" style="color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--status-green)'}">${collisionCount}</div>
|
||||
<div class="analytics-stat-value" style="color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--status-green)'}">${collisionCount}${collisionCount > 0 ? ' <span style="font-size:11px;opacity:0.7">▼</span>' : ''}</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
@@ -1362,7 +1401,7 @@
|
||||
// 3-byte: show a summary panel instead of a matrix
|
||||
if (bytes === 3) {
|
||||
el.innerHTML = hashStatCardsHtml(totalNodes, stats.using_this_size || 0, '3-byte', 16777216, stats.unique_prefixes || 0, stats.collision_count || 0) +
|
||||
`<p class="text-muted" style="margin:0;font-size:0.8em">The 3-byte prefix space (16.7M values) is too large to visualize as a grid.</p>` +
|
||||
`<p class="text-muted" style="margin:0;font-size:0.8em">The 3-byte prefix space (16.7M values) is too large to visualize as a grid.${(stats.collision_count || 0) > 0 ? ' See collision details below.' : ''}</p>` +
|
||||
`<p class="text-muted" style="margin:8px 0 0;font-size:0.8em">ℹ️ This tab only counts collisions among repeaters configured for this hash size. The <a href="#/analytics?tab=prefix-tool" style="color:var(--accent)">Prefix Tool</a> checks all repeaters regardless of configured hash size.</p>`;
|
||||
return;
|
||||
}
|
||||
@@ -1932,15 +1971,18 @@
|
||||
}
|
||||
|
||||
// Top hops leaderboard
|
||||
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
|
||||
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">Obs</th><th scope="col">Best SNR</th><th scope="col">Median SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
|
||||
const top20 = data.topHops.slice(0, 20);
|
||||
top20.forEach((h, i) => {
|
||||
const fromLink = h.fromPk ? `<a href="#/nodes/${encodeURIComponent(h.fromPk)}" class="analytics-link">${esc(h.fromName)}</a>` : esc(h.fromName || '?');
|
||||
const toLink = h.toPk ? `<a href="#/nodes/${encodeURIComponent(h.toPk)}" class="analytics-link">${esc(h.toName)}</a>` : esc(h.toName || '?');
|
||||
const snr = h.snr != null ? h.snr + ' dB' : '<span class="text-muted">—</span>';
|
||||
const bestSnr = h.bestSnr != null ? Number(h.bestSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
|
||||
const medianSnr = h.medianSnr != null ? Number(h.medianSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
|
||||
const obs = h.obsCount != null ? h.obsCount : 1;
|
||||
const pktLink = h.hash ? `<a href="#/packet/${encodeURIComponent(h.hash)}" class="analytics-link mono" style="font-size:0.85em">${esc(h.hash.slice(0, 12))}…</a>` : '—';
|
||||
const mapBtn = h.fromPk && h.toPk ? `<button class="btn-icon dist-map-hop" data-from="${esc(h.fromPk)}" data-to="${esc(h.toPk)}" title="View on map">🗺️</button>` : '';
|
||||
html += `<tr><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${snr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
|
||||
const tsTitle = h.timestamp ? `Best observation: ${h.timestamp}` : '';
|
||||
html += `<tr title="${esc(tsTitle)}"><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${obs}</td><td>${bestSnr}</td><td>${medianSnr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
|
||||
});
|
||||
html += `</tbody></table></div>`;
|
||||
|
||||
@@ -1995,6 +2037,8 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
window._analyticsRfNFColumnChart = rfNFColumnChart;
|
||||
window._analyticsRenderMultiByteCapability = renderMultiByteCapability;
|
||||
window._analyticsRenderMultiByteAdopters = renderMultiByteAdopters;
|
||||
window._analyticsHashStatCardsHtml = hashStatCardsHtml;
|
||||
window._analyticsRenderCollisionsFromServer = renderCollisionsFromServer;
|
||||
}
|
||||
|
||||
// ─── Neighbor Graph Tab ─────────────────────────────────────────────────────
|
||||
@@ -2009,8 +2053,8 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
<label style="font-size:13px">Roles:
|
||||
<span id="ngRoleChecks" style="margin-left:4px"></span>
|
||||
</label>
|
||||
<label style="font-size:13px">Min Score: <input type="range" id="ngMinScore" min="0" max="100" value="10" style="width:100px;vertical-align:middle">
|
||||
<span id="ngMinScoreVal">0.10</span>
|
||||
<label style="font-size:13px">Min Score: <input type="range" id="ngMinScore" min="0" max="100" value="70" style="width:100px;vertical-align:middle">
|
||||
<span id="ngMinScoreVal">0.70</span>
|
||||
</label>
|
||||
<label style="font-size:13px">Confidence:
|
||||
<select id="ngConfidence" style="font-size:12px;padding:2px 4px">
|
||||
@@ -2038,6 +2082,11 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
const color = (window.ROLE_COLORS || {})[r] || '#888';
|
||||
rcEl.innerHTML += `<label style="font-size:12px;margin-right:8px"><input type="checkbox" data-role="${r}" checked> <span style="color:${esc(color)}">${esc(r)}</span></label>`;
|
||||
});
|
||||
// Observer checkbox — unchecked by default (observers create hub-and-spoke noise)
|
||||
{
|
||||
const color = (window.ROLE_COLORS || {}).observer || '#8b5cf6';
|
||||
rcEl.innerHTML += `<label style="font-size:12px;margin-right:8px"><input type="checkbox" data-role="observer"> <span style="color:${esc(color)}">observer</span></label>`;
|
||||
}
|
||||
|
||||
// Load data
|
||||
const rqs = RegionFilter.regionQueryString();
|
||||
@@ -2055,8 +2104,17 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
startGraphRenderer();
|
||||
|
||||
// Filter listeners
|
||||
// Restore saved min score from localStorage
|
||||
var savedScore = localStorage.getItem('ng-min-score');
|
||||
if (savedScore !== null) {
|
||||
document.getElementById('ngMinScore').value = savedScore;
|
||||
document.getElementById('ngMinScoreVal').textContent = (savedScore / 100).toFixed(2);
|
||||
applyNGFilters();
|
||||
}
|
||||
|
||||
document.getElementById('ngMinScore').addEventListener('input', function() {
|
||||
document.getElementById('ngMinScoreVal').textContent = (this.value / 100).toFixed(2);
|
||||
localStorage.setItem('ng-min-score', this.value);
|
||||
applyNGFilters();
|
||||
});
|
||||
document.getElementById('ngConfidence').addEventListener('change', applyNGFilters);
|
||||
@@ -2095,7 +2153,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
// Filter nodes by role
|
||||
const visibleNodes = _ngState.allNodes.filter(n => {
|
||||
const role = (n.role || 'unknown').toLowerCase();
|
||||
return checkedRoles.has(role) || role === 'unknown' || role === 'observer';
|
||||
return checkedRoles.has(role) || role === 'unknown';
|
||||
});
|
||||
const visiblePKs = new Set(visibleNodes.map(n => n.pubkey));
|
||||
|
||||
@@ -3402,5 +3460,112 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
return svg;
|
||||
}
|
||||
|
||||
// #690 — Clock Health fleet view (M3)
|
||||
async function renderClockHealthTab(el) {
|
||||
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">Loading clock health data…</div>';
|
||||
try {
|
||||
var data = await (await fetch('/api/nodes/clock-skew')).json();
|
||||
if (!Array.isArray(data) || !data.length) {
|
||||
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">No clock skew data available. Nodes need recent adverts for clock analysis.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
// State
|
||||
var activeFilter = 'all';
|
||||
var sortKey = 'severity';
|
||||
var sortDir = 'asc'; // severity worst-first
|
||||
|
||||
function render() {
|
||||
// Filter
|
||||
var filtered = activeFilter === 'all' ? data : data.filter(function(n) { return n.severity === activeFilter; });
|
||||
|
||||
// Sort
|
||||
filtered = filtered.slice().sort(function(a, b) {
|
||||
var v;
|
||||
if (sortKey === 'severity') {
|
||||
v = (SKEW_SEVERITY_ORDER[a.severity] || 9) - (SKEW_SEVERITY_ORDER[b.severity] || 9);
|
||||
} else if (sortKey === 'skew') {
|
||||
v = Math.abs(window.currentSkewValue(b) || 0) - Math.abs(window.currentSkewValue(a) || 0);
|
||||
} else if (sortKey === 'name') {
|
||||
v = (a.nodeName || '').localeCompare(b.nodeName || '');
|
||||
} else if (sortKey === 'drift') {
|
||||
v = Math.abs(b.driftPerDaySec || 0) - Math.abs(a.driftPerDaySec || 0);
|
||||
}
|
||||
return sortDir === 'desc' ? -v : v;
|
||||
});
|
||||
|
||||
// Summary
|
||||
var counts = { ok: 0, warning: 0, critical: 0, absurd: 0 };
|
||||
data.forEach(function(n) { if (counts[n.severity] !== undefined) counts[n.severity]++; });
|
||||
|
||||
// Filter buttons (also serve as summary — no separate stats pills needed)
|
||||
var filterColors = { ok: 'var(--status-green)', warning: 'var(--status-yellow)', critical: 'var(--status-orange)', absurd: 'var(--status-purple)', no_clock: 'var(--text-muted)' };
|
||||
var filters = ['all', 'ok', 'warning', 'critical', 'absurd', 'no_clock'];
|
||||
var filterHtml = '<div style="margin-bottom:10px">' + filters.map(function(f) {
|
||||
var dot = f !== 'all' ? '<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:' + filterColors[f] + ';margin-right:4px;vertical-align:middle"></span>' : '';
|
||||
return '<button class="clock-filter-btn' + (activeFilter === f ? ' active' : '') + '" data-filter="' + f + '">' +
|
||||
dot + (f === 'all' ? 'All (' + data.length + ')' : (SKEW_SEVERITY_LABELS[f] || f) + ' (' + (counts[f] || 0) + ')') +
|
||||
'</button>';
|
||||
}).join('') + '</div>';
|
||||
|
||||
// Table
|
||||
var rowsHtml = filtered.map(function(n) {
|
||||
var rowClass = 'clock-fleet-row--' + (n.severity || 'ok');
|
||||
var lastAdv = n.lastObservedTS ? new Date(n.lastObservedTS * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC') : '—';
|
||||
var skewVal = window.currentSkewValue(n);
|
||||
var skewText = n.severity === 'no_clock' ? 'No Clock' : formatSkew(skewVal);
|
||||
var driftText = n.severity === 'no_clock' || !n.driftPerDaySec ? '–' : formatDrift(n.driftPerDaySec);
|
||||
return '<tr class="' + rowClass + '" data-pubkey="' + esc(n.pubkey) + '" style="cursor:pointer">' +
|
||||
'<td><strong>' + esc(n.nodeName || n.pubkey.slice(0, 12)) + '</strong></td>' +
|
||||
'<td style="font-family:var(--mono,monospace)">' + skewText + '</td>' +
|
||||
'<td>' + renderSkewBadge(n.severity, skewVal, n) + '</td>' +
|
||||
'<td style="font-family:var(--mono,monospace)">' + driftText + '</td>' +
|
||||
'<td style="font-size:11px">' + lastAdv + '</td>' +
|
||||
'</tr>';
|
||||
}).join('');
|
||||
|
||||
el.innerHTML = '<h3 style="margin:0 0 10px">⏰ Clock Health</h3>' +
|
||||
filterHtml +
|
||||
'<table class="data-table analytics-table" id="clock-health-table">' +
|
||||
'<thead><tr>' +
|
||||
'<th data-sort-col="name" style="cursor:pointer">Name</th>' +
|
||||
'<th data-sort-col="skew" style="cursor:pointer">Skew</th>' +
|
||||
'<th data-sort-col="severity" style="cursor:pointer">Severity</th>' +
|
||||
'<th data-sort-col="drift" style="cursor:pointer">Drift Rate</th>' +
|
||||
'<th>Last Advert</th>' +
|
||||
'</tr></thead><tbody>' + rowsHtml + '</tbody></table>';
|
||||
|
||||
// Bind filter clicks
|
||||
el.querySelectorAll('.clock-filter-btn').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
activeFilter = btn.dataset.filter;
|
||||
render();
|
||||
});
|
||||
});
|
||||
|
||||
// Bind header sort clicks
|
||||
el.querySelectorAll('[data-sort-col]').forEach(function(th) {
|
||||
th.addEventListener('click', function() {
|
||||
var col = th.dataset.sortCol;
|
||||
if (sortKey === col) { sortDir = sortDir === 'asc' ? 'desc' : 'asc'; }
|
||||
else { sortKey = col; sortDir = 'asc'; }
|
||||
render();
|
||||
});
|
||||
});
|
||||
|
||||
// Bind row clicks → navigate to node
|
||||
el.querySelectorAll('tr[data-pubkey]').forEach(function(tr) {
|
||||
tr.addEventListener('click', function() {
|
||||
location.hash = '#/nodes/' + encodeURIComponent(tr.dataset.pubkey);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
render();
|
||||
} catch (err) {
|
||||
el.innerHTML = '<div class="text-center" style="color:var(--status-red);padding:40px">Failed to load clock health data: ' + esc(String(err)) + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
registerPage('analytics', { init, destroy });
|
||||
})();
|
||||
|
||||
+2
-1
@@ -10,6 +10,8 @@ function routeTypeName(n) { return ROUTE_TYPES[n] || 'UNKNOWN'; }
|
||||
function payloadTypeName(n) { return PAYLOAD_TYPES[n] || 'UNKNOWN'; }
|
||||
function payloadTypeColor(n) { return PAYLOAD_COLORS[n] || 'unknown'; }
|
||||
function isTransportRoute(rt) { return rt === 0 || rt === 3; }
|
||||
/** Byte offset of path_len in raw_hex: 5 for transport routes (4 bytes of next/last hop codes precede it), 1 otherwise. */
|
||||
function getPathLenOffset(routeType) { return isTransportRoute(routeType) ? 5 : 1; }
|
||||
function transportBadge(rt) { return isTransportRoute(rt) ? ' <span class="badge badge-transport" title="' + routeTypeName(rt) + '">T</span>' : ''; }
|
||||
|
||||
// --- Utilities ---
|
||||
@@ -1027,7 +1029,6 @@ function makeColumnsResizable(tableSelector, storageKey) {
|
||||
// Add resize handles
|
||||
ths.forEach((th, i) => {
|
||||
if (i === ths.length - 1) return;
|
||||
th.style.position = 'relative';
|
||||
const handle = document.createElement('div');
|
||||
handle.className = 'col-resize-handle';
|
||||
handle.addEventListener('mousedown', (e) => {
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* Client-side MeshCore channel decryption module.
|
||||
*
|
||||
* Implements the same crypto as internal/channel/channel.go:
|
||||
* - Key derivation: SHA-256("#channelname")[:16]
|
||||
* - Channel hash: SHA-256(key)[0]
|
||||
* - MAC: HMAC-SHA256 with 32-byte secret (key + 16 zero bytes), truncated to 2 bytes
|
||||
* - Encryption: AES-128-ECB (block-by-block)
|
||||
* - Plaintext: timestamp(4 LE) + flags(1) + "sender: message\0"
|
||||
*
|
||||
* Keys NEVER leave the browser. No fetch/XHR/network calls in this module.
|
||||
*/
|
||||
/* eslint-disable no-var */
|
||||
window.ChannelDecrypt = (function () {
|
||||
'use strict';
|
||||
|
||||
var STORAGE_KEY = 'corescope_channel_keys';
|
||||
var CACHE_KEY = 'corescope_channel_cache';
|
||||
|
||||
// ---- Hex utilities ----
|
||||
|
||||
function bytesToHex(bytes) {
|
||||
var hex = '';
|
||||
for (var i = 0; i < bytes.length; i++) {
|
||||
hex += (bytes[i] < 16 ? '0' : '') + bytes[i].toString(16);
|
||||
}
|
||||
return hex;
|
||||
}
|
||||
|
||||
function hexToBytes(hex) {
|
||||
var bytes = new Uint8Array(hex.length / 2);
|
||||
for (var i = 0; i < hex.length; i += 2) {
|
||||
bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
// ---- Key derivation ----
|
||||
|
||||
/**
|
||||
* Derive AES-128 key from channel name: SHA-256("#channelname")[:16].
|
||||
* @param {string} channelName - e.g. "#LongFast"
|
||||
* @returns {Promise<Uint8Array>} 16-byte key
|
||||
*/
|
||||
async function deriveKey(channelName) {
|
||||
var enc = new TextEncoder();
|
||||
var hash = await crypto.subtle.digest('SHA-256', enc.encode(channelName));
|
||||
return new Uint8Array(hash).slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the 1-byte channel hash: SHA-256(key)[0].
|
||||
* @param {Uint8Array} key - 16-byte key
|
||||
* @returns {Promise<number>} single byte (0-255)
|
||||
*/
|
||||
async function computeChannelHash(key) {
|
||||
var hash = await crypto.subtle.digest('SHA-256', key);
|
||||
return new Uint8Array(hash)[0];
|
||||
}
|
||||
|
||||
// ---- AES-128-ECB via Web Crypto (CBC with zero IV, block-by-block) ----
|
||||
|
||||
/**
|
||||
* Decrypt AES-128-ECB by decrypting each 16-byte block independently
|
||||
* using AES-CBC with a zero IV (equivalent to ECB for single blocks).
|
||||
* @param {Uint8Array} key - 16-byte AES key
|
||||
* @param {Uint8Array} ciphertext - must be multiple of 16 bytes
|
||||
* @returns {Promise<Uint8Array>} plaintext
|
||||
*/
|
||||
async function decryptECB(key, ciphertext) {
|
||||
if (ciphertext.length === 0 || ciphertext.length % 16 !== 0) {
|
||||
return null;
|
||||
}
|
||||
var cryptoKey = await crypto.subtle.importKey(
|
||||
'raw', key, { name: 'AES-CBC' }, false, ['decrypt']
|
||||
);
|
||||
var zeroIV = new Uint8Array(16);
|
||||
var plaintext = new Uint8Array(ciphertext.length);
|
||||
|
||||
for (var i = 0; i < ciphertext.length; i += 16) {
|
||||
var block = ciphertext.slice(i, i + 16);
|
||||
// Append a dummy block (16 bytes of 0x10 = PKCS7 padding for empty next block)
|
||||
// so Web Crypto doesn't complain about padding
|
||||
var padded = new Uint8Array(32);
|
||||
padded.set(block, 0);
|
||||
// Second block is PKCS7 padding: 16 bytes of 0x10
|
||||
for (var j = 16; j < 32; j++) padded[j] = 16;
|
||||
|
||||
var decrypted = await crypto.subtle.decrypt(
|
||||
{ name: 'AES-CBC', iv: zeroIV }, cryptoKey, padded
|
||||
);
|
||||
var decBytes = new Uint8Array(decrypted);
|
||||
plaintext.set(decBytes.slice(0, 16), i);
|
||||
}
|
||||
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
// ---- MAC verification ----
|
||||
|
||||
/**
|
||||
* Verify HMAC-SHA256 MAC (first 2 bytes) using 32-byte secret (key + 16 zero bytes).
|
||||
* @param {Uint8Array} key - 16-byte AES key
|
||||
* @param {Uint8Array} ciphertext - encrypted data
|
||||
* @param {string} macHex - 4-char hex string (2 bytes)
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function verifyMAC(key, ciphertext, macHex) {
|
||||
// Build 32-byte channel secret: key + 16 zero bytes
|
||||
var secret = new Uint8Array(32);
|
||||
secret.set(key, 0);
|
||||
// remaining 16 bytes are already 0
|
||||
|
||||
var cryptoKey = await crypto.subtle.importKey(
|
||||
'raw', secret, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']
|
||||
);
|
||||
var sig = await crypto.subtle.sign('HMAC', cryptoKey, ciphertext);
|
||||
var sigBytes = new Uint8Array(sig);
|
||||
|
||||
var macBytes = hexToBytes(macHex);
|
||||
return sigBytes[0] === macBytes[0] && sigBytes[1] === macBytes[1];
|
||||
}
|
||||
|
||||
// ---- Plaintext parsing ----
|
||||
|
||||
/**
|
||||
* Parse decrypted plaintext: timestamp(4 LE) + flags(1) + "sender: message\0..."
|
||||
* @param {Uint8Array} plaintext
|
||||
* @returns {{ timestamp: number, flags: number, sender: string, message: string } | null}
|
||||
*/
|
||||
function parsePlaintext(plaintext) {
|
||||
if (!plaintext || plaintext.length < 5) return null;
|
||||
|
||||
var timestamp = plaintext[0] | (plaintext[1] << 8) | (plaintext[2] << 16) | ((plaintext[3] << 24) >>> 0);
|
||||
var flags = plaintext[4];
|
||||
|
||||
// Extract text up to first null byte
|
||||
var textBytes = plaintext.slice(5);
|
||||
var nullIdx = -1;
|
||||
for (var i = 0; i < textBytes.length; i++) {
|
||||
if (textBytes[i] === 0) { nullIdx = i; break; }
|
||||
}
|
||||
var text = new TextDecoder().decode(nullIdx >= 0 ? textBytes.slice(0, nullIdx) : textBytes);
|
||||
|
||||
// Count non-printable characters
|
||||
var nonPrintable = 0;
|
||||
for (var c = 0; c < text.length; c++) {
|
||||
var code = text.charCodeAt(c);
|
||||
if (code < 32 && code !== 10 && code !== 13 && code !== 9) nonPrintable++;
|
||||
}
|
||||
if (nonPrintable > 2) return null;
|
||||
|
||||
// Parse "sender: message" format
|
||||
var colonIdx = text.indexOf(': ');
|
||||
if (colonIdx > 0 && colonIdx < 50) {
|
||||
var potentialSender = text.substring(0, colonIdx);
|
||||
if (potentialSender.indexOf(':') < 0 && potentialSender.indexOf('[') < 0 && potentialSender.indexOf(']') < 0) {
|
||||
return { timestamp: timestamp, flags: flags, sender: potentialSender, message: text.substring(colonIdx + 2) };
|
||||
}
|
||||
}
|
||||
|
||||
return { timestamp: timestamp, flags: flags, sender: '', message: text };
|
||||
}
|
||||
|
||||
// ---- Full decrypt pipeline ----
|
||||
|
||||
/**
|
||||
* Verify MAC, decrypt, and parse a single packet.
|
||||
* @param {Uint8Array} keyBytes - 16-byte key
|
||||
* @param {string} macHex - 4-char hex MAC
|
||||
* @param {string} encryptedHex - hex-encoded ciphertext
|
||||
* @returns {Promise<{ sender: string, message: string, timestamp: number } | null>}
|
||||
*/
|
||||
async function decrypt(keyBytes, macHex, encryptedHex) {
|
||||
var ciphertext = hexToBytes(encryptedHex);
|
||||
if (ciphertext.length === 0 || ciphertext.length % 16 !== 0) return null;
|
||||
|
||||
var macOk = await verifyMAC(keyBytes, ciphertext, macHex);
|
||||
if (!macOk) return null;
|
||||
|
||||
var plaintext = await decryptECB(keyBytes, ciphertext);
|
||||
if (!plaintext) return null;
|
||||
|
||||
return parsePlaintext(plaintext);
|
||||
}
|
||||
|
||||
// Alias used by channels.js
|
||||
var decryptPacket = decrypt;
|
||||
|
||||
// ---- Key storage (localStorage) ----
|
||||
|
||||
function saveKey(channelName, keyHex) {
|
||||
var keys = getKeys();
|
||||
keys[channelName] = keyHex;
|
||||
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
|
||||
}
|
||||
|
||||
// Alias used by channels.js
|
||||
var storeKey = saveKey;
|
||||
|
||||
function getKeys() {
|
||||
try {
|
||||
var raw = localStorage.getItem(STORAGE_KEY);
|
||||
return raw ? JSON.parse(raw) : {};
|
||||
} catch (e) { return {}; }
|
||||
}
|
||||
|
||||
// Alias used by channels.js
|
||||
var getStoredKeys = getKeys;
|
||||
|
||||
function removeKey(channelName) {
|
||||
var keys = getKeys();
|
||||
delete keys[channelName];
|
||||
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
|
||||
// Also clear cached messages for this channel
|
||||
clearChannelCache(channelName);
|
||||
}
|
||||
|
||||
/** Remove cached messages for a specific channel (by name or hash). */
|
||||
function clearChannelCache(channelKey) {
|
||||
try {
|
||||
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
|
||||
delete cache[channelKey];
|
||||
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
|
||||
} catch (e) { /* quota */ }
|
||||
}
|
||||
|
||||
// ---- Message cache (localStorage) ----
|
||||
|
||||
function cacheMessages(channelHash, messages) {
|
||||
try {
|
||||
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
|
||||
cache[channelHash] = { messages: messages, ts: Date.now() };
|
||||
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
|
||||
} catch (e) { /* quota */ }
|
||||
}
|
||||
|
||||
function getCachedMessages(channelHash) {
|
||||
try {
|
||||
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
|
||||
var entry = cache[channelHash];
|
||||
return entry ? entry.messages : null;
|
||||
} catch (e) { return null; }
|
||||
}
|
||||
|
||||
// Cache with lastTimestamp and count (used by channels.js via getCache/setCache)
|
||||
var MAX_CACHED_MESSAGES = 1000;
|
||||
|
||||
function setCache(key, messages, lastTimestamp, totalCount) {
|
||||
try {
|
||||
// Enforce cache size limit: only keep most recent MAX_CACHED_MESSAGES
|
||||
var toStore = messages;
|
||||
if (messages.length > MAX_CACHED_MESSAGES) {
|
||||
toStore = messages.slice(messages.length - MAX_CACHED_MESSAGES);
|
||||
}
|
||||
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
|
||||
cache[key] = {
|
||||
messages: toStore,
|
||||
lastTimestamp: lastTimestamp,
|
||||
count: totalCount || toStore.length,
|
||||
ts: Date.now()
|
||||
};
|
||||
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
|
||||
} catch (e) { /* quota */ }
|
||||
}
|
||||
|
||||
function getCache(key) {
|
||||
try {
|
||||
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
|
||||
return cache[key] || null;
|
||||
} catch (e) { return null; }
|
||||
}
|
||||
|
||||
return {
|
||||
deriveKey: deriveKey,
|
||||
decrypt: decrypt,
|
||||
decryptPacket: decryptPacket,
|
||||
decryptECB: decryptECB,
|
||||
verifyMAC: verifyMAC,
|
||||
parsePlaintext: parsePlaintext,
|
||||
computeChannelHash: computeChannelHash,
|
||||
bytesToHex: bytesToHex,
|
||||
hexToBytes: hexToBytes,
|
||||
saveKey: saveKey,
|
||||
storeKey: storeKey,
|
||||
getKeys: getKeys,
|
||||
getStoredKeys: getStoredKeys,
|
||||
removeKey: removeKey,
|
||||
clearChannelCache: clearChannelCache,
|
||||
cacheMessages: cacheMessages,
|
||||
getCachedMessages: getCachedMessages,
|
||||
setCache: setCache,
|
||||
getCache: getCache
|
||||
};
|
||||
})();
|
||||
+491
-11
@@ -318,6 +318,290 @@
|
||||
|
||||
let regionChangeHandler = null;
|
||||
|
||||
// --- Client-side channel decryption (#725 M2) ---
|
||||
|
||||
// Check if input is a valid hex string (32 hex chars = 16 bytes)
|
||||
function isHexKey(val) {
|
||||
return /^[0-9a-fA-F]{32}$/.test(val);
|
||||
}
|
||||
|
||||
// Show status message in the add-channel form (#759)
|
||||
var statusTimer = null;
|
||||
function showAddStatus(msg, type) {
|
||||
var el = document.getElementById('chAddStatus');
|
||||
if (!el) return;
|
||||
el.textContent = msg;
|
||||
el.className = 'ch-add-status ch-add-status--' + (type || 'info');
|
||||
el.style.display = '';
|
||||
clearTimeout(statusTimer);
|
||||
if (type !== 'loading') {
|
||||
statusTimer = setTimeout(function () { el.style.display = 'none'; }, 5000);
|
||||
}
|
||||
}
|
||||
|
||||
// Add a user channel by name (#channelname) or hex key
|
||||
async function addUserChannel(val) {
|
||||
var displayName = val.startsWith('#') ? val : (isHexKey(val) ? val.substring(0, 8) + '…' : '#' + val);
|
||||
showAddStatus('Decrypting ' + displayName + ' messages…', 'loading');
|
||||
var channelName, keyHex;
|
||||
try {
|
||||
if (val.startsWith('#')) {
|
||||
channelName = val;
|
||||
var keyBytes = await ChannelDecrypt.deriveKey(channelName);
|
||||
keyHex = ChannelDecrypt.bytesToHex(keyBytes);
|
||||
} else if (isHexKey(val)) {
|
||||
keyHex = val.toLowerCase();
|
||||
channelName = 'psk:' + keyHex.substring(0, 8);
|
||||
} else {
|
||||
// Try with # prefix if user forgot
|
||||
channelName = '#' + val;
|
||||
var keyBytes2 = await ChannelDecrypt.deriveKey(channelName);
|
||||
keyHex = ChannelDecrypt.bytesToHex(keyBytes2);
|
||||
}
|
||||
|
||||
ChannelDecrypt.storeKey(channelName, keyHex);
|
||||
|
||||
// Compute channel hash byte to find matching encrypted channels
|
||||
var keyBytes3 = ChannelDecrypt.hexToBytes(keyHex);
|
||||
var hashByte = await ChannelDecrypt.computeChannelHash(keyBytes3);
|
||||
|
||||
// Add to sidebar or merge with existing encrypted channel
|
||||
mergeUserChannels();
|
||||
renderChannelList();
|
||||
|
||||
// Auto-select and start decrypting
|
||||
var targetHash = 'user:' + channelName;
|
||||
// Check if there's an existing encrypted channel with this hash byte
|
||||
var existingEncrypted = channels.find(function (ch) {
|
||||
return ch.encrypted && String(ch.hash) === String(hashByte);
|
||||
});
|
||||
if (existingEncrypted) {
|
||||
targetHash = existingEncrypted.hash;
|
||||
}
|
||||
await selectChannel(targetHash, { userKey: keyHex, channelHashByte: hashByte, channelName: channelName });
|
||||
|
||||
// Show success feedback (#759)
|
||||
var msgCount = document.querySelectorAll('#chMessages .ch-msg').length;
|
||||
var userDisplay = channelName.startsWith('psk:') ? 'Custom channel (' + channelName.substring(4) + ')' : channelName;
|
||||
if (msgCount > 0) {
|
||||
showAddStatus('Added ' + userDisplay + ' — ' + msgCount + ' messages decrypted', 'success');
|
||||
} else {
|
||||
showAddStatus('No messages found for ' + userDisplay, 'warn');
|
||||
}
|
||||
} catch (err) {
|
||||
showAddStatus('Failed to decrypt', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Merge user-stored keys into the channel list
|
||||
function mergeUserChannels() {
|
||||
var keys = ChannelDecrypt.getStoredKeys();
|
||||
var names = Object.keys(keys);
|
||||
for (var i = 0; i < names.length; i++) {
|
||||
var name = names[i];
|
||||
// Check if channel already exists by name
|
||||
var exists = channels.some(function (ch) {
|
||||
return ch.name === name || ch.hash === name || ch.hash === ('user:' + name);
|
||||
});
|
||||
if (!exists) {
|
||||
channels.push({
|
||||
hash: 'user:' + name,
|
||||
name: name,
|
||||
messageCount: 0,
|
||||
lastActivityMs: 0,
|
||||
lastSender: '',
|
||||
lastMessage: 'Encrypted — click to decrypt',
|
||||
encrypted: true,
|
||||
userAdded: true
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and decrypt GRP_TXT packets client-side (M5: delta fetch + cache)
|
||||
async function fetchAndDecryptChannel(keyHex, channelHashByte, channelName, opts) {
|
||||
opts = opts || {};
|
||||
var keyBytes = ChannelDecrypt.hexToBytes(keyHex);
|
||||
|
||||
// M5: Check cache first — serve cached messages immediately
|
||||
var cacheKey = channelName || String(channelHashByte);
|
||||
var cached = ChannelDecrypt.getCache(cacheKey);
|
||||
var cachedMsgs = cached ? cached.messages : [];
|
||||
var lastTs = cached ? cached.lastTimestamp : '';
|
||||
var cachedCount = cached ? (cached.count || 0) : 0;
|
||||
|
||||
// If we have cached messages and caller wants instant render, return them first
|
||||
if (cachedMsgs.length > 0 && !opts.forceFullDecrypt) {
|
||||
// Signal caller to render cache immediately, then do delta fetch
|
||||
if (opts.onCacheHit) opts.onCacheHit(cachedMsgs);
|
||||
}
|
||||
|
||||
// Fetch packets from API — get all payload_type=5 (GRP_TXT/CHAN)
|
||||
var rp = RegionFilter.getRegionParam();
|
||||
var qs = rp ? '®ion=' + encodeURIComponent(rp) : '';
|
||||
var data;
|
||||
try {
|
||||
data = await api('/packets?limit=1000&payloadType=5' + qs, { ttl: 10000 });
|
||||
} catch (e) {
|
||||
return { messages: cachedMsgs, error: 'Failed to fetch packets: ' + e.message, fromCache: cachedMsgs.length > 0 };
|
||||
}
|
||||
|
||||
var packets = data.packets || [];
|
||||
// Filter for GRP_TXT (encrypted) packets matching our channel hash byte
|
||||
var candidates = [];
|
||||
for (var i = 0; i < packets.length; i++) {
|
||||
var p = packets[i];
|
||||
var dj;
|
||||
try { dj = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json; }
|
||||
catch (e) { continue; }
|
||||
if (!dj) continue;
|
||||
|
||||
if (dj.type === 'CHAN' && dj.channel === channelName) {
|
||||
candidates.push({ type: 'already_decrypted', decoded: dj, packet: p });
|
||||
} else if (dj.type === 'GRP_TXT' && dj.encryptedData && dj.mac) {
|
||||
if (dj.channelHash === channelHashByte) {
|
||||
candidates.push({ type: 'encrypted', decoded: dj, packet: p });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// M5: Cache invalidation — if total candidate count changed, re-decrypt everything
|
||||
var totalCandidates = candidates.length;
|
||||
var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt;
|
||||
|
||||
// M5: Delta fetch — only decrypt packets newer than lastTs
|
||||
if (!needFullDecrypt && cachedMsgs.length > 0 && lastTs) {
|
||||
// Filter candidates to only those newer than cached lastTimestamp
|
||||
var newCandidates = candidates.filter(function (c) {
|
||||
var ts = c.packet.first_seen || c.packet.timestamp || '';
|
||||
return ts > lastTs;
|
||||
});
|
||||
|
||||
if (newCandidates.length === 0) {
|
||||
// Nothing new — return cache as-is
|
||||
return { messages: cachedMsgs, fromCache: true };
|
||||
}
|
||||
|
||||
// Decrypt only new candidates
|
||||
var newDecrypted = await decryptCandidates(keyBytes, newCandidates);
|
||||
if (newDecrypted.wrongKey) {
|
||||
return { messages: cachedMsgs, wrongKey: true };
|
||||
}
|
||||
|
||||
// Merge: cached + new, deduplicate by packetHash, sort chronologically
|
||||
var merged = deduplicateAndMerge(cachedMsgs, newDecrypted.messages);
|
||||
var newLastTs = merged.length ? merged[merged.length - 1].timestamp : lastTs;
|
||||
ChannelDecrypt.setCache(cacheKey, merged, newLastTs, totalCandidates);
|
||||
return { messages: merged, deltaCount: newDecrypted.messages.length };
|
||||
}
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return { messages: cachedMsgs, empty: true };
|
||||
}
|
||||
|
||||
// Full decrypt
|
||||
var result = await decryptCandidates(keyBytes, candidates);
|
||||
if (result.wrongKey) {
|
||||
return { messages: result.messages, wrongKey: true };
|
||||
}
|
||||
|
||||
var decrypted = result.messages;
|
||||
// Sort chronologically (oldest first)
|
||||
decrypted.sort(function (a, b) {
|
||||
var ta = a.timestamp || '';
|
||||
var tb = b.timestamp || '';
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
|
||||
// M5: Cache results
|
||||
var newLastTimestamp = decrypted.length ? decrypted[decrypted.length - 1].timestamp : '';
|
||||
ChannelDecrypt.setCache(cacheKey, decrypted, newLastTimestamp, totalCandidates);
|
||||
|
||||
return { messages: decrypted };
|
||||
}
|
||||
|
||||
/** Decrypt an array of candidate packets. Returns { messages, wrongKey }. */
|
||||
async function decryptCandidates(keyBytes, candidates) {
|
||||
// Sort newest first for progressive rendering
|
||||
candidates.sort(function (a, b) {
|
||||
var ta = a.packet.first_seen || a.packet.timestamp || '';
|
||||
var tb = b.packet.first_seen || b.packet.timestamp || '';
|
||||
return tb.localeCompare(ta);
|
||||
});
|
||||
|
||||
var decrypted = [];
|
||||
var macFailCount = 0;
|
||||
var macCheckCount = 0;
|
||||
|
||||
for (var j = 0; j < candidates.length; j++) {
|
||||
var c = candidates[j];
|
||||
|
||||
if (c.type === 'already_decrypted') {
|
||||
var d = c.decoded;
|
||||
var sender = d.sender || 'Unknown';
|
||||
var text = d.text || '';
|
||||
var ci = text.indexOf(': ');
|
||||
if (ci > 0 && ci < 50 && text.substring(0, ci) === sender) {
|
||||
text = text.substring(ci + 2);
|
||||
}
|
||||
decrypted.push({
|
||||
sender: sender, text: text,
|
||||
timestamp: c.packet.first_seen || c.packet.timestamp,
|
||||
sender_timestamp: d.sender_timestamp || null,
|
||||
packetHash: c.packet.hash, packetId: c.packet.id,
|
||||
hops: d.path_len || 0, snr: c.packet.snr || null,
|
||||
observers: c.packet.observer_name ? [c.packet.observer_name] : [],
|
||||
repeats: 1
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
macCheckCount++;
|
||||
var result = await ChannelDecrypt.decryptPacket(keyBytes, c.decoded.mac, c.decoded.encryptedData);
|
||||
if (result) {
|
||||
macFailCount = 0;
|
||||
decrypted.push({
|
||||
sender: result.sender, text: result.message,
|
||||
timestamp: c.packet.first_seen || c.packet.timestamp,
|
||||
sender_timestamp: result.timestamp || null,
|
||||
packetHash: c.packet.hash, packetId: c.packet.id,
|
||||
hops: 0, snr: c.packet.snr || null,
|
||||
observers: c.packet.observer_name ? [c.packet.observer_name] : [],
|
||||
repeats: 1
|
||||
});
|
||||
} else {
|
||||
macFailCount++;
|
||||
if (macCheckCount >= 10 && macFailCount >= macCheckCount) {
|
||||
return { messages: decrypted, wrongKey: true };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { messages: decrypted, wrongKey: false };
|
||||
}
|
||||
|
||||
/** Merge cached and new messages, deduplicate by packetHash, sort chronologically. */
|
||||
function deduplicateAndMerge(cached, newMsgs) {
|
||||
var seen = {};
|
||||
var merged = [];
|
||||
// Add cached first
|
||||
for (var i = 0; i < cached.length; i++) {
|
||||
var key = cached[i].packetHash || ('idx:' + i);
|
||||
if (!seen[key]) { seen[key] = true; merged.push(cached[i]); }
|
||||
}
|
||||
// Add new
|
||||
for (var j = 0; j < newMsgs.length; j++) {
|
||||
var key2 = newMsgs[j].packetHash || ('new:' + j);
|
||||
if (!seen[key2]) { seen[key2] = true; merged.push(newMsgs[j]); }
|
||||
}
|
||||
merged.sort(function (a, b) {
|
||||
var ta = a.timestamp || '';
|
||||
var tb = b.timestamp || '';
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
return merged;
|
||||
}
|
||||
|
||||
function init(app, routeParam) {
|
||||
var _initUrlParams = getHashParams();
|
||||
var _pendingNode = _initUrlParams.get('node');
|
||||
@@ -326,6 +610,21 @@
|
||||
<div class="ch-sidebar" aria-label="Channel list">
|
||||
<div class="ch-sidebar-header">
|
||||
<div class="ch-sidebar-title"><span class="ch-icon">💬</span> Channels</div>
|
||||
<label class="ch-encrypted-toggle" title="Show encrypted channels (no key configured)">
|
||||
<input type="checkbox" id="chShowEncrypted"> <span class="ch-toggle-label">🔒 No key</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="ch-key-input-wrap" style="padding:4px 8px">
|
||||
<form id="chKeyForm" autocomplete="off" class="ch-add-form">
|
||||
<div class="ch-add-row">
|
||||
<input type="text" id="chKeyInput" class="ch-key-input"
|
||||
placeholder="#channelname"
|
||||
aria-label="Channel name or hex key" spellcheck="false">
|
||||
<button type="submit" class="ch-add-btn" title="Add channel">+</button>
|
||||
</div>
|
||||
<div class="ch-add-hint">e.g. #LongFast or 32-char hex key — decrypted in your browser.</div>
|
||||
<div id="chAddStatus" class="ch-add-status" style="display:none"></div>
|
||||
</form>
|
||||
</div>
|
||||
<div id="chRegionFilter" class="region-filter-container" style="padding:0 8px"></div>
|
||||
<div class="ch-channel-list" id="chList" role="listbox" aria-label="Channels">
|
||||
@@ -347,6 +646,17 @@
|
||||
</div>`;
|
||||
|
||||
RegionFilter.init(document.getElementById('chRegionFilter'));
|
||||
|
||||
// Encrypted channels toggle (#727)
|
||||
var showEncryptedCb = document.getElementById('chShowEncrypted');
|
||||
var showEncrypted = localStorage.getItem('channels-show-encrypted') === 'true';
|
||||
showEncryptedCb.checked = showEncrypted;
|
||||
showEncryptedCb.addEventListener('change', function () {
|
||||
showEncrypted = showEncryptedCb.checked;
|
||||
localStorage.setItem('channels-show-encrypted', showEncrypted ? 'true' : 'false');
|
||||
loadChannels(true);
|
||||
});
|
||||
|
||||
regionChangeHandler = RegionFilter.onChange(function () {
|
||||
loadChannels(true).then(async function () {
|
||||
if (!selectedHash) return;
|
||||
@@ -354,8 +664,38 @@
|
||||
});
|
||||
});
|
||||
|
||||
// Channel key input handler (#725 M2, improved UX #759)
|
||||
var chKeyForm = document.getElementById('chKeyForm');
|
||||
if (chKeyForm) {
|
||||
var submitHandler = async function (e) {
|
||||
e.preventDefault();
|
||||
var input = document.getElementById('chKeyInput');
|
||||
var val = (input.value || '').trim();
|
||||
if (!val) return;
|
||||
input.value = '';
|
||||
await addUserChannel(val);
|
||||
};
|
||||
chKeyForm.addEventListener('submit', submitHandler);
|
||||
var chKeyInput = document.getElementById('chKeyInput');
|
||||
if (chKeyInput) {
|
||||
chKeyInput.addEventListener('focus', function () {
|
||||
var st = document.getElementById('chAddStatus');
|
||||
if (st) { st.style.display = 'none'; clearTimeout(statusTimer); statusTimer = null; }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-enable encrypted toggle if deep-linking to an encrypted channel
|
||||
if (routeParam && routeParam.startsWith('enc_') && !showEncrypted) {
|
||||
showEncrypted = true;
|
||||
showEncryptedCb.checked = true;
|
||||
localStorage.setItem('channels-show-encrypted', 'true');
|
||||
}
|
||||
|
||||
loadObserverRegions();
|
||||
loadChannels().then(async function () {
|
||||
// Also load user-added encrypted channels into the sidebar
|
||||
mergeUserChannels();
|
||||
if (routeParam) await selectChannel(routeParam);
|
||||
if (_pendingNode && _pendingNode.length < 200) await showNodeDetail(_pendingNode);
|
||||
});
|
||||
@@ -403,6 +743,29 @@
|
||||
|
||||
// Event delegation for channel selection (touch-friendly)
|
||||
document.getElementById('chList').addEventListener('click', (e) => {
|
||||
// M4: Remove channel button
|
||||
const removeBtn = e.target.closest('[data-remove-channel]');
|
||||
if (removeBtn) {
|
||||
e.stopPropagation();
|
||||
var channelHash = removeBtn.getAttribute('data-remove-channel');
|
||||
if (!channelHash) return;
|
||||
var chName = channelHash.startsWith('user:') ? channelHash.substring(5) : channelHash;
|
||||
if (!confirm('Remove channel "' + chName + '"? This will clear saved keys and cached messages.')) return;
|
||||
ChannelDecrypt.removeKey(chName);
|
||||
// Remove from channels array
|
||||
channels = channels.filter(function (c) { return c.hash !== channelHash; });
|
||||
if (selectedHash === channelHash) {
|
||||
selectedHash = null;
|
||||
messages = [];
|
||||
history.replaceState(null, '', '#/channels');
|
||||
var msgEl2 = document.getElementById('chMessages');
|
||||
if (msgEl2) msgEl2.innerHTML = '<div class="ch-empty">Choose a channel from the sidebar to view messages</div>';
|
||||
var header2 = document.getElementById('chHeader');
|
||||
if (header2) header2.querySelector('.ch-header-text').textContent = 'Select a channel';
|
||||
}
|
||||
renderChannelList();
|
||||
return;
|
||||
}
|
||||
// Color dot click — open picker, don't select channel
|
||||
const dot = e.target.closest('.ch-color-dot');
|
||||
if (dot && window.ChannelColorPicker) {
|
||||
@@ -652,7 +1015,11 @@
|
||||
async function loadChannels(silent) {
|
||||
try {
|
||||
const rp = RegionFilter.getRegionParam();
|
||||
const qs = rp ? '?region=' + encodeURIComponent(rp) : '';
|
||||
var showEnc = localStorage.getItem('channels-show-encrypted') === 'true';
|
||||
var params = [];
|
||||
if (rp) params.push('region=' + encodeURIComponent(rp));
|
||||
if (showEnc) params.push('includeEncrypted=true');
|
||||
const qs = params.length ? '?' + params.join('&') : '';
|
||||
const data = await api('/channels' + qs, { ttl: CLIENT_TTL.channels });
|
||||
channels = (data.channels || []).map(ch => {
|
||||
ch.lastActivityMs = ch.lastActivity ? new Date(ch.lastActivity).getTime() : 0;
|
||||
@@ -679,27 +1046,33 @@
|
||||
});
|
||||
|
||||
el.innerHTML = sorted.map(ch => {
|
||||
const name = ch.name || `Channel ${formatHashHex(ch.hash)}`;
|
||||
const color = getChannelColor(ch.hash);
|
||||
const isEncrypted = ch.encrypted === true;
|
||||
const name = isEncrypted ? (ch.name || 'Unknown') : (ch.name || `Channel ${formatHashHex(ch.hash)}`);
|
||||
const color = isEncrypted ? 'var(--text-muted, #6b7280)' : getChannelColor(ch.hash);
|
||||
const time = ch.lastActivityMs ? formatSecondsAgo(Math.floor((Date.now() - ch.lastActivityMs) / 1000)) : '';
|
||||
const preview = ch.lastSender && ch.lastMessage
|
||||
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
|
||||
: `${ch.messageCount} messages`;
|
||||
const preview = isEncrypted
|
||||
? `${ch.messageCount} encrypted messages (no key configured)`
|
||||
: ch.lastSender && ch.lastMessage
|
||||
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
|
||||
: `${ch.messageCount} messages`;
|
||||
const sel = selectedHash === ch.hash ? ' selected' : '';
|
||||
const abbr = name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase();
|
||||
const encClass = isEncrypted ? ' ch-encrypted' : '';
|
||||
const abbr = isEncrypted ? '🔒' : (name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase());
|
||||
// Channel color dot for color picker (#674)
|
||||
const chColor = window.ChannelColors ? window.ChannelColors.get(ch.hash) : null;
|
||||
const dotStyle = chColor ? ` style="background:${chColor}"` : '';
|
||||
// Left border for assigned color
|
||||
const borderStyle = chColor ? ` style="border-left:3px solid ${chColor}"` : '';
|
||||
// M4: Remove button for user-added channels
|
||||
const removeBtn = ch.userAdded ? ' <button class="ch-remove-btn" data-remove-channel="' + escapeHtml(ch.hash) + '" title="Remove channel" aria-label="Remove ' + escapeHtml(name) + '">✕</button>' : '';
|
||||
|
||||
return `<button class="ch-item${sel}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}">
|
||||
<div class="ch-badge" style="background:${color}" aria-hidden="true">${escapeHtml(abbr)}</div>
|
||||
return `<button class="ch-item${sel}${encClass}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}"${isEncrypted ? ' data-encrypted="true"' : ''}>
|
||||
<div class="ch-badge" style="background:${color}" aria-hidden="true">${isEncrypted ? '🔒' : escapeHtml(abbr)}</div>
|
||||
<div class="ch-item-body">
|
||||
<div class="ch-item-top">
|
||||
<span class="ch-item-name">${escapeHtml(name)}</span>
|
||||
<span class="ch-color-dot" data-channel="${escapeHtml(ch.hash)}"${dotStyle} title="Change channel color" aria-label="Change color for ${escapeHtml(name)}"></span>
|
||||
<span class="ch-item-time" data-channel-hash="${ch.hash}">${time}</span>
|
||||
<span class="ch-item-time" data-channel-hash="${ch.hash}">${time}</span>${removeBtn}
|
||||
</div>
|
||||
<div class="ch-item-preview">${escapeHtml(preview)}</div>
|
||||
</div>
|
||||
@@ -707,7 +1080,7 @@
|
||||
}).join('');
|
||||
}
|
||||
|
||||
async function selectChannel(hash) {
|
||||
async function selectChannel(hash, decryptOpts) {
|
||||
const rp = RegionFilter.getRegionParam() || '';
|
||||
const request = beginMessageRequest(hash, rp);
|
||||
selectedHash = hash;
|
||||
@@ -722,6 +1095,110 @@
|
||||
document.querySelector('.ch-layout')?.classList.add('ch-show-main');
|
||||
|
||||
const msgEl = document.getElementById('chMessages');
|
||||
|
||||
// Shared helper: fetch, decrypt, and render messages for a channel key (M5: cache-first)
|
||||
async function decryptAndRender(keyHex, channelHashByte, channelName) {
|
||||
msgEl.innerHTML = '<div class="ch-loading">Decrypting messages…</div>';
|
||||
var result = await fetchAndDecryptChannel(keyHex, channelHashByte, channelName, {
|
||||
onCacheHit: function (cachedMsgs) {
|
||||
// M5: Render cached messages immediately while delta fetch runs
|
||||
messages = cachedMsgs;
|
||||
if (messages.length > 0) {
|
||||
header.querySelector('.ch-header-text').textContent = name + ' — ' + messages.length + ' messages (cached)';
|
||||
renderMessages();
|
||||
scrollToBottom();
|
||||
}
|
||||
}
|
||||
});
|
||||
if (isStaleMessageRequest(request)) return true;
|
||||
if (result.wrongKey) {
|
||||
msgEl.innerHTML = '<div class="ch-empty ch-wrong-key">🔒 Key does not match — no messages could be decrypted</div>';
|
||||
return true;
|
||||
}
|
||||
if (result.error) {
|
||||
msgEl.innerHTML = '<div class="ch-empty">' + escapeHtml(result.error) + '</div>';
|
||||
return true;
|
||||
}
|
||||
messages = result.messages || [];
|
||||
if (messages.length === 0) {
|
||||
msgEl.innerHTML = '<div class="ch-empty">No encrypted messages found for this channel</div>';
|
||||
} else {
|
||||
header.querySelector('.ch-header-text').textContent = `${name} — ${messages.length} messages (decrypted)`;
|
||||
renderMessages();
|
||||
scrollToBottom();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Client-side decryption path (#725 M2)
|
||||
if (decryptOpts && decryptOpts.userKey) {
|
||||
await decryptAndRender(decryptOpts.userKey, decryptOpts.channelHashByte, decryptOpts.channelName);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if this is a user-added channel that needs decryption
|
||||
var storedKeys = typeof ChannelDecrypt !== 'undefined' ? ChannelDecrypt.getStoredKeys() : {};
|
||||
if (hash.startsWith('user:')) {
|
||||
var chName = hash.substring(5);
|
||||
if (storedKeys[chName]) {
|
||||
var keyHex = storedKeys[chName];
|
||||
var keyBytes = ChannelDecrypt.hexToBytes(keyHex);
|
||||
var hashByte = await ChannelDecrypt.computeChannelHash(keyBytes);
|
||||
await decryptAndRender(keyHex, hashByte, chName);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Also check if an encrypted channel hash matches a stored key
|
||||
if (ch && ch.encrypted) {
|
||||
for (var kn in storedKeys) {
|
||||
var kh = storedKeys[kn];
|
||||
var kb = ChannelDecrypt.hexToBytes(kh);
|
||||
var hb = await ChannelDecrypt.computeChannelHash(kb);
|
||||
if (String(hb) === String(hash) || String(ch.hash) === String(hb)) {
|
||||
await decryptAndRender(kh, hb, kn);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// #781: No matching key found — show lock message instead of fetching gibberish
|
||||
msgEl.innerHTML = '<div class="ch-empty">🔒 This channel is encrypted and no decryption key is configured</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
// #811: Deep link to a `#`-named channel that's not in the loaded list.
|
||||
// If a stored key matches, decrypt. Otherwise we must distinguish an
|
||||
// encrypted-no-key channel (show lock) from an unencrypted channel that
|
||||
// simply isn't in the toggle-off list (#825 — must fall through to REST).
|
||||
if (hash.charAt(0) === '#') {
|
||||
if (storedKeys[hash]) {
|
||||
var keyHex2 = storedKeys[hash];
|
||||
var keyBytes2 = ChannelDecrypt.hexToBytes(keyHex2);
|
||||
var hashByte2 = await ChannelDecrypt.computeChannelHash(keyBytes2);
|
||||
await decryptAndRender(keyHex2, hashByte2, hash);
|
||||
return;
|
||||
}
|
||||
// #825: confirm encrypted-ness via an encrypted-included channel list
|
||||
// before assuming a lock state. Conservative on error — fall through.
|
||||
// Show a loading affordance so cold deep links don't display stale content
|
||||
// for the duration of the metadata RTT (cached 15s thereafter).
|
||||
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
|
||||
try {
|
||||
var rpInc = RegionFilter.getRegionParam();
|
||||
var paramsInc = ['includeEncrypted=true'];
|
||||
if (rpInc) paramsInc.push('region=' + encodeURIComponent(rpInc));
|
||||
var allCh = await api('/channels?' + paramsInc.join('&'), { ttl: CLIENT_TTL.channels });
|
||||
if (isStaleMessageRequest(request)) return;
|
||||
var foundCh = (allCh.channels || []).find(function (c) { return c.hash === hash; });
|
||||
if (foundCh && foundCh.encrypted === true) {
|
||||
msgEl.innerHTML = '<div class="ch-empty">🔒 This channel is encrypted and no decryption key is configured</div>';
|
||||
return;
|
||||
}
|
||||
// Unencrypted (or unknown) — fall through to the REST fetch below.
|
||||
} catch (e) {
|
||||
// ignore — fall through to REST fetch
|
||||
}
|
||||
}
|
||||
|
||||
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
|
||||
|
||||
try {
|
||||
@@ -743,6 +1220,9 @@
|
||||
|
||||
async function refreshMessages(opts) {
|
||||
if (!selectedHash) return;
|
||||
// Skip refresh for encrypted channels — no messages to fetch
|
||||
var selCh = channels.find(function (c) { return c.hash === selectedHash; });
|
||||
if (selCh && selCh.encrypted) return;
|
||||
opts = opts || {};
|
||||
const msgEl = document.getElementById('chMessages');
|
||||
if (!msgEl) return;
|
||||
|
||||
@@ -1173,6 +1173,10 @@
|
||||
'<details style="margin-top:12px"><summary style="font-size:12px;font-weight:600;cursor:pointer;color:var(--text-muted)">Raw JSON</summary>' +
|
||||
'<textarea id="cv2ExportJson" style="width:100%;min-height:200px;font-family:var(--mono);font-size:12px;background:var(--surface-1);border:1px solid var(--border);border-radius:6px;padding:12px;color:var(--text);resize:vertical;box-sizing:border-box;margin-top:8px">' + esc(json) + '</textarea>' +
|
||||
'</details>' +
|
||||
'<p class="cust-section-title" style="margin-top:20px">Tools</p>' +
|
||||
'<p style="font-size:12px;color:var(--text-muted);margin-bottom:10px">Server-side configuration helpers.</p>' +
|
||||
'<a href="/geofilter-builder.html" target="_blank" style="display:inline-block;padding:7px 14px;background:var(--surface-1);border:1px solid var(--border);border-radius:6px;color:var(--accent);font-size:13px;text-decoration:none;font-weight:500">🗺️ GeoFilter Builder →</a>' +
|
||||
'<p style="font-size:11px;color:var(--text-muted);margin-top:6px">Draw a polygon on the map to generate a <code style="font-family:var(--mono)">geo_filter</code> block for <code style="font-family:var(--mono)">config.json</code>.</p>' +
|
||||
'</div>';
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>GeoFilter Builder — CoreScope</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"/>
|
||||
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { font-family: system-ui, sans-serif; background: #1a1a2e; color: #e0e0e0; height: 100vh; display: flex; flex-direction: column; }
|
||||
header { padding: 12px 16px; background: #0f0f23; border-bottom: 1px solid #333; display: flex; align-items: center; gap: 16px; flex-wrap: wrap; }
|
||||
header h1 { font-size: 1rem; font-weight: 600; color: #4a9eff; white-space: nowrap; }
|
||||
.controls { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
button { padding: 6px 14px; border: none; border-radius: 6px; cursor: pointer; font-size: 0.85rem; font-weight: 500; }
|
||||
#btnUndo { background: #333; color: #ccc; }
|
||||
#btnClear { background: #5a2020; color: #ffaaaa; }
|
||||
#btnUndo:hover { background: #444; }
|
||||
#btnClear:hover { background: #7a2020; }
|
||||
.hint { font-size: 0.8rem; color: #888; margin-left: auto; }
|
||||
#map { flex: 1; }
|
||||
#output-panel { background: #0f0f23; border-top: 1px solid #333; padding: 12px 16px; display: flex; gap: 12px; align-items: flex-start; }
|
||||
#output-panel label { font-size: 0.75rem; color: #888; white-space: nowrap; padding-top: 6px; }
|
||||
#output { flex: 1; background: #111; border: 1px solid #333; border-radius: 6px; padding: 10px 12px; font-family: monospace; font-size: 0.78rem; color: #7ec8e3; white-space: pre; overflow-x: auto; min-height: 54px; max-height: 140px; overflow-y: auto; cursor: text; }
|
||||
#output.empty { color: #555; font-style: italic; }
|
||||
#btnCopy { padding: 6px 14px; background: #1a4a7a; color: #7ec8e3; border-radius: 6px; border: none; cursor: pointer; font-size: 0.85rem; white-space: nowrap; align-self: flex-end; }
|
||||
#btnCopy:hover { background: #2a6aaa; }
|
||||
#btnCopy.copied { background: #1a6a3a; color: #7effa0; }
|
||||
#counter { font-size: 0.8rem; color: #888; padding-top: 6px; white-space: nowrap; }
|
||||
.bufferRow { display: flex; align-items: center; gap: 8px; }
|
||||
.bufferRow label { font-size: 0.85rem; color: #aaa; }
|
||||
.bufferRow input { width: 60px; padding: 5px 8px; background: #222; border: 1px solid #444; border-radius: 6px; color: #eee; font-size: 0.85rem; }
|
||||
#help-bar { background: #0f0f23; padding: 6px 16px; font-size: 0.75rem; color: #666; border-top: 1px solid #222; }
|
||||
#help-bar a { color: #4a9eff; text-decoration: none; }
|
||||
#help-bar a:hover { text-decoration: underline; }
|
||||
#back-link { font-size: 0.8rem; color: #4a9eff; text-decoration: none; white-space: nowrap; }
|
||||
#back-link:hover { text-decoration: underline; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a href="/" id="back-link">← CoreScope</a>
|
||||
<h1>GeoFilter Builder</h1>
|
||||
<div class="controls">
|
||||
<button id="btnUndo">↩ Undo</button>
|
||||
<button id="btnClear">✕ Clear</button>
|
||||
</div>
|
||||
<div class="bufferRow">
|
||||
<label for="bufferKm">Buffer km:</label>
|
||||
<!-- Extra margin (km) outside the polygon edge that still passes the filter -->
|
||||
<input type="number" id="bufferKm" value="20" min="0" max="500"/>
|
||||
</div>
|
||||
<span class="hint">Click on the map to add polygon points</span>
|
||||
</header>
|
||||
|
||||
<div id="map"></div>
|
||||
|
||||
<!-- Output panel: shows the geo_filter JSON block ready to paste into config.json -->
|
||||
<div id="output-panel">
|
||||
<label>config.json</label>
|
||||
<div id="output" class="empty">Add at least 3 points to generate config…</div>
|
||||
<div style="display:flex;flex-direction:column;gap:8px;align-items:flex-end">
|
||||
<span id="counter">0 points</span>
|
||||
<button id="btnCopy">Copy</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Instructions: paste the output into config.json as a top-level "geo_filter" key, then restart the server -->
|
||||
<div id="help-bar">
|
||||
Copy the JSON above → paste as a top-level key in <code>config.json</code> → restart the server.
|
||||
Nodes with no GPS fix always pass through. Remove the <code>geo_filter</code> block to disable filtering.
|
||||
· <a href="https://github.com/Kpa-clawbot/CoreScope/blob/master/docs/user-guide/geofilter.md" target="_blank">Documentation ↗</a>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const map = L.map('map').setView([50.5, 4.4], 8);
|
||||
|
||||
L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
|
||||
attribution: '© OpenStreetMap © CartoDB',
|
||||
maxZoom: 19
|
||||
}).addTo(map);
|
||||
|
||||
let points = [];
|
||||
let markers = [];
|
||||
let polygon = null;
|
||||
let closingLine = null;
|
||||
|
||||
function latLonPair(latlng) {
|
||||
return [parseFloat(latlng.lat.toFixed(6)), parseFloat(latlng.lng.toFixed(6))];
|
||||
}
|
||||
|
||||
function render() {
|
||||
// Remove existing polygon and closing line
|
||||
if (polygon) { map.removeLayer(polygon); polygon = null; }
|
||||
if (closingLine) { map.removeLayer(closingLine); closingLine = null; }
|
||||
|
||||
if (points.length >= 3) {
|
||||
polygon = L.polygon(points, {
|
||||
color: '#4a9eff', weight: 2, fillColor: '#4a9eff', fillOpacity: 0.12
|
||||
}).addTo(map);
|
||||
} else if (points.length === 2) {
|
||||
closingLine = L.polyline(points, { color: '#4a9eff', weight: 2, dashArray: '5,5' }).addTo(map);
|
||||
}
|
||||
|
||||
updateOutput();
|
||||
}
|
||||
|
||||
function updateOutput() {
|
||||
const el = document.getElementById('output');
|
||||
const counter = document.getElementById('counter');
|
||||
counter.textContent = points.length + ' point' + (points.length !== 1 ? 's' : '');
|
||||
|
||||
if (points.length < 3) {
|
||||
el.textContent = 'Add at least 3 points to generate config…';
|
||||
el.classList.add('empty');
|
||||
return;
|
||||
}
|
||||
el.classList.remove('empty');
|
||||
|
||||
const bufferKm = parseFloat(document.getElementById('bufferKm').value) || 0;
|
||||
// Output format: { "geo_filter": { "bufferKm": N, "polygon": [[lat,lon], ...] } }
|
||||
// Paste this as a top-level key in config.json
|
||||
const config = { bufferKm, polygon: points };
|
||||
el.textContent = JSON.stringify({ geo_filter: config }, null, 2);
|
||||
}
|
||||
|
||||
map.on('click', function(e) {
|
||||
const pt = latLonPair(e.latlng);
|
||||
points.push(pt);
|
||||
|
||||
const idx = points.length;
|
||||
const marker = L.circleMarker(e.latlng, {
|
||||
radius: 6, color: '#4a9eff', weight: 2, fillColor: '#4a9eff', fillOpacity: 0.9
|
||||
}).addTo(map).bindTooltip(String(idx), { permanent: true, direction: 'top', offset: [0, -8], className: 'pt-label' });
|
||||
markers.push(marker);
|
||||
|
||||
render();
|
||||
});
|
||||
|
||||
document.getElementById('btnUndo').addEventListener('click', function() {
|
||||
if (!points.length) return;
|
||||
points.pop();
|
||||
const m = markers.pop();
|
||||
if (m) map.removeLayer(m);
|
||||
render();
|
||||
});
|
||||
|
||||
document.getElementById('btnClear').addEventListener('click', function() {
|
||||
points = [];
|
||||
markers.forEach(m => map.removeLayer(m));
|
||||
markers = [];
|
||||
render();
|
||||
});
|
||||
|
||||
document.getElementById('bufferKm').addEventListener('input', updateOutput);
|
||||
|
||||
document.getElementById('btnCopy').addEventListener('click', function() {
|
||||
if (points.length < 3) return;
|
||||
const text = document.getElementById('output').textContent;
|
||||
navigator.clipboard.writeText(text).then(() => {
|
||||
const btn = document.getElementById('btnCopy');
|
||||
btn.textContent = 'Copied!';
|
||||
btn.classList.add('copied');
|
||||
setTimeout(() => { btn.textContent = 'Copy'; btn.classList.remove('copied'); }, 2000);
|
||||
});
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -81,9 +81,13 @@ window.HopDisplay = (function() {
|
||||
const regionalConflicts = conflicts.filter(c => c.regional);
|
||||
const badgeCount = regionalConflicts.length > 0 ? regionalConflicts.length : (globalFallback ? conflicts.length : 0);
|
||||
const conflictData = escapeHtml(JSON.stringify({ h, conflicts, globalFallback }));
|
||||
const warnBadge = badgeCount > 1
|
||||
const conflictBadge = badgeCount > 1
|
||||
? ` <button class="hop-conflict-btn" data-conflict='${conflictData}' onclick="event.preventDefault();event.stopPropagation();HopDisplay._showFromBtn(this)" title="${badgeCount} candidates — click for details">⚠${badgeCount}</button>`
|
||||
: '';
|
||||
const unreliableBadge = unreliable
|
||||
? ' <button class="hop-unreliable-btn" aria-label="Unreliable name resolution" title="Unreliable name resolution — this hash\u2192name match is geographically inconsistent with the surrounding path hops. The repeater itself may be fine; this specific hop assignment is uncertain.">⚠️</button>'
|
||||
: '';
|
||||
const warnBadge = conflictBadge + unreliableBadge;
|
||||
|
||||
const cls = [
|
||||
'hop',
|
||||
|
||||
@@ -95,6 +95,7 @@
|
||||
<script src="table-sort.js?v=__BUST__"></script>
|
||||
<script src="packet-filter.js?v=__BUST__"></script>
|
||||
<script src="packet-helpers.js?v=__BUST__"></script>
|
||||
<script src="channel-decrypt.js?v=__BUST__"></script>
|
||||
<script src="channel-colors.js?v=__BUST__"></script>
|
||||
<script src="channel-color-picker.js?v=__BUST__"></script>
|
||||
<script src="packets.js?v=__BUST__"></script>
|
||||
|
||||
+18
-3
@@ -132,7 +132,7 @@
|
||||
|
||||
/* ---- Node Detail Panel ---- */
|
||||
.live-node-detail {
|
||||
top: 60px;
|
||||
top: 64px;
|
||||
right: 12px;
|
||||
width: 320px;
|
||||
max-height: calc(100vh - 140px);
|
||||
@@ -325,11 +325,14 @@
|
||||
}
|
||||
.live-stats-row { flex-wrap: wrap; gap: 4px; }
|
||||
.live-stat-pill { font-size: 11px; padding: 2px 7px; }
|
||||
.live-toggles { font-size: 10px; gap: 6px; margin-left: 0; }
|
||||
.live-toggles { font-size: 10px; gap: 6px; margin-left: 0; overflow-x: auto; flex-wrap: nowrap; -webkit-overflow-scrolling: touch; width: 100%; min-width: 0; }
|
||||
.live-title { font-size: 12px; letter-spacing: 1px; }
|
||||
/* #203 — bottom-sheet node detail on mobile */
|
||||
.live-node-detail { width: 100%; right: 0; left: 0; top: auto; bottom: 0; max-height: 60vh; border-radius: 16px 16px 0 0; overflow-y: auto; }
|
||||
.live-node-detail { width: 100%; right: 0; left: 0; top: auto; bottom: 0; max-height: 60dvh; border-radius: 16px 16px 0 0; overflow-y: auto; z-index: 1050; }
|
||||
.live-node-detail.hidden { transform: translateY(100%); }
|
||||
/* Close button was unreachable: panel-header collapsed to 8px on mobile, panel-content
|
||||
scroll area started at y=8, overlapping the button's 36px tap target (y=6–42) */
|
||||
.live-node-detail .panel-header { min-height: 44px; }
|
||||
.feed-detail-card {
|
||||
position: fixed !important;
|
||||
right: 0 !important;
|
||||
@@ -689,6 +692,18 @@
|
||||
.live-feed { bottom: 68px; }
|
||||
.feed-show-btn { bottom: 68px !important; }
|
||||
|
||||
/* Backdrop for mobile tap-outside-to-close (#797) */
|
||||
.node-detail-backdrop {
|
||||
display: none;
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
z-index: 1049;
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
}
|
||||
@media (max-width: 640px) {
|
||||
.node-detail-backdrop.active { display: block; }
|
||||
}
|
||||
|
||||
/* Mobile VCR */
|
||||
@media (max-width: 640px) {
|
||||
/* Mobile VCR: two-row stacked layout */
|
||||
|
||||
+10
-3
@@ -849,6 +849,7 @@
|
||||
<div class="panel-content" aria-live="polite" aria-relevant="additions" role="log"></div>
|
||||
</div>
|
||||
<button class="feed-show-btn hidden" id="feedShowBtn" title="Show feed">📋</button>
|
||||
<div id="nodeDetailBackdrop" class="node-detail-backdrop"></div>
|
||||
<div class="live-overlay live-node-detail hidden" id="liveNodeDetail">
|
||||
<div class="panel-header">
|
||||
<button class="panel-corner-btn" data-panel="liveNodeDetail" title="Move panel to next corner" aria-label="Move panel to next corner">◫</button>
|
||||
@@ -1216,10 +1217,14 @@
|
||||
// Node detail panel
|
||||
const nodeDetailPanel = document.getElementById('liveNodeDetail');
|
||||
const nodeDetailContent = document.getElementById('nodeDetailContent');
|
||||
document.getElementById('nodeDetailClose').addEventListener('click', () => {
|
||||
const nodeDetailBackdrop = document.getElementById('nodeDetailBackdrop');
|
||||
function closeNodeDetail() {
|
||||
activeNodeDetailKey = null;
|
||||
nodeDetailPanel.classList.add('hidden');
|
||||
});
|
||||
nodeDetailBackdrop.classList.remove('active');
|
||||
}
|
||||
document.getElementById('nodeDetailClose').addEventListener('click', closeNodeDetail);
|
||||
nodeDetailBackdrop.addEventListener('click', closeNodeDetail);
|
||||
|
||||
// Feed panel resize handle (#27)
|
||||
const savedFeedWidth = localStorage.getItem('live-feed-width');
|
||||
@@ -1451,6 +1456,7 @@
|
||||
const panel = document.getElementById('liveNodeDetail');
|
||||
const content = document.getElementById('nodeDetailContent');
|
||||
panel.classList.remove('hidden');
|
||||
document.getElementById('nodeDetailBackdrop').classList.add('active');
|
||||
content.innerHTML = '<div style="padding:20px;color:var(--text-muted)">Loading…</div>';
|
||||
try {
|
||||
const [data, healthData] = await Promise.all([
|
||||
@@ -2730,6 +2736,7 @@
|
||||
const preview = text ? ' ' + (text.length > 35 ? text.slice(0, 35) + '…' : text) : '';
|
||||
const hopStr = hops.length ? `<span class="feed-hops">${hops.length}⇢</span>` : '';
|
||||
const obsBadge = pkt.observation_count > 1 ? `<span class="badge badge-obs" style="font-size:10px;margin-left:4px">👁 ${pkt.observation_count}</span>` : '';
|
||||
const anomalyIcon = (pkt.decoded && pkt.decoded.anomaly) ? '<span title="Anomaly detected" style="margin-left:4px">⚠️</span>' : '';
|
||||
var _ccPayload2 = (pkt.decoded || {}).payload || {};
|
||||
var _ccChan = (typeName === 'GRP_TXT' || typeName === 'CHAN') ? (_ccPayload2.channel || null) : null;
|
||||
var dotHtml = _ccChan ? _feedColorDot(_ccChan) : '';
|
||||
@@ -2744,7 +2751,7 @@
|
||||
item.innerHTML = `
|
||||
<span class="feed-icon" style="color:${color}">${icon}</span>
|
||||
<span class="feed-type" style="color:${color}">${typeName}</span>
|
||||
${dotHtml}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
|
||||
${dotHtml}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}${anomalyIcon}
|
||||
<span class="feed-text">${escapeHtml(preview)}</span>
|
||||
<span class="feed-time" data-ts="${pkt._ts || Date.now()}">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
|
||||
`;
|
||||
|
||||
+44
-10
@@ -25,7 +25,7 @@
|
||||
|
||||
// Roles loaded from shared roles.js (ROLE_STYLE, ROLE_LABELS, ROLE_COLORS globals)
|
||||
|
||||
function makeMarkerIcon(role, isStale) {
|
||||
function makeMarkerIcon(role, isStale, isAlsoObserver) {
|
||||
const s = ROLE_STYLE[role] || ROLE_STYLE.companion;
|
||||
const size = s.radius * 2 + 4;
|
||||
const c = size / 2;
|
||||
@@ -56,7 +56,22 @@
|
||||
default: // circle
|
||||
path = `<circle cx="${c}" cy="${c}" r="${c-2}" fill="${s.color}" stroke="#fff" stroke-width="2"/>`;
|
||||
}
|
||||
const svg = `<svg width="${size}" height="${size}" viewBox="0 0 ${size} ${size}" xmlns="http://www.w3.org/2000/svg">${path}</svg>`;
|
||||
// If this node is also an observer, add a small star overlay
|
||||
let obsOverlay = '';
|
||||
if (isAlsoObserver) {
|
||||
const starSize = 8;
|
||||
const sx = size - starSize, sy = 0;
|
||||
const scx = starSize / 2, scy = starSize / 2, so = starSize / 2 - 0.5, si = so * 0.4;
|
||||
let starPts = '';
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const aO = (i * 72 - 90) * Math.PI / 180;
|
||||
const aI = ((i * 72) + 36 - 90) * Math.PI / 180;
|
||||
starPts += `${scx + so * Math.cos(aO)},${scy + so * Math.sin(aO)} `;
|
||||
starPts += `${scx + si * Math.cos(aI)},${scy + si * Math.sin(aI)} `;
|
||||
}
|
||||
obsOverlay = `<g transform="translate(${sx},${sy})"><polygon points="${starPts.trim()}" fill="${ROLE_COLORS.observer || '#f1c40f'}" stroke="#fff" stroke-width="0.8"/></g>`;
|
||||
}
|
||||
const svg = `<svg width="${size}" height="${size}" viewBox="0 0 ${size} ${size}" xmlns="http://www.w3.org/2000/svg">${path}${obsOverlay}</svg>`;
|
||||
return L.divIcon({
|
||||
html: svg,
|
||||
className: 'meshcore-marker' + (isStale ? ' marker-stale' : ''),
|
||||
@@ -66,14 +81,16 @@
|
||||
});
|
||||
}
|
||||
|
||||
function makeRepeaterLabelIcon(node, isStale) {
|
||||
function makeRepeaterLabelIcon(node, isStale, isAlsoObserver) {
|
||||
var s = ROLE_STYLE['repeater'] || ROLE_STYLE.companion;
|
||||
var hs = node.hash_size || 1;
|
||||
// Show the short mesh hash ID (first N bytes of pubkey, uppercased)
|
||||
var shortHash = node.public_key ? node.public_key.slice(0, hs * 2).toUpperCase() : '??';
|
||||
var bgColor = s.color;
|
||||
// If this repeater is also an observer, show a star indicator inside the label
|
||||
var obsIndicator = isAlsoObserver ? ' <span style="color:' + (ROLE_COLORS.observer || '#f1c40f') + ';font-size:13px;line-height:1;" title="Also an observer">★</span>' : '';
|
||||
var html = '<div style="background:' + bgColor + ';color:#fff;font-weight:bold;font-size:11px;padding:2px 5px;border-radius:3px;border:2px solid #fff;box-shadow:0 1px 3px rgba(0,0,0,0.4);text-align:center;line-height:1.2;white-space:nowrap;">' +
|
||||
shortHash + '</div>';
|
||||
shortHash + obsIndicator + '</div>';
|
||||
return L.divIcon({
|
||||
html: html,
|
||||
className: 'meshcore-marker meshcore-label-marker' + (isStale ? ' marker-stale' : ''),
|
||||
@@ -547,7 +564,8 @@
|
||||
const el = document.getElementById('mcRoleChecks');
|
||||
if (!el) return;
|
||||
el.innerHTML = '';
|
||||
const obsCount = observers.filter(o => o.lat && o.lon).length;
|
||||
const nodePubkeys = new Set(nodes.map(n => (n.public_key || '').toLowerCase()));
|
||||
const obsCount = observers.filter(o => o.lat && o.lon && !(o.id && nodePubkeys.has(o.id.toLowerCase()))).length;
|
||||
const roles = ['repeater', 'companion', 'room', 'sensor', 'observer'];
|
||||
const shapeMap = { repeater: '◆', companion: '●', room: '■', sensor: '▲', observer: '★' };
|
||||
|
||||
@@ -638,6 +656,7 @@
|
||||
var _renderingMarkers = false;
|
||||
var _lastDeconflictZoom = null;
|
||||
var _currentMarkerData = []; // stored marker data for zoom-only repositioning
|
||||
var _observerByPubkey = new Map(); // observer id (pubkey) → observer object, rebuilt on each render
|
||||
var _zoomResizeTimer = null;
|
||||
|
||||
function deconflictLabels(markers, mapRef) {
|
||||
@@ -780,19 +799,31 @@
|
||||
|
||||
const allMarkers = [];
|
||||
|
||||
// Build a set of observer public keys for quick lookup
|
||||
_observerByPubkey = new Map();
|
||||
for (const obs of observers) {
|
||||
if (obs.id) _observerByPubkey.set(obs.id.toLowerCase(), obs);
|
||||
}
|
||||
|
||||
for (const node of filtered) {
|
||||
const lastSeenTime = node.last_heard || node.last_seen;
|
||||
const isStale = getNodeStatus(node.role || 'companion', lastSeenTime ? new Date(lastSeenTime).getTime() : 0) === 'stale';
|
||||
const pk = (node.public_key || '').toLowerCase();
|
||||
const isAlsoObserver = _observerByPubkey.has(pk);
|
||||
const useLabel = node.role === 'repeater' && filters.hashLabels;
|
||||
const icon = useLabel ? makeRepeaterLabelIcon(node, isStale) : makeMarkerIcon(node.role || 'companion', isStale);
|
||||
const icon = useLabel ? makeRepeaterLabelIcon(node, isStale, isAlsoObserver) : makeMarkerIcon(node.role || 'companion', isStale, isAlsoObserver);
|
||||
const latLng = L.latLng(node.lat, node.lon);
|
||||
allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: (node.name || 'Unknown') + ' (' + (node.role || 'node') + ')' });
|
||||
allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: (node.name || 'Unknown') + ' (' + (node.role || 'node') + (isAlsoObserver ? ' + observer' : '') + ')' });
|
||||
}
|
||||
|
||||
// Add observer markers
|
||||
// Add observer markers (skip observers already represented as a node marker)
|
||||
// Build set of node pubkeys that are displayed on the map
|
||||
const displayedNodePubkeys = new Set(filtered.map(n => (n.public_key || '').toLowerCase()));
|
||||
if (filters.observer) {
|
||||
for (const obs of observers) {
|
||||
if (!obs.lat || !obs.lon) continue;
|
||||
// Skip observers whose pubkey matches a displayed node — they're shown as combined markers
|
||||
if (obs.id && displayedNodePubkeys.has(obs.id.toLowerCase())) continue;
|
||||
const icon = makeMarkerIcon('observer');
|
||||
const latLng = L.latLng(obs.lat, obs.lon);
|
||||
allMarkers.push({ latLng, node: obs, icon, isLabel: false, popupFn: function() { return buildObserverPopup(obs); }, alt: (obs.name || obs.id || 'Unknown') + ' (observer)' });
|
||||
@@ -909,6 +940,9 @@
|
||||
const loc = (node.lat && node.lon) ? `${node.lat.toFixed(5)}, ${node.lon.toFixed(5)}` : '—';
|
||||
const lastAdvert = node.last_seen ? timeAgo(node.last_seen) : '—';
|
||||
const roleBadge = `<span style="display:inline-block;padding:2px 8px;border-radius:12px;font-size:11px;font-weight:600;background:${ROLE_COLORS[node.role] || '#4b5563'};color:#fff;">${(node.role || 'unknown').toUpperCase()}</span>`;
|
||||
// Check if this node is also an observer (combined repeater+observer)
|
||||
const matchingObs = node.public_key ? _observerByPubkey.get(node.public_key.toLowerCase()) : null;
|
||||
const obsBadge = matchingObs ? ` <span style="display:inline-block;padding:2px 8px;border-radius:12px;font-size:11px;font-weight:600;background:${ROLE_COLORS.observer || '#f1c40f'};color:#fff;">OBSERVER</span>` : '';
|
||||
const hs = node.hash_size || 1;
|
||||
const hashPrefix = node.public_key ? node.public_key.slice(0, hs * 2).toUpperCase() : '—';
|
||||
const hashPrefixRow = `<dt style="color:var(--text-muted);float:left;clear:left;width:80px;padding:2px 0;">Hash Prefix</dt>
|
||||
@@ -917,7 +951,7 @@
|
||||
return `
|
||||
<div class="map-popup" style="font-family:var(--font);min-width:180px;">
|
||||
<h3 style="font-weight:700;font-size:14px;margin:0 0 4px;">${safeEsc(node.name || 'Unknown')}</h3>
|
||||
${roleBadge}
|
||||
${roleBadge}${obsBadge}
|
||||
<dl style="margin-top:8px;font-size:12px;">
|
||||
${hashPrefixRow}
|
||||
<dt style="color:var(--text-muted);float:left;clear:left;width:80px;padding:2px 0;">Key</dt>
|
||||
@@ -931,7 +965,7 @@
|
||||
</dl>
|
||||
<div style="margin-top:8px;clear:both;">
|
||||
<a href="#/nodes/${node.public_key}" style="color:var(--accent);font-size:12px;">View Node →</a>
|
||||
${node.public_key ? ` · <a href="#" data-show-neighbors data-pubkey="${escapeHtml(node.public_key)}" data-name="${escapeHtml(node.name || 'Unknown')}" style="color:var(--accent);font-size:12px;">Show Neighbors</a>` : ''}
|
||||
${node.public_key ? ` · <a href="javascript:void(0)" role="button" data-show-neighbors data-pubkey="${escapeHtml(node.public_key)}" data-name="${escapeHtml(node.name || 'Unknown')}" style="color:var(--accent);font-size:12px;cursor:pointer;">Show Neighbors</a>` : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
+145
-14
@@ -286,11 +286,29 @@
|
||||
if (h) h.textContent = 'Neighbors (' + data.neighbors.length + ')';
|
||||
}
|
||||
var html = renderNeighborTable(data.neighbors, limit);
|
||||
if (limit && data.neighbors.length > limit && viewAllPubkey) {
|
||||
html += '<div style="margin-top:6px;text-align:right"><a href="#/nodes/' + encodeURIComponent(viewAllPubkey) + '?section=node-neighbors" style="font-size:12px">View all ' + data.neighbors.length + ' neighbors →</a></div>';
|
||||
if (limit && data.neighbors.length > limit) {
|
||||
html += '<div style="margin-top:6px;text-align:right"><button class="btn-link show-all-neighbors-btn" style="font-size:12px;cursor:pointer;background:none;border:none;color:var(--accent);padding:0">Show all ' + data.neighbors.length + ' neighbors ▼</button></div>';
|
||||
} else if (!limit && data.neighbors.length > 5) {
|
||||
// Collapse toggle when expanded (#855)
|
||||
html += '<div style="margin-top:6px;text-align:right"><button class="btn-link collapse-neighbors-btn" style="font-size:12px;cursor:pointer;background:none;border:none;color:var(--accent);padding:0">Show fewer ▲</button></div>';
|
||||
}
|
||||
el.innerHTML = html;
|
||||
|
||||
// Wire "Show all neighbors" expand button (#855)
|
||||
var expandBtn = el.querySelector('.show-all-neighbors-btn');
|
||||
if (expandBtn) {
|
||||
expandBtn.addEventListener('click', function() {
|
||||
renderNeighborData(data, containerId, 0, headerSelector, null);
|
||||
});
|
||||
}
|
||||
// Wire collapse button (#855)
|
||||
var collapseBtn = el.querySelector('.collapse-neighbors-btn');
|
||||
if (collapseBtn) {
|
||||
collapseBtn.addEventListener('click', function() {
|
||||
renderNeighborData(data, containerId, 5, headerSelector, null);
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize TableSort on neighbor table
|
||||
var neighborTable = el.querySelector('.neighbor-sort-table');
|
||||
if (neighborTable && window.TableSort) {
|
||||
@@ -319,7 +337,10 @@
|
||||
directNode = routeParam || null;
|
||||
|
||||
if (directNode) {
|
||||
// Full-screen single node view
|
||||
// Full-screen single node view (desktop + mobile).
|
||||
// Reached via the 🔍 Details link or a deep link to #/nodes/{pubkey}.
|
||||
// Row clicks use history.replaceState (no hashchange → no re-init),
|
||||
// so the split-panel UX on desktop is preserved.
|
||||
app.innerHTML = `<div class="node-fullscreen">
|
||||
<div class="node-full-header">
|
||||
<button class="detail-back-btn node-back-btn" id="nodeBackBtn" aria-label="Back to nodes">←</button>
|
||||
@@ -352,7 +373,7 @@
|
||||
|
||||
app.innerHTML = `<div class="nodes-page">
|
||||
<div class="nodes-topbar">
|
||||
<input type="text" class="nodes-search" id="nodeSearch" placeholder="Search nodes by name…" aria-label="Search nodes by name">
|
||||
<input type="text" class="nodes-search" id="nodeSearch" placeholder="Search by name or pubkey prefix…" aria-label="Search nodes by name or pubkey prefix">
|
||||
<div class="nodes-counts" id="nodeCounts"></div>
|
||||
</div>
|
||||
<div id="nodesRegionFilter" class="region-filter-container"></div>
|
||||
@@ -363,7 +384,7 @@
|
||||
</div>`;
|
||||
|
||||
RegionFilter.init(document.getElementById('nodesRegionFilter'));
|
||||
regionChangeHandler = RegionFilter.onChange(function () { _allNodes = null; loadNodes(); });
|
||||
regionChangeHandler = RegionFilter.onChange(function () { _allNodes = null; _fleetSkew = null; loadNodes(); });
|
||||
|
||||
if (search) {
|
||||
var _si = document.getElementById('nodeSearch');
|
||||
@@ -377,6 +398,7 @@
|
||||
}, 250));
|
||||
|
||||
loadNodes();
|
||||
if (directNode) selectNode(directNode);
|
||||
// Auto-refresh when ADVERT packets arrive via WebSocket (fixes #131)
|
||||
wsHandler = debouncedOnWS(function (msgs) {
|
||||
const advertMsgs = msgs.filter(isAdvertMessage);
|
||||
@@ -409,6 +431,7 @@
|
||||
|
||||
if (needReload) {
|
||||
_allNodes = null;
|
||||
_fleetSkew = null;
|
||||
invalidateApiCache('/nodes');
|
||||
}
|
||||
loadNodes(true);
|
||||
@@ -493,6 +516,8 @@
|
||||
<tr><td>Hash Prefix</td><td>${n.hash_size ? '<code style="font-family:var(--mono);font-weight:700">' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + '</code> (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' <span style="color:var(--status-yellow);cursor:help" title="Seen: ' + (Array.isArray(n.hash_sizes_seen) ? n.hash_sizes_seen : []).join(', ') + '-byte">⚠️ varies</span>' : ''}</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="node-full-card skew-detail-section" id="node-clock-skew" style="display:none"></div>
|
||||
|
||||
${observers.length ? `<div class="node-full-card" id="node-observers">
|
||||
${(() => { const regions = [...new Set(observers.map(o => o.iata).filter(Boolean))]; return regions.length ? `<div style="margin-bottom:8px"><strong>Regions:</strong> ${regions.map(r => '<span class="badge" style="margin:0 2px">' + escapeHtml(r) + '</span>').join(' ')}</div>` : ''; })()}
|
||||
<h4>Heard By (${observers.length} observer${observers.length > 1 ? 's' : ''})</h4>
|
||||
@@ -534,9 +559,10 @@
|
||||
</div>
|
||||
|
||||
<div class="node-full-card" id="node-packets">
|
||||
<h4>Recent Packets (${adverts.length})</h4>
|
||||
${(() => { const validPackets = adverts.filter(p => p.hash && p.timestamp); return `
|
||||
<h4>Recent Packets (${validPackets.length})</h4>
|
||||
<div class="node-activity-list">
|
||||
${adverts.length ? adverts.map(p => {
|
||||
${validPackets.length ? validPackets.map(p => {
|
||||
let decoded; try { decoded = JSON.parse(p.decoded_json); } catch {}
|
||||
const typeLabel = p.payload_type === 4 ? '📡 Advert' : p.payload_type === 5 ? '💬 Channel' : p.payload_type === 2 ? '✉️ DM' : '📦 Packet';
|
||||
const detail = decoded?.text ? ': ' + escapeHtml(truncate(decoded.text, 50)) : decoded?.name ? ' — ' + escapeHtml(decoded.name) : '';
|
||||
@@ -562,6 +588,7 @@
|
||||
</div>`;
|
||||
}).join('') : '<div class="text-muted">No recent packets</div>'}
|
||||
</div>
|
||||
`; })()}
|
||||
</div>`;
|
||||
|
||||
// Map
|
||||
@@ -624,6 +651,10 @@
|
||||
headerSelector: '#fullNeighborsHeader'
|
||||
});
|
||||
|
||||
// #690 — Clock Skew detail section (full-screen view)
|
||||
loadClockSkewInto(document.getElementById('node-clock-skew'), n.public_key);
|
||||
|
||||
|
||||
// Affinity debug panel — show if debugAffinity is enabled
|
||||
(function loadAffinityDebug() {
|
||||
var show = (window.CLIENT_CONFIG && window.CLIENT_CONFIG.debugAffinity) || localStorage.getItem('meshcore-affinity-debug') === 'true';
|
||||
@@ -777,6 +808,59 @@
|
||||
let _themeRefreshHandler = null;
|
||||
|
||||
let _allNodes = null; // cached full node list
|
||||
let _fleetSkew = null; // cached clock skew map: pubkey → {severity, recentMedianSkewSec, medianSkewSec, ...}
|
||||
|
||||
/**
|
||||
* Fetch per-node clock skew and render into the given container element.
|
||||
* Shared between the full-screen detail page and the side panel (#813, #690).
|
||||
* No-op if the container is missing, the API errors, or the response lacks severity.
|
||||
*/
|
||||
async function loadClockSkewInto(container, pubkey) {
|
||||
if (!container) return;
|
||||
try {
|
||||
var cs = await api('/nodes/' + encodeURIComponent(pubkey) + '/clock-skew', { ttl: 30000 });
|
||||
if (!cs || !cs.severity) return;
|
||||
container.style.display = '';
|
||||
var driftHtml = cs.driftPerDaySec ? '<div style="font-size:12px;color:var(--text-muted);margin-top:2px">Drift: ' + formatDrift(cs.driftPerDaySec) + '</div>' : '';
|
||||
var sparkHtml = renderSkewSparkline(cs.samples, 200, 32);
|
||||
var skewVal = window.currentSkewValue(cs);
|
||||
var skewDisplay = cs.severity === 'no_clock'
|
||||
? '<span style="font-size:18px;font-weight:700;color:var(--text-muted)">No Clock</span>'
|
||||
: '<span style="font-size:18px;font-weight:700;font-family:var(--mono)">' + formatSkew(skewVal) + '</span>';
|
||||
var bimodalWarning = '';
|
||||
if (cs.severity === 'bimodal_clock') {
|
||||
var totalRecent = cs.recentSampleCount || 0;
|
||||
bimodalWarning = '<div style="font-size:12px;color:var(--status-amber-text);margin-top:4px">⚠️ ' + (cs.recentBadSampleCount || '?') + ' of last ' + (totalRecent || '?') + ' adverts had nonsense timestamps (likely RTC reset)</div>';
|
||||
}
|
||||
container.innerHTML =
|
||||
'<h4 style="margin:0 0 6px">⏰ Clock Skew</h4>' +
|
||||
'<div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap">' +
|
||||
skewDisplay +
|
||||
renderSkewBadge(cs.severity, skewVal, cs) +
|
||||
(cs.calibrated ? ' <span style="font-size:10px;color:var(--text-muted)" title="Observer-calibrated">✓ calibrated</span>' : '') +
|
||||
'</div>' +
|
||||
driftHtml +
|
||||
(sparkHtml ? '<div class="skew-sparkline-wrap" style="margin-top:8px">' + sparkHtml + '<div style="font-size:10px;color:var(--text-muted)">Skew over time (' + (cs.samples || []).length + ' samples)</div></div>' : '') +
|
||||
bimodalWarning;
|
||||
} catch (e) {
|
||||
// Non-fatal — section stays hidden
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch fleet clock skew once, return map keyed by pubkey */
|
||||
async function getFleetSkew() {
|
||||
if (_fleetSkew) return _fleetSkew;
|
||||
try {
|
||||
const data = await api('/nodes/clock-skew', { ttl: 30000 });
|
||||
_fleetSkew = {};
|
||||
(Array.isArray(data) ? data : []).forEach(function(cs) {
|
||||
if (cs && cs.pubkey) _fleetSkew[cs.pubkey] = cs;
|
||||
});
|
||||
} catch (e) {
|
||||
_fleetSkew = {};
|
||||
}
|
||||
return _fleetSkew;
|
||||
}
|
||||
|
||||
// Build a map of lowercased name → count of distinct pubkeys sharing that name
|
||||
function buildDupNameMap(allNodes) {
|
||||
@@ -806,7 +890,10 @@
|
||||
const params = new URLSearchParams({ limit: '5000' });
|
||||
const rp = RegionFilter.getRegionParam();
|
||||
if (rp) params.set('region', rp);
|
||||
const data = await api('/nodes?' + params, { ttl: CLIENT_TTL.nodeList });
|
||||
const [data] = await Promise.all([
|
||||
api('/nodes?' + params, { ttl: CLIENT_TTL.nodeList }),
|
||||
getFleetSkew() // pre-fetch clock skew in parallel
|
||||
]);
|
||||
_allNodes = data.nodes || [];
|
||||
counts = data.counts || {};
|
||||
}
|
||||
@@ -815,8 +902,7 @@
|
||||
let filtered = _allNodes;
|
||||
if (activeTab !== 'all') filtered = filtered.filter(n => (n.role || '').toLowerCase() === activeTab);
|
||||
if (search) {
|
||||
const q = search.toLowerCase();
|
||||
filtered = filtered.filter(n => (n.name || '').toLowerCase().includes(q) || (n.public_key || '').toLowerCase().includes(q));
|
||||
filtered = filtered.filter(n => window._nodesMatchesSearch(n, search));
|
||||
}
|
||||
if (lastHeard) {
|
||||
const ms = { '1h': 3600000, '2h': 7200000, '6h': 21600000, '12h': 43200000, '24h': 86400000, '48h': 172800000, '3d': 259200000, '7d': 604800000, '14d': 1209600000, '30d': 2592000000 }[lastHeard];
|
||||
@@ -979,6 +1065,7 @@
|
||||
panel.classList.add('empty');
|
||||
panel.innerHTML = '<span>Select a node to view details</span>';
|
||||
selectedKey = null;
|
||||
history.replaceState(null, '', '#/nodes');
|
||||
renderRows();
|
||||
}
|
||||
}
|
||||
@@ -986,11 +1073,25 @@
|
||||
|
||||
// #630: Close button for node detail panel (important for mobile full-screen overlay)
|
||||
document.getElementById('nodesRight').addEventListener('click', function(e) {
|
||||
// #778/#856: Analytics link — force hashchange via replaceState + assign.
|
||||
// (Details button is handled separately via .node-detail-btn click listener)
|
||||
var link = e.target.closest('a.btn-primary[href^="#/nodes/"]');
|
||||
if (link) {
|
||||
e.preventDefault();
|
||||
var href = link.getAttribute('href');
|
||||
if (href.indexOf('/analytics') !== -1) {
|
||||
// Analytics link — different page, force hashchange via replaceState + assign
|
||||
history.replaceState(null, '', '#/');
|
||||
location.hash = href.substring(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (e.target.closest('.panel-close-btn')) {
|
||||
const panel = document.getElementById('nodesRight');
|
||||
panel.classList.add('empty');
|
||||
panel.innerHTML = '<span>Select a node to view details</span>';
|
||||
selectedKey = null;
|
||||
history.replaceState(null, '', '#/nodes');
|
||||
renderRows();
|
||||
}
|
||||
});
|
||||
@@ -1029,8 +1130,10 @@
|
||||
const lastSeenTime = n.last_heard || n.last_seen;
|
||||
const status = getNodeStatus(n.role || 'companion', lastSeenTime ? new Date(lastSeenTime).getTime() : 0);
|
||||
const lastSeenClass = status === 'active' ? 'last-seen-active' : 'last-seen-stale';
|
||||
const cs = _fleetSkew && _fleetSkew[n.public_key];
|
||||
const skewBadgeHtml = cs && cs.severity && cs.severity !== 'ok' ? renderSkewBadge(cs.severity, window.currentSkewValue(cs), cs) : '';
|
||||
return `<tr data-key="${n.public_key}" data-action="select" data-value="${n.public_key}" tabindex="0" role="row" class="${selectedKey === n.public_key ? 'selected' : ''}${isClaimed ? ' claimed-row' : ''}">
|
||||
<td>${favStar(n.public_key, 'node-fav')}${isClaimed ? '<span class="claimed-badge" title="My Mesh">★</span> ' : ''}<strong>${n.name || '(unnamed)'}</strong>${dupNameBadge(n.name, n.public_key, dupMap)}</td>
|
||||
<td>${favStar(n.public_key, 'node-fav')}${isClaimed ? '<span class="claimed-badge" title="My Mesh">★</span> ' : ''}<strong>${n.name || '(unnamed)'}</strong>${dupNameBadge(n.name, n.public_key, dupMap)}${skewBadgeHtml}</td>
|
||||
<td class="mono col-pubkey">${truncate(n.public_key, 16)}</td>
|
||||
<td><span class="badge" style="background:${roleColor}20;color:${roleColor}">${n.role}</span></td>
|
||||
<td class="${lastSeenClass}">${renderNodeTimestampHtml(n.last_heard || n.last_seen)}</td>
|
||||
@@ -1048,6 +1151,7 @@
|
||||
return;
|
||||
}
|
||||
selectedKey = pubkey;
|
||||
history.replaceState(null, '', '#/nodes/' + encodeURIComponent(pubkey));
|
||||
renderRows();
|
||||
const panel = document.getElementById('nodesRight');
|
||||
panel.classList.remove('empty');
|
||||
@@ -1086,7 +1190,7 @@
|
||||
<div class="node-detail">
|
||||
<div class="node-detail-name">${escapeHtml(n.name || '(unnamed)')}${dupBadge}</div>
|
||||
<div class="node-detail-role">${renderNodeBadges(n, roleColor)}
|
||||
<a href="#/nodes/${encodeURIComponent(n.public_key)}" class="btn-primary" style="display:inline-block;text-decoration:none;font-size:11px;padding:2px 8px;margin-left:8px">🔍 Details</a>
|
||||
<button class="btn-primary node-detail-btn" data-pubkey="${encodeURIComponent(n.public_key)}" aria-label="View details for ${escapeHtml(n.name || n.public_key)}" style="font-size:11px;padding:2px 8px;margin-left:8px;cursor:pointer">🔍 Details</button>
|
||||
<a href="#/nodes/${encodeURIComponent(n.public_key)}/analytics" class="btn-primary" style="display:inline-block;margin-left:4px;text-decoration:none;font-size:11px;padding:2px 8px">📊 Analytics</a>
|
||||
</div>
|
||||
${renderStatusExplanation(n)}
|
||||
@@ -1113,6 +1217,8 @@
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<div class="node-detail-section skew-detail-section" id="node-clock-skew" style="display:none"></div>
|
||||
|
||||
${observers.length ? `<div class="node-detail-section">
|
||||
${(() => { const regions = [...new Set(observers.map(o => o.iata).filter(Boolean))]; return regions.length ? `<div style="margin-bottom:6px;font-size:12px"><strong>Regions:</strong> ${regions.join(', ')}</div>` : ''; })()}
|
||||
<h4>Heard By (${observers.length} observer${observers.length > 1 ? 's' : ''})</h4>
|
||||
@@ -1135,9 +1241,10 @@
|
||||
</div>
|
||||
|
||||
<div class="node-detail-section">
|
||||
<h4>Recent Packets (${adverts.length})</h4>
|
||||
${(() => { const validPackets = adverts.filter(a => a.hash && a.timestamp); return `
|
||||
<h4>Recent Packets (${validPackets.length})</h4>
|
||||
<div id="advertTimeline">
|
||||
${adverts.length ? adverts.map(a => {
|
||||
${validPackets.length ? validPackets.map(a => {
|
||||
let decoded;
|
||||
try { decoded = JSON.parse(a.decoded_json); } catch {}
|
||||
const pType = PAYLOAD_TYPES[a.payload_type] || 'Packet';
|
||||
@@ -1156,6 +1263,7 @@
|
||||
</div>`;
|
||||
}).join('') : '<div class="text-muted" style="padding:8px">No recent packets</div>'}
|
||||
</div>
|
||||
`; })()}
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
@@ -1199,6 +1307,15 @@
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// #856: Wire "Details" button to navigate to full-screen node view
|
||||
var detailBtn = panel.querySelector('.node-detail-btn');
|
||||
if (detailBtn) {
|
||||
detailBtn.addEventListener('click', function() {
|
||||
var pk = detailBtn.getAttribute('data-pubkey');
|
||||
location.hash = '#/nodes/' + pk;
|
||||
});
|
||||
}
|
||||
|
||||
// Fetch neighbors for this node (condensed panel — top 5)
|
||||
fetchAndRenderNeighbors(n.public_key, 'panelNeighborsContent', {
|
||||
limit: 5,
|
||||
@@ -1206,6 +1323,10 @@
|
||||
viewAllPubkey: n.public_key
|
||||
});
|
||||
|
||||
// #813 — Clock Skew section in side panel (mirrors full-screen view)
|
||||
loadClockSkewInto(document.getElementById('node-clock-skew'), n.public_key);
|
||||
|
||||
|
||||
// Fetch paths through this node
|
||||
api('/nodes/' + encodeURIComponent(n.public_key) + '/paths', { ttl: CLIENT_TTL.nodeDetail }).then(pathData => {
|
||||
const el = document.getElementById('pathsContent');
|
||||
@@ -1304,4 +1425,14 @@
|
||||
window._nodesRenderNodeTimestampText = renderNodeTimestampText;
|
||||
window._nodesGetStatusInfo = getStatusInfo;
|
||||
window._nodesGetStatusTooltip = getStatusTooltip;
|
||||
|
||||
// #862: Expose search filter logic for testing
|
||||
window._nodesMatchesSearch = function(node, query) {
|
||||
if (!query) return true;
|
||||
var q = query.toLowerCase();
|
||||
var isHex = /^[0-9a-f]+$/i.test(q);
|
||||
if ((node.name || '').toLowerCase().includes(q)) return true;
|
||||
if (isHex && (node.public_key || '').toLowerCase().startsWith(q)) return true;
|
||||
return false;
|
||||
};
|
||||
})();
|
||||
|
||||
+236
-40
@@ -45,6 +45,11 @@
|
||||
var parts = [];
|
||||
if (timeWindowMin && timeWindowMin !== DEFAULT_TIME_WINDOW) parts.push('timeWindow=' + timeWindowMin);
|
||||
if (regionParam) parts.push('region=' + encodeURIComponent(regionParam));
|
||||
if (filters.hash) parts.push('hash=' + encodeURIComponent(filters.hash));
|
||||
if (filters.node) parts.push('node=' + encodeURIComponent(filters.node));
|
||||
if (filters.observer) parts.push('observer=' + encodeURIComponent(filters.observer));
|
||||
if (filters.channel) parts.push('channel=' + encodeURIComponent(filters.channel));
|
||||
if (filters._filterExpr) parts.push('filter=' + encodeURIComponent(filters._filterExpr));
|
||||
return parts.length ? '?' + parts.join('&') : '';
|
||||
}
|
||||
window.buildPacketsQuery = buildPacketsQuery;
|
||||
@@ -342,6 +347,16 @@
|
||||
}
|
||||
var _urlRegion = _initUrlParams.get('region');
|
||||
if (_urlRegion) _pendingUrlRegion = _urlRegion;
|
||||
var _urlHash = _initUrlParams.get('hash');
|
||||
if (_urlHash) filters.hash = _urlHash;
|
||||
var _urlNode = _initUrlParams.get('node');
|
||||
if (_urlNode) { filters.node = _urlNode; filters.nodeName = _urlNode.slice(0, 8); }
|
||||
var _urlObserver = _initUrlParams.get('observer');
|
||||
if (_urlObserver) filters.observer = _urlObserver;
|
||||
var _urlChannel = _initUrlParams.get('channel');
|
||||
if (_urlChannel) filters.channel = _urlChannel;
|
||||
var _urlFilterExpr = _initUrlParams.get('filter');
|
||||
if (_urlFilterExpr) filters._filterExpr = _urlFilterExpr;
|
||||
|
||||
app.innerHTML = `<div class="split-layout detail-collapsed">
|
||||
<div class="panel-left" id="pktLeft" aria-live="polite" aria-relevant="additions removals"></div>
|
||||
@@ -372,7 +387,7 @@
|
||||
const obs = data.observations.find(o => String(o.id) === String(obsTarget));
|
||||
if (obs) {
|
||||
expandedHashes.add(h);
|
||||
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, timestamp: obs.timestamp, first_seen: obs.timestamp};
|
||||
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, direction: obs.direction, timestamp: obs.timestamp, first_seen: obs.timestamp};
|
||||
clearParsedCache(obsPacket);
|
||||
selectPacket(obs.id, h, {packet: obsPacket, breakdown: data.breakdown, observations: data.observations}, obs.id);
|
||||
} else {
|
||||
@@ -610,6 +625,7 @@
|
||||
if (filters.hash) params.set('hash', filters.hash);
|
||||
if (filters.node) params.set('node', filters.node);
|
||||
if (filters.observer) params.set('observer', filters.observer);
|
||||
if (filters.channel) params.set('channel', filters.channel);
|
||||
if (groupByHash) {
|
||||
params.set('groupByHash', 'true');
|
||||
} else {
|
||||
@@ -738,6 +754,11 @@
|
||||
<button class="multi-select-trigger" id="typeTrigger" title="Filter by packet type">All Types ▾</button>
|
||||
<div class="multi-select-menu" id="typeMenu"></div>
|
||||
</div>
|
||||
<div class="filter-group" style="display:inline-flex;align-items:center;gap:4px">
|
||||
<select id="fChannel" class="filter-select" aria-label="Filter by channel" title="Filter Channel Messages (GRP_TXT) by channel">
|
||||
<option value="">All Channels</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="filter-group">
|
||||
<button class="btn ${groupByHash ? 'active' : ''}" id="fGroup" title="Collapse duplicate observations of the same packet into expandable groups">Group by Hash</button>
|
||||
@@ -797,6 +818,12 @@
|
||||
var pfError = document.getElementById('packetFilterError');
|
||||
var pfCount = document.getElementById('packetFilterCount');
|
||||
if (!pfInput || !window.PacketFilter) return;
|
||||
// Restore Wireshark filter expression from URL
|
||||
if (filters._filterExpr) {
|
||||
pfInput.value = filters._filterExpr;
|
||||
var _restored = PacketFilter.compile(filters._filterExpr);
|
||||
if (!_restored.error) { pfInput.classList.add('filter-active'); filters._packetFilter = _restored.filter; }
|
||||
}
|
||||
var pfTimer = null;
|
||||
pfInput.addEventListener('input', function() {
|
||||
clearTimeout(pfTimer);
|
||||
@@ -807,6 +834,8 @@
|
||||
pfError.style.display = 'none';
|
||||
pfCount.style.display = 'none';
|
||||
filters._packetFilter = null;
|
||||
filters._filterExpr = undefined;
|
||||
updatePacketsUrl();
|
||||
renderTableRows();
|
||||
return;
|
||||
}
|
||||
@@ -818,12 +847,16 @@
|
||||
pfError.style.display = 'block';
|
||||
pfCount.style.display = 'none';
|
||||
filters._packetFilter = null;
|
||||
filters._filterExpr = undefined;
|
||||
updatePacketsUrl();
|
||||
renderTableRows();
|
||||
} else {
|
||||
pfInput.classList.remove('filter-error');
|
||||
pfInput.classList.add('filter-active');
|
||||
pfError.style.display = 'none';
|
||||
filters._packetFilter = compiled.filter;
|
||||
filters._filterExpr = expr;
|
||||
updatePacketsUrl();
|
||||
renderTableRows();
|
||||
}
|
||||
}, 300);
|
||||
@@ -868,6 +901,7 @@
|
||||
if (filters.observer) localStorage.setItem('meshcore-observer-filter', filters.observer); else localStorage.removeItem('meshcore-observer-filter');
|
||||
buildObserverMenu();
|
||||
updateObsTrigger();
|
||||
updatePacketsUrl();
|
||||
renderTableRows();
|
||||
});
|
||||
|
||||
@@ -913,6 +947,63 @@
|
||||
renderTableRows();
|
||||
});
|
||||
|
||||
// --- Channel filter (#812) ---
|
||||
// Server-side filter: /api/packets?channel=<hash>. Triggers loadPackets()
|
||||
// (not just renderTableRows) so the filter applies before pagination.
|
||||
const channelSel = document.getElementById('fChannel');
|
||||
if (channelSel) {
|
||||
if (filters.channel) {
|
||||
// Pre-seed an option so the current filter shows as selected even
|
||||
// before the channels list arrives. Replaced when populateChannels resolves.
|
||||
const opt = document.createElement('option');
|
||||
opt.value = filters.channel;
|
||||
opt.textContent = filters.channel;
|
||||
opt.selected = true;
|
||||
channelSel.appendChild(opt);
|
||||
}
|
||||
api('/channels').then(data => {
|
||||
const channels = (data && data.channels) || [];
|
||||
// Build options via DOM API: channel names are network-supplied
|
||||
// and must NOT be interpolated into innerHTML (XSS, #812).
|
||||
// Sort alphabetically (case-insensitive) for predictable picker order;
|
||||
// the API returns last-activity order which is unstable for a dropdown.
|
||||
const sorted = channels.slice().sort((a, b) => {
|
||||
const an = (a.name || a.hash || '').toLowerCase();
|
||||
const bn = (b.name || b.hash || '').toLowerCase();
|
||||
return an < bn ? -1 : an > bn ? 1 : 0;
|
||||
});
|
||||
channelSel.textContent = '';
|
||||
const allOpt = document.createElement('option');
|
||||
allOpt.value = '';
|
||||
allOpt.textContent = 'All Channels';
|
||||
channelSel.appendChild(allOpt);
|
||||
let matched = false;
|
||||
for (const ch of sorted) {
|
||||
const v = ch.hash || ch.name || '';
|
||||
if (!v) continue;
|
||||
const opt = document.createElement('option');
|
||||
opt.value = v;
|
||||
opt.textContent = ch.name || v;
|
||||
if (v === filters.channel) { opt.selected = true; matched = true; }
|
||||
channelSel.appendChild(opt);
|
||||
}
|
||||
// If current filter isn't in the list (encrypted hash, stale, or
|
||||
// race with cache), keep it as a selected option so the UI reflects state.
|
||||
if (filters.channel && !matched) {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = filters.channel;
|
||||
opt.textContent = filters.channel;
|
||||
opt.selected = true;
|
||||
channelSel.appendChild(opt);
|
||||
}
|
||||
}).catch(() => {});
|
||||
channelSel.addEventListener('change', (e) => {
|
||||
filters.channel = e.target.value || undefined;
|
||||
updatePacketsUrl();
|
||||
loadPackets();
|
||||
});
|
||||
}
|
||||
|
||||
// Close multi-select menus on outside click
|
||||
bindDocumentHandler('menu', 'click', (e) => {
|
||||
const obsWrap = document.getElementById('observerFilterWrap');
|
||||
@@ -930,7 +1021,7 @@
|
||||
|
||||
// Filter event listeners
|
||||
document.getElementById('fHash').value = filters.hash || '';
|
||||
document.getElementById('fHash').addEventListener('input', debounce((e) => { filters.hash = e.target.value || undefined; loadPackets(); }, 300));
|
||||
document.getElementById('fHash').addEventListener('input', debounce((e) => { filters.hash = e.target.value || undefined; updatePacketsUrl(); loadPackets(); }, 300));
|
||||
|
||||
// Time window dropdown — restore from localStorage and bind change
|
||||
const fTimeWindow = document.getElementById('fTimeWindow');
|
||||
@@ -1065,7 +1156,7 @@
|
||||
if (!q) {
|
||||
fNodeDrop.classList.add('hidden');
|
||||
fNode.setAttribute('aria-expanded', 'false');
|
||||
if (filters.node) { filters.node = undefined; filters.nodeName = undefined; loadPackets(); }
|
||||
if (filters.node) { filters.node = undefined; filters.nodeName = undefined; updatePacketsUrl(); loadPackets(); }
|
||||
return;
|
||||
}
|
||||
try {
|
||||
@@ -1074,7 +1165,7 @@
|
||||
const nodes = data.nodes || [];
|
||||
if (nodes.length === 0) { fNodeDrop.classList.add('hidden'); fNode.setAttribute('aria-expanded', 'false'); return; }
|
||||
fNodeDrop.innerHTML = nodes.map((n, i) =>
|
||||
`<div class="node-filter-option" id="fNodeOpt-${i}" role="option" data-key="${n.public_key}" data-name="${escapeHtml(n.name || n.public_key.slice(0,8))}">${escapeHtml(n.name || n.public_key.slice(0,8))} <span style="color:var(--muted);font-size:0.8em">${n.public_key.slice(0,8)}</span></div>`
|
||||
`<div class="node-filter-option" id="fNodeOpt-${i}" role="option" data-key="${n.public_key}" data-name="${escapeHtml(n.name || n.public_key.slice(0,8))}">${escapeHtml(n.name || n.public_key.slice(0,8))} <span style="color:var(--text-muted);font-size:0.8em">${n.public_key.slice(0,8)}</span></div>`
|
||||
).join('');
|
||||
fNodeDrop.classList.remove('hidden');
|
||||
fNode.setAttribute('aria-expanded', 'true');
|
||||
@@ -1094,6 +1185,7 @@
|
||||
fNode.setAttribute('aria-expanded', 'false');
|
||||
fNode.setAttribute('aria-activedescendant', '');
|
||||
nodeActiveIdx = -1;
|
||||
updatePacketsUrl();
|
||||
loadPackets();
|
||||
}
|
||||
|
||||
@@ -1154,7 +1246,7 @@
|
||||
const child = group?._children?.find(c => String(c.id) === String(value));
|
||||
if (child) {
|
||||
const parentData = group._fetchedData;
|
||||
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, timestamp: child.timestamp, first_seen: child.timestamp} : child;
|
||||
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, direction: child.direction, timestamp: child.timestamp, first_seen: child.timestamp} : child;
|
||||
if (parentData) { clearParsedCache(obsPacket); }
|
||||
selectPacket(child.id, parentHash, {packet: obsPacket, breakdown: parentData?.breakdown, observations: parentData?.observations}, child.id);
|
||||
}
|
||||
@@ -1705,19 +1797,49 @@
|
||||
panel.innerHTML = isMobileNow ? '' : '<div class="panel-resize-handle" id="pktResizeHandle"></div>' + PANEL_CLOSE_HTML;
|
||||
const content = document.createElement('div');
|
||||
panel.appendChild(content);
|
||||
await renderDetail(content, data);
|
||||
await renderDetail(content, data, selectedObservationId);
|
||||
if (!isMobileNow) initPanelResize();
|
||||
} catch (e) {
|
||||
panel.innerHTML = `<div class="text-muted">Error: ${e.message}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
async function renderDetail(panel, data) {
|
||||
async function renderDetail(panel, data, chosenObsId) {
|
||||
const pkt = data.packet;
|
||||
const breakdown = data.breakdown || {};
|
||||
const ranges = breakdown.ranges || [];
|
||||
const decoded = getParsedDecoded(pkt) || {};
|
||||
const pathHops = getParsedPath(pkt) || [];
|
||||
const observations = data.observations || [];
|
||||
|
||||
// Per-observation rendering (issue #849):
|
||||
// When opened from a packet row (no specific observer), default to first observation.
|
||||
// When opened from an observation child row, use that observation.
|
||||
// Clicking a different observation row in the detail re-renders with that observation.
|
||||
let currentObs = null;
|
||||
const targetObsId = chosenObsId || selectedObservationId;
|
||||
if (targetObsId && observations.length) {
|
||||
currentObs = observations.find(o => String(o.id) === String(targetObsId));
|
||||
}
|
||||
if (!currentObs && observations.length) {
|
||||
currentObs = observations[0]; // fall back to first observation
|
||||
}
|
||||
|
||||
// If we have a current observation, build pkt fields from it so summary is per-observation
|
||||
const effectivePkt = currentObs ? clearParsedCache({...pkt, ...currentObs, _isObservation: true}) : pkt;
|
||||
const decoded = getParsedDecoded(effectivePkt) || {};
|
||||
const pathHops = getParsedPath(effectivePkt) || [];
|
||||
|
||||
// Cross-check: hop count from raw_hex path_len byte vs path_json length
|
||||
const obsRawHex = effectivePkt.raw_hex || pkt.raw_hex || '';
|
||||
let rawHopCount = null;
|
||||
if (obsRawHex.length >= 4) {
|
||||
// path_len byte position depends on route type
|
||||
const plOff = getPathLenOffset(pkt.route_type);
|
||||
const plByte = parseInt(obsRawHex.slice(plOff * 2, plOff * 2 + 2), 16);
|
||||
if (!isNaN(plByte)) rawHopCount = plByte & 0x3F;
|
||||
}
|
||||
if (rawHopCount != null && pathHops.length !== rawHopCount) {
|
||||
console.warn(`[CoreScope] Hop count inconsistency for packet ${pkt.hash}: path_json has ${pathHops.length} hops but raw_hex path_len has ${rawHopCount}. Trusting raw_hex.`);
|
||||
}
|
||||
|
||||
// Resolve sender GPS — from packet directly, or from known node in DB
|
||||
let senderLat = decoded.lat != null ? decoded.lat : (decoded.latitude || null);
|
||||
@@ -1761,15 +1883,16 @@
|
||||
}
|
||||
|
||||
// Parse hash size from path byte
|
||||
const rawPathByte = pkt.raw_hex ? parseInt(pkt.raw_hex.slice(2, 4), 16) : NaN;
|
||||
const plOff = getPathLenOffset(pkt.route_type);
|
||||
const rawPathByte = pkt.raw_hex ? parseInt(pkt.raw_hex.slice(plOff * 2, plOff * 2 + 2), 16) : NaN;
|
||||
const hashSize = (isNaN(rawPathByte) || (rawPathByte & 0x3F) === 0) ? null : ((rawPathByte >> 6) + 1);
|
||||
|
||||
const size = pkt.raw_hex ? Math.floor(pkt.raw_hex.length / 2) : 0;
|
||||
const size = effectivePkt.raw_hex ? Math.floor(effectivePkt.raw_hex.length / 2) : (pkt.raw_hex ? Math.floor(pkt.raw_hex.length / 2) : 0);
|
||||
const typeName = payloadTypeName(pkt.payload_type);
|
||||
|
||||
const snr = pkt.snr ?? decoded.SNR ?? decoded.snr ?? null;
|
||||
const rssi = pkt.rssi ?? decoded.RSSI ?? decoded.rssi ?? null;
|
||||
const hasRawHex = !!pkt.raw_hex;
|
||||
const snr = effectivePkt.snr ?? decoded.SNR ?? decoded.snr ?? null;
|
||||
const rssi = effectivePkt.rssi ?? decoded.RSSI ?? decoded.rssi ?? null;
|
||||
const hasRawHex = !!(effectivePkt.raw_hex || pkt.raw_hex);
|
||||
|
||||
// Build message preview
|
||||
let messageHtml = '';
|
||||
@@ -1780,17 +1903,16 @@
|
||||
const meta = [chLabel, hopLabel, snrLabel].filter(Boolean).join(' · ');
|
||||
messageHtml = `<div class="detail-message" style="padding:12px;margin:8px 0;background:var(--card-bg);border-radius:8px;border-left:3px solid var(--accent)">
|
||||
<div style="font-size:1.1em">${escapeHtml(decoded.text)}</div>
|
||||
${meta ? `<div style="font-size:0.85em;color:var(--muted);margin-top:4px">${meta}</div>` : ''}
|
||||
${meta ? `<div style="font-size:0.85em;color:var(--text-muted);margin-top:4px">${meta}</div>` : ''}
|
||||
</div>`;
|
||||
} else if (decoded.type === 'GRP_TXT' && decoded.channelHash != null) {
|
||||
const hashHex = decoded.channelHashHex || decoded.channelHash.toString(16).padStart(2, '0').toUpperCase();
|
||||
const statusLabel = decoded.decryptionStatus === 'no_key' ? 'no key' : 'decryption failed';
|
||||
messageHtml = `<div class="detail-message" style="padding:12px;margin:8px 0;background:var(--card-bg);border-radius:8px;border-left:3px solid var(--warning, #f0ad4e)">
|
||||
<div style="font-size:1.1em">🔒 Channel Hash: 0x${hashHex} <span style="color:var(--muted)">(${statusLabel})</span></div>
|
||||
<div style="font-size:1.1em">🔒 Channel Hash: 0x${hashHex} <span style="color:var(--text-muted)">(${statusLabel})</span></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
const observations = data.observations || [];
|
||||
const obsCount = data.observation_count || observations.length || 1;
|
||||
const uniqueObservers = new Set(observations.map(o => o.observer_id)).size;
|
||||
|
||||
@@ -1849,20 +1971,32 @@
|
||||
}
|
||||
}
|
||||
|
||||
const anomalyBanner = decoded.anomaly
|
||||
? `<div class="anomaly-banner" style="background:var(--warning, #f0ad4e); color:#000; padding:8px 12px; border-radius:4px; margin-bottom:8px; font-weight:600;">⚠️ Anomaly: ${escapeHtml(decoded.anomaly)}</div>`
|
||||
: '';
|
||||
|
||||
// Hop count display: trust raw_hex (firmware truth) over path_json
|
||||
const displayHopCount = rawHopCount != null ? rawHopCount : pathHops.length;
|
||||
const obsIndicator = currentObs && observations.length > 1
|
||||
? `<span style="font-size:0.8em;color:var(--text-muted);margin-left:6px">(observation ${observations.indexOf(currentObs) + 1} of ${observations.length})</span>`
|
||||
: '';
|
||||
|
||||
panel.innerHTML = `
|
||||
${anomalyBanner}
|
||||
<div class="detail-title">${hasRawHex ? `Packet Byte Breakdown (${size} bytes)` : typeName + ' Packet'}</div>
|
||||
<div class="detail-hash">${pkt.hash || 'Packet #' + pkt.id}</div>
|
||||
<div class="detail-hash">${pkt.hash || 'Packet #' + pkt.id}${obsIndicator}</div>
|
||||
${messageHtml}
|
||||
<dl class="detail-meta">
|
||||
<dt>Observer</dt><dd>${obsName(pkt.observer_id)}</dd>
|
||||
<dt>Observer</dt><dd>${obsName(effectivePkt.observer_id)}</dd>
|
||||
<dt>Location</dt><dd>${locationHtml}</dd>
|
||||
<dt>SNR / RSSI</dt><dd>${snr != null ? snr + ' dB' : '—'} / ${rssi != null ? rssi + ' dBm' : '—'}</dd>
|
||||
<dt>Route Type</dt><dd>${routeTypeName(pkt.route_type)}</dd>
|
||||
<dt>Payload Type</dt><dd><span class="badge badge-${payloadTypeColor(pkt.payload_type)}">${typeName}</span></dd>
|
||||
${hashSize ? `<dt>Hash Size</dt><dd>${hashSize} byte${hashSize !== 1 ? 's' : ''}</dd>` : ''}
|
||||
<dt>Timestamp</dt><dd>${renderTimestampCell(pkt.timestamp)}</dd>
|
||||
<dt>Timestamp</dt><dd>${renderTimestampCell(effectivePkt.timestamp)}</dd>
|
||||
<dt>Propagation</dt><dd>${propagationHtml}</dd>
|
||||
<dt>Path</dt><dd>${pathHops.length ? renderPath(pathHops, pkt.observer_id) : '—'}</dd>
|
||||
<dt>Path</dt><dd>${displayHopCount > 0 ? `<span class="badge badge-info">${displayHopCount} hop${displayHopCount !== 1 ? 's' : ''}</span> ` + renderPath(pathHops, effectivePkt.observer_id) : '— (direct)'}</dd>
|
||||
${effectivePkt.direction ? `<dt>Direction</dt><dd>${escapeHtml(effectivePkt.direction)}</dd>` : ''}
|
||||
</dl>
|
||||
<div class="detail-actions">
|
||||
<button class="copy-link-btn" data-packet-hash="${pkt.hash || ''}" data-packet-id="${pkt.id}" title="Copy link to this packet">🔗 Copy Link</button>
|
||||
@@ -1872,11 +2006,59 @@
|
||||
</div>
|
||||
|
||||
${hasRawHex ? `<div class="hex-legend">${buildHexLegend(ranges)}</div>
|
||||
<div class="hex-dump">${createColoredHexDump(pkt.raw_hex, ranges)}</div>` : ''}
|
||||
<div class="hex-dump">${createColoredHexDump(effectivePkt.raw_hex || pkt.raw_hex, ranges)}</div>` : ''}
|
||||
|
||||
${hasRawHex ? buildFieldTable(pkt, decoded, pathHops, ranges) : buildDecodedTable(decoded)}
|
||||
${hasRawHex ? buildFieldTable(effectivePkt.raw_hex ? effectivePkt : pkt, decoded, pathHops, ranges) : buildDecodedTable(decoded)}
|
||||
|
||||
${observations.length > 1 ? `
|
||||
<div class="detail-observations" style="margin-top:16px">
|
||||
<div style="font-weight:600;margin-bottom:6px">Observations (${observations.length})</div>
|
||||
<table class="detail-obs-table" style="width:100%;border-collapse:collapse;font-size:0.9em">
|
||||
<thead><tr style="border-bottom:1px solid var(--border)">
|
||||
<th style="padding:4px 6px;text-align:left">Observer</th>
|
||||
<th style="padding:4px 6px;text-align:left">Hops</th>
|
||||
<th style="padding:4px 6px;text-align:left">SNR</th>
|
||||
<th style="padding:4px 6px;text-align:left">RSSI</th>
|
||||
<th style="padding:4px 6px;text-align:left">Time</th>
|
||||
</tr></thead>
|
||||
<tbody>${observations.map(o => {
|
||||
const oPath = getParsedPath(o);
|
||||
const isCurrent = currentObs && String(o.id) === String(currentObs.id);
|
||||
return `<tr class="detail-obs-row${isCurrent ? ' observation-current' : ''}" data-obs-id="${o.id}" style="cursor:pointer;${isCurrent ? 'background:var(--accent-bg, rgba(0,122,255,0.1))' : ''}" title="Click to view this observation">
|
||||
<td style="padding:4px 6px">${obsName(o.observer_id)}</td>
|
||||
<td style="padding:4px 6px">${oPath.length}</td>
|
||||
<td style="padding:4px 6px">${o.snr != null ? o.snr + ' dB' : '—'}</td>
|
||||
<td style="padding:4px 6px">${o.rssi != null ? o.rssi + ' dBm' : '—'}</td>
|
||||
<td style="padding:4px 6px">${renderTimestampCell(o.timestamp)}</td>
|
||||
</tr>`;
|
||||
}).join('')}</tbody>
|
||||
</table>
|
||||
</div>` : ''}
|
||||
|
||||
${observations.length > 1 ? (() => {
|
||||
// Cross-observer aggregate (Option B): show longest observed path across all observers
|
||||
const aggregatePath = getParsedPath(pkt) || [];
|
||||
return `<div class="detail-aggregate" style="margin-top:12px;padding:10px;background:var(--card-bg);border-radius:6px;border:1px solid var(--border);font-size:0.9em">
|
||||
<div style="font-weight:600;margin-bottom:4px;color:var(--text-muted)">Cross-observer aggregate</div>
|
||||
<div>Longest observed path: ${aggregatePath.length ? `${aggregatePath.length} hops — ${renderPath(aggregatePath, pkt.observer_id)}` : '— (direct)'}</div>
|
||||
<div style="font-size:0.8em;color:var(--text-muted);margin-top:2px">Longest path seen across all ${uniqueObservers} observer${uniqueObservers !== 1 ? 's' : ''}</div>
|
||||
</div>`;
|
||||
})() : ''}
|
||||
`;
|
||||
|
||||
// Wire up observation row click handlers — re-render detail with clicked observation
|
||||
panel.querySelectorAll('.detail-obs-row').forEach(row => {
|
||||
row.addEventListener('click', () => {
|
||||
const obsId = row.dataset.obsId;
|
||||
selectedObservationId = obsId;
|
||||
// Update URL hash to reflect selected observation (deep linking)
|
||||
const pktHash = pkt.hash || pkt.id;
|
||||
const obsParam = obsId ? `?obs=${obsId}` : '';
|
||||
history.replaceState(null, '', `#/packets/${pktHash}${obsParam}`);
|
||||
renderDetail(panel, data, obsId);
|
||||
});
|
||||
});
|
||||
|
||||
// Wire up copy link button
|
||||
const copyLinkBtn = panel.querySelector('.copy-link-btn');
|
||||
if (copyLinkBtn) {
|
||||
@@ -1981,38 +2163,43 @@
|
||||
// Header section
|
||||
rows += sectionRow('Header', 'section-header');
|
||||
rows += fieldRow(0, 'Header Byte', '0x' + (buf.slice(0, 2) || '??'), `Route: ${routeTypeName(pkt.route_type)}, Payload: ${payloadTypeName(pkt.payload_type)}`);
|
||||
const pathByte0 = parseInt(buf.slice(2, 4), 16);
|
||||
const hashSizeVal = isNaN(pathByte0) ? '?' : ((pathByte0 >> 6) + 1);
|
||||
const hashCountVal = isNaN(pathByte0) ? '?' : (pathByte0 & 0x3F);
|
||||
rows += fieldRow(1, 'Path Length', '0x' + (buf.slice(2, 4) || '??'), hashCountVal === 0 ? `hash_count=0 (direct advert)` : `hash_size=${hashSizeVal} byte${hashSizeVal !== 1 ? 's' : ''}, hash_count=${hashCountVal}`);
|
||||
|
||||
// Transport codes
|
||||
let off = 2;
|
||||
if (pkt.route_type === 0 || pkt.route_type === 3) {
|
||||
// Transport codes come BEFORE path length for transport routes (bytes 1-4)
|
||||
let off = 1;
|
||||
if (isTransportRoute(pkt.route_type)) {
|
||||
rows += sectionRow('Transport Codes', 'section-transport');
|
||||
rows += fieldRow(off, 'Next Hop', buf.slice(off * 2, (off + 2) * 2), '');
|
||||
rows += fieldRow(off + 2, 'Last Hop', buf.slice((off + 2) * 2, (off + 4) * 2), '');
|
||||
off += 4;
|
||||
}
|
||||
|
||||
// Path
|
||||
if (pathHops.length > 0) {
|
||||
rows += sectionRow('Path (' + pathHops.length + ' hops)', 'section-path');
|
||||
const pathByte = parseInt(buf.slice(2, 4), 16);
|
||||
const hashSize = (pathByte >> 6) + 1;
|
||||
for (let i = 0; i < pathHops.length; i++) {
|
||||
const hopHtml = HopDisplay.renderHop(pathHops[i], hopNameCache[pathHops[i]]);
|
||||
// Path length byte is at current offset (byte 1 for non-transport, byte 5 for transport)
|
||||
const pathLenOffset = off;
|
||||
const pathByte0 = parseInt(buf.slice(off * 2, off * 2 + 2), 16);
|
||||
const hashSizeVal = isNaN(pathByte0) ? '?' : ((pathByte0 >> 6) + 1);
|
||||
const hashCountVal = isNaN(pathByte0) ? '?' : (pathByte0 & 0x3F);
|
||||
rows += fieldRow(off, 'Path Length', '0x' + (buf.slice(off * 2, off * 2 + 2) || '??'), hashCountVal === 0 ? `hash_count=0 (direct advert)` : `hash_size=${hashSizeVal} byte${hashSizeVal !== 1 ? 's' : ''}, hash_count=${hashCountVal}`);
|
||||
off += 1;
|
||||
|
||||
// Path — derive hop count from path_len byte (firmware truth), not aggregated _parsedPath
|
||||
const hashSize = isNaN(pathByte0) ? 1 : ((pathByte0 >> 6) + 1);
|
||||
if (typeof hashCountVal === 'number' && hashCountVal > 0) {
|
||||
rows += sectionRow('Path (' + hashCountVal + ' hops)', 'section-path');
|
||||
for (let i = 0; i < hashCountVal; i++) {
|
||||
const hopOff = off + i * hashSize;
|
||||
const hex = buf.slice(hopOff * 2, (hopOff + hashSize) * 2).toUpperCase();
|
||||
const hopHtml = HopDisplay.renderHop(hex, hopNameCache[hex]);
|
||||
const label = `Hop ${i} — ${hopHtml}`;
|
||||
rows += fieldRow(off + i * hashSize, label, pathHops[i], '');
|
||||
rows += fieldRow(hopOff, label, hex, '');
|
||||
}
|
||||
off += hashSize * pathHops.length;
|
||||
off += hashSize * hashCountVal;
|
||||
}
|
||||
|
||||
// Payload
|
||||
rows += sectionRow('Payload — ' + payloadTypeName(pkt.payload_type), 'section-payload');
|
||||
|
||||
if (decoded.type === 'ADVERT') {
|
||||
if (hashCountVal !== 0) rows += fieldRow(1, 'Advertised Hash Size', hashSizeVal + ' byte' + (hashSizeVal !== 1 ? 's' : ''), 'From path byte 0x' + (buf.slice(2, 4) || '??') + ' — bits 7-6 = ' + (hashSizeVal - 1));
|
||||
if (hashCountVal !== 0) rows += fieldRow(pathLenOffset, 'Advertised Hash Size', hashSizeVal + ' byte' + (hashSizeVal !== 1 ? 's' : ''), 'From path byte 0x' + (buf.slice(pathLenOffset * 2, pathLenOffset * 2 + 2) || '??') + ' — bits 7-6 = ' + (hashSizeVal - 1));
|
||||
rows += fieldRow(off, 'Public Key (32B)', truncate(decoded.pubKey || '', 24), '');
|
||||
rows += fieldRow(off + 32, 'Timestamp (4B)', decoded.timestampISO || '', 'Unix: ' + (decoded.timestamp || ''));
|
||||
rows += fieldRow(off + 36, 'Signature (64B)', truncate(decoded.signature || '', 24), '');
|
||||
@@ -2053,6 +2240,10 @@
|
||||
rows += fieldRow(off, 'Raw', truncate(buf.slice(off * 2), 40), '');
|
||||
}
|
||||
|
||||
if (decoded.anomaly) {
|
||||
rows += `<tr class="anomaly-row" style="background:var(--warning, #f0ad4e); color:#000; font-weight:600;"><td colspan="2">⚠️ Anomaly</td><td colspan="2">${escapeHtml(decoded.anomaly)}</td></tr>`;
|
||||
}
|
||||
|
||||
return `<table class="field-table">
|
||||
<thead><tr><th scope="col">Offset</th><th scope="col">Field</th><th scope="col">Value</th><th scope="col">Description</th></tr></thead>
|
||||
<tbody>${rows}</tbody>
|
||||
@@ -2144,6 +2335,11 @@
|
||||
|
||||
let html = '<div class="byop-decoded">';
|
||||
|
||||
// Anomaly banner
|
||||
if (d.anomaly) {
|
||||
html += '<div class="anomaly-banner" style="background:var(--warning, #f0ad4e); color:#000; padding:8px 12px; border-radius:4px; margin-bottom:8px; font-weight:600;">⚠️ Anomaly: ' + escapeHtml(d.anomaly) + '</div>';
|
||||
}
|
||||
|
||||
// Header section
|
||||
html += '<div class="byop-section">'
|
||||
+ '<div class="byop-section-title">Header</div>'
|
||||
|
||||
@@ -394,4 +394,83 @@
|
||||
});
|
||||
return html;
|
||||
};
|
||||
|
||||
// #690 — Clock Skew shared helpers
|
||||
var SKEW_SEVERITY_COLORS = {
|
||||
ok: 'var(--status-green)',
|
||||
warning: 'var(--status-yellow)',
|
||||
critical: 'var(--status-orange)',
|
||||
absurd: 'var(--status-purple)',
|
||||
bimodal_clock: 'var(--status-amber)',
|
||||
no_clock: 'var(--text-muted)'
|
||||
};
|
||||
var SKEW_SEVERITY_LABELS = {
|
||||
ok: 'OK', warning: 'Warning', critical: 'Critical', absurd: 'Absurd', bimodal_clock: 'Bimodal', no_clock: 'No Clock'
|
||||
};
|
||||
var SKEW_SEVERITY_ORDER = { no_clock: 0, bimodal_clock: 1, absurd: 2, critical: 3, warning: 4, ok: 5 };
|
||||
|
||||
window.SKEW_SEVERITY_COLORS = SKEW_SEVERITY_COLORS;
|
||||
window.SKEW_SEVERITY_LABELS = SKEW_SEVERITY_LABELS;
|
||||
window.SKEW_SEVERITY_ORDER = SKEW_SEVERITY_ORDER;
|
||||
|
||||
/** Format skew seconds into human-readable string like "+2m 34s" or "-15h 22m" */
|
||||
window.formatSkew = function(sec) {
|
||||
if (sec == null) return '—';
|
||||
var abs = Math.abs(sec);
|
||||
var sign = sec >= 0 ? '+' : '-';
|
||||
if (abs < 60) return sign + Math.round(abs) + 's';
|
||||
if (abs < 3600) return sign + Math.floor(abs / 60) + 'm ' + Math.round(abs % 60) + 's';
|
||||
if (abs < 86400) return sign + Math.floor(abs / 3600) + 'h ' + Math.round((abs % 3600) / 60) + 'm';
|
||||
return sign + Math.floor(abs / 86400) + 'd ' + Math.round((abs % 86400) / 3600) + 'h';
|
||||
};
|
||||
|
||||
/** Format drift rate as "+X.Xs/day" or "—" if falsy */
|
||||
window.formatDrift = function(secPerDay) {
|
||||
if (!secPerDay) return '—';
|
||||
return (secPerDay >= 0 ? '+' : '') + secPerDay.toFixed(1) + ' s/day';
|
||||
};
|
||||
|
||||
/** Pick the skew value that drives current-health UI: prefer the
|
||||
* recent-window median (#789, current health) over the all-time median
|
||||
* (poisoned by historical bad samples). Falls back gracefully if the
|
||||
* field isn't present (older API responses). */
|
||||
window.currentSkewValue = function(cs) {
|
||||
if (!cs) return null;
|
||||
return cs.recentMedianSkewSec != null ? cs.recentMedianSkewSec : cs.medianSkewSec;
|
||||
};
|
||||
|
||||
/** Render a clock skew badge HTML */
|
||||
window.renderSkewBadge = function(severity, skewSec, cs) {
|
||||
if (!severity) return '';
|
||||
var cls = 'skew-badge skew-badge--' + severity;
|
||||
if (severity === 'no_clock') {
|
||||
return '<span class="' + cls + '" title="Uninitialized RTC — no valid clock">🚫 No Clock</span>';
|
||||
}
|
||||
if (severity === 'bimodal_clock' && cs) {
|
||||
var badPct = cs.goodFraction != null ? Math.round((1 - cs.goodFraction) * 100) : '?';
|
||||
var label = '⏰ ' + window.formatSkew(skewSec);
|
||||
return '<span class="' + cls + '" title="Clock skew: ' + window.formatSkew(skewSec) + ' (bimodal: ' + badPct + '% of recent adverts have nonsense timestamps)">' + label + '</span>';
|
||||
}
|
||||
var label = severity === 'ok' ? '⏰' : '⏰ ' + window.formatSkew(skewSec);
|
||||
return '<span class="' + cls + '" title="Clock skew: ' + window.formatSkew(skewSec) + ' (' + (SKEW_SEVERITY_LABELS[severity] || severity) + ')">' + label + '</span>';
|
||||
};
|
||||
|
||||
/** Render a skew sparkline SVG (inline, word-sized) */
|
||||
window.renderSkewSparkline = function(samples, w, h) {
|
||||
w = w || 120; h = h || 24;
|
||||
if (!samples || samples.length < 2) return '';
|
||||
var values = samples.map(function(s) { return s.skew; });
|
||||
var max = Math.max.apply(null, values.map(function(v) { return Math.abs(v); }).concat([1]));
|
||||
var pts = values.map(function(v, i) {
|
||||
var x = i * (w / Math.max(values.length - 1, 1));
|
||||
var y = h / 2 - (v / max) * (h / 2 - 2);
|
||||
return x.toFixed(1) + ',' + y.toFixed(1);
|
||||
}).join(' ');
|
||||
// Zero line
|
||||
var zeroY = h / 2;
|
||||
return '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:' + w + 'px;height:' + h + 'px" role="img" aria-label="Clock skew sparkline">' +
|
||||
'<title>Clock skew over time</title>' +
|
||||
'<line x1="0" y1="' + zeroY + '" x2="' + w + '" y2="' + zeroY + '" stroke="var(--border)" stroke-width="0.5" stroke-dasharray="2"/>' +
|
||||
'<polyline points="' + pts + '" fill="none" stroke="var(--accent)" stroke-width="1.5"/></svg>';
|
||||
};
|
||||
})();
|
||||
|
||||
+101
-2
@@ -11,6 +11,12 @@
|
||||
--status-green: #22c55e;
|
||||
--status-yellow: #eab308;
|
||||
--status-red: #ef4444;
|
||||
--status-orange: #f97316;
|
||||
--status-purple: #a855f7;
|
||||
--status-amber: #f59e0b;
|
||||
--status-amber-light: #fef3c7;
|
||||
--status-amber-text: #92400e;
|
||||
--role-observer: #8b5cf6;
|
||||
--accent-hover: #6db3ff;
|
||||
--text: #1a1a2e;
|
||||
--text-muted: #5b6370;
|
||||
@@ -41,6 +47,11 @@
|
||||
--status-green: #22c55e;
|
||||
--status-yellow: #eab308;
|
||||
--status-red: #ef4444;
|
||||
--status-orange: #f97316;
|
||||
--status-purple: #a855f7;
|
||||
--status-amber: #f59e0b;
|
||||
--status-amber-light: #422006;
|
||||
--status-amber-text: #fcd34d;
|
||||
--surface-0: #0f0f23;
|
||||
--surface-1: #1a1a2e;
|
||||
--surface-2: #232340;
|
||||
@@ -65,6 +76,11 @@
|
||||
--status-green: #22c55e;
|
||||
--status-yellow: #eab308;
|
||||
--status-red: #ef4444;
|
||||
--status-orange: #f97316;
|
||||
--status-purple: #a855f7;
|
||||
--status-amber: #f59e0b;
|
||||
--status-amber-light: #422006;
|
||||
--status-amber-text: #fcd34d;
|
||||
--surface-0: #0f0f23;
|
||||
--surface-1: #1a1a2e;
|
||||
--surface-2: #232340;
|
||||
@@ -338,6 +354,9 @@ a:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible
|
||||
}
|
||||
.detail-meta dt { color: var(--text-muted); font-size: 11px; text-transform: uppercase; letter-spacing: .3px; }
|
||||
.detail-meta dd { font-weight: 500; margin-bottom: 4px; }
|
||||
.observation-current { background: var(--accent-bg, rgba(0,122,255,0.1)); font-weight: 600; }
|
||||
.detail-obs-row:hover { background: var(--hover-bg, rgba(255,255,255,0.05)); }
|
||||
.detail-obs-table th { font-size: 0.8em; text-transform: uppercase; color: var(--text-muted); }
|
||||
|
||||
/* === Hex Dump === */
|
||||
.hex-dump {
|
||||
@@ -463,6 +482,14 @@ fieldset.mc-section legend.mc-label { padding: 0; }
|
||||
.ch-sidebar-title {
|
||||
display: flex; align-items: center; gap: 8px; font-size: 16px; font-weight: 700; margin-bottom: 8px;
|
||||
}
|
||||
.ch-encrypted-toggle {
|
||||
display: flex; align-items: center; gap: 4px; font-size: 11px; color: var(--text-muted);
|
||||
cursor: pointer; user-select: none; margin-bottom: 4px;
|
||||
}
|
||||
.ch-encrypted-toggle input { margin: 0; cursor: pointer; }
|
||||
.ch-toggle-label { white-space: nowrap; }
|
||||
.ch-item.ch-encrypted { opacity: 0.55; }
|
||||
.ch-item.ch-encrypted .ch-item-name { font-style: italic; }
|
||||
.ch-icon { font-size: 20px; }
|
||||
.ch-sidebar-controls { display: flex; align-items: center; gap: 6px; }
|
||||
.ch-region-select {
|
||||
@@ -495,6 +522,9 @@ button.ch-item.selected { background: var(--selected-bg); }
|
||||
.ch-item-top { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 2px; }
|
||||
.ch-item-name { font-weight: 600; font-size: 14px; }
|
||||
.ch-item-time { font-size: 11px; color: var(--text-muted); white-space: nowrap; }
|
||||
.ch-remove-btn { background: none; border: none; color: var(--text-muted); cursor: pointer; font-size: 13px; padding: 0 2px; margin-left: 4px; opacity: 0; transition: opacity 0.15s; line-height: 1; }
|
||||
button.ch-item:hover .ch-remove-btn { opacity: 0.6; }
|
||||
.ch-remove-btn:hover { opacity: 1 !important; color: var(--danger, #dc2626); }
|
||||
.ch-item-preview { font-size: 12px; color: var(--text-muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
|
||||
.ch-main { flex: 1; display: flex; flex-direction: column; overflow: hidden; position: relative; }
|
||||
@@ -679,7 +709,9 @@ button.ch-item.selected { background: var(--selected-bg); }
|
||||
.advert-dot {
|
||||
width: 10px; height: 10px; border-radius: 50%; flex-shrink: 0; margin-top: 4px;
|
||||
}
|
||||
.advert-info { font-size: 12px; line-height: 1.5; }
|
||||
/* #829: explicit color so text stays readable when inherited color matches card-bg */
|
||||
.advert-info { font-size: 12px; line-height: 1.5; color: var(--text); }
|
||||
.advert-info a { color: var(--accent); }
|
||||
|
||||
/* === Traces Page === */
|
||||
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; }
|
||||
@@ -1097,6 +1129,49 @@ button.ch-item.ch-item-encrypted:hover { opacity: 0.7; }
|
||||
button.ch-item.ch-item-encrypted.selected { opacity: 0.8; }
|
||||
button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
|
||||
|
||||
/* Channel key input (#725 M2) */
|
||||
.ch-key-input {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
box-sizing: border-box;
|
||||
padding: 6px 8px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px 0 0 6px;
|
||||
background: var(--card-bg);
|
||||
color: var(--text);
|
||||
font-size: 12px;
|
||||
font-family: inherit;
|
||||
}
|
||||
.ch-key-input:focus {
|
||||
outline: 2px solid var(--accent, #3b82f6);
|
||||
outline-offset: -1px;
|
||||
border-color: var(--accent, #3b82f6);
|
||||
}
|
||||
.ch-key-input::placeholder { color: var(--text-muted); }
|
||||
.ch-key-input-wrap { margin-bottom: 4px; }
|
||||
.ch-wrong-key { color: var(--danger, #ef4444); font-weight: 500; }
|
||||
|
||||
/* Add channel form (#759) */
|
||||
.ch-add-form { margin: 0; }
|
||||
.ch-add-label { display: block; font-weight: 600; font-size: 13px; color: var(--text); margin-bottom: 4px; }
|
||||
.ch-key-input, .ch-add-btn { height: 32px; box-sizing: border-box; }
|
||||
.ch-add-row { display: flex; align-items: stretch; }
|
||||
.ch-add-btn {
|
||||
width: 32px; height: 32px; flex-shrink: 0;
|
||||
border: 1px solid var(--accent, #3b82f6); border-left: none;
|
||||
border-radius: 0 6px 6px 0;
|
||||
background: var(--accent, #3b82f6); color: #fff;
|
||||
font-size: 18px; font-weight: 700; line-height: 1;
|
||||
cursor: pointer; display: flex; align-items: center; justify-content: center;
|
||||
}
|
||||
.ch-add-btn:hover { opacity: 0.85; }
|
||||
.ch-add-hint { font-size: 11px; color: var(--text-muted); margin-top: 4px; line-height: 1.3; }
|
||||
.ch-add-status { font-size: 12px; margin-top: 4px; padding: 4px 6px; border-radius: 4px; }
|
||||
.ch-add-status--loading { color: var(--text-muted); }
|
||||
.ch-add-status--success { color: var(--success, #22c55e); }
|
||||
.ch-add-status--warn { color: var(--warning, #eab308); }
|
||||
.ch-add-status--error { color: var(--danger, #ef4444); }
|
||||
|
||||
/* Touch-friendly tappable elements */
|
||||
.ch-tappable {
|
||||
cursor: pointer;
|
||||
@@ -1362,7 +1437,9 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
|
||||
.hop-conflict-name { font-weight: 600; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.hop-conflict-dist { font-size: 11px; color: var(--text-muted); font-family: var(--mono); white-space: nowrap; }
|
||||
.hop-conflict-pk { font-size: 10px; color: var(--text-muted); font-family: var(--mono); }
|
||||
.hop-unreliable { opacity: 0.5; text-decoration: line-through; }
|
||||
.hop-unreliable { opacity: 0.85; }
|
||||
.hop-unreliable-btn { background: none; border: none; color: var(--status-yellow, #f59e0b); font-size: 13px;
|
||||
cursor: help; vertical-align: middle; margin-left: 2px; padding: 0 2px; line-height: 1; }
|
||||
.hop-global-fallback { border-bottom: 1px dashed var(--status-red); }
|
||||
.hop-current { font-weight: 700 !important; color: var(--accent) !important; }
|
||||
|
||||
@@ -2211,3 +2288,25 @@ th[data-sort-key] { cursor: pointer; user-select: none; }
|
||||
th[data-sort-key]:hover { background: var(--hover-bg, rgba(255,255,255,0.05)); }
|
||||
th.sort-active { color: var(--accent, #60a5fa); }
|
||||
.sort-arrow { font-size: 0.75em; opacity: 0.8; }
|
||||
|
||||
/* #690 — Clock Skew badges & fleet table */
|
||||
.skew-badge { display: inline-block; font-size: 10px; padding: 1px 5px; border-radius: 3px; margin-left: 4px; font-weight: 600; white-space: nowrap; }
|
||||
.skew-badge--ok { background: var(--status-green); color: #fff; }
|
||||
.skew-badge--warning { background: var(--status-yellow); color: #000; }
|
||||
.skew-badge--critical { background: var(--status-orange); color: #fff; }
|
||||
.skew-badge--absurd { background: var(--status-purple); color: #fff; }
|
||||
.skew-badge--no_clock { background: var(--text-muted); color: #fff; }
|
||||
.skew-badge--bimodal_clock { background: var(--status-amber-light); color: var(--status-amber-text); border: 1px solid var(--status-amber); }
|
||||
|
||||
.skew-detail-section { padding: 10px 16px; margin-bottom: 8px; }
|
||||
.skew-sparkline-wrap { margin-top: 6px; }
|
||||
.skew-sparkline-wrap svg { display: block; }
|
||||
|
||||
|
||||
.clock-fleet-row--warning { background: color-mix(in srgb, var(--status-yellow) 10%, transparent); }
|
||||
.clock-fleet-row--critical { background: color-mix(in srgb, var(--status-orange) 10%, transparent); }
|
||||
.clock-fleet-row--absurd { background: color-mix(in srgb, var(--status-purple) 10%, transparent); }
|
||||
.clock-fleet-row--no_clock { background: color-mix(in srgb, var(--text-muted) 10%, transparent); }
|
||||
|
||||
.clock-filter-btn { font-size: 12px; padding: 3px 8px; border: 1px solid var(--border); border-radius: 4px; background: var(--card-bg, #fff); color: var(--text); cursor: pointer; margin-right: 4px; }
|
||||
.clock-filter-btn.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# CoreScope QA artifacts
|
||||
|
||||
Project-specific assets for the [`qa-suite`](https://github.com/Kpa-clawbot/ai-sdlc/tree/master/skills/qa-suite) skill.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
qa/
|
||||
├── README.md ← this file
|
||||
├── plans/
|
||||
│ └── <release>.md ← per-release test plans (one file per RC)
|
||||
└── scripts/
|
||||
└── api-contract-diff.sh ← CoreScope-tuned API contract diff
|
||||
```
|
||||
|
||||
## How to run
|
||||
|
||||
```
|
||||
qa staging # use the latest plans/v*-rc.md against staging
|
||||
qa pr 806 # use plans/pr-806.md if it exists, else latest plans/v*-rc.md
|
||||
qa v3.6.0-rc # use plans/v3.6.0-rc.md
|
||||
```
|
||||
|
||||
The parent agent loads the qa-suite skill, which reads:
|
||||
1. The plan file from `qa/plans/`
|
||||
2. Bundled scripts from `qa/scripts/`
|
||||
3. The reusable engine + qa-engineer persona from the skill itself
|
||||
|
||||
## Adding a new plan
|
||||
|
||||
For each release candidate, copy the latest `plans/v*-rc.md` to `plans/<new-tag>.md` and update:
|
||||
- The commit-range header (`vN.M..master`)
|
||||
- Any new sections for new features in the release
|
||||
- The "Test data" section if new fixture types are needed
|
||||
- The GO criteria (which sections are blockers)
|
||||
|
||||
## Adding a new script
|
||||
|
||||
Custom scripts go in `qa/scripts/` with `mode=auto: <script-name>` referenced from the plan. The qa-engineer subagent runs them with two args: `BASELINE_URL TARGET_URL`.
|
||||
|
||||
Authoring rules from the qa-suite skill:
|
||||
- 4-way error classification: `curl-failed` / `parse-empty` / `shape-diff` / field-missing
|
||||
- Distinguish HTTP errors from jq parse failures
|
||||
- Don't silence stderr — script bugs must surface
|
||||
- Exit code = number of failures
|
||||
@@ -0,0 +1,108 @@
|
||||
# Plan: v3.6.0-rc
|
||||
|
||||
Targets the changes between v3.5.1 and v3.6.0 candidate (~34 commits).
|
||||
|
||||
## Test data
|
||||
The qa-engineer should pick concrete test fixtures at run time and include them in the report:
|
||||
- **Pivot node pubkey**: pick the top-result from `/api/nodes?limit=20&sort=advert_count` that has `role=repeater` AND a non-zero `totalPaths` from `/api/nodes/{pk}/paths`. Used for sections 5.1, 8.1, 8.2.
|
||||
- **Multi-role pubkey** (section 8.6): pick a node whose pubkey appears in BOTH `/api/observers` and `/api/nodes?role=repeater`. If none → mark 8.6 `needs-human`.
|
||||
- **Sample packet hash**: `/api/packets?limit=1` → `.packets[0].hash`. Used for sections 3.x.
|
||||
- **Channel sample**: pick a channel name from `/api/channels` (if endpoint exists) or scrape `/#channels` page.
|
||||
|
||||
Record every fixture used in the final report so failures are reproducible.
|
||||
|
||||
## Sections
|
||||
|
||||
### 1. Memory & Load
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 1.1 | Container with **3 GB** limit starts on heaviest available DB | No OOM, steady state under limit. Note: 1 GB cap is unrealistic without `GOMEMLIMIT` and bounded cold-load — see #836 | #806/#836 | human |
|
||||
| 1.2 | Hit `/debug/pprof/heap` after Load completes; run `pprof-snapshot.sh` | `unmarshalResolvedPath` absent from top-15 inuse_space; `Load()`-attributed inuse_space ≤ 250 MB on staging-sized DB (~1.5M obs); total heap < 1 GB | #806 | auto: pprof-snapshot.sh |
|
||||
| 1.3 | Set tight `MaxLoadMemMB`, restart | Load stops gracefully at budget; server still serves `/api/stats` 200 | #790 | human |
|
||||
| 1.4 | Watch `processRSSMB` (from `/api/stats`) vs procfs RSS over ingest+eviction cycles | `processRSSMB` tracks `cat /proc/$(pidof corescope)/status | awk '/VmRSS/{print $2}'` (kB → MB) within ±20% across one full eviction cycle. Note: `storeDataMB` (formerly `trackedMB`) is the in-store packet byte estimate and is expected to be a **subset** of RSS, not equal to it. | #751, #832 | human |
|
||||
| 1.5 | Run 30 min under live ingest | Sawtooth heap pattern (≥1 eviction-driven dip), not monotonic ramp | #806/#807 | human |
|
||||
|
||||
### 2. API contract
|
||||
Run `scripts/api-contract-diff.sh BASELINE_URL TARGET_URL` once. Report the script's exit code; nonzero = failures.
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 2.1 | api-contract-diff baseline vs target | Exit code 0; all endpoints carry `resolved_path` where expected | #806 | auto: api-contract-diff.sh |
|
||||
| 2.2 | WebSocket `/ws` carries `resolved_path` on broadcasts | Run JS hook in browser console: `(function(){let n=0,r=0; const W=WebSocket; window.WebSocket=function(...a){const s=new W(...a); s.addEventListener('message',e=>{n++; try{const m=JSON.parse(e.data); if(m && (m.resolved_path !== undefined || (m.observations||[]).some(o=>o.resolved_path!==undefined))) r++;}catch{}}); return s;}; window.__wsCount=()=>({n,r});})()` then navigate to `/`, wait 30s, eval `__wsCount()` — `r` must be ≥ 1 if `n` ≥ 1 | #806 | browser |
|
||||
|
||||
### 3. Decoder & hashing
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 3.1 | Recompute content hashes for sample of recent packets vs stored | All match (hash uses payload-type bits only) | #787 | human |
|
||||
| 3.2 | Inspect a TRACE packet detail panel | path_json length matches path_sz from flags byte | #732 | browser |
|
||||
| 3.3 | Check `hash_size` on transport-route packet AND zero-hop advert | Correct hash_size detected | #747 | browser |
|
||||
| 3.4 | Field-table column offsets for transport-route packet | Snapshot of detail panel: each field row has nonzero `offset`/`length` cells AND offsets monotonically increase | #766 | browser |
|
||||
| 3.5 | Corrupt advert ingest log check | Rejected, counted, no DB entry | #794 | human |
|
||||
| 3.6 | Public channel packet rendering | No empty/garbled decode | #761 | browser |
|
||||
|
||||
### 4. Channels (#725)
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 4.1 | Channel list — full message history loads from DB | Past messages persist across reload | #726 | browser |
|
||||
| 4.2 | Add custom channel via UI (then remove it as teardown) | Channel appears, encrypted msgs decrypt; teardown removes it cleanly. STAGING ONLY. | #733 | browser |
|
||||
| 4.3 | PSK channel add + channel removal (already a self-teardown) | Both work, UI state correct after | #750 | browser |
|
||||
| 4.4 | Deep link to encrypted channel without key | Lock message shows | #783 | browser |
|
||||
| 4.5 | Undecryptable msgs hidden by default + toggle | Hidden default; toggle shows | #728 | browser |
|
||||
| 4.6 | Add-channel button + hint + status feedback | All present | #760 | browser |
|
||||
| 4.7 | Filter packets by channel | Functional: filter applies, packet count drops; performance: response time ≤ 500 ms for `/api/packets?channel=<name>&limit=100` (timed via `curl -w '%{time_total}'`) | #762/#763 | browser+auto |
|
||||
|
||||
### 5. Clock skew (#690)
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 5.1 | Node detail clock-skew badge + sparkline | Both render | #746/#752 | browser |
|
||||
| 5.2 | Analytics fleet clock-skew page | Renders, epoch-0 filtered | #769 | browser |
|
||||
| 5.3 | Outlier sample doesn't poison median | Sanity caps respected; severity uses `recentMedianSkewSec` (#789), not all-time `medianSkewSec` | #769/#789 | human |
|
||||
| 5.4 | Roles page clock-skew indicator | Renders | #752 | browser |
|
||||
|
||||
### 6. Observers
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 6.1 | Observer with no packets in N days disappears after retention sweep | Removed | #764 | human |
|
||||
| 6.2 | Analytics observer-graph (M1+M2) | Renders (`#observerGraph` element present at `public/analytics.js:2048-2051`) | #774 | browser |
|
||||
|
||||
### 7. Multi-byte hash adopters
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 7.1 | Hash Usage Matrix collision details for all hash sizes | Click cell → colliding pubkeys shown | #758 | browser |
|
||||
| 7.2 | Multi-byte adopter table includes all node types | Repeaters, room servers, sensors all present | #767 | browser |
|
||||
| 7.3 | Role column reflects multi-byte adoption + advert precedence | For 3 sample multi-byte adopter pubkeys (from #758 matrix), the Role column on `/#nodes` matches the role inferred from their latest advert flags via `/api/nodes/{pk}/health` | #767 | browser |
|
||||
|
||||
### 8. Frontend nav & deep linking
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 8.1 | Click node on map/list — URL hash updates, panel opens | Hash matches | #739 | browser |
|
||||
| 8.2 | Open saved deep-link to a node | Full-screen detail view opens (post-#823: desktop deep links match the Details-link path) | #739/#823 | browser |
|
||||
| 8.3 | Packets page filter URL hash | Reload preserves filters | #740 | browser |
|
||||
| 8.4 | Details/Analytics links in node detail panel | Navigate without router glitch | #779/#785 | browser |
|
||||
| 8.5 | Neighbor graph slider | Persists across reloads, default 0.7 | #776 | browser |
|
||||
| 8.6 | Repeater that's also observer | Single map marker | #745 | browser |
|
||||
| 8.7 | Side-panel "Recent Packets" — click any entry, lands on packet detail (no 404), entry text is readable in current theme | DB-fallback works (#827); `.advert-info` has explicit color (#829) | #827/#829 | browser |
|
||||
|
||||
### 9. Geofilter & customizer
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 9.1 | Customize → "Open geofilter builder" link | Opens app-served builder | #735 | browser |
|
||||
| 9.2 | Build a filter, save, reload (STAGING ONLY; teardown: delete the saved filter) | Persists across reload; teardown removes it | #735 | browser |
|
||||
| 9.3 | Geofilter docs page | Renders, content matches behavior | #734 | browser |
|
||||
|
||||
### 10. Node blacklist
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 10.1 | Add node pubkey to nodeBlacklist config; restart | Hidden from listings/map/neighbor graph | #742 | auto: blacklist-test.sh |
|
||||
| 10.2 | Packets still in DB | Yes (filter not delete) | #742 | auto: blacklist-test.sh |
|
||||
|
||||
`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Optional: `TARGET_DB_PATH` or `ADMIN_API_TOKEN` for §10.2 probe; `TARGET_SSH_KEY` (default `/root/.ssh/id_ed25519`). Mandatory teardown removes the pubkey and verifies the node returns to listings.
|
||||
|
||||
### 11. Deploy/ops
|
||||
| # | Step | Pass criteria | Source | Mode |
|
||||
|---|---|---|---|---|
|
||||
| 11.1 | Force-redeploy staging | Container removed cleanly even if `docker run`, not compose. Playwright E2E `Desktop: deep link #/nodes/{pubkey} opens full-screen detail view` passes (updated #833 — was asserting old pre-#823 split-panel behavior). | fa348ef/#833 | human |
|
||||
|
||||
## GO criteria
|
||||
- Sections 1.2, 2, 3 must all pass — release blockers
|
||||
- Section 4 (channels) — any visible regression must be fixed before tag
|
||||
- Other sections: file follow-up issues; decide per-item whether to tag with known issues
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
# api-contract-diff.sh — diff CoreScope API endpoints between two deployments.
|
||||
# Usage: api-contract-diff.sh BASELINE_URL TARGET_URL [-k AUTH_HEADER]
|
||||
#
|
||||
# Compares JSON shape (recursive key set) per endpoint and asserts presence of
|
||||
# `resolved_path` where contract requires it. Prints a per-endpoint result line
|
||||
# (✅/❌) and a summary. Exit code = number of failures.
|
||||
#
|
||||
# Distinguishes:
|
||||
# curl-failed → HTTP error or network timeout (real outage)
|
||||
# parse-empty → curl succeeded but response shape unexpected (probable
|
||||
# contract drift in this script or in the API)
|
||||
# shape-diff → recursive key set differs between baseline and target
|
||||
# rp-missing → resolved_path absent on target where it was promised
|
||||
#
|
||||
# PUBLIC repo: do not commit URLs or keys here. Caller passes them.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
OLD="${1:-}"; NEW="${2:-}"
|
||||
[[ -z "$OLD" || -z "$NEW" ]] && { echo "usage: $0 BASELINE_URL TARGET_URL [-k AUTH_HEADER]" >&2; exit 2; }
|
||||
shift 2 || true
|
||||
AUTH=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-k) AUTH="$2"; shift 2 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# Wrapper: fetch URL, return body on stdout, exit 1 on HTTP error / timeout.
|
||||
fetch() {
|
||||
local url="$1" out="$2"
|
||||
local code
|
||||
code=$(curl -s -m 30 -o "$out" -w "%{http_code}" ${AUTH:+-H "$AUTH"} "$url" 2>/dev/null) || code="000"
|
||||
if [[ "$code" != "2"* ]]; then
|
||||
echo " HTTP $code"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Seed lookups from TARGET (so the picked IDs are guaranteed present there).
|
||||
seed_packets="$TMP/seed_packets.json"
|
||||
seed_observers="$TMP/seed_observers.json"
|
||||
seed_nodes="$TMP/seed_nodes.json"
|
||||
|
||||
if ! fetch "$NEW/api/packets?limit=1" "$seed_packets"; then echo "seed /api/packets failed" >&2; fi
|
||||
if ! fetch "$NEW/api/observers" "$seed_observers"; then echo "seed /api/observers failed" >&2; fi
|
||||
if ! fetch "$NEW/api/nodes?limit=1" "$seed_nodes"; then echo "seed /api/nodes failed" >&2; fi
|
||||
|
||||
HASH=$(jq -r '.packets[0].hash // empty' "$seed_packets" 2>/dev/null || true)
|
||||
OBSID=$(jq -r '.observers[0].id // empty' "$seed_observers" 2>/dev/null || true)
|
||||
NODEPK=$(jq -r '.nodes[0].public_key // empty' "$seed_nodes" 2>/dev/null || true)
|
||||
|
||||
[[ -z "$HASH" ]] && echo "warn: no packet hash from /api/packets — packet-detail endpoints will be skipped" >&2
|
||||
[[ -z "$OBSID" ]] && echo "warn: no observer id from /api/observers — observer-detail endpoints will be skipped" >&2
|
||||
[[ -z "$NODEPK" ]] && echo "warn: no node pubkey from /api/nodes — node-detail endpoints will be skipped" >&2
|
||||
|
||||
# Endpoints to diff: path | jq filter (selects subobject to compare) | RP-required(yes/no)
|
||||
declare -a ENDPOINTS
|
||||
ENDPOINTS+=("/api/packets?limit=20|.packets[0]|yes")
|
||||
ENDPOINTS+=("/api/packets?limit=20&expandObservations=true|.packets[0]|yes")
|
||||
ENDPOINTS+=("/api/observers|.observers[0]|no")
|
||||
[[ -n "$HASH" ]] && ENDPOINTS+=("/api/packets/$HASH|.|yes")
|
||||
[[ -n "$OBSID" ]] && ENDPOINTS+=("/api/observers/$OBSID|.|no")
|
||||
[[ -n "$OBSID" ]] && ENDPOINTS+=("/api/observers/$OBSID/analytics|.|no")
|
||||
[[ -n "$NODEPK" ]] && ENDPOINTS+=("/api/nodes/$NODEPK/health|.recentPackets[0]|yes")
|
||||
[[ -n "$NODEPK" ]] && ENDPOINTS+=("/api/nodes/$NODEPK/paths|.|no")
|
||||
|
||||
# Strip volatile fields (timestamps + counters) from a JSON value.
|
||||
STRIP='walk(if type=="object" then del(.timestamp, .first_seen, .last_seen, .last_heard, .updated_at, .server_time, .packet_count, .packetsLastHour, .uptime_secs, .battery_mv, .noise_floor, .observation_count, .advert_count) else . end)'
|
||||
|
||||
fails=0
|
||||
for ep in "${ENDPOINTS[@]}"; do
|
||||
IFS='|' read -r path filter need_rp <<<"$ep"
|
||||
echo "=== $path (resolved_path required: $need_rp) ==="
|
||||
|
||||
oldfile="$TMP/old.json"; newfile="$TMP/new.json"
|
||||
if ! fetch "$OLD$path" "$oldfile"; then echo " ❌ baseline curl-failed"; fails=$((fails+1)); continue; fi
|
||||
if ! fetch "$NEW$path" "$newfile"; then echo " ❌ target curl-failed"; fails=$((fails+1)); continue; fi
|
||||
|
||||
# Selector + strip on each side. jq stderr is preserved so script bugs surface.
|
||||
oldj=$(jq "$filter | $STRIP" "$oldfile")
|
||||
jq_old_rc=$?
|
||||
newj=$(jq "$filter | $STRIP" "$newfile")
|
||||
jq_new_rc=$?
|
||||
|
||||
if [[ $jq_old_rc -ne 0 ]]; then
|
||||
echo " ❌ baseline jq-error (filter='$filter') — likely script bug or API shape changed"
|
||||
fails=$((fails+1)); continue
|
||||
fi
|
||||
if [[ $jq_new_rc -ne 0 ]]; then
|
||||
echo " ❌ target jq-error (filter='$filter') — likely script bug or API shape changed"
|
||||
fails=$((fails+1)); continue
|
||||
fi
|
||||
if [[ -z "$oldj" || "$oldj" == "null" ]]; then
|
||||
echo " ❌ baseline parse-empty (filter returned empty/null; check API shape)"
|
||||
fails=$((fails+1)); continue
|
||||
fi
|
||||
if [[ -z "$newj" || "$newj" == "null" ]]; then
|
||||
echo " ❌ target parse-empty (filter returned empty/null; check API shape)"
|
||||
fails=$((fails+1)); continue
|
||||
fi
|
||||
|
||||
# Recursive key-set diff. Canonicalize array indices (numbers) → "[]" so two
|
||||
# different sample responses with different array lengths don't false-positive.
|
||||
KEYS_FILTER='[paths(scalars or type=="null" or (type=="array" and length==0) or (type=="object" and length==0)) | map(if type=="number" then "[]" else . end) | join(".")] | unique | .[]'
|
||||
oldkeys=$(echo "$oldj" | jq -r "$KEYS_FILTER" | sort -u)
|
||||
newkeys=$(echo "$newj" | jq -r "$KEYS_FILTER" | sort -u)
|
||||
if ! diff <(echo "$oldkeys") <(echo "$newkeys") >/dev/null; then
|
||||
echo " ❌ shape-diff (key set differs):"
|
||||
diff <(echo "$oldkeys") <(echo "$newkeys") | sed 's/^/ /'
|
||||
fails=$((fails+1))
|
||||
continue
|
||||
fi
|
||||
|
||||
# If RP expected, assert present on target (any value, may be null).
|
||||
if [[ "$need_rp" == "yes" ]]; then
|
||||
if ! echo "$newj" | jq -e '.. | objects | select(has("resolved_path")) | .resolved_path' >/dev/null 2>&1; then
|
||||
echo " ❌ rp-missing (resolved_path not present anywhere in selector)"
|
||||
fails=$((fails+1))
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " ✅ ok"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "failures: $fails / ${#ENDPOINTS[@]}"
|
||||
exit $fails
|
||||
Executable
+271
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env bash
|
||||
# blacklist-test.sh — verify nodeBlacklist hides a pubkey from API surface
|
||||
# while retaining its packets in the DB. Implements QA plan §10.1 + §10.2.
|
||||
#
|
||||
# Usage:
|
||||
# blacklist-test.sh BASELINE_URL TARGET_URL
|
||||
#
|
||||
# BASELINE_URL is currently unused for assertions but kept as a positional
|
||||
# arg for parity with other qa-suite scripts (always called with two URLs).
|
||||
#
|
||||
# Required env (target host control + test data):
|
||||
# TEST_NODE_PUBKEY — hex pubkey of a real, currently-visible node on TARGET_URL
|
||||
# TARGET_SSH_HOST — e.g. runner@example
|
||||
# TARGET_SSH_KEY — path to ssh private key (default: /root/.ssh/id_ed25519)
|
||||
# TARGET_CONFIG_PATH — absolute path to config.json on the target
|
||||
# TARGET_CONTAINER — docker container name on the target
|
||||
# Optional env:
|
||||
# TARGET_DB_PATH — sqlite db path on the target (for §10.2 sqlite probe)
|
||||
# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of ssh+sqlite
|
||||
# (read from env, not argv — never appears in ps)
|
||||
# CURL_TIMEOUT — per-request curl timeout, seconds (default 60)
|
||||
# RESTART_WAIT_S — max wait for /api/stats after restart (default 120)
|
||||
#
|
||||
# Distinguishes:
|
||||
# ssh-failed → cannot reach/control target
|
||||
# restart-stuck → /api/stats not 200 within RESTART_WAIT_S
|
||||
# hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail)
|
||||
# retain-failed → blacklisted pubkey absent from DB (§10.2 fail)
|
||||
# teardown-failed→ post-test removal did not restore listing
|
||||
#
|
||||
# Exit code = number of failures (0 = pass).
|
||||
# PUBLIC repo: zero PII — no real pubkeys, IPs, or hostnames as defaults.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
BASELINE_URL="${1:-}"
|
||||
TARGET_URL="${2:-}"
|
||||
if [[ -z "$BASELINE_URL" || -z "$TARGET_URL" ]]; then
|
||||
echo "usage: $0 BASELINE_URL TARGET_URL (TEST_NODE_PUBKEY+TARGET_* via env)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
TEST_PUBKEY="${TEST_NODE_PUBKEY:-}"
|
||||
TARGET_SSH_HOST="${TARGET_SSH_HOST:-}"
|
||||
TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}"
|
||||
TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}"
|
||||
TARGET_CONTAINER="${TARGET_CONTAINER:-}"
|
||||
TARGET_DB_PATH="${TARGET_DB_PATH:-}"
|
||||
ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}"
|
||||
|
||||
if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then
|
||||
echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Hard input validation — these strings are interpolated into remote shell/SQL.
|
||||
# Pubkey must be hex (MeshCore pubkeys are hex-encoded ed25519 prefixes).
|
||||
if ! [[ "$TEST_PUBKEY" =~ ^[0-9a-fA-F]+$ ]]; then
|
||||
echo "error: TEST_NODE_PUBKEY must be hex (got: redacted)" >&2
|
||||
exit 2
|
||||
fi
|
||||
# Container name must match docker's allowed chars: [a-zA-Z0-9][a-zA-Z0-9_.-]*
|
||||
if ! [[ "$TARGET_CONTAINER" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then
|
||||
echo "error: TARGET_CONTAINER has illegal chars" >&2
|
||||
exit 2
|
||||
fi
|
||||
# Config path must be an absolute, sane path (no spaces, quotes, $, ;, etc.).
|
||||
if ! [[ "$TARGET_CONFIG_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then
|
||||
echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then
|
||||
echo "error: TARGET_DB_PATH must be a sane absolute path" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
CURL_TIMEOUT="${CURL_TIMEOUT:-60}"
|
||||
RESTART_WAIT_S="${RESTART_WAIT_S:-120}"
|
||||
|
||||
SSH_OPTS=(-i "$TARGET_SSH_KEY" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o BatchMode=yes)
|
||||
ssh_t() { ssh "${SSH_OPTS[@]}" "$TARGET_SSH_HOST" "$@"; }
|
||||
|
||||
TMP=$(mktemp -d)
|
||||
fails=0
|
||||
TEARDOWN_DONE=0
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Teardown — MANDATORY in all exit paths.
|
||||
# -----------------------------------------------------------------------------
|
||||
teardown() {
|
||||
local rc=$?
|
||||
if [[ "$TEARDOWN_DONE" == "1" ]]; then rm -rf "$TMP"; exit "$rc"; fi
|
||||
TEARDOWN_DONE=1
|
||||
echo "=== teardown: removing $TEST_PUBKEY from nodeBlacklist ==="
|
||||
if remove_from_blacklist && restart_target && wait_for_stats; then
|
||||
if node_visible; then
|
||||
echo " ✅ teardown ok — node returned to listings"
|
||||
else
|
||||
echo " ❌ teardown-failed: node still hidden after removal"
|
||||
rc=$((rc + 1))
|
||||
fi
|
||||
else
|
||||
echo " ❌ teardown-failed: could not restore config / restart / stats"
|
||||
rc=$((rc + 1))
|
||||
fi
|
||||
rm -rf "$TMP"
|
||||
exit "$rc"
|
||||
}
|
||||
trap teardown EXIT INT TERM
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# -----------------------------------------------------------------------------
|
||||
fetch_code() {
|
||||
local url="$1" out="$2"
|
||||
curl -s -m "$CURL_TIMEOUT" -o "$out" -w "%{http_code}" "$url" 2>/dev/null || echo "000"
|
||||
}
|
||||
|
||||
wait_for_stats() {
|
||||
local deadline code
|
||||
echo " waiting up to ${RESTART_WAIT_S}s for $TARGET_URL/api/stats ..."
|
||||
deadline=$(( $(date +%s) + RESTART_WAIT_S ))
|
||||
while (( $(date +%s) < deadline )); do
|
||||
code=$(fetch_code "$TARGET_URL/api/stats" "$TMP/stats.json")
|
||||
if [[ "$code" == "200" ]]; then echo " stats OK"; return 0; fi
|
||||
sleep 3
|
||||
done
|
||||
echo " ❌ restart-stuck: /api/stats never returned 200"
|
||||
return 1
|
||||
}
|
||||
|
||||
restart_target() {
|
||||
echo " restarting container $TARGET_CONTAINER ..."
|
||||
# TARGET_CONTAINER is validated above; still quote defensively.
|
||||
if ! ssh_t "docker restart $(printf %q "$TARGET_CONTAINER")" >/dev/null; then
|
||||
echo " ❌ ssh-failed: docker restart failed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Mutate config.json on target. Values pass via env (printf %q + single-quoted
|
||||
# heredoc) so $TEST_PUBKEY etc. never enter the remote shell as code.
|
||||
set_blacklist_state() {
|
||||
local mode="$1" # add | remove
|
||||
ssh_t "CFG=$(printf %q "$TARGET_CONFIG_PATH") PK=$(printf %q "$TEST_PUBKEY") MODE=$(printf %q "$mode") bash -s" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
TMP="$(mktemp)"
|
||||
trap 'rm -f "$TMP"' EXIT
|
||||
if command -v jq >/dev/null; then
|
||||
if [ "$MODE" = "add" ]; then
|
||||
jq --arg pk "$PK" '.nodeBlacklist = ((.nodeBlacklist // []) + [$pk] | unique)' "$CFG" > "$TMP"
|
||||
else
|
||||
jq --arg pk "$PK" '.nodeBlacklist = ((.nodeBlacklist // []) - [$pk])' "$CFG" > "$TMP"
|
||||
fi
|
||||
else
|
||||
python3 - "$CFG" "$PK" "$MODE" "$TMP" <<'PY'
|
||||
import json, sys
|
||||
cfg, pk, mode, out = sys.argv[1:]
|
||||
with open(cfg) as f: d = json.load(f)
|
||||
bl = list(dict.fromkeys(d.get("nodeBlacklist") or []))
|
||||
if mode == "add":
|
||||
if pk not in bl: bl.append(pk)
|
||||
else:
|
||||
bl = [x for x in bl if x != pk]
|
||||
d["nodeBlacklist"] = bl
|
||||
with open(out, "w") as f: json.dump(d, f, indent=2)
|
||||
PY
|
||||
fi
|
||||
# Preserve mode and ownership; mv across same FS is atomic.
|
||||
chmod --reference="$CFG" "$TMP" 2>/dev/null || true
|
||||
chown --reference="$CFG" "$TMP" 2>/dev/null || true
|
||||
mv "$TMP" "$CFG"
|
||||
trap - EXIT
|
||||
REMOTE
|
||||
local rc=$?
|
||||
if (( rc != 0 )); then
|
||||
echo " ❌ ssh-failed: could not edit $TARGET_CONFIG_PATH ($mode)"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
add_to_blacklist() { set_blacklist_state add; }
|
||||
remove_from_blacklist() { set_blacklist_state remove; }
|
||||
|
||||
node_visible() {
|
||||
# Returns 0 if the pubkey is currently visible via API.
|
||||
local code
|
||||
code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/node.json")
|
||||
if [[ "$code" == "200" ]]; then return 0; fi
|
||||
fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/nodes.json" >/dev/null
|
||||
if grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/nodes.json" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# §10.1 — hide
|
||||
# -----------------------------------------------------------------------------
|
||||
echo "=== §10.1 add $TEST_PUBKEY to nodeBlacklist ==="
|
||||
if ! add_to_blacklist; then fails=$((fails+1)); exit "$fails"; fi
|
||||
if ! restart_target; then fails=$((fails+1)); exit "$fails"; fi
|
||||
if ! wait_for_stats; then fails=$((fails+1)); exit "$fails"; fi
|
||||
|
||||
detail_code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/detail.json")
|
||||
list_code=$(fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/list.json")
|
||||
in_list=0
|
||||
if [[ "$list_code" == "200" ]] && grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/list.json"; then
|
||||
in_list=1
|
||||
fi
|
||||
if [[ "$detail_code" == "404" || "$in_list" == "0" ]]; then
|
||||
echo " ✅ hide ok: detail=$detail_code in_list=$in_list"
|
||||
else
|
||||
echo " ❌ hide-failed: detail=$detail_code in_list=$in_list — pubkey still surfaced"
|
||||
fails=$((fails+1))
|
||||
fi
|
||||
|
||||
topo_code=$(fetch_code "$TARGET_URL/api/topology" "$TMP/topo.json")
|
||||
if [[ "$topo_code" != "200" ]]; then
|
||||
echo " ⚠️ /api/topology HTTP $topo_code — skipping topology assertion"
|
||||
elif grep -qF -- "$TEST_PUBKEY" "$TMP/topo.json"; then
|
||||
echo " ❌ hide-failed: /api/topology references blacklisted pubkey"
|
||||
fails=$((fails+1))
|
||||
else
|
||||
echo " ✅ topology clean"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# §10.2 — DB retain
|
||||
# -----------------------------------------------------------------------------
|
||||
echo "=== §10.2 verify packets retained in DB ==="
|
||||
count=""
|
||||
if [[ -n "$ADMIN_API_TOKEN" ]]; then
|
||||
# Read auth header from stdin so the token never enters argv (ps-safe).
|
||||
code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \
|
||||
curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \
|
||||
"$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000")
|
||||
if [[ "$code" == "200" ]]; then
|
||||
count=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "")
|
||||
fi
|
||||
fi
|
||||
if [[ -z "$count" ]]; then
|
||||
if [[ -z "$TARGET_DB_PATH" ]]; then
|
||||
echo " ❌ retain-failed: TARGET_DB_PATH unset and no ADMIN_API_TOKEN — cannot probe"
|
||||
fails=$((fails+1))
|
||||
else
|
||||
# TEST_PUBKEY is hex-validated → safe to inline single-quoted in SQL.
|
||||
# Container/db path also validated; printf %q for defense in depth.
|
||||
q="SELECT COUNT(*) FROM transmissions WHERE from_node = '$TEST_PUBKEY';"
|
||||
qq=$(printf %q "$q")
|
||||
if ! count=$(ssh_t "docker exec $(printf %q "$TARGET_CONTAINER") sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null); then
|
||||
count=$(ssh_t "sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null || echo "")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$count" ]]; then
|
||||
echo " ❌ retain-failed: could not read transmissions count"
|
||||
fails=$((fails+1))
|
||||
elif [[ "$count" =~ ^[0-9]+$ ]] && (( count > 0 )); then
|
||||
echo " ✅ DB retains $count packets from $TEST_PUBKEY"
|
||||
else
|
||||
echo " ❌ retain-failed: count=$count (expected > 0)"
|
||||
fails=$((fails+1))
|
||||
fi
|
||||
|
||||
echo "=== summary: $fails failure(s) before teardown ==="
|
||||
# trap handles teardown + exit
|
||||
exit "$fails"
|
||||
@@ -0,0 +1,136 @@
|
||||
// migrate-fixture-hashes recomputes content hashes in a fixture DB using the
|
||||
// current ComputeContentHash formula. Run once; idempotent.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func computeContentHash(rawHex string) string {
|
||||
buf, err := hex.DecodeString(rawHex)
|
||||
if err != nil || len(buf) < 2 {
|
||||
if len(rawHex) >= 16 {
|
||||
return rawHex[:16]
|
||||
}
|
||||
return rawHex
|
||||
}
|
||||
|
||||
headerByte := buf[0]
|
||||
offset := 1
|
||||
routeType := int(headerByte & 0x03)
|
||||
if routeType == 2 || routeType == 3 { // transport
|
||||
offset += 4
|
||||
}
|
||||
if offset >= len(buf) {
|
||||
if len(rawHex) >= 16 {
|
||||
return rawHex[:16]
|
||||
}
|
||||
return rawHex
|
||||
}
|
||||
pathByte := buf[offset]
|
||||
offset++
|
||||
hashSize := int((pathByte>>6)&0x3) + 1
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
pathBytes := hashSize * hashCount
|
||||
|
||||
payloadStart := offset + pathBytes
|
||||
if payloadStart > len(buf) {
|
||||
if len(rawHex) >= 16 {
|
||||
return rawHex[:16]
|
||||
}
|
||||
return rawHex
|
||||
}
|
||||
|
||||
payload := buf[payloadStart:]
|
||||
payloadType := (headerByte >> 2) & 0x0F
|
||||
toHash := []byte{payloadType}
|
||||
|
||||
// TRACE = payload type 7
|
||||
if int(payloadType) == 7 {
|
||||
toHash = append(toHash, pathByte, 0x00)
|
||||
}
|
||||
toHash = append(toHash, payload...)
|
||||
|
||||
h := sha256.Sum256(toHash)
|
||||
return hex.EncodeToString(h[:])[:16]
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintf(os.Stderr, "usage: %s <db-path>\n", os.Args[0])
|
||||
os.Exit(1)
|
||||
}
|
||||
dbPath := os.Args[1]
|
||||
|
||||
db, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query("SELECT id, raw_hex, hash FROM transmissions")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
type update struct {
|
||||
id int
|
||||
newHash string
|
||||
}
|
||||
var updates []update
|
||||
|
||||
for rows.Next() {
|
||||
var id int
|
||||
var rawHex, oldHash string
|
||||
if err := rows.Scan(&id, &rawHex, &oldHash); err != nil {
|
||||
log.Printf("scan: %v", err)
|
||||
continue
|
||||
}
|
||||
newHash := computeContentHash(rawHex)
|
||||
if newHash != oldHash {
|
||||
updates = append(updates, update{id, newHash})
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if len(updates) == 0 {
|
||||
fmt.Println("All hashes already match current formula.")
|
||||
return
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
stmt, err := tx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
merged := 0
|
||||
for _, u := range updates {
|
||||
if _, err := stmt.Exec(u.newHash, u.id); err != nil {
|
||||
// UNIQUE constraint = duplicate (same content, different old hash).
|
||||
// Move observations to the surviving tx, then delete the dup.
|
||||
log.Printf("update id %d: %v — merging duplicate", u.id, err)
|
||||
// Find surviving tx id
|
||||
var survID int
|
||||
if err2 := tx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil {
|
||||
tx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.id)
|
||||
tx.Exec("DELETE FROM transmissions WHERE id = ?", u.id)
|
||||
merged++
|
||||
}
|
||||
}
|
||||
}
|
||||
stmt.Close()
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("Migrated %d hashes, merged %d duplicates.\n", len(updates)-merged, merged)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Tests for #759 — Add channel UX: button, hint, status feedback.
|
||||
* Validates the HTML structure rendered by channels.js init.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function assert(cond, msg) {
|
||||
if (cond) { passed++; console.log(' ✓ ' + msg); }
|
||||
else { failed++; console.error(' ✗ ' + msg); }
|
||||
}
|
||||
|
||||
function assertIncludes(html, substr, msg) {
|
||||
assert(html.includes(substr), msg);
|
||||
}
|
||||
|
||||
// Read the channels.js source to extract the HTML template
|
||||
const src = fs.readFileSync(__dirname + '/public/channels.js', 'utf8');
|
||||
|
||||
// Extract the sidebar HTML from the template literal
|
||||
const htmlMatch = src.match(/app\.innerHTML\s*=\s*`([\s\S]*?)`;/);
|
||||
const html = htmlMatch ? htmlMatch[1] : '';
|
||||
|
||||
console.log('Test: Add channel UX (#759)');
|
||||
|
||||
// 1. Button renders in the form
|
||||
assertIncludes(html, 'class="ch-add-btn"', 'Add button has ch-add-btn class');
|
||||
assertIncludes(html, 'type="submit"', 'Button is type=submit');
|
||||
assertIncludes(html, '>+</button>', 'Button shows + text');
|
||||
|
||||
// 2. Form has proper structure
|
||||
assertIncludes(html, 'class="ch-add-form"', 'Form has ch-add-form class');
|
||||
assertIncludes(html, 'class="ch-add-row"', 'Row wrapper present');
|
||||
assert(!html.includes('class="ch-add-label"'), 'Label removed (redundant with hint)');
|
||||
|
||||
// 3. Hint text present
|
||||
assertIncludes(html, 'class="ch-add-hint"', 'Hint div present');
|
||||
assertIncludes(html, 'e.g. #LongFast or 32-char hex key', 'Hint text correct');
|
||||
|
||||
// 4. Status div present
|
||||
assertIncludes(html, 'id="chAddStatus"', 'Status div has correct id');
|
||||
assertIncludes(html, 'class="ch-add-status"', 'Status div has correct class');
|
||||
assertIncludes(html, 'style="display:none"', 'Status div hidden by default');
|
||||
|
||||
// 5. showAddStatus function exists in source
|
||||
assert(src.includes('function showAddStatus('), 'showAddStatus function defined');
|
||||
assert(src.includes("'success'"), 'Success status type referenced');
|
||||
assert(src.includes("'error'"), 'Error status type referenced');
|
||||
|
||||
// 6. CSS classes exist
|
||||
const css = fs.readFileSync(__dirname + '/public/style.css', 'utf8');
|
||||
assert(css.includes('.ch-add-form'), 'CSS: .ch-add-form defined');
|
||||
assert(css.includes('.ch-add-btn'), 'CSS: .ch-add-btn defined');
|
||||
assert(css.includes('.ch-add-hint'), 'CSS: .ch-add-hint defined');
|
||||
assert(css.includes('.ch-add-status'), 'CSS: .ch-add-status defined');
|
||||
assert(css.includes('.ch-add-row'), 'CSS: .ch-add-row defined');
|
||||
// .ch-add-label CSS kept for backward compat but label removed from HTML
|
||||
|
||||
console.log('\n' + passed + ' passed, ' + failed + ' failed');
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Tests for #725 M3 (PSK hex key), M4 (channel removal), M5 (message caching).
|
||||
* Runs in Node.js via vm.createContext to simulate browser environment.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const vm = require('vm');
|
||||
const fs = require('fs');
|
||||
const { subtle } = require('crypto').webcrypto;
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function assert(cond, msg) {
|
||||
if (cond) { passed++; console.log(' ✓ ' + msg); }
|
||||
else { failed++; console.error(' ✗ ' + msg); }
|
||||
}
|
||||
|
||||
// Build a minimal browser-like sandbox
|
||||
function createSandbox() {
|
||||
const storage = {};
|
||||
const localStorage = {
|
||||
getItem: (k) => storage[k] !== undefined ? storage[k] : null,
|
||||
setItem: (k, v) => { storage[k] = String(v); },
|
||||
removeItem: (k) => { delete storage[k]; },
|
||||
_data: storage
|
||||
};
|
||||
|
||||
const ctx = {
|
||||
window: {},
|
||||
crypto: { subtle },
|
||||
TextEncoder: TextEncoder,
|
||||
TextDecoder: TextDecoder,
|
||||
Uint8Array,
|
||||
localStorage,
|
||||
console,
|
||||
Date,
|
||||
JSON,
|
||||
parseInt,
|
||||
Math,
|
||||
String,
|
||||
Number,
|
||||
Object,
|
||||
Array,
|
||||
RegExp,
|
||||
Error,
|
||||
Promise,
|
||||
setTimeout,
|
||||
btoa: (s) => Buffer.from(s, 'binary').toString('base64'),
|
||||
atob: (s) => Buffer.from(s, 'base64').toString('binary'),
|
||||
};
|
||||
ctx.window = ctx;
|
||||
ctx.self = ctx;
|
||||
return ctx;
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== M3: PSK hex key detection ===');
|
||||
|
||||
// Load channel-decrypt.js in sandbox
|
||||
const cdSrc = fs.readFileSync(__dirname + '/public/channel-decrypt.js', 'utf8');
|
||||
const sandbox = createSandbox();
|
||||
const context = vm.createContext(sandbox);
|
||||
vm.runInContext(cdSrc, context);
|
||||
const CD = sandbox.window.ChannelDecrypt;
|
||||
|
||||
// Test: isHexKey detection (via channels.js logic)
|
||||
// We test the pattern directly since isHexKey is inside channels.js IIFE
|
||||
const isHexKey = (val) => /^[0-9a-fA-F]{32}$/.test(val);
|
||||
|
||||
assert(isHexKey('0123456789abcdef0123456789abcdef'), 'Valid 32-char hex detected');
|
||||
assert(isHexKey('AABBCCDD11223344AABBCCDD11223344'), 'Valid uppercase hex detected');
|
||||
assert(!isHexKey('#LongFast'), 'Hashtag name NOT detected as hex');
|
||||
assert(!isHexKey('0123456789abcdef'), 'Short hex (16 chars) NOT detected');
|
||||
assert(!isHexKey('0123456789abcdef0123456789abcdefXX'), 'Too long NOT detected');
|
||||
assert(!isHexKey('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'), 'Non-hex chars NOT detected');
|
||||
|
||||
// Test: PSK decrypt with known key bytes
|
||||
console.log('\n=== M3: PSK decrypt produces correct plaintext ===');
|
||||
|
||||
// Derive a key from #LongFast for testing
|
||||
const keyBytes = await CD.deriveKey('#LongFast');
|
||||
assert(keyBytes.length === 16, 'Derived key is 16 bytes');
|
||||
|
||||
const keyHex = CD.bytesToHex(keyBytes);
|
||||
assert(keyHex.length === 32, 'Key hex is 32 chars');
|
||||
|
||||
// Round-trip: hex → bytes → hex
|
||||
const roundTrip = CD.bytesToHex(CD.hexToBytes(keyHex));
|
||||
assert(roundTrip === keyHex, 'Hex round-trip preserves key');
|
||||
|
||||
// Channel hash computation works
|
||||
const hashByte = await CD.computeChannelHash(keyBytes);
|
||||
assert(typeof hashByte === 'number' && hashByte >= 0 && hashByte <= 255, 'Channel hash byte is valid (0-255)');
|
||||
|
||||
// PSK key (raw hex) stored and retrieved correctly
|
||||
const pskHex = 'aabbccdd11223344aabbccdd11223344';
|
||||
CD.storeKey('psk:aabbccdd', pskHex);
|
||||
const keys = CD.getStoredKeys();
|
||||
assert(keys['psk:aabbccdd'] === pskHex, 'PSK key stored and retrieved correctly');
|
||||
|
||||
console.log('\n=== M4: Channel removal clears key + cache ===');
|
||||
|
||||
// Store a key and some cached messages
|
||||
CD.storeKey('#TestChannel', 'deadbeefdeadbeefdeadbeefdeadbeef');
|
||||
CD.setCache('#TestChannel', [{ sender: 'A', text: 'hello', timestamp: '2026-01-01T00:00:00Z', packetHash: 'h1' }], '2026-01-01T00:00:00Z', 1);
|
||||
|
||||
// Verify they exist
|
||||
var storedKeys = CD.getStoredKeys();
|
||||
assert(storedKeys['#TestChannel'] === 'deadbeefdeadbeefdeadbeefdeadbeef', 'Key exists before removal');
|
||||
var cachedBefore = CD.getCache('#TestChannel');
|
||||
assert(cachedBefore && cachedBefore.messages.length === 1, 'Cache exists before removal');
|
||||
|
||||
// Remove the key (also clears cache)
|
||||
CD.removeKey('#TestChannel');
|
||||
var storedAfter = CD.getStoredKeys();
|
||||
assert(!storedAfter['#TestChannel'], 'Key cleared after removal');
|
||||
var cachedAfter = CD.getCache('#TestChannel');
|
||||
assert(!cachedAfter, 'Cache cleared after removal');
|
||||
|
||||
console.log('\n=== M5: Cache operations ===');
|
||||
|
||||
// Test: setCache with count and size limit
|
||||
var bigMessages = [];
|
||||
for (var i = 0; i < 1200; i++) {
|
||||
bigMessages.push({ sender: 'S', text: 'msg' + i, timestamp: '2026-01-01T00:00:' + String(i).padStart(2, '0') + 'Z', packetHash: 'h' + i });
|
||||
}
|
||||
CD.setCache('bigchannel', bigMessages, '2026-01-01T00:20:00Z', 1200);
|
||||
var bigCached = CD.getCache('bigchannel');
|
||||
assert(bigCached.messages.length <= 1000, 'Cache enforces 1000 message limit (got ' + bigCached.messages.length + ')');
|
||||
assert(bigCached.count === 1200, 'Cache stores total count');
|
||||
assert(bigCached.lastTimestamp === '2026-01-01T00:20:00Z', 'Cache stores lastTimestamp');
|
||||
// Should keep most recent 1000
|
||||
assert(bigCached.messages[0].packetHash === 'h200', 'Cache keeps most recent 1000 (first is h200)');
|
||||
|
||||
// Test: cache hit (delta fetch scenario)
|
||||
CD.setCache('deltatest', [
|
||||
{ sender: 'A', text: 'old', timestamp: '2026-01-01T00:00:00Z', packetHash: 'p1' }
|
||||
], '2026-01-01T00:00:00Z', 1);
|
||||
|
||||
var deltaCache = CD.getCache('deltatest');
|
||||
assert(deltaCache.messages.length === 1, 'Delta cache has 1 message');
|
||||
assert(deltaCache.lastTimestamp === '2026-01-01T00:00:00Z', 'Delta cache lastTimestamp correct');
|
||||
assert(deltaCache.count === 1, 'Delta cache count correct');
|
||||
|
||||
// Test: clearChannelCache
|
||||
CD.setCache('clearthis', [{ sender: 'X', text: 'y' }], 'ts', 1);
|
||||
assert(CD.getCache('clearthis') !== null, 'Cache exists before clear');
|
||||
CD.clearChannelCache('clearthis');
|
||||
assert(CD.getCache('clearthis') === null, 'Cache cleared by clearChannelCache');
|
||||
|
||||
console.log('\n=== Results ===');
|
||||
console.log('Passed: ' + passed + ', Failed: ' + failed);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests().catch(e => { console.error(e); process.exit(1); });
|
||||
+96
-4
@@ -231,6 +231,30 @@ async function run() {
|
||||
assert(hasStatus, 'No status indicator found in node detail');
|
||||
});
|
||||
|
||||
// Test: Node side panel Details link navigates to full detail page (#778)
|
||||
await test('Node side panel Details link navigates', async () => {
|
||||
await page.goto(`${BASE}/#/nodes`, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('table tbody tr');
|
||||
// Click first row to open side panel
|
||||
const firstRow = await page.$('table tbody tr');
|
||||
assert(firstRow, 'No node rows found');
|
||||
await firstRow.click();
|
||||
await page.waitForSelector('.node-detail');
|
||||
// Find the Details link in the side panel
|
||||
const detailsLink = await page.$('#nodesRight a.btn-primary[href^="#/nodes/"]');
|
||||
assert(detailsLink, 'Details link not found in side panel');
|
||||
const href = await detailsLink.getAttribute('href');
|
||||
// Click the Details link — this should navigate to the full detail page
|
||||
await detailsLink.click();
|
||||
// Wait for navigation — the full detail page has sections like neighbors/packets
|
||||
await page.waitForFunction((expectedHash) => {
|
||||
return location.hash === expectedHash;
|
||||
}, href, { timeout: 5000 });
|
||||
// Verify we're on the full detail page (should have section tabs or detail content)
|
||||
const hash = await page.evaluate(() => location.hash);
|
||||
assert(hash === href, `Expected hash "${href}" but got "${hash}"`);
|
||||
});
|
||||
|
||||
// Test: Nodes page has WebSocket auto-update listener (#131)
|
||||
await test('Nodes page has WebSocket auto-update', async () => {
|
||||
await page.goto(`${BASE}/#/nodes`, { waitUntil: 'domcontentloaded' });
|
||||
@@ -398,8 +422,13 @@ async function run() {
|
||||
}
|
||||
}, { timeout: 10000 });
|
||||
|
||||
// Full reload on the packets page — scripts re-execute, IIFE reads localStorage
|
||||
await page.reload({ waitUntil: 'load' });
|
||||
// Force a full page reload to reset module-level state (savedTimeWindowMin is
|
||||
// read from localStorage once at IIFE time). Navigating from /#/packets to /#/packets
|
||||
// is a hash-only change — no reload, so the IIFE never re-reads localStorage.
|
||||
// Going to / first forces a fresh page load, then the hash change to /#/packets
|
||||
// calls init() with the freshly-read savedTimeWindowMin = 60.
|
||||
await page.goto(`${BASE}/`, { waitUntil: 'load' });
|
||||
await page.goto(`${BASE}/#/packets`, { waitUntil: 'load' });
|
||||
await page.waitForSelector('#fTimeWindow', { timeout: 10000 });
|
||||
const timeWindowValue = await page.$eval('#fTimeWindow', (el) => el.value);
|
||||
assert(timeWindowValue === '60', `Expected time window dropdown to restore 60, got ${timeWindowValue}`);
|
||||
@@ -1470,13 +1499,18 @@ async function run() {
|
||||
// ─── Neighbor section tests ───────────────────────────────────────────────
|
||||
|
||||
await test('Node detail: neighbors section exists with correct columns', async () => {
|
||||
// Full-screen node view (with #node-neighbors) is mobile-only since #676 fix.
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
// Navigate to a node detail page (use the first node in the list)
|
||||
await page.goto(BASE + '/#/nodes');
|
||||
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
|
||||
// Get the first node's pubkey from the row's data-key attribute
|
||||
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
|
||||
await page.goto(BASE + '/#/nodes/' + pubkey);
|
||||
await page.waitForSelector('#node-neighbors', { timeout: 10000 });
|
||||
// Use evaluate to change hash (reliable same-document navigation)
|
||||
await page.evaluate((pk) => { location.hash = '#/nodes/' + pk; }, pubkey);
|
||||
// Wait for the full-node view to render (async API fetch populates body)
|
||||
await page.waitForSelector('.node-fullscreen', { timeout: 10000 });
|
||||
await page.waitForSelector('#node-neighbors', { timeout: 25000 });
|
||||
// Check the section exists
|
||||
const header = await page.$eval('#fullNeighborsHeader', el => el.textContent);
|
||||
assert(header.startsWith('Neighbors'), 'Header should start with "Neighbors", got: ' + header);
|
||||
@@ -1500,6 +1534,7 @@ async function run() {
|
||||
const text = await page.$eval('#fullNeighborsContent', el => el.textContent);
|
||||
assert(text.includes('No neighbor data') || text.includes('Could not load'), 'Should show empty or error state');
|
||||
}
|
||||
await page.setViewportSize({ width: 1280, height: 800 });
|
||||
});
|
||||
|
||||
|
||||
@@ -1653,6 +1688,33 @@ async function run() {
|
||||
assert(url.includes('tab=room'), `URL should contain tab=room after click, got: ${url}`);
|
||||
});
|
||||
|
||||
// Test: clicking a node on desktop updates URL hash (#676)
|
||||
await test('Desktop: clicking a node updates URL to #/nodes/{pubkey}', async () => {
|
||||
await page.setViewportSize({ width: 1280, height: 800 });
|
||||
await page.goto(BASE + '#/nodes', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
|
||||
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
|
||||
await page.click('#nodesBody tr[data-key]');
|
||||
await page.waitForTimeout(300);
|
||||
const url = page.url();
|
||||
assert(url.includes(encodeURIComponent(pubkey)), `URL should contain pubkey after click, got: ${url}`);
|
||||
assert(!url.includes('node-fullscreen') || await page.$('#nodesRight:not(.empty)'), 'Split panel should be visible on desktop');
|
||||
});
|
||||
|
||||
// Test: loading #/nodes/{pubkey} on desktop opens full-screen detail view (#823)
|
||||
// Updated from #676's earlier "split panel on desktop" assertion. The Details
|
||||
// link now opens the full-screen single-node view on desktop too — see PR #824.
|
||||
await test('Desktop: deep link #/nodes/{pubkey} opens full-screen detail view', async () => {
|
||||
await page.setViewportSize({ width: 1280, height: 800 });
|
||||
await page.goto(BASE + '#/nodes', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
|
||||
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
|
||||
await page.goto(BASE + '#/nodes/' + encodeURIComponent(pubkey), { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForTimeout(500);
|
||||
const hasFullScreen = await page.$('.node-fullscreen');
|
||||
assert(hasFullScreen, 'Full-screen detail view should be open on desktop deep link (#823)');
|
||||
});
|
||||
|
||||
// Test: packets timeWindow deep link
|
||||
await test('Packets timeWindow deep link restores dropdown', async () => {
|
||||
await page.goto(BASE + '#/packets?timeWindow=60', { waitUntil: 'domcontentloaded' });
|
||||
@@ -1663,6 +1725,36 @@ async function run() {
|
||||
assert(url.includes('timeWindow=60'), `URL should still contain timeWindow=60, got: ${url}`);
|
||||
});
|
||||
|
||||
// Test: hash filter updates URL and is restored (#682)
|
||||
await test('Packets hash filter updates URL and restores on reload', async () => {
|
||||
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#fHash', { timeout: 8000 });
|
||||
await page.fill('#fHash', 'abc123');
|
||||
await page.waitForTimeout(500);
|
||||
const url = page.url();
|
||||
assert(url.includes('hash=abc123'), `URL should contain hash=abc123, got: ${url}`);
|
||||
// Reload and check input restored
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#fHash', { timeout: 8000 });
|
||||
const val = await page.$eval('#fHash', el => el.value);
|
||||
assert(val === 'abc123', `fHash should be restored to abc123, got: ${val}`);
|
||||
});
|
||||
|
||||
// Test: Wireshark filter expression updates URL and is restored (#682)
|
||||
await test('Packets filter expression updates URL and restores on reload', async () => {
|
||||
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#packetFilterInput', { timeout: 8000 });
|
||||
await page.fill('#packetFilterInput', 'type == ADVERT');
|
||||
await page.waitForTimeout(500);
|
||||
const url = page.url();
|
||||
assert(url.includes('filter=') && url.includes('ADVERT'), `URL should contain filter=type%3D%3DADVERT, got: ${url}`);
|
||||
// Reload and check expression restored
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#packetFilterInput', { timeout: 8000 });
|
||||
const val = await page.$eval('#packetFilterInput', el => el.value);
|
||||
assert(val === 'type == ADVERT', `packetFilterInput should be restored, got: ${val}`);
|
||||
});
|
||||
|
||||
// Test: timeWindow change updates URL
|
||||
await test('Packets timeWindow change updates URL', async () => {
|
||||
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
|
||||
|
||||
Binary file not shown.
+1176
-1
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user