mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-07-02 05:11:37 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 you
|
7ec2b77ccd | ||
|
you
|
8d89f7d3e9 | ||
|
you
|
7abd05ff7f |
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"93 passed","color":"brightgreen"}
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"45 passed","color":"brightgreen"}
|
||||
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"40.01%","color":"red"}
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"39.68%","color":"red"}
|
||||
@@ -79,12 +79,6 @@ jobs:
|
||||
go test ./...
|
||||
echo "--- Decrypt CLI tests passed ---"
|
||||
|
||||
- name: Run JS unit tests (packet-filter)
|
||||
run: |
|
||||
set -e
|
||||
node test-packet-filter.js
|
||||
node test-channel-decrypt-insecure-context.js
|
||||
|
||||
- name: Verify proto syntax
|
||||
run: |
|
||||
set -e
|
||||
@@ -141,7 +135,7 @@ jobs:
|
||||
e2e-test:
|
||||
name: "🎭 Playwright E2E Tests"
|
||||
needs: [go-test]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, Linux]
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -151,6 +145,13 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Free disk space
|
||||
run: |
|
||||
# Prune old runner diagnostic logs (can accumulate 50MB+)
|
||||
find ~/actions-runner/_diag/ -name '*.log' -mtime +3 -delete 2>/dev/null || true
|
||||
# Show available disk space
|
||||
df -h / | tail -1
|
||||
|
||||
- name: Set up Node.js 22
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
@@ -182,9 +183,6 @@ jobs:
|
||||
- name: Instrument frontend JS for coverage
|
||||
run: sh scripts/instrument-frontend.sh
|
||||
|
||||
- name: Freshen fixture timestamps
|
||||
run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db
|
||||
|
||||
- name: Start Go server with fixture DB
|
||||
run: |
|
||||
fuser -k 13581/tcp 2>/dev/null || true
|
||||
@@ -192,7 +190,7 @@ jobs:
|
||||
./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented &
|
||||
echo $! > .server.pid
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then
|
||||
if curl -sf http://localhost:13581/api/stats > /dev/null 2>&1; then
|
||||
echo "Server ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
@@ -254,11 +252,17 @@ jobs:
|
||||
build-and-publish:
|
||||
name: "🏗️ Build & Publish Docker Image"
|
||||
needs: [e2e-test]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, meshcore-runner-2]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Free disk space
|
||||
run: |
|
||||
docker system prune -af 2>/dev/null || true
|
||||
docker builder prune -af 2>/dev/null || true
|
||||
df -h /
|
||||
|
||||
- name: Compute build metadata
|
||||
id: meta
|
||||
run: |
|
||||
@@ -286,10 +290,6 @@ jobs:
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Set up QEMU (arm64 runtime stage)
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
@@ -317,7 +317,7 @@ jobs:
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
tags: ${{ steps.docker-meta.outputs.tags }}
|
||||
labels: ${{ steps.docker-meta.outputs.labels }}
|
||||
build-args: |
|
||||
@@ -432,11 +432,10 @@ jobs:
|
||||
|
||||
- name: Smoke test staging API
|
||||
run: |
|
||||
PORT="${STAGING_GO_HTTP_PORT:-80}"
|
||||
if curl -sf "http://localhost:${PORT}/api/stats" | grep -q engine; then
|
||||
if curl -sf http://localhost:82/api/stats | grep -q engine; then
|
||||
echo "Staging verified — engine field present ✅"
|
||||
else
|
||||
echo "Staging /api/stats did not return engine field (port ${PORT})"
|
||||
echo "Staging /api/stats did not return engine field"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -458,7 +457,7 @@ jobs:
|
||||
name: "📝 Publish Badges & Summary"
|
||||
if: github.event_name == 'push'
|
||||
needs: [deploy]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, Linux]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [3.7.2] — 2026-05-06
|
||||
|
||||
Hotfix release branched from `v3.7.1`. Cherry-picks PR #1121 only — no other changes.
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
- **Ingestor: backfill infinite loop on `path_json='[]'` rows** (#1119, #1121) — `BackfillPathJSONAsync` re-selected observations whose `path_json` was already `'[]'`, rewrote them to `'[]'`, and looped forever. The migration marker was never recorded and the ingestor sustained 2–3 MB/s WAL writes at idle (~76% CPU in `sqlite.Exec`). Fix: drop `'[]'` from the WHERE clause so the loop terminates after one full pass and the `backfill_path_json_from_raw_hex_v1` marker is written.
|
||||
|
||||
## [2.5.0] "Digital Rain" — 2026-03-22
|
||||
|
||||
### ✨ Matrix Mode — Full Cyberpunk Map Theme
|
||||
|
||||
+7
-17
@@ -1,37 +1,28 @@
|
||||
# Build stage always runs natively on the builder's arch ($BUILDPLATFORM)
|
||||
# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
|
||||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache build-base
|
||||
|
||||
ARG APP_VERSION=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_TIME=unknown
|
||||
# Provided by buildx for multi-arch builds
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly)
|
||||
# Build server
|
||||
WORKDIR /build/server
|
||||
COPY cmd/server/go.mod cmd/server/go.sum ./
|
||||
COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
COPY internal/packetpath/ ../../internal/packetpath/
|
||||
COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
RUN go mod download
|
||||
COPY cmd/server/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
|
||||
# Build ingestor
|
||||
WORKDIR /build/ingestor
|
||||
COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./
|
||||
COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
COPY internal/packetpath/ ../../internal/packetpath/
|
||||
COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
RUN go mod download
|
||||
COPY cmd/ingestor/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -o /corescope-ingestor .
|
||||
RUN go build -o /corescope-ingestor .
|
||||
|
||||
# Build decrypt CLI
|
||||
WORKDIR /build/decrypt
|
||||
@@ -39,8 +30,7 @@ COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./
|
||||
COPY internal/channel/ ../../internal/channel/
|
||||
RUN go mod download
|
||||
COPY cmd/decrypt/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags="-s -w" -o /corescope-decrypt .
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /corescope-decrypt .
|
||||
|
||||
# Runtime image
|
||||
FROM alpine:3.20
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
# v3.6.0 - The Forensics
|
||||
|
||||
CoreScope just got eyes everywhere. This release drops **path inspection**, **color-by-hash markers**, **clock skew detection**, **full channel encryption**, an **observer graph**, and a pile of robustness fixes that make your mesh network feel like it's being watched by someone who actually cares.
|
||||
|
||||
134 commits, 105 PRs merged, 18K+ lines added. Here's what shipped.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 New Features
|
||||
|
||||
### Path-Prefix Candidate Inspector (#944, #945)
|
||||
The marquee feature. Click any path segment and CoreScope opens an interactive inspector showing every candidate node that could match that hop prefix - plotted on a map with scoring by neighbor-graph affinity and geographic centroid. Ambiguous hops? Now you can see *why* they're ambiguous and pick the right one.
|
||||
|
||||
**Why you'll love it:** No more guessing which `0xA3` is the real repeater. The inspector lays out every candidate, scores them, and lets you drill in visually.
|
||||
|
||||
### Color-by-Hash Packet Markers (#948, #951)
|
||||
Every packet type gets a vivid, hash-derived color - on the live feed, map polylines, and flying-packet animations. Bright fill with dark outline for contrast. No more monochrome blobs - you can visually track packet flows by color at a glance.
|
||||
|
||||
### Node Filter on Live Page (#924, #771)
|
||||
Filter the live packet stream to show only traffic flowing through a specific node. Pick a repeater, see exactly what it's carrying. That simple.
|
||||
|
||||
### Clock Skew Detection (#746, #752, #828, #850)
|
||||
Full pipeline: backend computes drift using Theil-Sen regression with outlier rejection (#828), the UI shows per-node badges, detail sparklines, and fleet-wide analytics (#752). Bimodal clock severity (#850) surfaces flaky-RTC nodes that toggle between accurate and drifted - instead of hiding them as "No Clock."
|
||||
|
||||
**Why you'll love it:** Nodes with bad clocks silently corrupt your timeline. Now they glow red before they ruin your analysis.
|
||||
|
||||
### Observer Graph (M1+M2) (#774)
|
||||
Observers are now first-class graph citizens. CoreScope builds a neighbor graph from observation overlaps, scores hop-resolver candidates by graph edges (#876), and uses geographic centroid for tiebreaking. The observer topology is visible and queryable.
|
||||
|
||||
### Channel Encryption - Full Stack (#726, #733, #750, #760)
|
||||
Three milestones landed as one: DB-backed channel message history (#726), client-side PSK decryption in the browser (#733), and PSK channel management with add/remove UX and message caching (#750). Add a channel key in the UI, and CoreScope decrypts messages client-side - no server-side key storage. The add-channel button (#760) makes it dead simple.
|
||||
|
||||
**Why you'll love it:** Encrypted channels are no longer black boxes. Add your PSK, see the messages, search history - all without exposing keys to the server.
|
||||
|
||||
### Hash Collision Inspector (#758)
|
||||
The Hash Usage Matrix now shows collision details for all hash sizes. When two nodes share a prefix, you see exactly who collides and at what size.
|
||||
|
||||
### Geofilter Builder - In-App (#735, #900)
|
||||
The geofilter polygon builder is now served directly from CoreScope with a full docs page (#900). No more hunting for external tools. Link from the customizer, draw your polygon, done.
|
||||
|
||||
### Node Blacklist (#742)
|
||||
`nodeBlacklist` in config hides abusive or troll nodes from all views. They're gone.
|
||||
|
||||
### Observer Retention (#764)
|
||||
Stale observers are automatically pruned after a configurable number of days. Your observer list stays clean without manual intervention.
|
||||
|
||||
### Advert Signature Validation (#794)
|
||||
Corrupt packets with invalid advert signatures are now rejected at ingest. Bad data never hits your store.
|
||||
|
||||
### Bounded Cold Load (#790)
|
||||
`Load()` now respects a memory budget - no more OOM on cold start with a fat database. Combined with retention-hours cutoff (#917), cold start is safe on constrained hardware.
|
||||
|
||||
### Multi-Arch Docker Images (#869)
|
||||
Official images now publish `amd64` + `arm64` in a single multi-arch manifest. Raspberry Pi operators: pull and run. No special tags needed.
|
||||
|
||||
### /nodes Detail Panel + Search (#868)
|
||||
The nodes detail panel ships with search improvements (#862) - find nodes fast, see their full detail in a slide-out panel.
|
||||
|
||||
### Deduplicated Top Longest Hops (#848)
|
||||
Longest hops are now deduplicated by pair with observation count and SNR cues. No more seeing the same link 47 times.
|
||||
|
||||
---
|
||||
|
||||
## 🔥 Performance Wins
|
||||
|
||||
### StoreTx ResolvedPath Elimination (#806)
|
||||
The per-transaction `ResolvedPath` computation is gone - replaced by a membership index with on-demand decode. This was one of the hottest paths in the ingestor.
|
||||
|
||||
### Node Packet Queries (#803)
|
||||
Raw JSON text search for node packets replaced with a proper `byNode` index (#673). Night and day.
|
||||
|
||||
### Channel Query Performance (#762, #763)
|
||||
New `channel_hash` column enables SQL-level channel filtering. No more full-table scan to find messages in a channel.
|
||||
|
||||
### SQLite Auto-Vacuum (#919, #920)
|
||||
Incremental auto-vacuum enabled - the database file actually shrinks after retention pruning. No more 2GB database holding 200MB of live data.
|
||||
|
||||
### Retention-Hours Cutoff on Load (#917)
|
||||
`Load()` now applies `retentionHours` at read time, preventing OOM when the DB has more history than memory allows.
|
||||
|
||||
---
|
||||
|
||||
## 🛡️ Security & Robustness
|
||||
|
||||
### MQTT Reconnect with Bounded Backoff (#947, #949)
|
||||
The ingestor now reconnects to MQTT brokers with exponential backoff, observability logging, and bounded retry. No more silent disconnects that kill your data stream.
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Bugs Squashed
|
||||
|
||||
This release exterminates **40+ bugs** — from protocol-level hash mismatches to pixel-level CSS breakage. Operators told us what hurt; we listened.
|
||||
|
||||
- **Path inspector "Show on Map" missed origin and first hop** (#950) - map view now includes all hops
|
||||
- **Content hash used full header byte** (#787) - content hashing now uses payload type bits only, fixing hash collisions between packets that differ only in header flags
|
||||
- **Encrypted channel deep links showed broken UI** (#825, #826, #815) - deep links to encrypted channels now show a lock message instead of broken UI when you don't have the key
|
||||
- **Geofilter longitude wrapping** (#925) - geofilter builder wraps longitude to [-180, 180]; southern hemisphere polygons no longer invert
|
||||
- **Hash filter bypasses saved region filter** (#939) - hash lookups now skip the geo filter as intended
|
||||
- **Companion-as-repeater excluded from path hops** (#935, #936) - non-repeater nodes no longer pollute hop resolution
|
||||
- **Customize panel re-renders while typing** (#927) - text fields keep focus during config changes
|
||||
- **Per-observation raw_hex** (#881, #882) - each observer's hex dump now shows what *that observer* actually received
|
||||
- **Per-observation children in packet groups** (#866, #880) - expanded groups show per-obs data, not cross-observer aggregates
|
||||
- **Full-page obs-switch** (#866, #870) - switching observers updates hex, path, and direction correctly
|
||||
- **Packet detail shows wrong observation** (#849, #851) - clicking a specific observation opens *that* observation
|
||||
- **Byte breakdown hop count** (#844, #846) - derived from `path_len`, not aggregated `_parsedPath`
|
||||
- **Transport-route path_len offset** (#852, #853) - correct offset calculation + CSS variable fix
|
||||
- **Packets/hour chart bars + x-axis** (#858, #865) - bars render correctly, x-axis labels properly decimated
|
||||
- **Channel timeline capped to top 8** (#860, #864) - no more 47-channel chart spaghetti
|
||||
- **Reachability row opacity removed** (#859, #863) - clean rows without misleading gradient
|
||||
- **Sticky table headers on mobile** (#861, #867) - restored after regression
|
||||
- **Map popup 'Show Neighbors' on iOS Safari** (#840, #841) - link actually works now
|
||||
- **Node detail Recent Packets invisible text** (#829, #830) - CSS fix
|
||||
- **/api/packets/{hash} falls back to DB** (#827, #831) - when in-memory store misses, DB catches it
|
||||
- **IATA filter bypass for status messages** (#694, #802) - status packets no longer filtered out by airport codes
|
||||
- **Desktop node click URL hash** (#676, #739) - clicking a node updates the URL for deep linking
|
||||
- **Filter params in URL hash** (#682, #740) - all filter state serialized for shareable links
|
||||
- **Hide undecryptable channel messages** (#727, #728) - clean default view
|
||||
- **TRACE path_json uses path_sz** (#732) - correct field from flags byte, not header hash_size
|
||||
- **Multi-byte adopters** (#754, #767) - all node types, role column, advert precedence
|
||||
- **Channel key case sensitivity** (#761) - Public decode works correctly
|
||||
- **Transport route field offsets** (#766) - correct offsets in field table
|
||||
- **Clock skew sanity checks** (#769) - filter epoch-0, cap drift, require minimum samples
|
||||
- **Neighbor graph slider persistence** (#776) - default 0.7, persisted to localStorage
|
||||
- **Node detail panel navigation** (#779, #785) - Details/Analytics links actually navigate
|
||||
- **Channel key removal** (#898) - user-added keys for server-known channels can be removed
|
||||
- **Side-panel Details on desktop** (#892) - opens full-screen correctly
|
||||
- **Hex-dump byte ranges client-side** (#891) - computed from per-obs raw_hex
|
||||
- **path_json derived from raw_hex at ingest** (#886, #887) - single source of truth
|
||||
- **Path pill and byte breakdown hop agreement** (#885) - they match now
|
||||
- **Mobile close button + toolbar scroll** (#797, #805) - accessible and scrollable
|
||||
- **/health.recentPackets resolved_path fallback** (#810, #821) - falls back to longest sibling observation
|
||||
- **Channel filter on Packets page** (#812, #816) - UI and API both fixed
|
||||
- **Clock-skew section in side panel** (#813, #814) - renders correctly
|
||||
- **Real RSS in /api/stats** (#832, #835) - surface actual RSS alongside tracked store bytes
|
||||
- **Hash size detection for transport routes + zero-hop adverts** (#747) - correct detection
|
||||
- **Repeater+observer merged map marker** (#745) - single marker, not two overlapping
|
||||
|
||||
---
|
||||
|
||||
## 🎨 UI Polish
|
||||
|
||||
- QA findings applied across the board (#832, #833, #836, #837, #838) - dozens of small UX fixes from systematic QA pass
|
||||
|
||||
---
|
||||
|
||||
## 📦 Upgrading
|
||||
|
||||
```bash
|
||||
git pull
|
||||
docker compose down
|
||||
docker compose build prod
|
||||
docker compose up -d prod
|
||||
```
|
||||
|
||||
Your existing `config.json` works as-is. New optional config keys:
|
||||
- `nodeBlacklist` - array of node hashes to hide
|
||||
- `observerRetentionDays` - days before stale observers are pruned
|
||||
- `memoryBudgetMB` - cap on in-memory packet store
|
||||
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
curl -s http://localhost/api/health | jq .version
|
||||
# "3.6.0"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🙏 External Contributors
|
||||
|
||||
- **#735** ([@efiten](https://github.com/efiten)) - Serve geofilter builder from app, link from customizer
|
||||
- **#739** ([@efiten](https://github.com/efiten)) - Desktop node click updates URL hash for deep linking
|
||||
- **#740** ([@efiten](https://github.com/efiten)) - Serialize filter params in URL hash for shareable links
|
||||
- **#742** ([@Joel-Claw](https://github.com/Joel-Claw)) - Add nodeBlacklist config to hide abusive/troll nodes
|
||||
- **#761** ([@copelaje](https://github.com/copelaje)) - Fix channel key case sensitivity for Public decode
|
||||
- **#764** ([@Joel-Claw](https://github.com/Joel-Claw)) - Add observer retention - prune stale observers after configurable days
|
||||
- **#802** ([@efiten](https://github.com/efiten)) - Bypass IATA filter for status messages, fill SNR on duplicate observations
|
||||
- **#803** ([@efiten](https://github.com/efiten)) - Replace raw JSON text search with byNode index for node packet queries
|
||||
- **#805** ([@efiten](https://github.com/efiten)) - Mobile close button accessible + toolbar scrollable
|
||||
- **#900** ([@efiten](https://github.com/efiten)) - App-served geofilter docs page
|
||||
- **#917** ([@efiten](https://github.com/efiten)) - Apply retentionHours cutoff in Load() to prevent OOM on cold start
|
||||
- **#924** ([@efiten](https://github.com/efiten)) - Node filter on live page - show only traffic through a specific node
|
||||
- **#925** ([@efiten](https://github.com/efiten)) - Fix geobuilder longitude wrapping for southern hemisphere polygons
|
||||
- **#927** ([@efiten](https://github.com/efiten)) - Skip customize panel re-render while text field has focus
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Breaking Changes
|
||||
|
||||
**None.** All API endpoints remain backwards-compatible. New fields are additive only.
|
||||
|
||||
---
|
||||
|
||||
## 📊 By the Numbers
|
||||
|
||||
| Stat | Count |
|
||||
|------|-------|
|
||||
| Commits | 134 |
|
||||
| PRs merged | 105 |
|
||||
| Lines added | 18,480 |
|
||||
| Lines removed | 1,632 |
|
||||
| Files changed | 110 |
|
||||
| Contributors | 4 |
|
||||
|
||||
---
|
||||
|
||||
*Previous release: [v3.5.2](https://github.com/Kpa-clawbot/CoreScope/releases/tag/v3.5.2)*
|
||||
@@ -7,9 +7,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
)
|
||||
|
||||
@@ -22,17 +20,6 @@ type MQTTSource struct {
|
||||
RejectUnauthorized *bool `json:"rejectUnauthorized,omitempty"`
|
||||
Topics []string `json:"topics"`
|
||||
IATAFilter []string `json:"iataFilter,omitempty"`
|
||||
ConnectTimeoutSec int `json:"connectTimeoutSec,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
}
|
||||
|
||||
// ConnectTimeoutOrDefault returns the per-source connect timeout in seconds,
|
||||
// or 30 if not set (matching the WaitTimeout default from #926).
|
||||
func (s MQTTSource) ConnectTimeoutOrDefault() int {
|
||||
if s.ConnectTimeoutSec > 0 {
|
||||
return s.ConnectTimeoutSec
|
||||
}
|
||||
return 30
|
||||
}
|
||||
|
||||
// MQTTLegacy is the old single-broker config format.
|
||||
@@ -54,26 +41,6 @@ type Config struct {
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
|
||||
// ObserverIATAWhitelist restricts which observer IATA regions are processed.
|
||||
// When non-empty, only observers whose IATA code (from the MQTT topic) matches
|
||||
// one of these entries are accepted. Case-insensitive. An empty list means all
|
||||
// IATA codes are allowed. This applies globally, unlike the per-source iataFilter.
|
||||
ObserverIATAWhitelist []string `json:"observerIATAWhitelist,omitempty"`
|
||||
|
||||
// obsIATAWhitelistCached is the lazily-built uppercase set for O(1) lookups.
|
||||
obsIATAWhitelistCached map[string]bool
|
||||
obsIATAWhitelistOnce sync.Once
|
||||
|
||||
// ObserverBlacklist is a list of observer public keys to drop at ingest.
|
||||
// Messages from blacklisted observers are silently discarded — no DB writes,
|
||||
// no UpsertObserver, no observations, no metrics.
|
||||
ObserverBlacklist []string `json:"observerBlacklist,omitempty"`
|
||||
|
||||
// obsBlacklistSetCached is the lazily-built lowercase set for O(1) lookups.
|
||||
obsBlacklistSetCached map[string]bool
|
||||
obsBlacklistOnce sync.Once
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
@@ -91,17 +58,6 @@ type MetricsConfig struct {
|
||||
SampleIntervalSec int `json:"sampleIntervalSec"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
type DBConfig = dbconfig.DBConfig
|
||||
|
||||
// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
|
||||
func (c *Config) IncrementalVacuumPages() int {
|
||||
if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
|
||||
return c.DB.IncrementalVacuumPages
|
||||
}
|
||||
return 1024
|
||||
}
|
||||
|
||||
// ShouldValidateSignatures returns true (default) unless explicitly disabled.
|
||||
func (c *Config) ShouldValidateSignatures() bool {
|
||||
if c.ValidateSignatures != nil {
|
||||
@@ -143,43 +99,6 @@ func (c *Config) ObserverDaysOrDefault() int {
|
||||
return 14
|
||||
}
|
||||
|
||||
// IsObserverBlacklisted returns true if the given observer ID is in the observerBlacklist.
|
||||
func (c *Config) IsObserverBlacklisted(id string) bool {
|
||||
if c == nil || len(c.ObserverBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
c.obsBlacklistOnce.Do(func() {
|
||||
m := make(map[string]bool, len(c.ObserverBlacklist))
|
||||
for _, pk := range c.ObserverBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsBlacklistSetCached = m
|
||||
})
|
||||
return c.obsBlacklistSetCached[strings.ToLower(strings.TrimSpace(id))]
|
||||
}
|
||||
|
||||
// IsObserverIATAAllowed returns true if the given IATA code is permitted.
|
||||
// When ObserverIATAWhitelist is empty, all codes are allowed.
|
||||
func (c *Config) IsObserverIATAAllowed(iata string) bool {
|
||||
if c == nil || len(c.ObserverIATAWhitelist) == 0 {
|
||||
return true
|
||||
}
|
||||
c.obsIATAWhitelistOnce.Do(func() {
|
||||
m := make(map[string]bool, len(c.ObserverIATAWhitelist))
|
||||
for _, code := range c.ObserverIATAWhitelist {
|
||||
trimmed := strings.ToUpper(strings.TrimSpace(code))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsIATAWhitelistCached = m
|
||||
})
|
||||
return c.obsIATAWhitelistCached[strings.ToUpper(strings.TrimSpace(iata))]
|
||||
}
|
||||
|
||||
// LoadConfig reads configuration from a JSON file, with env var overrides.
|
||||
// If the config file does not exist, sensible defaults are used (zero-config startup).
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
|
||||
@@ -284,113 +284,3 @@ func TestLoadConfigWithAllFields(t *testing.T) {
|
||||
t.Errorf("iataFilter=%v", src.IATAFilter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTimeoutOrDefault(t *testing.T) {
|
||||
// Default when unset
|
||||
s := MQTTSource{}
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
||||
t.Errorf("default: got %d, want 30", got)
|
||||
}
|
||||
|
||||
// Custom value
|
||||
s.ConnectTimeoutSec = 5
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 5 {
|
||||
t.Errorf("custom: got %d, want 5", got)
|
||||
}
|
||||
|
||||
// Zero treated as unset
|
||||
s.ConnectTimeoutSec = 0
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
||||
t.Errorf("zero: got %d, want 30", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTimeoutFromJSON(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := dir + "/config.json"
|
||||
os.WriteFile(cfgPath, []byte(`{"mqttSources":[{"name":"s1","broker":"tcp://b:1883","topics":["#"],"connectTimeoutSec":5}]}`), 0644)
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.MQTTSources[0].ConnectTimeoutOrDefault(); got != 5 {
|
||||
t.Errorf("from JSON: got %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelist(t *testing.T) {
|
||||
// Config with whitelist set
|
||||
cfg := Config{
|
||||
ObserverIATAWhitelist: []string{"ARN", "got"},
|
||||
}
|
||||
|
||||
// Matching (case-insensitive)
|
||||
if !cfg.IsObserverIATAAllowed("ARN") {
|
||||
t.Error("ARN should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("arn") {
|
||||
t.Error("arn (lowercase) should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("GOT") {
|
||||
t.Error("GOT should be allowed")
|
||||
}
|
||||
|
||||
// Non-matching
|
||||
if cfg.IsObserverIATAAllowed("SJC") {
|
||||
t.Error("SJC should NOT be allowed")
|
||||
}
|
||||
|
||||
// Empty string not allowed
|
||||
if cfg.IsObserverIATAAllowed("") {
|
||||
t.Error("empty IATA should NOT be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelistEmpty(t *testing.T) {
|
||||
// No whitelist = allow all
|
||||
cfg := Config{}
|
||||
if !cfg.IsObserverIATAAllowed("SJC") {
|
||||
t.Error("with no whitelist, all IATAs should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("") {
|
||||
t.Error("with no whitelist, even empty IATA should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelistJSON(t *testing.T) {
|
||||
json := `{
|
||||
"dbPath": "test.db",
|
||||
"observerIATAWhitelist": ["ARN", "GOT"]
|
||||
}`
|
||||
tmp := t.TempDir() + "/config.json"
|
||||
os.WriteFile(tmp, []byte(json), 0644)
|
||||
cfg, err := LoadConfig(tmp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.ObserverIATAWhitelist) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(cfg.ObserverIATAWhitelist))
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("ARN") {
|
||||
t.Error("ARN should be allowed after loading from JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTSourceRegionField(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
os.WriteFile(cfgPath, []byte(`{
|
||||
"dbPath": "/tmp/test.db",
|
||||
"mqttSources": [
|
||||
{"name": "cascadia", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"], "region": "PDX"}
|
||||
]
|
||||
}`), 0o644)
|
||||
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.MQTTSources[0].Region != "PDX" {
|
||||
t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region)
|
||||
}
|
||||
}
|
||||
|
||||
+11
-232
@@ -8,11 +8,9 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
@@ -45,7 +43,6 @@ type Store struct {
|
||||
stmtUpsertMetrics *sql.Stmt
|
||||
|
||||
sampleIntervalSec int
|
||||
backfillWg sync.WaitGroup
|
||||
}
|
||||
|
||||
// OpenStore opens or creates a SQLite DB at the given path, applying the
|
||||
@@ -61,7 +58,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
|
||||
return nil, fmt.Errorf("creating data dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)")
|
||||
db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening db: %w", err)
|
||||
}
|
||||
@@ -87,9 +84,6 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
|
||||
}
|
||||
|
||||
func applySchema(db *sql.DB) error {
|
||||
// auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode).
|
||||
// Logging of current mode is handled by CheckAutoVacuum — no duplicate log here.
|
||||
|
||||
schema := `
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
public_key TEXT PRIMARY KEY,
|
||||
@@ -118,8 +112,7 @@ func applySchema(db *sql.DB) error {
|
||||
battery_mv INTEGER,
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT DEFAULT NULL
|
||||
inactive INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen);
|
||||
@@ -196,7 +189,7 @@ func applySchema(db *sql.DB) error {
|
||||
db.Exec(`DROP VIEW IF EXISTS packets_v`)
|
||||
_, vErr := db.Exec(`
|
||||
CREATE VIEW packets_v AS
|
||||
SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex,
|
||||
SELECT o.id, t.raw_hex,
|
||||
datetime(o.timestamp, 'unixepoch') AS timestamp,
|
||||
obs.id AS observer_id, obs.name AS observer_name,
|
||||
o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type,
|
||||
@@ -415,54 +408,6 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] dropped_packets table created")
|
||||
}
|
||||
|
||||
// Migration: add raw_hex column to observations (#881)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observations_raw_hex_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding raw_hex column to observations...")
|
||||
db.Exec(`ALTER TABLE observations ADD COLUMN raw_hex TEXT`)
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('observations_raw_hex_v1')`)
|
||||
log.Println("[migration] observations.raw_hex column added")
|
||||
}
|
||||
|
||||
// Migration: add last_packet_at column to observers (#last-packet-at)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding last_packet_at column to observers...")
|
||||
_, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`)
|
||||
if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") {
|
||||
return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr)
|
||||
}
|
||||
// Backfill: set last_packet_at = last_seen only for observers that actually have
|
||||
// observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1
|
||||
// on INSERT even for status-only observers).
|
||||
res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen
|
||||
WHERE last_packet_at IS NULL
|
||||
AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`)
|
||||
if err == nil {
|
||||
n, _ := res.RowsAffected()
|
||||
log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n)
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`)
|
||||
log.Println("[migration] observers.last_packet_at column added")
|
||||
}
|
||||
|
||||
// Migration: backfill observations.path_json from raw_hex (#888)
|
||||
// NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup.
|
||||
// See staging outage where ~502K rows blocked ingest for 15+ hours.
|
||||
|
||||
// One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994)
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...")
|
||||
db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`)
|
||||
res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`)
|
||||
if err == nil {
|
||||
deleted, _ := res.RowsAffected()
|
||||
log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted)
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -488,13 +433,8 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtInsertObservation, err = s.db.Prepare(`
|
||||
INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET
|
||||
snr = COALESCE(excluded.snr, snr),
|
||||
rssi = COALESCE(excluded.rssi, rssi),
|
||||
score = COALESCE(excluded.score, score),
|
||||
raw_hex = COALESCE(excluded.raw_hex, raw_hex)
|
||||
INSERT OR IGNORE INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -546,7 +486,7 @@ func (s *Store) prepareStatements() error {
|
||||
return err
|
||||
}
|
||||
|
||||
s.stmtUpdateObserverLastSeen, err = s.db.Prepare("UPDATE observers SET last_seen = ?, last_packet_at = ? WHERE rowid = ?")
|
||||
s.stmtUpdateObserverLastSeen, err = s.db.Prepare("UPDATE observers SET last_seen = ? WHERE rowid = ?")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -625,9 +565,9 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
err := s.stmtGetObserverRowid.QueryRow(data.ObserverID).Scan(&rowid)
|
||||
if err == nil {
|
||||
observerIdx = &rowid
|
||||
// Update observer last_seen and last_packet_at on every packet to prevent
|
||||
// Update observer last_seen on every packet to prevent
|
||||
// low-traffic observers from appearing offline (#463)
|
||||
_, _ = s.stmtUpdateObserverLastSeen.Exec(now, now, rowid)
|
||||
_, _ = s.stmtUpdateObserverLastSeen.Exec(now, rowid)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -640,7 +580,7 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
_, err = s.stmtInsertObservation.Exec(
|
||||
txID, observerIdx, data.Direction,
|
||||
data.SNR, data.RSSI, data.Score,
|
||||
data.PathJSON, epochTs, nilIfEmpty(data.RawHex),
|
||||
data.PathJSON, epochTs,
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
@@ -756,7 +696,6 @@ func (s *Store) UpsertObserver(id, name, iata string, meta *ObserverMeta) error
|
||||
|
||||
// Close checkpoints the WAL and closes the database.
|
||||
func (s *Store) Close() error {
|
||||
s.backfillWg.Wait()
|
||||
s.Checkpoint()
|
||||
return s.db.Close()
|
||||
}
|
||||
@@ -834,58 +773,6 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning
|
||||
// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919).
|
||||
func (s *Store) CheckAutoVacuum(cfg *Config) {
|
||||
var autoVacuum int
|
||||
if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
|
||||
log.Printf("[db] warning: could not read auto_vacuum: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if autoVacuum == 2 {
|
||||
log.Printf("[db] auto_vacuum=INCREMENTAL")
|
||||
return
|
||||
}
|
||||
|
||||
modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"}
|
||||
mode := modes[autoVacuum]
|
||||
if mode == "" {
|
||||
mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum)
|
||||
}
|
||||
|
||||
log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+
|
||||
"Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+
|
||||
"See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode)
|
||||
|
||||
if cfg.DB != nil && cfg.DB.VacuumOnStartup {
|
||||
// WARNING: Full VACUUM creates a temporary copy of the entire DB file.
|
||||
// Requires ~2× the DB file size in free disk space or it will fail.
|
||||
log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...")
|
||||
start := time.Now()
|
||||
|
||||
if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err)
|
||||
return
|
||||
}
|
||||
if _, err := s.db.Exec("VACUUM"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
|
||||
// RunIncrementalVacuum returns free pages to the OS (#919).
|
||||
// Safe to call on auto_vacuum=NONE databases (noop).
|
||||
func (s *Store) RunIncrementalVacuum(pages int) {
|
||||
if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
|
||||
log.Printf("[vacuum] incremental_vacuum error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoint forces a WAL checkpoint to release the WAL lock file,
|
||||
// preventing lock contention with a new process starting up.
|
||||
func (s *Store) Checkpoint() {
|
||||
@@ -896,94 +783,6 @@ func (s *Store) Checkpoint() {
|
||||
}
|
||||
}
|
||||
|
||||
// BackfillPathJSONAsync launches the path_json backfill in a background goroutine.
|
||||
// It processes observations with NULL/empty path_json that have raw_hex available,
|
||||
// decoding hop paths and updating the column. Safe to run concurrently with ingest
|
||||
// because new observations get path_json at write time; this only touches NULL rows.
|
||||
// Idempotent: skips if migration already recorded.
|
||||
func (s *Store) BackfillPathJSONAsync() {
|
||||
s.backfillWg.Add(1)
|
||||
go func() {
|
||||
defer s.backfillWg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[backfill] path_json async panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
var migDone int
|
||||
row := s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'")
|
||||
if row.Scan(&migDone) == nil {
|
||||
return // already done
|
||||
}
|
||||
|
||||
log.Println("[backfill] Starting async path_json backfill from raw_hex...")
|
||||
updated := 0
|
||||
errored := false
|
||||
const batchSize = 1000
|
||||
batchNum := 0
|
||||
for {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT o.id, o.raw_hex
|
||||
FROM observations o
|
||||
JOIN transmissions t ON o.transmission_id = t.id
|
||||
WHERE o.raw_hex IS NOT NULL AND o.raw_hex != ''
|
||||
-- NB: '[]' is the "already attempted, no hops" sentinel; excluded
|
||||
-- to prevent the infinite re-UPDATE loop fixed in #1119.
|
||||
AND (o.path_json IS NULL OR o.path_json = '')
|
||||
AND t.payload_type != 9
|
||||
LIMIT ?`, batchSize)
|
||||
if err != nil {
|
||||
log.Printf("[backfill] path_json query error: %v", err)
|
||||
errored = true
|
||||
break
|
||||
}
|
||||
type pendingRow struct {
|
||||
id int64
|
||||
rawHex string
|
||||
}
|
||||
var batch []pendingRow
|
||||
for rows.Next() {
|
||||
var r pendingRow
|
||||
if err := rows.Scan(&r.id, &r.rawHex); err == nil {
|
||||
batch = append(batch, r)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
for _, r := range batch {
|
||||
hops, err := packetpath.DecodePathFromRawHex(r.rawHex)
|
||||
if err != nil || len(hops) == 0 {
|
||||
if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = ?`, r.id); execErr != nil {
|
||||
log.Printf("[backfill] write error (id=%d): %v", r.id, execErr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
b, _ := json.Marshal(hops)
|
||||
if _, execErr := s.db.Exec(`UPDATE observations SET path_json = ? WHERE id = ?`, string(b), r.id); execErr != nil {
|
||||
log.Printf("[backfill] write error (id=%d): %v", r.id, execErr)
|
||||
} else {
|
||||
updated++
|
||||
}
|
||||
}
|
||||
batchNum++
|
||||
if batchNum%50 == 0 {
|
||||
log.Printf("[backfill] progress: %d observations updated so far (%d batches)", updated, batchNum)
|
||||
}
|
||||
// Throttle: yield to ingest writers between batches
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
log.Printf("[backfill] Async path_json backfill complete: %d observations updated", updated)
|
||||
if !errored {
|
||||
s.db.Exec(`INSERT INTO _migrations (name) VALUES ('backfill_path_json_from_raw_hex_v1')`)
|
||||
} else {
|
||||
log.Printf("[backfill] NOT recording migration due to errors — will retry on next restart")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// LogStats logs current operational metrics.
|
||||
func (s *Store) LogStats() {
|
||||
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d sig_drops=%d",
|
||||
@@ -1107,7 +906,6 @@ type PacketData struct {
|
||||
PathJSON string
|
||||
DecodedJSON string
|
||||
ChannelHash string // grouping key for channel queries (#762)
|
||||
Region string // observer region: payload > topic > source config (#788)
|
||||
}
|
||||
|
||||
// nilIfEmpty returns nil for empty strings (for nullable DB columns).
|
||||
@@ -1126,26 +924,14 @@ type MQTTPacketMessage struct {
|
||||
Score *float64 `json:"score"`
|
||||
Direction *string `json:"direction"`
|
||||
Origin string `json:"origin"`
|
||||
Region string `json:"region,omitempty"` // optional region override (#788)
|
||||
}
|
||||
|
||||
// BuildPacketData constructs a PacketData from a decoded packet and MQTT message.
|
||||
// path_json is derived directly from raw_hex header bytes (not decoded.Path.Hops)
|
||||
// to guarantee the stored path always matches the raw bytes. This matters for
|
||||
// TRACE packets where decoded.Path.Hops is overwritten with payload hops (#886).
|
||||
func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string) *PacketData {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
pathJSON := "[]"
|
||||
// For TRACE packets, path_json must be the payload-decoded route hops
|
||||
// (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values.
|
||||
// For all other packet types, derive path from raw_hex (#886).
|
||||
if !packetpath.PathBytesAreHops(byte(decoded.Header.PayloadType)) {
|
||||
if len(decoded.Path.Hops) > 0 {
|
||||
b, _ := json.Marshal(decoded.Path.Hops)
|
||||
pathJSON = string(b)
|
||||
}
|
||||
} else if hops, err := packetpath.DecodePathFromRawHex(msg.Raw); err == nil && len(hops) > 0 {
|
||||
b, _ := json.Marshal(hops)
|
||||
if len(decoded.Path.Hops) > 0 {
|
||||
b, _ := json.Marshal(decoded.Path.Hops)
|
||||
pathJSON = string(b)
|
||||
}
|
||||
|
||||
@@ -1166,13 +952,6 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
|
||||
DecodedJSON: PayloadJSON(&decoded.Payload),
|
||||
}
|
||||
|
||||
// Region priority: payload field > topic-derived parameter (#788)
|
||||
if msg.Region != "" {
|
||||
pd.Region = msg.Region
|
||||
} else {
|
||||
pd.Region = region
|
||||
}
|
||||
|
||||
// Populate channel_hash for fast channel queries (#762)
|
||||
if decoded.Header.PayloadType == PayloadGRP_TXT {
|
||||
if decoded.Payload.Type == "CHAN" && decoded.Payload.Channel != "" {
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -11,8 +10,6 @@ import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
)
|
||||
|
||||
func tempDBPath(t *testing.T) string {
|
||||
@@ -569,61 +566,6 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Insert observer via status path — last_packet_at should be NULL
|
||||
if err := s.UpsertObserver("obs1", "Observer1", "SJC", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var lastPacketAt sql.NullString
|
||||
s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt)
|
||||
if lastPacketAt.Valid {
|
||||
t.Fatalf("expected last_packet_at to be NULL after UpsertObserver, got %s", lastPacketAt.String)
|
||||
}
|
||||
|
||||
// Insert a packet from this observer — last_packet_at should be set
|
||||
data := &PacketData{
|
||||
RawHex: "0A00D69F",
|
||||
Timestamp: "2026-04-24T12:00:00Z",
|
||||
ObserverID: "obs1",
|
||||
Hash: "lastpackettest123456",
|
||||
RouteType: 2,
|
||||
PayloadType: 2,
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt)
|
||||
if !lastPacketAt.Valid {
|
||||
t.Fatal("expected last_packet_at to be non-NULL after InsertTransmission")
|
||||
}
|
||||
// InsertTransmission uses `now = data.Timestamp || time.Now()`, so last_packet_at
|
||||
// should match the packet's Timestamp when provided (same source-of-truth as last_seen).
|
||||
if lastPacketAt.String != "2026-04-24T12:00:00Z" {
|
||||
t.Errorf("expected last_packet_at=2026-04-24T12:00:00Z, got %s", lastPacketAt.String)
|
||||
}
|
||||
|
||||
// UpsertObserver again (status path) — last_packet_at should NOT change
|
||||
if err := s.UpsertObserver("obs1", "Observer1", "SJC", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var lastPacketAtAfterStatus sql.NullString
|
||||
s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAtAfterStatus)
|
||||
if !lastPacketAtAfterStatus.Valid || lastPacketAtAfterStatus.String != lastPacketAt.String {
|
||||
t.Errorf("UpsertObserver should not change last_packet_at; expected %s, got %v", lastPacketAt.String, lastPacketAtAfterStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndToEndIngest(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
@@ -1940,781 +1882,3 @@ func TestExtractObserverMetaNewFields(t *testing.T) {
|
||||
t.Errorf("RecvErrors = %v, want 3", meta.RecvErrors)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInsertObservationSNRFillIn verifies that when the same observation is
|
||||
// received twice — first without SNR, then with SNR — the SNR is filled in
|
||||
// rather than silently discarded. The unique dedup index is
|
||||
// (transmission_id, observer_idx, COALESCE(path_json, '')); observer_idx must
|
||||
// be non-NULL for the conflict to fire (SQLite treats NULL != NULL).
|
||||
func TestInsertObservationSNRFillIn(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Register the observer so observer_idx is non-NULL (required for dedup).
|
||||
if err := s.UpsertObserver("pymc-obs1", "PyMC Observer", "SJC", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// First arrival: same observer, no SNR/RSSI (e.g. broker replay without RF fields).
|
||||
data1 := &PacketData{
|
||||
RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976",
|
||||
Timestamp: "2026-04-20T00:00:00Z",
|
||||
Hash: "snrfillin0001hash",
|
||||
RouteType: 1,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: nil,
|
||||
RSSI: nil,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr1, rssi1 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr1, &rssi1)
|
||||
if snr1 != nil || rssi1 != nil {
|
||||
t.Fatalf("precondition: first insert should have nil SNR/RSSI, got snr=%v rssi=%v", snr1, rssi1)
|
||||
}
|
||||
|
||||
// Second arrival: same packet, same observer, now WITH SNR/RSSI.
|
||||
snr := 10.5
|
||||
rssi := -88.0
|
||||
data2 := &PacketData{
|
||||
RawHex: data1.RawHex,
|
||||
Timestamp: data1.Timestamp,
|
||||
Hash: data1.Hash,
|
||||
RouteType: data1.RouteType,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: &snr,
|
||||
RSSI: &rssi,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr2, rssi2 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr2, &rssi2)
|
||||
if snr2 == nil || *snr2 != snr {
|
||||
t.Errorf("SNR not filled in by second arrival: got %v, want %v", snr2, snr)
|
||||
}
|
||||
if rssi2 == nil || *rssi2 != rssi {
|
||||
t.Errorf("RSSI not filled in by second arrival: got %v, want %v", rssi2, rssi)
|
||||
}
|
||||
|
||||
// Third arrival: same packet again, SNR absent — must NOT overwrite existing SNR.
|
||||
data3 := &PacketData{
|
||||
RawHex: data1.RawHex,
|
||||
Timestamp: data1.Timestamp,
|
||||
Hash: data1.Hash,
|
||||
RouteType: data1.RouteType,
|
||||
ObserverID: "pymc-obs1",
|
||||
SNR: nil,
|
||||
RSSI: nil,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var snr3, rssi3 *float64
|
||||
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr3, &rssi3)
|
||||
if snr3 == nil || *snr3 != snr {
|
||||
t.Errorf("SNR overwritten by null arrival: got %v, want %v", snr3, snr)
|
||||
}
|
||||
if rssi3 == nil || *rssi3 != rssi {
|
||||
t.Errorf("RSSI overwritten by null arrival: got %v, want %v", rssi3, rssi)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPerObservationRawHex verifies that two MQTT packets for the same hash
|
||||
// from different observers store distinct raw_hex per observation (#881).
|
||||
func TestPerObservationRawHex(t *testing.T) {
|
||||
store, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Register two observers
|
||||
store.UpsertObserver("obs-A", "Observer A", "", nil)
|
||||
store.UpsertObserver("obs-B", "Observer B", "", nil)
|
||||
|
||||
hash := "abc123def456"
|
||||
rawA := "c0ffee01"
|
||||
rawB := "c0ffee0201aa"
|
||||
dir := "RX"
|
||||
|
||||
// First observation from observer A
|
||||
pdA := &PacketData{
|
||||
RawHex: rawA,
|
||||
Hash: hash,
|
||||
Timestamp: "2026-04-21T10:00:00Z",
|
||||
ObserverID: "obs-A",
|
||||
Direction: &dir,
|
||||
PathJSON: "[]",
|
||||
}
|
||||
isNew, err := store.InsertTransmission(pdA)
|
||||
if err != nil {
|
||||
t.Fatalf("insert A: %v", err)
|
||||
}
|
||||
if !isNew {
|
||||
t.Fatal("expected new transmission")
|
||||
}
|
||||
|
||||
// Second observation from observer B (same hash, different raw bytes)
|
||||
pdB := &PacketData{
|
||||
RawHex: rawB,
|
||||
Hash: hash,
|
||||
Timestamp: "2026-04-21T10:00:01Z",
|
||||
ObserverID: "obs-B",
|
||||
Direction: &dir,
|
||||
PathJSON: `["aabb"]`,
|
||||
}
|
||||
isNew2, err := store.InsertTransmission(pdB)
|
||||
if err != nil {
|
||||
t.Fatalf("insert B: %v", err)
|
||||
}
|
||||
if isNew2 {
|
||||
t.Fatal("expected duplicate transmission")
|
||||
}
|
||||
|
||||
// Query observations and verify per-observation raw_hex
|
||||
rows, err := store.db.Query(`
|
||||
SELECT o.raw_hex, obs.id
|
||||
FROM observations o
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
ORDER BY o.id ASC
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type obsResult struct {
|
||||
rawHex string
|
||||
observerID string
|
||||
}
|
||||
var results []obsResult
|
||||
for rows.Next() {
|
||||
var rh, oid sql.NullString
|
||||
if err := rows.Scan(&rh, &oid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results = append(results, obsResult{
|
||||
rawHex: rh.String,
|
||||
observerID: oid.String,
|
||||
})
|
||||
}
|
||||
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("expected 2 observations, got %d", len(results))
|
||||
}
|
||||
if results[0].rawHex != rawA {
|
||||
t.Errorf("obs A raw_hex: got %q, want %q", results[0].rawHex, rawA)
|
||||
}
|
||||
if results[1].rawHex != rawB {
|
||||
t.Errorf("obs B raw_hex: got %q, want %q", results[1].rawHex, rawB)
|
||||
}
|
||||
if results[0].rawHex == results[1].rawHex {
|
||||
t.Error("both observations have same raw_hex — should differ")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPacketData_TraceUsesPayloadHops verifies that TRACE packets use
|
||||
// payload-decoded route hops in path_json (NOT the raw_hex header SNR bytes).
|
||||
// Issue #886 / #887.
|
||||
func TestBuildPacketData_TraceUsesPayloadHops(t *testing.T) {
|
||||
// TRACE packet: header path has SNR bytes [30,2D,0D,23], but decoded.Path.Hops
|
||||
// is overwritten to payload hops [67,33,D6,33,67].
|
||||
rawHex := "2604302D0D2359FEE7B100000000006733D63367"
|
||||
decoded, err := DecodePacket(rawHex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// decoded.Path.Hops should be the TRACE-replaced hops (payload hops)
|
||||
if len(decoded.Path.Hops) != 5 {
|
||||
t.Fatalf("expected 5 decoded hops, got %d", len(decoded.Path.Hops))
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pd := BuildPacketData(msg, decoded, "test-obs", "TST")
|
||||
|
||||
// For TRACE: path_json MUST be the payload-decoded route hops, NOT the SNR bytes
|
||||
expectedPathJSON := `["67","33","D6","33","67"]`
|
||||
if pd.PathJSON != expectedPathJSON {
|
||||
t.Errorf("path_json = %s, want %s (TRACE must use payload hops)", pd.PathJSON, expectedPathJSON)
|
||||
}
|
||||
|
||||
// Verify that DecodePathFromRawHex returns the SNR bytes (header path) which differ
|
||||
headerHops, herr := packetpath.DecodePathFromRawHex(rawHex)
|
||||
if herr != nil {
|
||||
t.Fatal(herr)
|
||||
}
|
||||
headerJSON, _ := json.Marshal(headerHops)
|
||||
if string(headerJSON) == expectedPathJSON {
|
||||
t.Error("header path (SNR) should differ from payload hops for TRACE")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPacketData_NonTracePathJSON verifies non-TRACE packets also derive path from raw_hex.
|
||||
func TestBuildPacketData_NonTracePathJSON(t *testing.T) {
|
||||
// A simple ADVERT packet (payload type 0) with 2 hops, hash_size 1
|
||||
// Header 0x09 = FLOOD(1), ADVERT(2), version 0
|
||||
// Path byte 0x02 = hash_size 1, hash_count 2
|
||||
// Path bytes: AA BB
|
||||
rawHex := "0902AABB" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
|
||||
decoded, err := DecodePacket(rawHex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pd := BuildPacketData(msg, decoded, "obs1", "TST")
|
||||
|
||||
expectedPathJSON := `["AA","BB"]`
|
||||
if pd.PathJSON != expectedPathJSON {
|
||||
t.Errorf("path_json = %s, want %s", pd.PathJSON, expectedPathJSON)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Issue #888: Backfill path_json from raw_hex ---
|
||||
|
||||
func TestBackfillPathJsonFromRawHex(t *testing.T) {
|
||||
dbPath := tempDBPath(t)
|
||||
s, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Insert a transmission with payload_type != TRACE (e.g. 0x01)
|
||||
// raw_hex: header 0x05 (route FLOOD, payload 0x01), path byte 0x42 (hash_size=2, count=2),
|
||||
// hops: AABB, CCDD, then some payload bytes
|
||||
rawHex := "0542AABBCCDD0000000000000000000000000000"
|
||||
s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex)
|
||||
|
||||
// Insert observation with raw_hex but empty path_json
|
||||
s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, ?, '[]')`, rawHex)
|
||||
// Insert observation with raw_hex and NULL path_json
|
||||
s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, ?, NULL)`, rawHex)
|
||||
// Insert observation with existing path_json (should NOT be overwritten)
|
||||
s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, ?, '["XX","YY"]')`, rawHex)
|
||||
|
||||
// Insert a TRACE transmission (payload_type = 0x09) — should be skipped
|
||||
traceRaw := "2604302D0D2359FEE7B100000000006733D63367"
|
||||
s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw)
|
||||
s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, ?, '[]')`, traceRaw)
|
||||
|
||||
// Remove the migration marker so it runs again on reopen
|
||||
s.db.Exec(`DELETE FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'`)
|
||||
s.Close()
|
||||
|
||||
// Reopen — backfill is now async, must trigger explicitly
|
||||
s2, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
// Trigger async backfill and wait for completion
|
||||
s2.BackfillPathJSONAsync()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
var migCount int
|
||||
for time.Now().Before(deadline) {
|
||||
s2.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'").Scan(&migCount)
|
||||
if migCount == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
if migCount != 1 {
|
||||
t.Fatalf("migration not recorded")
|
||||
}
|
||||
|
||||
// Row 1 (was '[]') is NOT re-processed by the backfill — '[]' means
|
||||
// "already attempted, no hops" and is excluded by the WHERE to avoid the
|
||||
// infinite-loop bug fixed in #1119. It must remain '[]'.
|
||||
var pj1 string
|
||||
s2.db.QueryRow("SELECT path_json FROM observations WHERE id = 1").Scan(&pj1)
|
||||
if pj1 != "[]" {
|
||||
t.Errorf("row 1 path_json = %q, want %q (must not re-process '[]' rows after #1119)", pj1, "[]")
|
||||
}
|
||||
|
||||
// Row 2 (was NULL) should now have decoded hops
|
||||
var pj2 string
|
||||
s2.db.QueryRow("SELECT path_json FROM observations WHERE id = 2").Scan(&pj2)
|
||||
if pj2 != `["AABB","CCDD"]` {
|
||||
t.Errorf("row 2 path_json = %q, want %q", pj2, `["AABB","CCDD"]`)
|
||||
}
|
||||
|
||||
// Row 3 (had existing data) should NOT be overwritten
|
||||
var pj3 string
|
||||
s2.db.QueryRow("SELECT path_json FROM observations WHERE id = 3").Scan(&pj3)
|
||||
if pj3 != `["XX","YY"]` {
|
||||
t.Errorf("row 3 path_json = %q, want %q (should not be overwritten)", pj3, `["XX","YY"]`)
|
||||
}
|
||||
|
||||
// Row 4 (TRACE) should NOT be updated
|
||||
var pj4 string
|
||||
s2.db.QueryRow("SELECT path_json FROM observations WHERE id = 4").Scan(&pj4)
|
||||
if pj4 != "[]" {
|
||||
t.Errorf("row 4 (TRACE) path_json = %q, want %q (should be skipped)", pj4, "[]")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupLegacyNullHashTimestamp(t *testing.T) {
|
||||
path := tempDBPath(t)
|
||||
|
||||
// Create a bare-bones DB with legacy bad data
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Exec(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
first_seen TEXT NOT NULL,
|
||||
route_type INTEGER,
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
channel_hash TEXT DEFAULT NULL
|
||||
)`)
|
||||
db.Exec(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_idx INTEGER,
|
||||
direction TEXT,
|
||||
snr REAL,
|
||||
rssi REAL,
|
||||
score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL
|
||||
)`)
|
||||
db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`)
|
||||
db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`)
|
||||
db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`)
|
||||
|
||||
// Insert good transmission
|
||||
db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`)
|
||||
db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`)
|
||||
|
||||
// Insert bad: empty hash
|
||||
db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`)
|
||||
db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`)
|
||||
|
||||
// Insert bad: empty first_seen
|
||||
db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`)
|
||||
db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`)
|
||||
|
||||
db.Close()
|
||||
|
||||
// Now open via OpenStore which should run the migration
|
||||
s, err := OpenStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// Good transmission should remain
|
||||
var count int
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 1").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Error("good transmission should not be deleted")
|
||||
}
|
||||
|
||||
// Bad transmissions should be gone
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 2").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("transmission with empty hash should be deleted, got count=%d", count)
|
||||
}
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 3").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("transmission with empty first_seen should be deleted, got count=%d", count)
|
||||
}
|
||||
|
||||
// Observations for bad transmissions should be gone
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id IN (2, 3)").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("observations for bad transmissions should be deleted, got count=%d", count)
|
||||
}
|
||||
|
||||
// Observation for good transmission should remain
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id = 1").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Error("observation for good transmission should remain")
|
||||
}
|
||||
|
||||
// Migration marker should exist
|
||||
var migCount int
|
||||
s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'").Scan(&migCount)
|
||||
if migCount != 1 {
|
||||
t.Error("migration marker cleanup_legacy_null_hash_ts should be recorded")
|
||||
}
|
||||
|
||||
// Idempotent: opening again should not error
|
||||
s.Close()
|
||||
s2, err := OpenStore(path)
|
||||
if err != nil {
|
||||
t.Fatal("second open should not fail:", err)
|
||||
}
|
||||
s2.Close()
|
||||
}
|
||||
|
||||
func TestBuildPacketDataRegionFromPayload(t *testing.T) {
|
||||
msg := &MQTTPacketMessage{Raw: "0102030405060708", Region: "PDX"}
|
||||
decoded := &DecodedPacket{
|
||||
Header: Header{RouteType: 1, PayloadType: 3},
|
||||
}
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
// When payload has region, it should override the topic-derived region
|
||||
if pkt.Region != "PDX" {
|
||||
t.Fatalf("expected region PDX from payload, got %q", pkt.Region)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPacketDataRegionFallsBackToTopic(t *testing.T) {
|
||||
msg := &MQTTPacketMessage{Raw: "0102030405060708"}
|
||||
decoded := &DecodedPacket{
|
||||
Header: Header{RouteType: 1, PayloadType: 3},
|
||||
}
|
||||
pkt := BuildPacketData(msg, decoded, "obs1", "SJC")
|
||||
if pkt.Region != "SJC" {
|
||||
t.Fatalf("expected region SJC from topic, got %q", pkt.Region)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestBackfillPathJSONAsync verifies that the path_json backfill does NOT block
|
||||
// OpenStore from returning. MQTT connect happens immediately after OpenStore;
|
||||
// if the backfill is synchronous, MQTT would be delayed indefinitely on large DBs.
|
||||
// This test creates pending backfill rows, opens the store, and asserts that
|
||||
// OpenStore returns before the migration is recorded — proving async execution.
|
||||
func TestBackfillPathJSONAsync(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "async_test.db")
|
||||
|
||||
// Bootstrap schema manually so we can insert test data BEFORE OpenStore
|
||||
db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create tables manually (minimal schema for this test)
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE _migrations (name TEXT PRIMARY KEY);
|
||||
CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
route_type INTEGER,
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
channel_hash TEXT
|
||||
);
|
||||
CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
iata TEXT,
|
||||
last_seen TEXT,
|
||||
first_seen TEXT,
|
||||
packet_count INTEGER DEFAULT 0,
|
||||
model TEXT,
|
||||
firmware TEXT,
|
||||
client_version TEXT,
|
||||
radio TEXT,
|
||||
battery_mv INTEGER,
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT
|
||||
);
|
||||
CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY,
|
||||
name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0,
|
||||
battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE inactive_nodes (
|
||||
public_key TEXT PRIMARY KEY,
|
||||
name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0,
|
||||
battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_idx INTEGER,
|
||||
direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL,
|
||||
raw_hex TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, ''));
|
||||
CREATE INDEX idx_observations_transmission_id ON observations(transmission_id);
|
||||
CREATE INDEX idx_observations_observer_idx ON observations(observer_idx);
|
||||
CREATE INDEX idx_observations_timestamp ON observations(timestamp);
|
||||
CREATE TABLE observer_metrics (
|
||||
observer_id TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER,
|
||||
recv_errors INTEGER, battery_mv INTEGER,
|
||||
packets_sent INTEGER, packets_recv INTEGER,
|
||||
PRIMARY KEY (observer_id, timestamp)
|
||||
);
|
||||
CREATE TABLE dropped_packets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT, raw_hex TEXT, reason TEXT NOT NULL,
|
||||
observer_id TEXT, observer_name TEXT,
|
||||
node_pubkey TEXT, node_name TEXT,
|
||||
dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal("bootstrap schema:", err)
|
||||
}
|
||||
|
||||
// Mark all migrations as done EXCEPT the path_json backfill
|
||||
for _, m := range []string{
|
||||
"advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1",
|
||||
"obs_timestamp_index_v1", "observer_metrics_v1", "observer_metrics_ts_idx",
|
||||
"observers_inactive_v1", "observer_metrics_packets_v1", "channel_hash_v1",
|
||||
"dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1",
|
||||
"cleanup_legacy_null_hash_ts",
|
||||
} {
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m)
|
||||
}
|
||||
|
||||
// Insert a transmission + observations with NULL path_json and valid raw_hex
|
||||
// raw_hex "0102AABBCCDD0000" has 2-hop path decodable by packetpath
|
||||
rawHex := "41020304AABBCCDD05060708"
|
||||
_, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex)
|
||||
if err != nil {
|
||||
t.Fatal("insert tx:", err)
|
||||
}
|
||||
// Insert 100 observations needing backfill
|
||||
for i := 0; i < 100; i++ {
|
||||
_, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, NULL)`,
|
||||
i+1, 1700000000+i, rawHex)
|
||||
if err != nil {
|
||||
// dedup index might fire — use unique observer_idx
|
||||
t.Fatalf("insert obs %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
db.Close()
|
||||
|
||||
// Now open store via OpenStore — this must return QUICKLY (non-blocking)
|
||||
start := time.Now()
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatal("OpenStore:", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// OpenStore must return in under 2 seconds (backfill is no longer in applySchema)
|
||||
if elapsed > 2*time.Second {
|
||||
t.Fatalf("OpenStore blocked for %v — backfill must not run in applySchema", elapsed)
|
||||
}
|
||||
|
||||
// Backfill must NOT be recorded yet — it hasn't been triggered
|
||||
var done int
|
||||
err = store.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'").Scan(&done)
|
||||
if err == nil {
|
||||
t.Fatal("migration recorded during OpenStore — backfill must be async via BackfillPathJSONAsync()")
|
||||
}
|
||||
|
||||
// Now trigger the async backfill (simulates what main.go does after OpenStore)
|
||||
store.BackfillPathJSONAsync()
|
||||
|
||||
// Wait for backfill to complete (should be very fast with 100 rows)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
err = store.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'").Scan(&done)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal("backfill never completed within 10s")
|
||||
}
|
||||
|
||||
// Verify backfill actually worked — observations should have non-NULL path_json
|
||||
var nullCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observations WHERE path_json IS NULL").Scan(&nullCount)
|
||||
if nullCount > 0 {
|
||||
t.Errorf("backfill left %d observations with NULL path_json", nullCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackfillPathJSONAsyncMethodExists verifies the async backfill API surface
|
||||
// exists — BackfillPathJSONAsync must be callable independently from OpenStore.
|
||||
func TestBackfillPathJSONAsyncMethodExists(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "method_test.db")
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// BackfillPathJSONAsync must exist as a method on *Store
|
||||
// This is a compile-time check — if the method doesn't exist, the test won't compile.
|
||||
store.BackfillPathJSONAsync()
|
||||
}
|
||||
|
||||
// TestBackfillPathJSONAsync_BracketRowsTerminate exercises the infinite-loop bug
|
||||
// from issue #1119. Observations whose path_json is already '[]' (meaning a prior
|
||||
// backfill pass attempted to decode them and found no hops) must NOT be re-selected
|
||||
// by the WHERE clause — otherwise the loop rewrites the same '[]' value forever
|
||||
// and never records the migration marker.
|
||||
//
|
||||
// This test seeds N rows with path_json='[]' and a raw_hex that DecodePathFromRawHex
|
||||
// resolves to zero hops. With the bug, the backfill loops infinitely re-UPDATEing
|
||||
// the same rows back to '[]', batch is never empty, migration marker is never
|
||||
// written. With the fix, no rows match → the very first batch is empty → migration
|
||||
// is recorded immediately.
|
||||
func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "bracket_terminate.db")
|
||||
|
||||
// Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows
|
||||
// before OpenStore runs.
|
||||
db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE _migrations (name TEXT PRIMARY KEY);
|
||||
CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
route_type INTEGER,
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
channel_hash TEXT
|
||||
);
|
||||
CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY, name TEXT, iata TEXT,
|
||||
last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0,
|
||||
model TEXT, firmware TEXT, client_version TEXT, radio TEXT,
|
||||
battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0, last_packet_at TEXT
|
||||
);
|
||||
CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
|
||||
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT,
|
||||
advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE inactive_nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
|
||||
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT,
|
||||
advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_idx INTEGER, direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL,
|
||||
raw_hex TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, ''));
|
||||
CREATE INDEX idx_observations_transmission_id ON observations(transmission_id);
|
||||
CREATE INDEX idx_observations_observer_idx ON observations(observer_idx);
|
||||
CREATE INDEX idx_observations_timestamp ON observations(timestamp);
|
||||
CREATE TABLE observer_metrics (
|
||||
observer_id TEXT NOT NULL, timestamp TEXT NOT NULL,
|
||||
noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER,
|
||||
recv_errors INTEGER, battery_mv INTEGER,
|
||||
packets_sent INTEGER, packets_recv INTEGER,
|
||||
PRIMARY KEY (observer_id, timestamp)
|
||||
);
|
||||
CREATE TABLE dropped_packets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
hash TEXT, raw_hex TEXT, reason TEXT NOT NULL,
|
||||
observer_id TEXT, observer_name TEXT,
|
||||
node_pubkey TEXT, node_name TEXT,
|
||||
dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal("bootstrap schema:", err)
|
||||
}
|
||||
|
||||
// Mark all migrations done EXCEPT backfill_path_json_from_raw_hex_v1.
|
||||
for _, m := range []string{
|
||||
"advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1",
|
||||
"obs_timestamp_index_v1", "observer_metrics_v1", "observer_metrics_ts_idx",
|
||||
"observers_inactive_v1", "observer_metrics_packets_v1", "channel_hash_v1",
|
||||
"dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1",
|
||||
"cleanup_legacy_null_hash_ts",
|
||||
} {
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m)
|
||||
}
|
||||
|
||||
// raw_hex producing ZERO hops via DecodePathFromRawHex:
|
||||
// DIRECT route (type=2), payload_type=2, version=0 → header 0x0A; path byte 0x00.
|
||||
// (See internal/packetpath/path_test.go: TestDecodePathFromRawHex_ZeroHops.)
|
||||
rawHex := "0A00DEADBEEF"
|
||||
_, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex)
|
||||
if err != nil {
|
||||
t.Fatal("insert tx:", err)
|
||||
}
|
||||
const seedCount = 100
|
||||
for i := 0; i < seedCount; i++ {
|
||||
_, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, '[]')`,
|
||||
i+1, 1700000000+i, rawHex)
|
||||
if err != nil {
|
||||
t.Fatalf("insert obs %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
db.Close()
|
||||
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal("OpenStore:", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Trigger backfill. With the bug, every iteration re-fetches all 100 rows
|
||||
// (because '[]' matches the WHERE), rewrites them to '[]', sleeps 50ms, repeats.
|
||||
// The loop never terminates and the migration marker is never written.
|
||||
store.BackfillPathJSONAsync()
|
||||
|
||||
// Generous deadline: with the fix the marker is written essentially immediately.
|
||||
// With the bug the marker is never written within any bounded time.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var done int
|
||||
for time.Now().Before(deadline) {
|
||||
err = store.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'").Scan(&done)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("issue #1119: backfill never recorded migration marker within 5s — infinite loop on path_json='[]' rows")
|
||||
}
|
||||
|
||||
// Verify the seeded '[]' rows still have '[]' (sanity — neither bug nor fix
|
||||
// should change their value), and that there are no NULL/empty path_json rows
|
||||
// the backfill should have processed.
|
||||
var bracketCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observations WHERE path_json = '[]'").Scan(&bracketCount)
|
||||
if bracketCount != seedCount {
|
||||
t.Errorf("expected %d rows with path_json='[]', got %d", seedCount, bracketCount)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-22
@@ -12,7 +12,6 @@ import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/sigvalidate"
|
||||
)
|
||||
|
||||
@@ -131,7 +130,6 @@ type Payload struct {
|
||||
SenderTimestamp uint32 `json:"sender_timestamp,omitempty"`
|
||||
EphemeralPubKey string `json:"ephemeralPubKey,omitempty"`
|
||||
PathData string `json:"pathData,omitempty"`
|
||||
SNRValues []float64 `json:"snrValues,omitempty"`
|
||||
Tag uint32 `json:"tag,omitempty"`
|
||||
AuthCode uint32 `json:"authCode,omitempty"`
|
||||
TraceFlags *int `json:"traceFlags,omitempty"`
|
||||
@@ -194,9 +192,8 @@ func decodePath(pathByte byte, buf []byte, offset int) (Path, int) {
|
||||
}, totalBytes
|
||||
}
|
||||
|
||||
// isTransportRoute delegates to packetpath.IsTransportRoute.
|
||||
func isTransportRoute(routeType int) bool {
|
||||
return packetpath.IsTransportRoute(routeType)
|
||||
return routeType == RouteTransportFlood || routeType == RouteTransportDirect
|
||||
}
|
||||
|
||||
func decodeEncryptedPayload(typeName string, buf []byte) Payload {
|
||||
@@ -600,9 +597,6 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
|
||||
// how far the trace got vs the full intended route.
|
||||
var anomaly string
|
||||
if header.PayloadType == PayloadTRACE && payload.Error != "" {
|
||||
anomaly = fmt.Sprintf("TRACE payload decode failed: %s", payload.Error)
|
||||
}
|
||||
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
|
||||
// Flag anomalous routing — firmware only sends TRACE as DIRECT
|
||||
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
|
||||
@@ -610,21 +604,6 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
}
|
||||
// The header path hops count represents SNR entries = completed hops
|
||||
hopsCompleted := path.HashCount
|
||||
// Extract per-hop SNR from header path bytes (int8, quarter-dB encoding).
|
||||
// Mirrors cmd/server/decoder.go — must be done at ingest time so SNR
|
||||
// values are persisted in decoded_json (server endpoint serves DB as-is).
|
||||
if hopsCompleted > 0 && len(path.Hops) >= hopsCompleted {
|
||||
snrVals := make([]float64, 0, hopsCompleted)
|
||||
for i := 0; i < hopsCompleted; i++ {
|
||||
b, err := hex.DecodeString(path.Hops[i])
|
||||
if err == nil && len(b) == 1 {
|
||||
snrVals = append(snrVals, float64(int8(b[0]))/4.0)
|
||||
}
|
||||
}
|
||||
if len(snrVals) > 0 {
|
||||
payload.SNRValues = snrVals
|
||||
}
|
||||
}
|
||||
pathBytes, err := hex.DecodeString(payload.PathData)
|
||||
if err == nil && payload.TraceFlags != nil {
|
||||
// path_sz from flags byte is a power-of-two exponent per firmware:
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/sigvalidate"
|
||||
)
|
||||
|
||||
@@ -1823,156 +1822,3 @@ func TestDecodeAdvertWithSignatureValidation(t *testing.T) {
|
||||
t.Error("SignatureValid should be nil when validation disabled")
|
||||
}
|
||||
}
|
||||
|
||||
// === Tests for DecodePathFromRawHex (issue #886) ===
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize1(t *testing.T) {
|
||||
// Header byte 0x26 = route_type DIRECT, payload TRACE
|
||||
// Path byte 0x04 = hash_size 1 (bits 7-6 = 00 → 0+1=1), hash_count 4
|
||||
// Path bytes: 30 2D 0D 23
|
||||
raw := "2604302D0D2359FEE7B100000000006733D63367"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"30", "2D", "0D", "23"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize2(t *testing.T) {
|
||||
// Path byte 0x42 = hash_size 2 (bits 7-6 = 01 → 1+1=2), hash_count 2
|
||||
// Header 0x09 = FLOOD route (rt=1), payload ADVERT (pt=2)
|
||||
// Path bytes: AABB CCDD (4 bytes = 2 hops * 2 bytes)
|
||||
raw := "0942AABBCCDD" + "00000000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"AABB", "CCDD"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize3(t *testing.T) {
|
||||
// Path byte 0x81 = hash_size 3 (bits 7-6 = 10 → 2+1=3), hash_count 1
|
||||
// Header 0x09 = FLOOD route (rt=1), payload ADVERT
|
||||
raw := "0981AABBCC" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 1 || hops[0] != "AABBCC" {
|
||||
t.Fatalf("got %v, want [AABBCC]", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize4(t *testing.T) {
|
||||
// Path byte 0xC1 = hash_size 4 (bits 7-6 = 11 → 3+1=4), hash_count 1
|
||||
// Header 0x09 = FLOOD route (rt=1)
|
||||
raw := "09C1AABBCCDD" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 1 || hops[0] != "AABBCCDD" {
|
||||
t.Fatalf("got %v, want [AABBCCDD]", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_DirectZeroHops(t *testing.T) {
|
||||
// Path byte 0x00 = hash_size 1, hash_count 0
|
||||
// Header 0x0A = DIRECT route (rt=2), payload ADVERT
|
||||
raw := "0A00" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 0 {
|
||||
t.Fatalf("got %d hops, want 0", len(hops))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_Transport(t *testing.T) {
|
||||
// Route type 3 = TRANSPORT_DIRECT → 4 transport code bytes before path byte
|
||||
// Header 0x27 = route_type 3, payload TRACE
|
||||
// Transport codes: 1122 3344
|
||||
// Path byte 0x02 = hash_size 1, hash_count 2
|
||||
// Path bytes: AA BB
|
||||
raw := "2711223344" + "02AABB" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"AA", "BB"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeTracePayloadFailSetsAnomaly(t *testing.T) {
|
||||
// Issue #889: TRACE packet with payload too short to decode (< 9 bytes)
|
||||
// should still return a DecodedPacket (observation stored) but with Anomaly
|
||||
// set to warn operators that the decode was degraded.
|
||||
// Packet: header 0x26 (TRACE+DIRECT), pathByte 0x00, payload 4 bytes (too short).
|
||||
pkt, err := DecodePacket("2600aabbccdd", nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.Payload.Type != "TRACE" {
|
||||
t.Fatalf("payload type=%s, want TRACE", pkt.Payload.Type)
|
||||
}
|
||||
if pkt.Payload.Error == "" {
|
||||
t.Fatal("expected payload.Error to indicate decode failure")
|
||||
}
|
||||
// The key assertion: Anomaly must be set when TRACE decode fails
|
||||
if pkt.Anomaly == "" {
|
||||
t.Error("expected Anomaly to be set when TRACE payload decode fails but observation is stored")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeTraceExtractsSNRValues verifies that for TRACE packets, the header
|
||||
// path bytes are interpreted as int8 SNR values (quarter-dB) and exposed via
|
||||
// payload.SNRValues. Mirrors logic in cmd/server/decoder.go (issue: SNR values
|
||||
// extracted by server but never written into decoded_json by ingestor).
|
||||
//
|
||||
// Packet 26022FF8116A23A80000000001C0DE1000DEDE:
|
||||
// header 0x26 → TRACE (pt=9), DIRECT (rt=2)
|
||||
// pathByte 0x02 → hash_size=1, hash_count=2
|
||||
// header path: 2F F8 → SNR = [int8(0x2F)/4, int8(0xF8)/4] = [11.75, -2.0]
|
||||
// payload (15B): tag=116A23A8 auth=00000000 flags=0x01 pathData=C0DE1000DEDE
|
||||
func TestDecodeTraceExtractsSNRValues(t *testing.T) {
|
||||
pkt, err := DecodePacket("26022FF8116A23A80000000001C0DE1000DEDE", nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.Payload.Type != "TRACE" {
|
||||
t.Fatalf("payload type=%s, want TRACE", pkt.Payload.Type)
|
||||
}
|
||||
if len(pkt.Payload.SNRValues) != 2 {
|
||||
t.Fatalf("len(SNRValues)=%d, want 2 (got %v)", len(pkt.Payload.SNRValues), pkt.Payload.SNRValues)
|
||||
}
|
||||
if pkt.Payload.SNRValues[0] != 11.75 {
|
||||
t.Errorf("SNRValues[0]=%v, want 11.75", pkt.Payload.SNRValues[0])
|
||||
}
|
||||
if pkt.Payload.SNRValues[1] != -2.0 {
|
||||
t.Errorf("SNRValues[1]=%v, want -2.0", pkt.Payload.SNRValues[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,6 @@ replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter
|
||||
|
||||
replace github.com/meshcore-analyzer/sigvalidate => ../../internal/sigvalidate
|
||||
|
||||
require github.com/meshcore-analyzer/packetpath v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath
|
||||
|
||||
require github.com/meshcore-analyzer/dbconfig v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
|
||||
+38
-120
@@ -57,12 +57,6 @@ func main() {
|
||||
defer store.Close()
|
||||
log.Printf("SQLite opened: %s", cfg.DBPath)
|
||||
|
||||
// Async backfill: path_json from raw_hex (#888) — must not block MQTT startup
|
||||
store.BackfillPathJSONAsync()
|
||||
|
||||
// Check auto_vacuum mode and optionally migrate (#919)
|
||||
store.CheckAutoVacuum(cfg)
|
||||
|
||||
// Node retention: move stale nodes to inactive_nodes on startup
|
||||
nodeDays := cfg.NodeDaysOrDefault()
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
@@ -75,15 +69,12 @@ func main() {
|
||||
metricsDays := cfg.MetricsRetentionDays()
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
vacuumPages := cfg.IncrementalVacuumPages()
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
|
||||
// Daily ticker for node retention
|
||||
retentionTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
for range retentionTicker.C {
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -92,10 +83,8 @@ func main() {
|
||||
go func() {
|
||||
time.Sleep(90 * time.Second) // stagger after metrics prune
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
for range observerRetentionTicker.C {
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -105,7 +94,6 @@ func main() {
|
||||
for range metricsRetentionTicker.C {
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -126,16 +114,29 @@ func main() {
|
||||
|
||||
// Connect to each MQTT source
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
for _, source := range sources {
|
||||
tag := source.Name
|
||||
if tag == "" {
|
||||
tag = source.Broker
|
||||
}
|
||||
|
||||
opts := buildMQTTOpts(source)
|
||||
connectTimeout := source.ConnectTimeoutOrDefault()
|
||||
log.Printf("MQTT [%s] connect timeout: %ds", tag, connectTimeout)
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(source.Broker).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetOrderMatters(true)
|
||||
|
||||
if source.Username != "" {
|
||||
opts.SetUsername(source.Username)
|
||||
}
|
||||
if source.Password != "" {
|
||||
opts.SetPassword(source.Password)
|
||||
}
|
||||
if source.RejectUnauthorized != nil && !*source.RejectUnauthorized {
|
||||
opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") {
|
||||
opts.SetTLSConfig(&tls.Config{})
|
||||
}
|
||||
|
||||
opts.SetOnConnectHandler(func(c mqtt.Client) {
|
||||
log.Printf("MQTT [%s] connected to %s", tag, source.Broker)
|
||||
@@ -155,11 +156,7 @@ func main() {
|
||||
})
|
||||
|
||||
opts.SetConnectionLostHandler(func(c mqtt.Client, err error) {
|
||||
log.Printf("MQTT [%s] disconnected from %s: %v", tag, source.Broker, err)
|
||||
})
|
||||
|
||||
opts.SetReconnectingHandler(func(c mqtt.Client, options *mqtt.ClientOptions) {
|
||||
log.Printf("MQTT [%s] reconnecting to %s", tag, source.Broker)
|
||||
log.Printf("MQTT [%s] disconnected: %v", tag, err)
|
||||
})
|
||||
|
||||
// Capture source for closure
|
||||
@@ -170,43 +167,19 @@ func main() {
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
token := client.Connect()
|
||||
// With ConnectRetry=true, token.Wait() blocks forever for unreachable brokers.
|
||||
// WaitTimeout lets startup proceed; the client keeps retrying in the background
|
||||
// and OnConnect fires (subscribing) when it eventually connects (#910).
|
||||
if !token.WaitTimeout(time.Duration(connectTimeout) * time.Second) {
|
||||
log.Printf("MQTT [%s] initial connection timed out — retrying in background", tag)
|
||||
clients = append(clients, client)
|
||||
continue
|
||||
}
|
||||
token.Wait()
|
||||
if token.Error() != nil {
|
||||
log.Printf("MQTT [%s] connection failed (non-fatal): %v", tag, token.Error())
|
||||
// BL1 fix: Disconnect to stop Paho's internal retry goroutines.
|
||||
// With ConnectRetry=true, Connect() spawns background goroutines
|
||||
// that leak if the client is simply discarded.
|
||||
client.Disconnect(0)
|
||||
continue
|
||||
}
|
||||
connectedCount++
|
||||
clients = append(clients, client)
|
||||
}
|
||||
|
||||
// BL2 fix: require at least one immediately-connected source. Timed-out
|
||||
// clients are retrying in background (tracked in clients) but don't count
|
||||
// as "connected" — a single unreachable broker must not silently run with
|
||||
// zero active connections.
|
||||
if connectedCount == 0 {
|
||||
// Clean up any timed-out clients still retrying
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
log.Fatal("no MQTT sources connected — all timed out or failed. Check broker is running (default: mqtt://localhost:1883). Set MQTT_BROKER env var or configure mqttSources in config.json")
|
||||
if len(clients) == 0 {
|
||||
log.Fatal("no MQTT connections established — check broker is running (default: mqtt://localhost:1883). Set MQTT_BROKER env var or configure mqttSources in config.json")
|
||||
}
|
||||
|
||||
if connectedCount < len(clients) {
|
||||
log.Printf("Running — %d MQTT source(s) connected, %d retrying in background", connectedCount, len(clients)-connectedCount)
|
||||
} else {
|
||||
log.Printf("Running — %d MQTT source(s) connected", connectedCount)
|
||||
}
|
||||
log.Printf("Running — %d MQTT source(s) connected", len(clients))
|
||||
|
||||
// Wait for shutdown signal
|
||||
sig := make(chan os.Signal, 1)
|
||||
@@ -224,32 +197,6 @@ func main() {
|
||||
log.Println("Done.")
|
||||
}
|
||||
|
||||
// buildMQTTOpts creates MQTT client options for a source with bounded reconnect
|
||||
// backoff, connect timeout, and TLS/auth configuration.
|
||||
func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(source.Broker).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetOrderMatters(true).
|
||||
SetMaxReconnectInterval(30 * time.Second).
|
||||
SetConnectTimeout(10 * time.Second).
|
||||
SetWriteTimeout(10 * time.Second)
|
||||
|
||||
if source.Username != "" {
|
||||
opts.SetUsername(source.Username)
|
||||
}
|
||||
if source.Password != "" {
|
||||
opts.SetPassword(source.Password)
|
||||
}
|
||||
if source.RejectUnauthorized != nil && !*source.RejectUnauthorized {
|
||||
opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") {
|
||||
opts.SetTLSConfig(&tls.Config{})
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, cfg *Config) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -260,6 +207,21 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
topic := m.Topic()
|
||||
parts := strings.Split(topic, "/")
|
||||
|
||||
// IATA filter
|
||||
if len(source.IATAFilter) > 0 && len(parts) > 1 {
|
||||
region := parts[1]
|
||||
matched := false
|
||||
for _, f := range source.IATAFilter {
|
||||
if f == region {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
|
||||
return
|
||||
@@ -270,23 +232,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
return
|
||||
}
|
||||
|
||||
// Observer blacklist: drop ALL messages from blacklisted observers before any
|
||||
// DB writes (status, metrics, packets). Trumps IATA filter.
|
||||
if len(parts) > 2 && cfg.IsObserverBlacklisted(parts[2]) {
|
||||
log.Printf("MQTT [%s] observer %.8s blacklisted, dropping", tag, parts[2])
|
||||
return
|
||||
}
|
||||
|
||||
// Global observer IATA whitelist: if configured, drop messages from observers
|
||||
// in non-whitelisted IATA regions. Applies to ALL message types (status + packets).
|
||||
if len(parts) > 1 && !cfg.IsObserverIATAAllowed(parts[1]) {
|
||||
return
|
||||
}
|
||||
|
||||
// Status topic: meshcore/<region>/<observer_id>/status
|
||||
// Per-source IATA filter does NOT apply here — observer metadata (noise_floor, battery, etc.)
|
||||
// is region-independent and should be accepted from all observers regardless of
|
||||
// which IATA regions are configured for packet ingestion.
|
||||
if len(parts) >= 4 && parts[3] == "status" {
|
||||
observerID := parts[2]
|
||||
name, _ := msg["origin"].(string)
|
||||
@@ -315,21 +261,6 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
return
|
||||
}
|
||||
|
||||
// IATA filter applies to packet messages only — not status messages above.
|
||||
if len(source.IATAFilter) > 0 && len(parts) > 1 {
|
||||
region := parts[1]
|
||||
matched := false
|
||||
for _, f := range source.IATAFilter {
|
||||
if f == region {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Format 1: Raw packet (meshcoretomqtt / Cisien format)
|
||||
rawHex, _ := msg["raw"].(string)
|
||||
if rawHex != "" {
|
||||
@@ -348,16 +279,8 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
if len(parts) > 1 {
|
||||
region = parts[1]
|
||||
}
|
||||
// Fallback to source-level region config when topic has no region (#788)
|
||||
if region == "" && source.Region != "" {
|
||||
region = source.Region
|
||||
}
|
||||
|
||||
mqttMsg := &MQTTPacketMessage{Raw: rawHex}
|
||||
// Parse optional region from JSON payload (#788)
|
||||
if v, ok := msg["region"].(string); ok && v != "" {
|
||||
mqttMsg.Region = v
|
||||
}
|
||||
if v, ok := msg["SNR"]; ok {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
mqttMsg.SNR = &f
|
||||
@@ -457,12 +380,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
// Upsert observer
|
||||
if observerID != "" {
|
||||
origin, _ := msg["origin"].(string)
|
||||
// Use effective region: payload > topic > source config (#788)
|
||||
effectiveRegion := region
|
||||
if mqttMsg.Region != "" {
|
||||
effectiveRegion = mqttMsg.Region
|
||||
}
|
||||
if err := store.UpsertObserver(observerID, origin, effectiveRegion, nil); err != nil {
|
||||
if err := store.UpsertObserver(observerID, origin, region, nil); err != nil {
|
||||
log.Printf("MQTT [%s] observer upsert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,8 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
)
|
||||
|
||||
func TestToFloat64(t *testing.T) {
|
||||
@@ -742,196 +739,3 @@ func TestToFloat64WithUnits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIATAFilterDoesNotDropStatusMessages verifies that status messages from
|
||||
// out-of-region observers are still processed (noise_floor, battery, etc.)
|
||||
// even when an IATA filter is configured for packet data.
|
||||
func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test", IATAFilter: []string{"SJC"}}
|
||||
|
||||
// BFL observer sends a status message with noise_floor — outside the IATA filter.
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/status",
|
||||
payload: []byte(`{"origin":"BFLObserver","stats":{"noise_floor":-105.0}}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, &Config{})
|
||||
|
||||
var name string
|
||||
var noiseFloor *float64
|
||||
err := store.db.QueryRow("SELECT name, noise_floor FROM observers WHERE id = 'bfl-obs1'").Scan(&name, &noiseFloor)
|
||||
if err != nil {
|
||||
t.Fatalf("observer not found after status from out-of-region observer: %v", err)
|
||||
}
|
||||
if name != "BFLObserver" {
|
||||
t.Errorf("name=%q, want BFLObserver", name)
|
||||
}
|
||||
if noiseFloor == nil || *noiseFloor != -105.0 {
|
||||
t.Errorf("noise_floor=%v, want -105.0 — status message was dropped by IATA filter when it should not be", noiseFloor)
|
||||
}
|
||||
|
||||
// Verify that a packet from BFL is still filtered.
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
pktMsg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, pktMsg, nil, &Config{})
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("packet from out-of-region BFL should still be filtered by IATA")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTConnectRetryTimeoutDoesNotBlock verifies that WaitTimeout returns within
|
||||
// the deadline for an unreachable broker when ConnectRetry=true (#910). Previously,
|
||||
// token.Wait() would block forever in this configuration.
|
||||
func TestMQTTConnectRetryTimeoutDoesNotBlock(t *testing.T) {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1"). // port 1 — nothing listening, fast refusal
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true)
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
token := client.Connect()
|
||||
defer client.Disconnect(100)
|
||||
|
||||
start := time.Now()
|
||||
connected := token.WaitTimeout(3 * time.Second)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if connected {
|
||||
t.Skip("port 1 unexpectedly accepted a connection — skipping")
|
||||
}
|
||||
if elapsed > 4*time.Second {
|
||||
t.Errorf("WaitTimeout blocked for %v — token.Wait() would block forever with ConnectRetry=true", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBL1_GoroutineLeakOnHardFailure reproduces BLOCKER 1: without Disconnect()
|
||||
// on the error path, Paho's internal retry goroutines leak when a client is
|
||||
// discarded after Connect() with ConnectRetry=true.
|
||||
//
|
||||
// We prove the leak by creating N clients WITHOUT Disconnect — goroutines grow
|
||||
// proportionally. The fix (client.Disconnect(0) before continue) prevents this.
|
||||
func TestBL1_GoroutineLeakOnHardFailure(t *testing.T) {
|
||||
runtime.GC()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
baseline := runtime.NumGoroutine()
|
||||
|
||||
// Create multiple clients connected to unreachable broker, WITHOUT disconnecting.
|
||||
// Each one spawns Paho retry goroutines that accumulate.
|
||||
const numClients = 10
|
||||
clients := make([]mqtt.Client, numClients)
|
||||
for i := 0; i < numClients; i++ {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1").
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectTimeout(500 * time.Millisecond)
|
||||
c := mqtt.NewClient(opts)
|
||||
tok := c.Connect()
|
||||
tok.WaitTimeout(1 * time.Second)
|
||||
clients[i] = c
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
leaked := runtime.NumGoroutine()
|
||||
goroutineGrowth := leaked - baseline
|
||||
|
||||
// Clean up to not actually leak in test
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
|
||||
t.Logf("baseline=%d, after %d undisconnected clients=%d, growth=%d",
|
||||
baseline, numClients, leaked, goroutineGrowth)
|
||||
|
||||
// With ConnectRetry=true, each Connect() spawns retry goroutines.
|
||||
// Without Disconnect, these accumulate. Verify growth is meaningful.
|
||||
if goroutineGrowth < 3 {
|
||||
t.Skip("Connect didn't spawn enough extra goroutines to measure leak")
|
||||
}
|
||||
|
||||
// The fix: calling client.Disconnect(0) on the error path prevents accumulation.
|
||||
// Anti-tautology: removing the Disconnect(0) call from main.go's error path
|
||||
// would cause goroutine accumulation proportional to failed broker count.
|
||||
t.Logf("CONFIRMED: %d leaked goroutines from %d clients without Disconnect — fix adds Disconnect(0) on error path", goroutineGrowth, numClients)
|
||||
}
|
||||
|
||||
// TestBL2_ZeroConnectedFatals verifies BLOCKER 2: when all brokers are unreachable,
|
||||
// connectedCount==0 must be detected. We test the logic directly — if only timed-out
|
||||
// clients exist (appended to clients slice) but connectedCount is 0, the guard triggers.
|
||||
func TestBL2_ZeroConnectedFatals(t *testing.T) {
|
||||
// Simulate the connection loop result: 1 timed-out client, 0 connected
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
|
||||
// Create a client that times out (unreachable broker)
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1").
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true)
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
token := client.Connect()
|
||||
if !token.WaitTimeout(2 * time.Second) {
|
||||
// Timed out — PR #926 appends to clients
|
||||
clients = append(clients, client)
|
||||
}
|
||||
defer func() {
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
}()
|
||||
|
||||
// OLD bug: len(clients) == 0 would be false (1 timed-out client in list)
|
||||
// → ingestor would silently run with zero connections
|
||||
if len(clients) == 0 {
|
||||
t.Fatal("expected timed-out client to be in clients slice")
|
||||
}
|
||||
|
||||
// NEW fix: connectedCount == 0 catches this
|
||||
if connectedCount != 0 {
|
||||
t.Errorf("connectedCount should be 0, got %d", connectedCount)
|
||||
}
|
||||
|
||||
// The real code does: if connectedCount == 0 { log.Fatal(...) }
|
||||
// This test proves len(clients) > 0 but connectedCount == 0 — the old guard
|
||||
// would have missed it.
|
||||
if len(clients) > 0 && connectedCount == 0 {
|
||||
t.Log("BL2 confirmed: old guard len(clients)==0 would NOT fatal; new guard connectedCount==0 correctly catches zero-connected state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleMessageObserverIATAWhitelist(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{
|
||||
ObserverIATAWhitelist: []string{"ARN"},
|
||||
}
|
||||
|
||||
// Message from non-whitelisted region GOT — should be dropped
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/GOT/obs1/status",
|
||||
payload: []byte(`{"origin":"node1","noise_floor":-110}`),
|
||||
}, nil, cfg)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs1'").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("observer from non-whitelisted IATA GOT should be dropped")
|
||||
}
|
||||
|
||||
// Message from whitelisted region ARN — should be accepted
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/ARN/obs2/status",
|
||||
payload: []byte(`{"origin":"node2","noise_floor":-105}`),
|
||||
}, nil, cfg)
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs2'").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Errorf("observer from whitelisted IATA ARN should be accepted, got count=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBuildMQTTOpts_ReconnectSettings(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "tcp://localhost:1883",
|
||||
Name: "test",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.MaxReconnectInterval != 30*time.Second {
|
||||
t.Errorf("MaxReconnectInterval = %v, want 30s", opts.MaxReconnectInterval)
|
||||
}
|
||||
if opts.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout = %v, want 10s", opts.ConnectTimeout)
|
||||
}
|
||||
if opts.WriteTimeout != 10*time.Second {
|
||||
t.Errorf("WriteTimeout = %v, want 10s", opts.WriteTimeout)
|
||||
}
|
||||
if !opts.AutoReconnect {
|
||||
t.Error("AutoReconnect should be true")
|
||||
}
|
||||
if !opts.ConnectRetry {
|
||||
t.Error("ConnectRetry should be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_Credentials(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "tcp://broker:1883",
|
||||
Username: "user1",
|
||||
Password: "pass1",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.Username != "user1" {
|
||||
t.Errorf("Username = %q, want %q", opts.Username, "user1")
|
||||
}
|
||||
if opts.Password != "pass1" {
|
||||
t.Errorf("Password = %q, want %q", opts.Password, "pass1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_TLS_InsecureSkipVerify(t *testing.T) {
|
||||
f := false
|
||||
source := MQTTSource{
|
||||
Broker: "ssl://broker:8883",
|
||||
RejectUnauthorized: &f,
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig should be set")
|
||||
}
|
||||
if !opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify should be true when RejectUnauthorized=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_TLS_SSL_Prefix(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "ssl://broker:8883",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig should be set for ssl:// brokers")
|
||||
}
|
||||
if opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify should be false by default")
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIngestorIsObserverBlacklisted(t *testing.T) {
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"OBS1", "obs2"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{"OBS1", true},
|
||||
{"obs1", true},
|
||||
{"OBS2", true},
|
||||
{"obs3", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := cfg.IsObserverBlacklisted(tt.id)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsObserverBlacklisted(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestorIsObserverBlacklistedEmpty(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("empty blacklist should not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestorIsObserverBlacklistedNil(t *testing.T) {
|
||||
var cfg *Config
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("nil config should not match")
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// handleBackup streams a consistent SQLite snapshot of the analyzer DB.
|
||||
//
|
||||
// Requires API-key authentication (mounted via requireAPIKey in routes.go).
|
||||
//
|
||||
// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented
|
||||
// copy of the current database into a new file. It runs at READ ISOLATION
|
||||
// against the source DB (works on our read-only connection) and never
|
||||
// blocks concurrent writers — the ingestor keeps writing to the WAL while
|
||||
// the snapshot is taken from a consistent read transaction.
|
||||
//
|
||||
// Response:
|
||||
//
|
||||
// 200 OK
|
||||
// Content-Type: application/octet-stream
|
||||
// Content-Disposition: attachment; filename="corescope-backup-<unix>.db"
|
||||
// <body: complete SQLite database file>
|
||||
//
|
||||
// The temp file is removed after the response is fully written, regardless
|
||||
// of whether the client successfully consumed the stream.
|
||||
func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "database unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now().UTC().Unix()
|
||||
clientIP := r.Header.Get("X-Forwarded-For")
|
||||
if clientIP == "" {
|
||||
clientIP = r.RemoteAddr
|
||||
}
|
||||
log.Printf("[backup] generating backup for client %s", clientIP)
|
||||
|
||||
// Stage the snapshot in the OS temp dir so we never touch the live DB
|
||||
// directory (avoids confusing operators / accidental WAL clobber).
|
||||
tmpDir, err := os.MkdirTemp("", "corescope-backup-")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "tempdir failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if rmErr := os.RemoveAll(tmpDir); rmErr != nil {
|
||||
log.Printf("[backup] cleanup error: %v", rmErr)
|
||||
}
|
||||
}()
|
||||
|
||||
snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts))
|
||||
|
||||
// SQLite parses the path literal — escape any single quotes defensively.
|
||||
// (mkdtemp output won't contain quotes, but be paranoid for future-proofing.)
|
||||
escaped := strings.ReplaceAll(snapshotPath, "'", "''")
|
||||
if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(snapshotPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "open snapshot failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
stat, err := f.Stat()
|
||||
if err == nil {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
// Headers already flushed; just log. Client will see truncated stream.
|
||||
log.Printf("[backup] stream error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database.
|
||||
// See https://www.sqlite.org/fileformat.html#magic_header_string
|
||||
const sqliteMagic = "SQLite format 3\x00"
|
||||
|
||||
func TestBackupRequiresAPIKey(t *testing.T) {
|
||||
_, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough")
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/backup", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("expected 401 without API key, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) {
|
||||
const apiKey = "test-secret-key-strong-enough"
|
||||
_, router := setupTestServerWithAPIKey(t, apiKey)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/backup", nil)
|
||||
req.Header.Set("X-API-Key", apiKey)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
ct := w.Header().Get("Content-Type")
|
||||
if ct != "application/octet-stream" {
|
||||
t.Errorf("expected Content-Type application/octet-stream, got %q", ct)
|
||||
}
|
||||
|
||||
cd := w.Header().Get("Content-Disposition")
|
||||
if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") {
|
||||
t.Errorf("expected Content-Disposition attachment with corescope-backup-<ts>.db filename, got %q", cd)
|
||||
}
|
||||
|
||||
body := w.Body.Bytes()
|
||||
if len(body) < len(sqliteMagic) {
|
||||
t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body))
|
||||
}
|
||||
if got := string(body[:len(sqliteMagic)]); got != sqliteMagic {
|
||||
t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got)
|
||||
}
|
||||
}
|
||||
@@ -127,92 +127,6 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// loadStoreWithRetention creates a PacketStore with retentionHours set.
|
||||
func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore {
|
||||
t.Helper()
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{RetentionHours: retentionHours}
|
||||
store := NewPacketStore(db, cfg)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// createTestDBWithAgedPackets inserts numRecent packets with timestamps within
|
||||
// the last hour and numOld packets with timestamps 48 hours ago.
|
||||
func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(s string) {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
t.Fatalf("setup: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)
|
||||
execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`)
|
||||
execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`)
|
||||
execOrFail(`CREATE TABLE schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := 1
|
||||
// Insert old packets (48 hours ago)
|
||||
for i := 0; i < numOld; i++ {
|
||||
ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339)
|
||||
conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`)
|
||||
conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "")
|
||||
id++
|
||||
}
|
||||
// Insert recent packets (within last hour)
|
||||
for i := 0; i < numRecent; i++ {
|
||||
ts := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second).Format(time.RFC3339)
|
||||
conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`)
|
||||
conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "")
|
||||
id++
|
||||
}
|
||||
return dbPath
|
||||
}
|
||||
|
||||
func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) {
|
||||
dbPath := createTestDBWithAgedPackets(t, 50, 100)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// retention = 2 hours — should load only the 50 recent packets, not the 100 old ones
|
||||
store := loadStoreWithRetention(t, dbPath, 2)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 50 {
|
||||
t.Errorf("expected 50 recent packets, got %d (old packets should be excluded by retentionHours)", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) {
|
||||
dbPath := createTestDBWithAgedPackets(t, 50, 100)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// retention = 0 (unlimited) — should load all 150 packets
|
||||
store := loadStoreWithRetention(t, dbPath, 0)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 150 {
|
||||
t.Errorf("expected all 150 packets with retentionHours=0, got %d", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEstimateStoreTxBytesTypical(t *testing.T) {
|
||||
est := estimateStoreTxBytesTypical(10)
|
||||
if est < 1000 {
|
||||
@@ -315,7 +229,7 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) {
|
||||
id INTEGER PRIMARY KEY,
|
||||
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
@@ -366,7 +280,7 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) {
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ = time.Second // suppress unused import
|
||||
|
||||
// Helper to create a minimal PacketStore with GRP_TXT packets for channel analytics testing.
|
||||
func newChannelTestStore(packets []*StoreTx) *PacketStore {
|
||||
ps := &PacketStore{
|
||||
packets: packets,
|
||||
byHash: make(map[string]*StoreTx),
|
||||
byTxID: make(map[int]*StoreTx),
|
||||
byObsID: make(map[int]*StoreObs),
|
||||
byObserver: make(map[string][]*StoreObs),
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
byPathHop: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
byPayloadType: make(map[int][]*StoreTx),
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
collisionCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
spIndex: make(map[string]int),
|
||||
spTxIndex: make(map[string][]*StoreTx),
|
||||
advertPubkeys: make(map[string]int),
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
clockSkew: NewClockSkewEngine(),
|
||||
}
|
||||
ps.byPayloadType[5] = packets
|
||||
return ps
|
||||
}
|
||||
|
||||
func makeGrpTx(channelHash int, channel, text, sender string) *StoreTx {
|
||||
decoded := map[string]interface{}{
|
||||
"type": "CHAN",
|
||||
"channelHash": float64(channelHash),
|
||||
"channel": channel,
|
||||
"text": text,
|
||||
"sender": sender,
|
||||
}
|
||||
b, _ := json.Marshal(decoded)
|
||||
pt := 5
|
||||
return &StoreTx{
|
||||
ID: 1,
|
||||
DecodedJSON: string(b),
|
||||
FirstSeen: "2026-05-01T12:00:00Z",
|
||||
PayloadType: &pt,
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_MergesEncryptedAndDecrypted verifies that packets
|
||||
// with the same hash byte but different decryption status merge into ONE bucket.
|
||||
func TestComputeAnalyticsChannels_MergesEncryptedAndDecrypted(t *testing.T) {
|
||||
// Hash 129 is the real hash for #wardriving: SHA256(SHA256("#wardriving")[:16])[0] = 129
|
||||
// Some packets are decrypted (have channel name), some are not (encrypted)
|
||||
packets := []*StoreTx{
|
||||
makeGrpTx(129, "#wardriving", "hello", "alice"),
|
||||
makeGrpTx(129, "#wardriving", "world", "bob"),
|
||||
makeGrpTx(129, "", "", ""), // encrypted — no channel name
|
||||
makeGrpTx(129, "", "", ""), // encrypted
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected 1 channel bucket, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
ch := channels[0]
|
||||
if ch["name"] != "#wardriving" {
|
||||
t.Errorf("expected name '#wardriving', got %q", ch["name"])
|
||||
}
|
||||
if ch["messages"] != 4 {
|
||||
t.Errorf("expected 4 messages, got %v", ch["messages"])
|
||||
}
|
||||
if ch["encrypted"] != false {
|
||||
t.Errorf("expected encrypted=false (some packets decrypted), got %v", ch["encrypted"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_RejectsRainbowTableMismatch verifies that a packet
|
||||
// with channelHash=72 but channel="#wardriving" (mismatch) does NOT create a
|
||||
// "#wardriving" bucket — it falls into "ch72" instead.
|
||||
func TestComputeAnalyticsChannels_RejectsRainbowTableMismatch(t *testing.T) {
|
||||
// Hash 72 is NOT the correct hash for #wardriving (which is 129).
|
||||
// This simulates a rainbow-table collision/mismatch.
|
||||
packets := []*StoreTx{
|
||||
makeGrpTx(72, "#wardriving", "ghost", "eve"), // mismatch: hash 72 != wardriving's real hash
|
||||
makeGrpTx(129, "#wardriving", "real", "alice"), // correct match
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 2 {
|
||||
t.Fatalf("expected 2 channel buckets, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
|
||||
// Find the buckets
|
||||
var ch72, ch129 map[string]interface{}
|
||||
for _, ch := range channels {
|
||||
if ch["hash"] == "72" {
|
||||
ch72 = ch
|
||||
} else if ch["hash"] == "129" {
|
||||
ch129 = ch
|
||||
}
|
||||
}
|
||||
|
||||
if ch72 == nil {
|
||||
t.Fatal("expected a bucket for hash 72")
|
||||
}
|
||||
if ch129 == nil {
|
||||
t.Fatal("expected a bucket for hash 129")
|
||||
}
|
||||
|
||||
// ch72 should NOT be named "#wardriving" — it should be the placeholder
|
||||
if ch72["name"] == "#wardriving" {
|
||||
t.Errorf("hash 72 bucket should NOT be named '#wardriving' (rainbow-table mismatch rejected)")
|
||||
}
|
||||
if ch72["name"] != "ch72" {
|
||||
t.Errorf("expected hash 72 bucket named 'ch72', got %q", ch72["name"])
|
||||
}
|
||||
|
||||
// ch129 should be named "#wardriving"
|
||||
if ch129["name"] != "#wardriving" {
|
||||
t.Errorf("expected hash 129 bucket named '#wardriving', got %q", ch129["name"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelNameMatchesHash verifies the hash validation function.
|
||||
func TestChannelNameMatchesHash(t *testing.T) {
|
||||
// #wardriving hashes to 129
|
||||
if !channelNameMatchesHash("#wardriving", "129") {
|
||||
t.Error("expected #wardriving to match hash 129")
|
||||
}
|
||||
if channelNameMatchesHash("#wardriving", "72") {
|
||||
t.Error("expected #wardriving to NOT match hash 72")
|
||||
}
|
||||
// Without leading # should also work
|
||||
if !channelNameMatchesHash("wardriving", "129") {
|
||||
t.Error("expected wardriving (without #) to match hash 129")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsPlaceholderName verifies placeholder detection.
|
||||
func TestIsPlaceholderName(t *testing.T) {
|
||||
if !isPlaceholderName("ch129") {
|
||||
t.Error("ch129 should be placeholder")
|
||||
}
|
||||
if !isPlaceholderName("ch0") {
|
||||
t.Error("ch0 should be placeholder")
|
||||
}
|
||||
if isPlaceholderName("#wardriving") {
|
||||
t.Error("#wardriving should NOT be placeholder")
|
||||
}
|
||||
if isPlaceholderName("Public") {
|
||||
t.Error("Public should NOT be placeholder")
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPacketsChannelFilter verifies /api/packets?channel=... actually filters
|
||||
// (regression test for #812).
|
||||
func TestPacketsChannelFilter(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
get := func(url string) map[string]interface{} {
|
||||
req := httptest.NewRequest("GET", url, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s: expected 200, got %d", url, w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode %s: %v", url, err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
all := get("/api/packets?limit=50")
|
||||
allTotal := int(all["total"].(float64))
|
||||
if allTotal < 2 {
|
||||
t.Fatalf("expected baseline >= 2 packets, got %d", allTotal)
|
||||
}
|
||||
|
||||
test := get("/api/packets?limit=50&channel=%23test")
|
||||
testTotal := int(test["total"].(float64))
|
||||
if testTotal == 0 {
|
||||
t.Fatalf("channel=#test: expected >= 1 match, got 0 (filter ignored?)")
|
||||
}
|
||||
if testTotal >= allTotal {
|
||||
t.Fatalf("channel=#test: expected fewer packets than baseline (%d), got %d", allTotal, testTotal)
|
||||
}
|
||||
|
||||
// Every returned packet must be a CHAN/GRP_TXT (payload_type=5) on #test.
|
||||
pkts, _ := test["packets"].([]interface{})
|
||||
for _, p := range pkts {
|
||||
m := p.(map[string]interface{})
|
||||
if pt, _ := m["payload_type"].(float64); int(pt) != 5 {
|
||||
t.Errorf("channel=#test: returned non-GRP_TXT packet (payload_type=%v)", m["payload_type"])
|
||||
}
|
||||
}
|
||||
|
||||
none := get("/api/packets?limit=50&channel=nonexistentchannel")
|
||||
if int(none["total"].(float64)) != 0 {
|
||||
t.Fatalf("channel=nonexistentchannel: expected total=0, got %v", none["total"])
|
||||
}
|
||||
}
|
||||
+37
-324
@@ -16,8 +16,7 @@ const (
|
||||
SkewWarning SkewSeverity = "warning" // 5 min – 1 hour
|
||||
SkewCritical SkewSeverity = "critical" // 1 hour – 30 days
|
||||
SkewAbsurd SkewSeverity = "absurd" // > 30 days
|
||||
SkewNoClock SkewSeverity = "no_clock" // > 365 days — uninitialized RTC
|
||||
SkewBimodalClock SkewSeverity = "bimodal_clock" // mixed good+bad recent samples (flaky RTC)
|
||||
SkewNoClock SkewSeverity = "no_clock" // > 365 days — uninitialized RTC
|
||||
)
|
||||
|
||||
// Default thresholds in seconds.
|
||||
@@ -34,38 +33,6 @@ const (
|
||||
// maxReasonableDriftPerDay caps drift display. Physically impossible
|
||||
// drift rates (> 1 day/day) indicate insufficient or outlier samples.
|
||||
maxReasonableDriftPerDay = 86400.0
|
||||
|
||||
// recentSkewWindowCount is the number of most-recent advert samples
|
||||
// used to derive the "current" skew for severity classification (see
|
||||
// issue #789). The all-time median is poisoned by historical bad
|
||||
// samples (e.g. a node that was off and then GPS-corrected); severity
|
||||
// must reflect current health, not lifetime statistics.
|
||||
recentSkewWindowCount = 5
|
||||
|
||||
// recentSkewWindowSec bounds the recent-window in time as well: only
|
||||
// samples from the last N seconds count as "recent" for severity.
|
||||
// The effective window is min(recentSkewWindowCount, samples in 1h).
|
||||
recentSkewWindowSec = 3600
|
||||
|
||||
// bimodalSkewThresholdSec is the absolute skew threshold (1 hour)
|
||||
// above which a sample is considered "bad" — likely firmware emitting
|
||||
// a nonsense timestamp from an uninitialized RTC, not real drift.
|
||||
// Chosen to match the warning/critical severity boundary: real clock
|
||||
// drift rarely exceeds 1 hour, while epoch-0 RTCs produce ~1.7B sec.
|
||||
bimodalSkewThresholdSec = 3600.0
|
||||
|
||||
// maxPlausibleSkewJumpSec is the largest skew change between
|
||||
// consecutive samples that we treat as physical drift. Anything larger
|
||||
// (e.g. a GPS sync that jumps the clock by minutes/days) is rejected
|
||||
// as an outlier when computing drift. Real microcontroller drift is
|
||||
// fractions of a second per advert; 60s is a generous safety factor.
|
||||
maxPlausibleSkewJumpSec = 60.0
|
||||
|
||||
// theilSenMaxPoints caps the number of points fed to Theil-Sen
|
||||
// regression (O(n²) in pairs). For nodes with thousands of samples we
|
||||
// keep the most-recent points, which are also the most relevant for
|
||||
// current drift.
|
||||
theilSenMaxPoints = 200
|
||||
)
|
||||
|
||||
// classifySkew maps absolute skew (seconds) to a severity level.
|
||||
@@ -109,7 +76,6 @@ type NodeClockSkew struct {
|
||||
MeanSkewSec float64 `json:"meanSkewSec"` // corrected mean skew (positive = node ahead)
|
||||
MedianSkewSec float64 `json:"medianSkewSec"` // corrected median skew
|
||||
LastSkewSec float64 `json:"lastSkewSec"` // most recent corrected skew
|
||||
RecentMedianSkewSec float64 `json:"recentMedianSkewSec"` // median across most-recent samples (drives severity, see #789)
|
||||
DriftPerDaySec float64 `json:"driftPerDaySec"` // linear drift rate (sec/day)
|
||||
Severity SkewSeverity `json:"severity"`
|
||||
SampleCount int `json:"sampleCount"`
|
||||
@@ -117,11 +83,6 @@ type NodeClockSkew struct {
|
||||
LastAdvertTS int64 `json:"lastAdvertTS"` // most recent advert timestamp
|
||||
LastObservedTS int64 `json:"lastObservedTS"` // most recent observation timestamp
|
||||
Samples []SkewSample `json:"samples,omitempty"` // time-series for sparklines
|
||||
GoodFraction float64 `json:"goodFraction"` // fraction of recent samples with |skew| <= 1h
|
||||
RecentBadSampleCount int `json:"recentBadSampleCount"` // count of recent samples with |skew| > 1h
|
||||
RecentSampleCount int `json:"recentSampleCount"` // total recent samples in window
|
||||
RecentHashEvidence []HashEvidence `json:"recentHashEvidence,omitempty"`
|
||||
CalibrationSummary *CalibrationSummary `json:"calibrationSummary,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"` // populated in fleet responses
|
||||
NodeRole string `json:"nodeRole,omitempty"` // populated in fleet responses
|
||||
}
|
||||
@@ -132,31 +93,6 @@ type SkewSample struct {
|
||||
SkewSec float64 `json:"skew"` // corrected skew in seconds
|
||||
}
|
||||
|
||||
// HashEvidenceObserver is one observer's contribution to a per-hash evidence entry.
|
||||
type HashEvidenceObserver struct {
|
||||
ObserverID string `json:"observerID"`
|
||||
ObserverName string `json:"observerName"`
|
||||
RawSkewSec float64 `json:"rawSkewSec"`
|
||||
CorrectedSkewSec float64 `json:"correctedSkewSec"`
|
||||
ObserverOffsetSec float64 `json:"observerOffsetSec"`
|
||||
Calibrated bool `json:"calibrated"`
|
||||
}
|
||||
|
||||
// HashEvidence is per-hash clock skew evidence showing individual observer contributions.
|
||||
type HashEvidence struct {
|
||||
Hash string `json:"hash"`
|
||||
Observers []HashEvidenceObserver `json:"observers"`
|
||||
MedianCorrectedSkewSec float64 `json:"medianCorrectedSkewSec"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// CalibrationSummary counts how many samples were corrected via observer calibration.
|
||||
type CalibrationSummary struct {
|
||||
TotalSamples int `json:"totalSamples"`
|
||||
CalibratedSamples int `json:"calibratedSamples"`
|
||||
UncalibratedSamples int `json:"uncalibratedSamples"`
|
||||
}
|
||||
|
||||
// txSkewResult maps tx hash → per-transmission skew stats. This is an
|
||||
// intermediate result keyed by hash (not pubkey); the store maps hash → pubkey
|
||||
// when building the final per-node view.
|
||||
@@ -170,27 +106,15 @@ type ClockSkewEngine struct {
|
||||
observerOffsets map[string]float64 // observerID → calibrated offset (seconds)
|
||||
observerSamples map[string]int // observerID → number of multi-observer packets used
|
||||
nodeSkew txSkewResult
|
||||
hashEvidence map[string][]hashEvidenceEntry // hash → per-observer raw/corrected data
|
||||
lastComputed time.Time
|
||||
computeInterval time.Duration
|
||||
}
|
||||
|
||||
// hashEvidenceEntry stores raw evidence per observer per hash, cached during Recompute.
|
||||
type hashEvidenceEntry struct {
|
||||
observerID string
|
||||
rawSkew float64
|
||||
corrected float64
|
||||
offset float64
|
||||
calibrated bool
|
||||
observedTS int64
|
||||
}
|
||||
|
||||
func NewClockSkewEngine() *ClockSkewEngine {
|
||||
return &ClockSkewEngine{
|
||||
observerOffsets: make(map[string]float64),
|
||||
observerSamples: make(map[string]int),
|
||||
nodeSkew: make(txSkewResult),
|
||||
hashEvidence: make(map[string][]hashEvidenceEntry),
|
||||
computeInterval: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
@@ -215,16 +139,14 @@ func (e *ClockSkewEngine) Recompute(store *PacketStore) {
|
||||
var newOffsets map[string]float64
|
||||
var newSamples map[string]int
|
||||
var newNodeSkew txSkewResult
|
||||
var newHashEvidence map[string][]hashEvidenceEntry
|
||||
|
||||
if len(samples) > 0 {
|
||||
newOffsets, newSamples = calibrateObservers(samples)
|
||||
newNodeSkew, newHashEvidence = computeNodeSkew(samples, newOffsets)
|
||||
newNodeSkew = computeNodeSkew(samples, newOffsets)
|
||||
} else {
|
||||
newOffsets = make(map[string]float64)
|
||||
newSamples = make(map[string]int)
|
||||
newNodeSkew = make(txSkewResult)
|
||||
newHashEvidence = make(map[string][]hashEvidenceEntry)
|
||||
}
|
||||
|
||||
// Swap results under brief write lock.
|
||||
@@ -237,7 +159,6 @@ func (e *ClockSkewEngine) Recompute(store *PacketStore) {
|
||||
e.observerOffsets = newOffsets
|
||||
e.observerSamples = newSamples
|
||||
e.nodeSkew = newNodeSkew
|
||||
e.hashEvidence = newHashEvidence
|
||||
e.lastComputed = time.Now()
|
||||
e.mu.Unlock()
|
||||
}
|
||||
@@ -374,7 +295,7 @@ func calibrateObservers(samples []skewSample) (map[string]float64, map[string]in
|
||||
// ── Phase 3: Per-Node Skew ─────────────────────────────────────────────────────
|
||||
|
||||
// computeNodeSkew calculates corrected skew statistics for each node.
|
||||
func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) (txSkewResult, map[string][]hashEvidenceEntry) {
|
||||
func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) txSkewResult {
|
||||
// Compute corrected skew per sample, grouped by hash (each hash = one
|
||||
// node's advert transmission). The caller maps hash → pubkey via byNode.
|
||||
type correctedSample struct {
|
||||
@@ -385,7 +306,6 @@ func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) (txSke
|
||||
|
||||
byHash := make(map[string][]correctedSample)
|
||||
hashAdvertTS := make(map[string]int64)
|
||||
evidence := make(map[string][]hashEvidenceEntry) // hash → per-observer evidence
|
||||
|
||||
for _, s := range samples {
|
||||
obsOffset, hasCal := obsOffsets[s.observerID]
|
||||
@@ -402,14 +322,6 @@ func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) (txSke
|
||||
calibrated: hasCal,
|
||||
})
|
||||
hashAdvertTS[s.hash] = s.advertTS
|
||||
evidence[s.hash] = append(evidence[s.hash], hashEvidenceEntry{
|
||||
observerID: s.observerID,
|
||||
rawSkew: round(rawSkew, 1),
|
||||
corrected: round(corrected, 1),
|
||||
offset: round(obsOffset, 1),
|
||||
calibrated: hasCal,
|
||||
observedTS: s.observedTS,
|
||||
})
|
||||
}
|
||||
|
||||
// Each hash represents one advert from one node. Compute median corrected
|
||||
@@ -448,7 +360,7 @@ func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) (txSke
|
||||
LastObservedTS: latestObsTS,
|
||||
}
|
||||
}
|
||||
return result, evidence
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Integration with PacketStore ───────────────────────────────────────────────
|
||||
@@ -507,95 +419,12 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
|
||||
medSkew := median(allSkews)
|
||||
meanSkew := mean(allSkews)
|
||||
absMedian := math.Abs(medSkew)
|
||||
severity := classifySkew(absMedian)
|
||||
|
||||
// Severity is derived from RECENT samples only (issue #789). The
|
||||
// all-time median is poisoned by historical bad data — a node that
|
||||
// was off for hours and then GPS-corrected can have median = -59M sec
|
||||
// while its current skew is -0.8s. Operators need severity to reflect
|
||||
// current health, so they trust the dashboard.
|
||||
//
|
||||
// Sort tsSkews by time and take the last recentSkewWindowCount samples
|
||||
// (or all samples within recentSkewWindowSec of the latest, whichever
|
||||
// gives FEWER samples — we want the more-current view; a chatty node
|
||||
// can fit dozens of samples in 1h, in which case the count cap wins).
|
||||
sort.Slice(tsSkews, func(i, j int) bool { return tsSkews[i].ts < tsSkews[j].ts })
|
||||
|
||||
recentSkew := lastSkew
|
||||
var recentVals []float64
|
||||
if n := len(tsSkews); n > 0 {
|
||||
latestTS := tsSkews[n-1].ts
|
||||
// Index-based window: last K samples.
|
||||
startByCount := n - recentSkewWindowCount
|
||||
if startByCount < 0 {
|
||||
startByCount = 0
|
||||
}
|
||||
// Time-based window: samples newer than latestTS - windowSec.
|
||||
startByTime := n - 1
|
||||
for i := n - 1; i >= 0; i-- {
|
||||
if latestTS-tsSkews[i].ts <= recentSkewWindowSec {
|
||||
startByTime = i
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Pick the narrower (larger-index) of the two windows — the most
|
||||
// current view of the node's clock health.
|
||||
start := startByCount
|
||||
if startByTime > start {
|
||||
start = startByTime
|
||||
}
|
||||
recentVals = make([]float64, 0, n-start)
|
||||
for i := start; i < n; i++ {
|
||||
recentVals = append(recentVals, tsSkews[i].skew)
|
||||
}
|
||||
if len(recentVals) > 0 {
|
||||
recentSkew = median(recentVals)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Bimodal detection (#845) ─────────────────────────────────────────
|
||||
// Split recent samples into "good" (|skew| <= 1h, real clock) and
|
||||
// "bad" (|skew| > 1h, firmware nonsense from uninitialized RTC).
|
||||
// Classification order (first match wins):
|
||||
// no_clock — goodFraction < 0.10 (essentially no real clock)
|
||||
// bimodal_clock — 0.10 <= goodFraction < 0.80 AND badCount > 0
|
||||
// ok/warn/etc. — goodFraction >= 0.80 (normal, outliers filtered)
|
||||
var goodSamples []float64
|
||||
for _, v := range recentVals {
|
||||
if math.Abs(v) <= bimodalSkewThresholdSec {
|
||||
goodSamples = append(goodSamples, v)
|
||||
}
|
||||
}
|
||||
recentSampleCount := len(recentVals)
|
||||
recentBadCount := recentSampleCount - len(goodSamples)
|
||||
var goodFraction float64
|
||||
if recentSampleCount > 0 {
|
||||
goodFraction = float64(len(goodSamples)) / float64(recentSampleCount)
|
||||
}
|
||||
|
||||
var severity SkewSeverity
|
||||
if goodFraction < 0.10 {
|
||||
// Essentially no real clock — classify as no_clock regardless
|
||||
// of the raw skew magnitude.
|
||||
severity = SkewNoClock
|
||||
} else if goodFraction < 0.80 && recentBadCount > 0 {
|
||||
// Bimodal: use median of GOOD samples as the "real" skew.
|
||||
severity = SkewBimodalClock
|
||||
if len(goodSamples) > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
} else {
|
||||
// Normal path: if there are good samples, use their median
|
||||
// (filters out rare outliers in ≥80% good case).
|
||||
if len(goodSamples) > 0 && recentBadCount > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
severity = classifySkew(math.Abs(recentSkew))
|
||||
}
|
||||
|
||||
// For no_clock / bimodal_clock nodes, skip drift when data is unreliable.
|
||||
// For no_clock nodes (uninitialized RTC), skip drift — data is meaningless.
|
||||
var drift float64
|
||||
if severity != SkewNoClock && severity != SkewBimodalClock && len(tsSkews) >= minDriftSamples {
|
||||
if severity != SkewNoClock && len(tsSkews) >= minDriftSamples {
|
||||
drift = computeDrift(tsSkews)
|
||||
// Cap physically impossible drift rates.
|
||||
if math.Abs(drift) > maxReasonableDriftPerDay {
|
||||
@@ -603,94 +432,25 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
}
|
||||
}
|
||||
|
||||
// Build sparkline samples from tsSkews (already sorted by time above).
|
||||
// Build sparkline samples from tsSkews (sorted by time).
|
||||
sort.Slice(tsSkews, func(i, j int) bool { return tsSkews[i].ts < tsSkews[j].ts })
|
||||
samples := make([]SkewSample, len(tsSkews))
|
||||
for i, p := range tsSkews {
|
||||
samples[i] = SkewSample{Timestamp: p.ts, SkewSec: round(p.skew, 1)}
|
||||
}
|
||||
|
||||
// Build per-hash evidence (most recent 10 hashes with ≥1 observer).
|
||||
// Observer name lookup from store observations.
|
||||
obsNameMap := make(map[string]string)
|
||||
type hashMeta struct {
|
||||
hash string
|
||||
ts int64
|
||||
}
|
||||
var evidenceHashes []hashMeta
|
||||
for _, tx := range txs {
|
||||
if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
|
||||
continue
|
||||
}
|
||||
ev, ok := s.clockSkew.hashEvidence[tx.Hash]
|
||||
if !ok || len(ev) == 0 {
|
||||
continue
|
||||
}
|
||||
// Collect observer names from tx observations.
|
||||
for _, obs := range tx.Observations {
|
||||
if obs.ObserverID != "" && obs.ObserverName != "" {
|
||||
obsNameMap[obs.ObserverID] = obs.ObserverName
|
||||
}
|
||||
}
|
||||
evidenceHashes = append(evidenceHashes, hashMeta{hash: tx.Hash, ts: ev[0].observedTS})
|
||||
}
|
||||
// Sort by timestamp descending, take most recent 10.
|
||||
sort.Slice(evidenceHashes, func(i, j int) bool { return evidenceHashes[i].ts > evidenceHashes[j].ts })
|
||||
if len(evidenceHashes) > 10 {
|
||||
evidenceHashes = evidenceHashes[:10]
|
||||
}
|
||||
var recentEvidence []HashEvidence
|
||||
var calSummary CalibrationSummary
|
||||
for _, eh := range evidenceHashes {
|
||||
entries := s.clockSkew.hashEvidence[eh.hash]
|
||||
var observers []HashEvidenceObserver
|
||||
var corrSkews []float64
|
||||
for _, e := range entries {
|
||||
name := obsNameMap[e.observerID]
|
||||
if name == "" {
|
||||
name = e.observerID
|
||||
}
|
||||
observers = append(observers, HashEvidenceObserver{
|
||||
ObserverID: e.observerID,
|
||||
ObserverName: name,
|
||||
RawSkewSec: e.rawSkew,
|
||||
CorrectedSkewSec: e.corrected,
|
||||
ObserverOffsetSec: e.offset,
|
||||
Calibrated: e.calibrated,
|
||||
})
|
||||
corrSkews = append(corrSkews, e.corrected)
|
||||
calSummary.TotalSamples++
|
||||
if e.calibrated {
|
||||
calSummary.CalibratedSamples++
|
||||
} else {
|
||||
calSummary.UncalibratedSamples++
|
||||
}
|
||||
}
|
||||
recentEvidence = append(recentEvidence, HashEvidence{
|
||||
Hash: eh.hash,
|
||||
Observers: observers,
|
||||
MedianCorrectedSkewSec: round(median(corrSkews), 1),
|
||||
Timestamp: eh.ts,
|
||||
})
|
||||
}
|
||||
|
||||
return &NodeClockSkew{
|
||||
Pubkey: pubkey,
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(lastSkew, 1),
|
||||
RecentMedianSkewSec: round(recentSkew, 1),
|
||||
DriftPerDaySec: round(drift, 2),
|
||||
Severity: severity,
|
||||
SampleCount: totalSamples,
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: lastAdvTS,
|
||||
LastObservedTS: lastObsTS,
|
||||
Samples: samples,
|
||||
GoodFraction: round(goodFraction, 2),
|
||||
RecentBadSampleCount: recentBadCount,
|
||||
RecentSampleCount: recentSampleCount,
|
||||
RecentHashEvidence: recentEvidence,
|
||||
CalibrationSummary: &calSummary,
|
||||
Pubkey: pubkey,
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(lastSkew, 1),
|
||||
DriftPerDaySec: round(drift, 2),
|
||||
Severity: severity,
|
||||
SampleCount: totalSamples,
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: lastAdvTS,
|
||||
LastObservedTS: lastObsTS,
|
||||
Samples: samples,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,10 +478,8 @@ func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew {
|
||||
cs.NodeName = ni.Name
|
||||
cs.NodeRole = ni.Role
|
||||
}
|
||||
// Omit samples and evidence in fleet response (too much data).
|
||||
// Omit samples in fleet response (too much data).
|
||||
cs.Samples = nil
|
||||
cs.RecentHashEvidence = nil
|
||||
cs.CalibrationSummary = nil
|
||||
results = append(results, cs)
|
||||
}
|
||||
return results
|
||||
@@ -786,18 +544,7 @@ type tsSkewPair struct {
|
||||
}
|
||||
|
||||
// computeDrift estimates linear drift in seconds per day from time-ordered
|
||||
// (timestamp, skew) pairs. Issue #789: a single GPS-correction event (huge
|
||||
// skew jump in seconds) used to dominate ordinary least squares and produce
|
||||
// absurd drift like 1.7M sec/day. We now:
|
||||
//
|
||||
// 1. Drop pairs whose consecutive skew jump exceeds maxPlausibleSkewJumpSec
|
||||
// (clock corrections, not physical drift). This protects both OLS-style
|
||||
// consumers and Theil-Sen.
|
||||
// 2. Use Theil-Sen regression — the slope is the median of all pairwise
|
||||
// slopes, naturally robust to remaining outliers (breakdown point ~29%).
|
||||
//
|
||||
// For very small samples after filtering we fall back to a simple slope
|
||||
// between first and last calibrated samples.
|
||||
// (timestamp, skew) pairs using simple linear regression.
|
||||
func computeDrift(pairs []tsSkewPair) float64 {
|
||||
if len(pairs) < 2 {
|
||||
return 0
|
||||
@@ -813,55 +560,21 @@ func computeDrift(pairs []tsSkewPair) float64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Outlier filter: drop samples where the skew jumps more than
|
||||
// maxPlausibleSkewJumpSec from the running "stable" baseline.
|
||||
// We anchor on the first sample, then accept each subsequent point
|
||||
// that's within the threshold of the most recent accepted point —
|
||||
// this preserves a slow drift while rejecting correction events.
|
||||
filtered := make([]tsSkewPair, 0, len(pairs))
|
||||
filtered = append(filtered, pairs[0])
|
||||
for i := 1; i < len(pairs); i++ {
|
||||
prev := filtered[len(filtered)-1]
|
||||
if math.Abs(pairs[i].skew-prev.skew) <= maxPlausibleSkewJumpSec {
|
||||
filtered = append(filtered, pairs[i])
|
||||
}
|
||||
// Simple linear regression: skew = a + b*t
|
||||
n := float64(len(pairs))
|
||||
var sumX, sumY, sumXY, sumX2 float64
|
||||
for _, p := range pairs {
|
||||
x := float64(p.ts - pairs[0].ts) // normalize to avoid large numbers
|
||||
y := p.skew
|
||||
sumX += x
|
||||
sumY += y
|
||||
sumXY += x * y
|
||||
sumX2 += x * x
|
||||
}
|
||||
// If the filter killed too much (e.g. unstable node), fall back to the
|
||||
// raw series so we at least produce *something* — it'll be capped by
|
||||
// maxReasonableDriftPerDay downstream.
|
||||
if len(filtered) < 2 || float64(filtered[len(filtered)-1].ts-filtered[0].ts) < 3600 {
|
||||
filtered = pairs
|
||||
}
|
||||
|
||||
// Cap point count for Theil-Sen (O(n²) on pairs). Keep most-recent.
|
||||
if len(filtered) > theilSenMaxPoints {
|
||||
filtered = filtered[len(filtered)-theilSenMaxPoints:]
|
||||
}
|
||||
|
||||
return theilSenSlope(filtered) * 86400 // sec/sec → sec/day
|
||||
}
|
||||
|
||||
// theilSenSlope returns the Theil-Sen estimator: median of all pairwise
|
||||
// slopes (yj - yi) / (tj - ti) for i < j. Naturally robust to outliers.
|
||||
// Pairs must be sorted by timestamp ascending.
|
||||
func theilSenSlope(pairs []tsSkewPair) float64 {
|
||||
n := len(pairs)
|
||||
if n < 2 {
|
||||
denom := n*sumX2 - sumX*sumX
|
||||
if denom == 0 {
|
||||
return 0
|
||||
}
|
||||
// Pre-allocate: n*(n-1)/2 pairs.
|
||||
slopes := make([]float64, 0, n*(n-1)/2)
|
||||
for i := 0; i < n; i++ {
|
||||
for j := i + 1; j < n; j++ {
|
||||
dt := float64(pairs[j].ts - pairs[i].ts)
|
||||
if dt <= 0 {
|
||||
continue
|
||||
}
|
||||
slopes = append(slopes, (pairs[j].skew-pairs[i].skew)/dt)
|
||||
}
|
||||
}
|
||||
if len(slopes) == 0 {
|
||||
return 0
|
||||
}
|
||||
return median(slopes)
|
||||
slope := (n*sumXY - sumX*sumY) / denom // seconds of drift per second
|
||||
return slope * 86400 // convert to seconds per day
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ func TestComputeNodeSkew_BasicCorrection(t *testing.T) {
|
||||
// So the median approach finds obs2 is +5 ahead (relative to median)
|
||||
|
||||
// Now compute node skew with those offsets:
|
||||
nodeSkew, _ := computeNodeSkew(samples, offsets)
|
||||
nodeSkew := computeNodeSkew(samples, offsets)
|
||||
cs, ok := nodeSkew["h1"]
|
||||
if !ok {
|
||||
t.Fatal("expected skew data for hash h1")
|
||||
@@ -220,7 +220,7 @@ func TestComputeNodeSkew_ThreeObservers(t *testing.T) {
|
||||
t.Errorf("obs3 offset = %v, want 30", offsets["obs3"])
|
||||
}
|
||||
|
||||
nodeSkew, _ := computeNodeSkew(samples, offsets)
|
||||
nodeSkew := computeNodeSkew(samples, offsets)
|
||||
cs := nodeSkew["h1"]
|
||||
if cs == nil {
|
||||
t.Fatal("expected skew data for h1")
|
||||
@@ -544,514 +544,3 @@ func TestGetNodeClockSkew_NormalNodeWithDrift(t *testing.T) {
|
||||
func formatInt64(n int64) string {
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
|
||||
// ── #789: Recent-window severity & robust drift ───────────────────────────────
|
||||
|
||||
// TestSeverityUsesRecentNotMedian: 100 historical bad samples (skew=-60s,
|
||||
// each ~5min apart) followed by 5 fresh good samples (skew=-1s). All-time
|
||||
// median is still huge-ish but recent-window severity must reflect the
|
||||
// current healthy state.
|
||||
func TestSeverityUsesRecentNotMedian(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 105; i++ {
|
||||
obsTS := baseObs + int64(i)*300 // 5 min apart
|
||||
var skew int64 = -60
|
||||
if i >= 100 {
|
||||
skew = -1 // good samples at the tail
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("recent-h%03d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["RECENT"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("RECENT")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok (recent samples are healthy)", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec) > 5 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ~-1", r.RecentMedianSkewSec)
|
||||
}
|
||||
// Historical median should still be retained for context.
|
||||
if math.Abs(r.MedianSkewSec) < 30 {
|
||||
t.Errorf("medianSkewSec = %v, expected historical median to remain large", r.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriftRejectsCorrectionJump: 30 minutes of clean linear drift, then a
|
||||
// single 60-second skew jump. The pre-jump slope should win — drift must
|
||||
// not be catastrophically inflated by the correction event.
|
||||
func TestDriftRejectsCorrectionJump(t *testing.T) {
|
||||
pairs := []tsSkewPair{}
|
||||
// 30 min of stable, ~12 sec/day drift: 1s per 7200s.
|
||||
for i := 0; i < 12; i++ {
|
||||
ts := int64(i) * 300
|
||||
skew := float64(i) * (1.0 / 24.0) // ~0.04s per 5min step → 12 s/day
|
||||
pairs = append(pairs, tsSkewPair{ts: ts, skew: skew})
|
||||
}
|
||||
// Wait an hour, then a single 1000-sec correction jump (clearly outlier).
|
||||
pairs = append(pairs, tsSkewPair{ts: 3600 + 12*300, skew: 1000})
|
||||
|
||||
drift := computeDrift(pairs)
|
||||
// Without rejection this would be ~ (1000-0)/(end-0) * 86400 = enormous.
|
||||
if math.Abs(drift) > 100 {
|
||||
t.Errorf("drift = %v, expected small (~12 s/day), correction jump should be filtered", drift)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheilSenMatchesOLSWhenClean: on clean linear data Theil-Sen should
|
||||
// produce essentially the OLS answer.
|
||||
func TestTheilSenMatchesOLSWhenClean(t *testing.T) {
|
||||
// 1 sec drift per hour = 24 sec/day, 20 evenly-spaced samples.
|
||||
pairs := []tsSkewPair{}
|
||||
for i := 0; i < 20; i++ {
|
||||
pairs = append(pairs, tsSkewPair{
|
||||
ts: int64(i) * 600,
|
||||
skew: float64(i) * (600.0 / 3600.0),
|
||||
})
|
||||
}
|
||||
drift := computeDrift(pairs)
|
||||
if math.Abs(drift-24.0) > 0.25 { // ~1%
|
||||
t.Errorf("drift = %v, want ~24", drift)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReporterScenario_789: reproduce the exact scenario from issue #789.
|
||||
// Reporter saw mean=-52565156, median=-59063561, last=-0.8, sample count
|
||||
// 1662, drift +1793549.9 s/day, severity=absurd. After the fix, severity
|
||||
// must be ok (recent samples are healthy) and drift must be sane.
|
||||
func TestReporterScenario_789(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 1657 samples with the bad ~-683-day skew (the historical poison),
|
||||
// then 5 freshly corrected samples at -0.8s — totals 1662.
|
||||
for i := 0; i < 1662; i++ {
|
||||
obsTS := baseObs + int64(i)*60 // 1 min apart
|
||||
var skew int64
|
||||
if i < 1657 {
|
||||
skew = -59063561 // ~ -683 days
|
||||
} else {
|
||||
skew = -1 // corrected (rounded; reporter saw -0.8)
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("rep-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["REPNODE"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("REPNODE")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
// Severity must reflect current health, not the all-time median.
|
||||
if r.Severity != SkewOK && r.Severity != SkewWarning {
|
||||
t.Errorf("severity = %v, want ok/warning (recent samples are healthy)", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec) > 5 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want near 0", r.RecentMedianSkewSec)
|
||||
}
|
||||
// Drift must not be absurd. The historical jump is one event between
|
||||
// the 1657th and 1658th sample; outlier rejection must contain it.
|
||||
if math.Abs(r.DriftPerDaySec) > maxReasonableDriftPerDay {
|
||||
t.Errorf("drift = %v, must be <= cap %v", r.DriftPerDaySec, maxReasonableDriftPerDay)
|
||||
}
|
||||
// And it should be close to zero (stable historical + stable corrected).
|
||||
if math.Abs(r.DriftPerDaySec) > 1000 {
|
||||
t.Errorf("drift = %v, expected near zero after outlier rejection", r.DriftPerDaySec)
|
||||
}
|
||||
// Historical median is preserved as context.
|
||||
if math.Abs(r.MedianSkewSec) < 1e6 {
|
||||
t.Errorf("medianSkewSec = %v, expected historical poison preserved as context", r.MedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBimodalClock_845: 60% good samples → bimodal_clock severity.
|
||||
func TestBimodalClock_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 6 good samples (-5s each), 4 bad samples (-50000000s each) = 60% good
|
||||
// Interleave so the recent window (last 5) captures both good and bad.
|
||||
skews := []int64{-5, -5, -50000000, -5, -50000000, -5, -50000000, -5, -50000000, -5}
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS + skews[i]
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("bimodal-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["BIMODAL"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("BIMODAL")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewBimodalClock {
|
||||
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ≈ -5 (median of good samples)", r.RecentMedianSkewSec)
|
||||
}
|
||||
if r.GoodFraction < 0.5 || r.GoodFraction > 0.7 {
|
||||
t.Errorf("goodFraction = %v, want ~0.6", r.GoodFraction)
|
||||
}
|
||||
if r.RecentBadSampleCount < 1 {
|
||||
t.Errorf("recentBadSampleCount = %v, want > 0", r.RecentBadSampleCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllBad_NoClock_845: all samples bad → no_clock.
|
||||
func TestAllBad_NoClock_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS - 50000000
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("allbad-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["ALLBAD"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("ALLBAD")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewNoClock {
|
||||
t.Errorf("severity = %v, want no_clock", r.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMostlyGood_OK_845: 90% good 10% bad → ok (outlier filtered).
|
||||
func TestMostlyGood_OK_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
// 9 good at -5s, 1 bad at -50000000s
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
var skew int64
|
||||
if i < 9 {
|
||||
skew = -5
|
||||
} else {
|
||||
skew = -50000000
|
||||
}
|
||||
advTS := obsTS + skew
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("mostly-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["MOSTLY"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("MOSTLY")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
// 90% good → normal classification path, median of good samples = -5s → ok
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
|
||||
t.Errorf("recentMedianSkewSec = %v, want ≈ -5", r.RecentMedianSkewSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleSample_845: one good sample → ok.
|
||||
func TestSingleSample_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
obsTS := int64(1700000000)
|
||||
advTS := obsTS - 30 // 30s skew
|
||||
tx := &StoreTx{
|
||||
Hash: "single-0001",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["SINGLE"] = []*StoreTx{tx}
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("SINGLE")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if r.RecentSampleCount != 1 {
|
||||
t.Errorf("recentSampleCount = %d, want 1", r.RecentSampleCount)
|
||||
}
|
||||
if r.GoodFraction != 1.0 {
|
||||
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFiftyFifty_Bimodal_845: 50% good / 50% bad → bimodal_clock.
|
||||
func TestFiftyFifty_Bimodal_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
var skew int64
|
||||
if i%2 == 0 {
|
||||
skew = -10
|
||||
} else {
|
||||
skew = -50000000
|
||||
}
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("fifty-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS+skew) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["FIFTY"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("FIFTY")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewBimodalClock {
|
||||
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
|
||||
}
|
||||
if r.GoodFraction < 0.4 || r.GoodFraction > 0.6 {
|
||||
t.Errorf("goodFraction = %v, want ~0.5", r.GoodFraction)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllGood_OK_845: all samples good → ok, no bimodal.
|
||||
func TestAllGood_OK_845(t *testing.T) {
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4
|
||||
baseObs := int64(1700000000)
|
||||
var txs []*StoreTx
|
||||
for i := 0; i < 10; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
tx := &StoreTx{
|
||||
Hash: fmt.Sprintf("allgood-%04d", i),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS-3) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
ps.mu.Lock()
|
||||
ps.byNode["ALLGOOD"] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("ALLGOOD")
|
||||
if r == nil {
|
||||
t.Fatal("nil result")
|
||||
}
|
||||
if r.Severity != SkewOK {
|
||||
t.Errorf("severity = %v, want ok", r.Severity)
|
||||
}
|
||||
if r.GoodFraction != 1.0 {
|
||||
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
|
||||
}
|
||||
if r.RecentBadSampleCount != 0 {
|
||||
t.Errorf("recentBadSampleCount = %v, want 0", r.RecentBadSampleCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeClockSkew_EvidencePayload(t *testing.T) {
|
||||
// 3-observer scenario: obs1 ahead by +2s, obs2 on time, obs3 behind by -1s.
|
||||
// Node clock is 60s ahead. Raw skew = advertTS - obsTS.
|
||||
// Hash has 3 observations, each observer sees same advert.
|
||||
ps := NewPacketStore(nil, nil)
|
||||
|
||||
pt := 4 // ADVERT
|
||||
// Advert timestamp: 1700000060 (node 60s ahead of true time 1700000000)
|
||||
// obs1 sees at 1700000002 (2s ahead of true time) → raw = 60 - 2 = 58
|
||||
// obs2 sees at 1700000000 (on time) → raw = 60 - 0 = 60
|
||||
// obs3 sees at 1699999999 (-1s, behind) → raw = 60 + 1 = 61
|
||||
// Median obsTS = 1700000000, so:
|
||||
// obs1 offset = 1700000002 - 1700000000 = +2
|
||||
// obs2 offset = 0
|
||||
// obs3 offset = 1699999999 - 1700000000 = -1
|
||||
// Corrected: raw + offset → obs1: 58+2=60, obs2: 60+0=60, obs3: 61+(-1)=60
|
||||
|
||||
tx1 := &StoreTx{
|
||||
Hash: "evidence_hash_1",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":1700000060}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", ObserverName: "Observer Alpha", Timestamp: "2023-11-14T22:13:22Z"},
|
||||
{ObserverID: "obs2", ObserverName: "Observer Beta", Timestamp: "2023-11-14T22:13:20Z"},
|
||||
{ObserverID: "obs3", ObserverName: "Observer Gamma", Timestamp: "2023-11-14T22:13:19Z"},
|
||||
},
|
||||
}
|
||||
// Second hash to ensure we get multiple evidence entries.
|
||||
tx2 := &StoreTx{
|
||||
Hash: "evidence_hash_2",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":1700003660}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", ObserverName: "Observer Alpha", Timestamp: "2023-11-14T23:13:22Z"},
|
||||
{ObserverID: "obs2", ObserverName: "Observer Beta", Timestamp: "2023-11-14T23:13:20Z"},
|
||||
{ObserverID: "obs3", ObserverName: "Observer Gamma", Timestamp: "2023-11-14T23:13:19Z"},
|
||||
},
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode["NODETEST"] = []*StoreTx{tx1, tx2}
|
||||
ps.byPayloadType[4] = []*StoreTx{tx1, tx2}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
|
||||
r := ps.GetNodeClockSkew("NODETEST")
|
||||
if r == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
|
||||
// Check recentHashEvidence exists.
|
||||
if len(r.RecentHashEvidence) == 0 {
|
||||
t.Fatal("expected recentHashEvidence to be populated")
|
||||
}
|
||||
if len(r.RecentHashEvidence) != 2 {
|
||||
t.Errorf("recentHashEvidence length = %d, want 2", len(r.RecentHashEvidence))
|
||||
}
|
||||
|
||||
// Check first evidence entry has 3 observers.
|
||||
ev := r.RecentHashEvidence[0]
|
||||
if len(ev.Observers) != 3 {
|
||||
t.Fatalf("evidence observers = %d, want 3", len(ev.Observers))
|
||||
}
|
||||
|
||||
// Verify corrected = raw + offset for each observer.
|
||||
for _, o := range ev.Observers {
|
||||
expected := o.RawSkewSec + o.ObserverOffsetSec
|
||||
if math.Abs(o.CorrectedSkewSec-expected) > 0.2 {
|
||||
t.Errorf("observer %s: corrected=%.1f, expected raw(%.1f)+offset(%.1f)=%.1f",
|
||||
o.ObserverID, o.CorrectedSkewSec, o.RawSkewSec, o.ObserverOffsetSec, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// All corrected values should be ~60s (node is 60s ahead).
|
||||
if math.Abs(ev.MedianCorrectedSkewSec-60) > 1 {
|
||||
t.Errorf("median corrected = %.1f, want ~60", ev.MedianCorrectedSkewSec)
|
||||
}
|
||||
|
||||
// Check calibration summary.
|
||||
if r.CalibrationSummary == nil {
|
||||
t.Fatal("expected calibrationSummary")
|
||||
}
|
||||
if r.CalibrationSummary.TotalSamples != 6 { // 3 observers × 2 hashes
|
||||
t.Errorf("calibration total = %d, want 6", r.CalibrationSummary.TotalSamples)
|
||||
}
|
||||
if r.CalibrationSummary.CalibratedSamples != 6 {
|
||||
t.Errorf("calibrated = %d, want 6 (all multi-observer)", r.CalibrationSummary.CalibratedSamples)
|
||||
}
|
||||
|
||||
// Check observer names are populated.
|
||||
nameFound := false
|
||||
for _, o := range ev.Observers {
|
||||
if o.ObserverName == "Observer Alpha" || o.ObserverName == "Observer Beta" {
|
||||
nameFound = true
|
||||
}
|
||||
}
|
||||
if !nameFound {
|
||||
t.Error("expected observer names to be populated from tx observations")
|
||||
}
|
||||
}
|
||||
|
||||
+1
-56
@@ -8,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
)
|
||||
|
||||
@@ -63,30 +62,14 @@ type Config struct {
|
||||
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
|
||||
PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
|
||||
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
|
||||
Timestamps *TimestampConfig `json:"timestamps,omitempty"`
|
||||
|
||||
// CORSAllowedOrigins is the list of origins permitted to make cross-origin
|
||||
// requests. When empty (default), no Access-Control-* headers are sent,
|
||||
// so browsers enforce same-origin policy. Set to ["*"] to allow all origins.
|
||||
CORSAllowedOrigins []string `json:"corsAllowedOrigins,omitempty"`
|
||||
|
||||
DebugAffinity bool `json:"debugAffinity,omitempty"`
|
||||
|
||||
// ObserverBlacklist is a list of observer public keys to exclude from API
|
||||
// responses (defense in depth — ingestor drops at ingest, server filters
|
||||
// any that slipped through from a prior unblocked window).
|
||||
ObserverBlacklist []string `json:"observerBlacklist,omitempty"`
|
||||
|
||||
// obsBlacklistSetCached is the lazily-built set version of ObserverBlacklist.
|
||||
obsBlacklistSetCached map[string]bool
|
||||
obsBlacklistOnce sync.Once
|
||||
|
||||
ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"`
|
||||
NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"`
|
||||
}
|
||||
@@ -132,8 +115,7 @@ type NeighborGraphConfig struct {
|
||||
// PacketStoreConfig controls in-memory packet store limits.
|
||||
type PacketStoreConfig struct {
|
||||
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
MaxResolvedPubkeyIndexEntries int `json:"maxResolvedPubkeyIndexEntries"` // warning threshold for index size (0 = 5M default)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
@@ -146,17 +128,6 @@ type RetentionConfig struct {
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
type DBConfig = dbconfig.DBConfig
|
||||
|
||||
// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
|
||||
func (c *Config) IncrementalVacuumPages() int {
|
||||
if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
|
||||
return c.DB.IncrementalVacuumPages
|
||||
}
|
||||
return 1024
|
||||
}
|
||||
|
||||
// MetricsRetentionDays returns configured metrics retention or 30 days default.
|
||||
func (c *Config) MetricsRetentionDays() int {
|
||||
if c.Retention != nil && c.Retention.MetricsDays > 0 {
|
||||
@@ -416,29 +387,3 @@ func (c *Config) IsBlacklisted(pubkey string) bool {
|
||||
}
|
||||
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
|
||||
}
|
||||
|
||||
// obsBlacklistSet lazily builds and caches the observerBlacklist as a set for O(1) lookups.
|
||||
func (c *Config) obsBlacklistSet() map[string]bool {
|
||||
c.obsBlacklistOnce.Do(func() {
|
||||
if len(c.ObserverBlacklist) == 0 {
|
||||
return
|
||||
}
|
||||
m := make(map[string]bool, len(c.ObserverBlacklist))
|
||||
for _, pk := range c.ObserverBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsBlacklistSetCached = m
|
||||
})
|
||||
return c.obsBlacklistSetCached
|
||||
}
|
||||
|
||||
// IsObserverBlacklisted returns true if the given observer ID is in the observerBlacklist.
|
||||
func (c *Config) IsObserverBlacklisted(id string) bool {
|
||||
if c == nil || len(c.ObserverBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
return c.obsBlacklistSet()[strings.ToLower(strings.TrimSpace(id))]
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import "net/http"
|
||||
|
||||
// corsMiddleware returns a middleware that sets CORS headers based on the
|
||||
// configured allowed origins. When CORSAllowedOrigins is empty (default),
|
||||
// no Access-Control-* headers are added, preserving browser same-origin policy.
|
||||
func (s *Server) corsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
origins := s.cfg.CORSAllowedOrigins
|
||||
if len(origins) == 0 {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
reqOrigin := r.Header.Get("Origin")
|
||||
if reqOrigin == "" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if origin is allowed
|
||||
allowed := false
|
||||
wildcard := false
|
||||
for _, o := range origins {
|
||||
if o == "*" {
|
||||
allowed = true
|
||||
wildcard = true
|
||||
break
|
||||
}
|
||||
if o == reqOrigin {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !allowed {
|
||||
// Origin not in allowlist — don't add CORS headers
|
||||
if r.Method == http.MethodOptions {
|
||||
// Still reject preflight with 403
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Set CORS headers
|
||||
if wildcard {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
} else {
|
||||
w.Header().Set("Access-Control-Allow-Origin", reqOrigin)
|
||||
w.Header().Set("Vary", "Origin")
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key")
|
||||
|
||||
// Handle preflight
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newTestServerWithCORS creates a minimal Server with the given CORS config.
|
||||
func newTestServerWithCORS(origins []string) *Server {
|
||||
cfg := &Config{CORSAllowedOrigins: origins}
|
||||
srv := &Server{cfg: cfg}
|
||||
return srv
|
||||
}
|
||||
|
||||
// dummyHandler is a simple handler that writes 200 OK.
|
||||
var dummyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
func TestCORS_DefaultNoHeaders(t *testing.T) {
|
||||
srv := newTestServerWithCORS(nil)
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO header, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_AllowlistMatch(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://good.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "https://good.example" {
|
||||
t.Fatalf("expected origin echo, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Methods"); v != "GET, POST, OPTIONS" {
|
||||
t.Fatalf("expected methods header, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Headers"); v != "Content-Type, X-API-Key" {
|
||||
t.Fatalf("expected headers header, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Vary"); v != "Origin" {
|
||||
t.Fatalf("expected Vary: Origin, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_AllowlistNoMatch(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO header for non-matching origin, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_PreflightAllowed(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("OPTIONS", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://good.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("expected 204, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "https://good.example" {
|
||||
t.Fatalf("expected origin echo, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_PreflightRejected(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("OPTIONS", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected 403, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_Wildcard(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"*"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://anything.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "*" {
|
||||
t.Fatalf("expected *, got %q", v)
|
||||
}
|
||||
// Wildcard should NOT set Vary: Origin
|
||||
if v := rr.Header().Get("Vary"); v == "Origin" {
|
||||
t.Fatalf("wildcard should not set Vary: Origin")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_NoOriginHeader(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
// No Origin header
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO without Origin header, got %q", v)
|
||||
}
|
||||
}
|
||||
+78
-54
@@ -35,8 +35,7 @@ func setupTestDBv2(t *testing.T) *DB {
|
||||
CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT,
|
||||
packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT,
|
||||
client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0
|
||||
client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL
|
||||
);
|
||||
CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL,
|
||||
@@ -48,7 +47,7 @@ func setupTestDBv2(t *testing.T) *DB {
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_id TEXT, observer_name TEXT, direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT
|
||||
snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL
|
||||
);
|
||||
`
|
||||
if _, err := conn.Exec(schema); err != nil {
|
||||
@@ -586,15 +585,12 @@ func TestHandlePacketsMultiNodeWithStore(t *testing.T) {
|
||||
func TestHandlePacketDetailNoStore(t *testing.T) {
|
||||
_, router := setupNoStoreServer(t)
|
||||
|
||||
// With no in-memory store, handlePacketDetail now falls back to the DB
|
||||
// (#827). The seeded transmissions are present in the DB, so by-hash and
|
||||
// by-ID lookups succeed; only truly absent IDs return 404.
|
||||
t.Run("by hash", func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/api/packets/abc123def4567890", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -602,8 +598,8 @@ func TestHandlePacketDetailNoStore(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/api/packets/1", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
|
||||
if w.Code != 404 {
|
||||
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -764,9 +760,9 @@ func TestGetChannelsFromStore(t *testing.T) {
|
||||
|
||||
func TestPrefixMapResolve(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0},
|
||||
{Role: "repeater", PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false},
|
||||
{Role: "repeater", PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0},
|
||||
{PublicKey: "aabbccdd11223344", Name: "NodeA", HasGPS: true, Lat: 37.5, Lon: -122.0},
|
||||
{PublicKey: "aabbccdd55667788", Name: "NodeB", HasGPS: false},
|
||||
{PublicKey: "eeff0011aabbccdd", Name: "NodeC", HasGPS: true, Lat: 38.0, Lon: -121.0},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
@@ -806,8 +802,8 @@ func TestPrefixMapResolve(t *testing.T) {
|
||||
|
||||
t.Run("multiple candidates no GPS", func(t *testing.T) {
|
||||
noGPSNodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aa11bb22", Name: "X", HasGPS: false},
|
||||
{Role: "repeater", PublicKey: "aa11cc33", Name: "Y", HasGPS: false},
|
||||
{PublicKey: "aa11bb22", Name: "X", HasGPS: false},
|
||||
{PublicKey: "aa11cc33", Name: "Y", HasGPS: false},
|
||||
}
|
||||
pm2 := buildPrefixMap(noGPSNodes)
|
||||
n := pm2.resolve("aa11")
|
||||
@@ -821,8 +817,8 @@ func TestPrefixMapResolve(t *testing.T) {
|
||||
func TestPrefixMapCap(t *testing.T) {
|
||||
// 16-char pubkey — longer than maxPrefixLen
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aabbccdd11223344", Name: "LongKey"},
|
||||
{Role: "repeater", PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars
|
||||
{PublicKey: "aabbccdd11223344", Name: "LongKey"},
|
||||
{PublicKey: "eeff0011", Name: "ShortKey"}, // exactly 8 chars
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
@@ -2326,24 +2322,12 @@ func TestSubpathTxIndexPopulated(t *testing.T) {
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
// spTxIndex must be populated alongside spIndex
|
||||
if len(store.spTxIndex) == 0 {
|
||||
t.Fatal("expected spTxIndex to be populated after Load()")
|
||||
// spIndex must be populated after Load()
|
||||
if len(store.spIndex) == 0 {
|
||||
t.Fatal("expected spIndex to be populated after Load()")
|
||||
}
|
||||
|
||||
// Every key in spIndex must also exist in spTxIndex with matching count
|
||||
for key, count := range store.spIndex {
|
||||
txs, ok := store.spTxIndex[key]
|
||||
if !ok {
|
||||
t.Errorf("spTxIndex missing key %q that exists in spIndex", key)
|
||||
continue
|
||||
}
|
||||
if len(txs) != count {
|
||||
t.Errorf("spTxIndex[%q] has %d txs, spIndex count is %d", key, len(txs), count)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSubpathDetail should return correct match count via indexed lookup
|
||||
// GetSubpathDetail should return correct match count via scan fallback
|
||||
detail := store.GetSubpathDetail([]string{"eeff", "0011"})
|
||||
if detail == nil {
|
||||
t.Fatal("expected non-nil detail for existing subpath")
|
||||
@@ -2498,9 +2482,9 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) {
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch)
|
||||
|
||||
// Also a decrypted CHAN with numeric channelHash — use hash 198 which is the real hash for #general
|
||||
// Also a decrypted CHAN with numeric channelHash
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('DD03', 'chan_num_hash_3', ?, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent)
|
||||
VALUES ('DD03', 'chan_num_hash_3', ?, 1, 5, '{"type":"CHAN","channel":"general","channelHash":97,"channelHashHex":"61","text":"hello","sender":"Alice"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (6, 1, 12.0, -88, '[]', ?)`, recentEpoch)
|
||||
|
||||
@@ -2509,8 +2493,8 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) {
|
||||
result := store.GetAnalyticsChannels("")
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) < 3 {
|
||||
t.Errorf("expected at least 3 channels (hash 97 + hash 42 + hash 198), got %d", len(channels))
|
||||
if len(channels) < 2 {
|
||||
t.Errorf("expected at least 2 channels (hash 97 + hash 42), got %d", len(channels))
|
||||
}
|
||||
|
||||
// Verify the numeric-hash channels we inserted have proper hashes (not "?")
|
||||
@@ -2531,13 +2515,13 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) {
|
||||
t.Error("expected to find channel with hash '42' (numeric channelHash parsing)")
|
||||
}
|
||||
|
||||
// Verify the decrypted CHAN channel has the correct name (now at hash 198)
|
||||
// Verify the decrypted CHAN channel has the correct name
|
||||
foundGeneral := false
|
||||
for _, ch := range channels {
|
||||
if ch["name"] == "general" {
|
||||
foundGeneral = true
|
||||
if ch["hash"] != "198" {
|
||||
t.Errorf("expected hash '198' for general channel, got %v", ch["hash"])
|
||||
if ch["hash"] != "97" {
|
||||
t.Errorf("expected hash '97' for general channel, got %v", ch["hash"])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4320,48 +4304,88 @@ func TestIndexByNodePreCheck(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestIndexByNodeResolvedPath tests that indexByNode only indexes decoded JSON pubkeys.
|
||||
// After #800, resolved_path entries are handled via the decode-window, not indexByNode.
|
||||
// TestIndexByNodeResolvedPath tests that resolved_path entries are indexed in byNode.
|
||||
func TestIndexByNodeResolvedPath(t *testing.T) {
|
||||
store := &PacketStore{
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
}
|
||||
|
||||
t.Run("decoded JSON pubkeys still indexed", func(t *testing.T) {
|
||||
pk := "aabb1122334455ff"
|
||||
t.Run("indexes resolved path pubkeys from observations", func(t *testing.T) {
|
||||
relayPK := "aabb1122334455ff"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp1",
|
||||
DecodedJSON: `{"pubKey":"` + pk + `"}`,
|
||||
DecodedJSON: `{"type":"CHAN","text":"hello"}`, // no pubKey fields
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&relayPK}},
|
||||
},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[relayPK]) != 1 {
|
||||
t.Errorf("expected relay pubkey indexed, got %d", len(store.byNode[relayPK]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("skips null entries in resolved path", func(t *testing.T) {
|
||||
pk := "cc11dd22ee33ff44"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp2",
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{nil, &pk, nil}},
|
||||
},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[pk]) != 1 {
|
||||
t.Errorf("expected decoded pubkey indexed, got %d", len(store.byNode[pk]))
|
||||
t.Errorf("expected resolved pubkey indexed, got %d", len(store.byNode[pk]))
|
||||
}
|
||||
// Verify nil entries didn't create empty-string keys
|
||||
if _, exists := store.byNode[""]; exists {
|
||||
t.Error("nil/empty resolved path entries should not create byNode entries")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("resolved path pubkeys NOT indexed by indexByNode", func(t *testing.T) {
|
||||
// After #800, indexByNode only handles decoded JSON fields.
|
||||
// Resolved path pubkeys are handled by the decode-window.
|
||||
t.Run("relay-only node appears in byNode", func(t *testing.T) {
|
||||
// A packet with no decoded pubkey fields, only a relay in resolved path
|
||||
relayOnly := "relay0only0pubkey"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp2",
|
||||
DecodedJSON: `{"type":"CHAN","text":"hello"}`, // no pubKey fields
|
||||
Hash: "rp3",
|
||||
// No DecodedJSON at all — pure relay
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&relayOnly}},
|
||||
},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
// No new entries expected since there are no decoded pubkeys
|
||||
if len(store.byNode[relayOnly]) != 1 {
|
||||
t.Errorf("expected relay-only node indexed, got %d", len(store.byNode[relayOnly]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dedup within decoded JSON", func(t *testing.T) {
|
||||
t.Run("dedup between decoded JSON and resolved path", func(t *testing.T) {
|
||||
pk := "dedup0test0pk1234"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp4",
|
||||
DecodedJSON: `{"pubKey":"` + pk + `","destPubKey":"` + pk + `"}`,
|
||||
DecodedJSON: `{"pubKey":"` + pk + `"}`,
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&pk}},
|
||||
},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[pk]) != 1 {
|
||||
t.Errorf("expected dedup to keep 1 entry, got %d", len(store.byNode[pk]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("indexes tx.ResolvedPath when observations empty", func(t *testing.T) {
|
||||
rpPK := "txlevel0resolved1"
|
||||
tx := &StoreTx{
|
||||
Hash: "rp5",
|
||||
ResolvedPath: []*string{&rpPK},
|
||||
}
|
||||
store.indexByNode(tx)
|
||||
if len(store.byNode[rpPK]) != 1 {
|
||||
t.Errorf("expected tx-level resolved path indexed, got %d", len(store.byNode[rpPK]))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkIndexByNode measures indexByNode performance with and without pubkey
|
||||
|
||||
+14
-100
@@ -20,7 +20,6 @@ type DB struct {
|
||||
path string // filesystem path to the database file
|
||||
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
|
||||
hasResolvedPath bool // observations table has resolved_path column
|
||||
hasObsRawHex bool // observations table has raw_hex column (#881)
|
||||
|
||||
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
|
||||
channelsCacheMu sync.Mutex
|
||||
@@ -77,9 +76,6 @@ func (db *DB) detectSchema() {
|
||||
if colName == "resolved_path" {
|
||||
db.hasResolvedPath = true
|
||||
}
|
||||
if colName == "raw_hex" {
|
||||
db.hasObsRawHex = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -170,7 +166,6 @@ type Observer struct {
|
||||
BatteryMv *int `json:"battery_mv"`
|
||||
UptimeSecs *int64 `json:"uptime_secs"`
|
||||
NoiseFloor *float64 `json:"noise_floor"`
|
||||
LastPacketAt *string `json:"last_packet_at"`
|
||||
}
|
||||
|
||||
// Transmission represents a row from the transmissions table.
|
||||
@@ -232,7 +227,7 @@ func (db *DB) GetStats() (*Stats, error) {
|
||||
sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339)
|
||||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE last_seen > ?", sevenDaysAgo).Scan(&s.TotalNodes)
|
||||
db.conn.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&s.TotalNodesAllTime)
|
||||
db.conn.QueryRow("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers)
|
||||
db.conn.QueryRow("SELECT COUNT(*) FROM observers").Scan(&s.TotalObservers)
|
||||
|
||||
oneHourAgo := time.Now().Add(-1 * time.Hour).Unix()
|
||||
db.conn.QueryRow("SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneHourAgo).Scan(&s.PacketsLastHour)
|
||||
@@ -389,7 +384,6 @@ type PacketQuery struct {
|
||||
Until string
|
||||
Region string
|
||||
Node string
|
||||
Channel string // channel_hash filter (#812). Plain names like "#test"/"public" or "enc_<HEX>" for encrypted
|
||||
Order string // ASC or DESC
|
||||
ExpandObservations bool // when true, include observation sub-maps in txToMap output
|
||||
}
|
||||
@@ -626,11 +620,6 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
|
||||
where = append(where, "t.decoded_json LIKE ?")
|
||||
args = append(args, "%"+pk+"%")
|
||||
}
|
||||
if q.Channel != "" {
|
||||
// channel_hash column is indexed for payload_type = 5; filter is exact match.
|
||||
where = append(where, "t.channel_hash = ?")
|
||||
args = append(args, q.Channel)
|
||||
}
|
||||
if q.Observer != "" {
|
||||
ids := strings.Split(q.Observer, ",")
|
||||
placeholders := strings.Repeat("?,", len(ids))
|
||||
@@ -697,20 +686,6 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// GetObservationsForHash returns all observations for the transmission with
|
||||
// the given content hash. Used as a fallback by the packet-detail handler
|
||||
// when the in-memory PacketStore has pruned the entry but the DB still has it.
|
||||
func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} {
|
||||
var txID int
|
||||
err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?",
|
||||
strings.ToLower(hash)).Scan(&txID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
obsByTx := db.getObservationsForTransmissions([]int{txID})
|
||||
return obsByTx[txID]
|
||||
}
|
||||
|
||||
|
||||
// GetNodes returns filtered, paginated node list.
|
||||
func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortBy, region string) ([]map[string]interface{}, int, map[string]int, error) {
|
||||
@@ -831,55 +806,6 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// GetNodeByPrefix resolves a hex prefix (>=8 chars) to a unique node.
|
||||
// Returns (node, ambiguous, error). When multiple nodes share the prefix,
|
||||
// returns (nil, true, nil). Used by the short-URL feature (issue #772).
|
||||
//
|
||||
// Trade-off vs an opaque ID lookup table: prefixes are stable across
|
||||
// restarts, self-describing (no allocator needed), and resolve to the
|
||||
// authoritative pubkey on the server. Cost: ambiguity grows with the
|
||||
// node directory; we mitigate with a hard 8-hex-char (32-bit) minimum
|
||||
// and surface 409 Conflict when collisions occur.
|
||||
func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, error) {
|
||||
if len(prefix) < 8 {
|
||||
return nil, false, nil
|
||||
}
|
||||
// Validate hex (avoid SQL LIKE wildcards leaking through).
|
||||
for _, c := range prefix {
|
||||
isHex := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
|
||||
if !isHex {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
rows, err := db.conn.Query(
|
||||
`SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c
|
||||
FROM nodes WHERE public_key LIKE ? LIMIT 2`,
|
||||
prefix+"%",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var first map[string]interface{}
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
n := scanNodeRow(rows)
|
||||
if n == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
if count == 1 {
|
||||
first = n
|
||||
} else {
|
||||
return nil, true, nil
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
return first, false, nil
|
||||
}
|
||||
|
||||
// GetNodeByPubkey returns a single node.
|
||||
func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) {
|
||||
rows, err := db.conn.Query("SELECT public_key, name, role, lat, lon, last_seen, first_seen, advert_count, battery_mv, temperature_c FROM nodes WHERE public_key = ?", pubkey)
|
||||
@@ -1020,9 +946,9 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string]
|
||||
return result
|
||||
}
|
||||
|
||||
// GetObservers returns active observers (not soft-deleted) sorted by last_seen DESC.
|
||||
// GetObservers returns all observers sorted by last_seen DESC.
|
||||
func (db *DB) GetObservers() ([]Observer, error) {
|
||||
rows, err := db.conn.Query("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC")
|
||||
rows, err := db.conn.Query("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor FROM observers ORDER BY last_seen DESC")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1033,7 +959,7 @@ func (db *DB) GetObservers() ([]Observer, error) {
|
||||
var o Observer
|
||||
var batteryMv, uptimeSecs sql.NullInt64
|
||||
var noiseFloor sql.NullFloat64
|
||||
if err := rows.Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt); err != nil {
|
||||
if err := rows.Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor); err != nil {
|
||||
continue
|
||||
}
|
||||
if batteryMv.Valid {
|
||||
@@ -1056,8 +982,8 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) {
|
||||
var o Observer
|
||||
var batteryMv, uptimeSecs sql.NullInt64
|
||||
var noiseFloor sql.NullFloat64
|
||||
err := db.conn.QueryRow("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at FROM observers WHERE id = ?", id).
|
||||
Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt)
|
||||
err := db.conn.QueryRow("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor FROM observers WHERE id = ?", id).
|
||||
Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1105,17 +1031,6 @@ func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) {
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// normalizeRegionCodes parses a region query parameter into a list of upper-case
|
||||
// IATA codes. Returns nil to signal "no filter" (match all regions).
|
||||
//
|
||||
// Sentinel handling (issue #770): the frontend region filter dropdown labels its
|
||||
// catch-all option "All". When that option is selected the UI may send
|
||||
// ?region=All; older code interpreted that literally and tried to match an
|
||||
// IATA code "ALL", which never exists, returning an empty result set. Treat
|
||||
// "All" / "ALL" / "all" (case-insensitive, optionally surrounded by whitespace
|
||||
// or mixed with empty CSV slots) as equivalent to an empty value.
|
||||
//
|
||||
// Real IATA codes (e.g. "SJC", "PDX") still pass through unchanged.
|
||||
func normalizeRegionCodes(regionParam string) []string {
|
||||
if regionParam == "" {
|
||||
return nil
|
||||
@@ -1124,13 +1039,9 @@ func normalizeRegionCodes(regionParam string) []string {
|
||||
codes := make([]string, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
code := strings.TrimSpace(strings.ToUpper(token))
|
||||
if code == "" || code == "ALL" {
|
||||
continue
|
||||
if code != "" {
|
||||
codes = append(codes, code)
|
||||
}
|
||||
codes = append(codes, code)
|
||||
}
|
||||
if len(codes) == 0 {
|
||||
return nil
|
||||
}
|
||||
return codes
|
||||
}
|
||||
@@ -1937,10 +1848,11 @@ func nullInt(ni sql.NullInt64) interface{} {
|
||||
// Returns the number of transmissions deleted.
|
||||
// Opens a separate read-write connection since the main connection is read-only.
|
||||
func (db *DB) PruneOldPackets(days int) (int64, error) {
|
||||
rw, err := cachedRW(db.path)
|
||||
rw, err := openRW(db.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)
|
||||
tx, err := rw.Begin()
|
||||
@@ -2283,10 +2195,11 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) {
|
||||
|
||||
// PruneOldMetrics deletes observer_metrics rows older than retentionDays.
|
||||
func (db *DB) PruneOldMetrics(retentionDays int) (int64, error) {
|
||||
rw, err := cachedRW(db.path)
|
||||
rw, err := openRW(db.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339)
|
||||
res, err := rw.Exec(`DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff)
|
||||
@@ -2309,10 +2222,11 @@ func (db *DB) RemoveStaleObservers(observerDays int) (int64, error) {
|
||||
if observerDays <= -1 {
|
||||
return 0, nil // keep forever
|
||||
}
|
||||
rw, err := cachedRW(db.path)
|
||||
rw, err := openRW(db.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
|
||||
res, err := rw.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
|
||||
|
||||
+4
-136
@@ -48,9 +48,7 @@ func setupTestDB(t *testing.T) *DB {
|
||||
radio TEXT,
|
||||
battery_mv INTEGER,
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT DEFAULT NULL
|
||||
noise_floor REAL
|
||||
);
|
||||
|
||||
CREATE TABLE transmissions (
|
||||
@@ -76,8 +74,7 @@ func setupTestDB(t *testing.T) *DB {
|
||||
score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL,
|
||||
resolved_path TEXT,
|
||||
raw_hex TEXT
|
||||
resolved_path TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS observer_metrics (
|
||||
@@ -357,35 +354,6 @@ func TestGetObservers(t *testing.T) {
|
||||
if observers[0].ID != "obs1" {
|
||||
t.Errorf("expected obs1 first (most recent), got %s", observers[0].ID)
|
||||
}
|
||||
// last_packet_at should be nil since seedTestData doesn't set it
|
||||
if observers[0].LastPacketAt != nil {
|
||||
t.Errorf("expected nil LastPacketAt for obs1 from seed, got %v", *observers[0].LastPacketAt)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: GetObservers must exclude soft-deleted (inactive=1) rows.
|
||||
// Stale observers were appearing in /api/observers despite the auto-prune
|
||||
// marking them inactive, because the SELECT query had no WHERE filter.
|
||||
func TestGetObservers_ExcludesInactive(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
// Mark obs2 inactive — soft delete simulating a stale-observer prune.
|
||||
if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs2"); err != nil {
|
||||
t.Fatalf("update inactive: %v", err)
|
||||
}
|
||||
observers, err := db.GetObservers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(observers) != 1 {
|
||||
t.Errorf("expected 1 observer (obs1) after marking obs2 inactive, got %d", len(observers))
|
||||
}
|
||||
for _, o := range observers {
|
||||
if o.ID == "obs2" {
|
||||
t.Errorf("inactive observer obs2 should be excluded")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObserverByID(t *testing.T) {
|
||||
@@ -400,48 +368,6 @@ func TestGetObserverByID(t *testing.T) {
|
||||
if obs.ID != "obs1" {
|
||||
t.Errorf("expected obs1, got %s", obs.ID)
|
||||
}
|
||||
// Verify last_packet_at is nil by default
|
||||
if obs.LastPacketAt != nil {
|
||||
t.Errorf("expected nil LastPacketAt, got %v", *obs.LastPacketAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObserverLastPacketAt(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// Set last_packet_at for obs1
|
||||
ts := "2026-04-24T12:00:00Z"
|
||||
db.conn.Exec(`UPDATE observers SET last_packet_at = ? WHERE id = ?`, ts, "obs1")
|
||||
|
||||
// Verify via GetObservers
|
||||
observers, err := db.GetObservers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var obs1 *Observer
|
||||
for i := range observers {
|
||||
if observers[i].ID == "obs1" {
|
||||
obs1 = &observers[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if obs1 == nil {
|
||||
t.Fatal("obs1 not found")
|
||||
}
|
||||
if obs1.LastPacketAt == nil || *obs1.LastPacketAt != ts {
|
||||
t.Errorf("expected LastPacketAt=%s via GetObservers, got %v", ts, obs1.LastPacketAt)
|
||||
}
|
||||
|
||||
// Verify via GetObserverByID
|
||||
obs, err := db.GetObserverByID("obs1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if obs.LastPacketAt == nil || *obs.LastPacketAt != ts {
|
||||
t.Errorf("expected LastPacketAt=%s via GetObserverByID, got %v", ts, obs.LastPacketAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObserverByIDNotFound(t *testing.T) {
|
||||
@@ -1182,8 +1108,7 @@ func setupTestDBV2(t *testing.T) *DB {
|
||||
iata TEXT,
|
||||
last_seen TEXT,
|
||||
first_seen TEXT,
|
||||
packet_count INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT DEFAULT NULL
|
||||
packet_count INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE transmissions (
|
||||
@@ -1209,8 +1134,7 @@ func setupTestDBV2(t *testing.T) *DB {
|
||||
rssi REAL,
|
||||
score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL,
|
||||
raw_hex TEXT
|
||||
timestamp INTEGER NOT NULL
|
||||
);
|
||||
`
|
||||
if _, err := conn.Exec(schema); err != nil {
|
||||
@@ -2051,59 +1975,3 @@ func TestParseWindowDuration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPerObservationRawHexEnrich verifies enrichObs returns per-observation raw_hex
|
||||
// when available, falling back to transmission raw_hex when NULL (#881).
|
||||
func TestPerObservationRawHexEnrich(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
// Insert observers
|
||||
db.conn.Exec(`INSERT INTO observers (id, name) VALUES ('obs-a', 'Observer A')`)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name) VALUES ('obs-b', 'Observer B')`)
|
||||
|
||||
var rowA, rowB int64
|
||||
db.conn.QueryRow(`SELECT rowid FROM observers WHERE id='obs-a'`).Scan(&rowA)
|
||||
db.conn.QueryRow(`SELECT rowid FROM observers WHERE id='obs-b'`).Scan(&rowB)
|
||||
|
||||
// Insert transmission with raw_hex
|
||||
txHex := "deadbeef"
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, 'hash1', '2026-04-21T10:00:00Z')`, txHex)
|
||||
|
||||
// Insert two observations: A has its own raw_hex, B has NULL (historical)
|
||||
obsAHex := "c0ffee01"
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex)
|
||||
VALUES (1, ?, -5.0, -90.0, '[]', 1745236800, ?)`, rowA, obsAHex)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, ?, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store load: %v", err)
|
||||
}
|
||||
|
||||
tx := store.byHash["hash1"]
|
||||
if tx == nil {
|
||||
t.Fatal("transmission not loaded")
|
||||
}
|
||||
if len(tx.Observations) < 2 {
|
||||
t.Fatalf("expected 2 observations, got %d", len(tx.Observations))
|
||||
}
|
||||
|
||||
// Check enriched observations
|
||||
for _, obs := range tx.Observations {
|
||||
m := store.enrichObs(obs)
|
||||
rh, _ := m["raw_hex"].(string)
|
||||
if obs.RawHex != "" {
|
||||
// Observer A: should get per-observation raw_hex
|
||||
if rh != obsAHex {
|
||||
t.Errorf("obs with own raw_hex: got %q, want %q", rh, obsAHex)
|
||||
}
|
||||
} else {
|
||||
// Observer B: should fall back to transmission raw_hex
|
||||
if rh != txHex {
|
||||
t.Errorf("obs without raw_hex: got %q, want %q (tx fallback)", rh, txHex)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic
|
||||
// (simulated here) with auto_vacuum=INCREMENTAL set before tables.
|
||||
func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB {
|
||||
t.Helper()
|
||||
// auto_vacuum must be set via DSN before journal_mode creates the DB file
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
// Create minimal schema
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_hex TEXT NOT NULL,
|
||||
hash TEXT NOT NULL UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
route_type INTEGER,
|
||||
payload_type INTEGER,
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
channel_hash TEXT
|
||||
);
|
||||
CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transmission_id INTEGER NOT NULL REFERENCES transmissions(id),
|
||||
observer_idx INTEGER,
|
||||
direction TEXT,
|
||||
snr REAL,
|
||||
rssi REAL,
|
||||
score INTEGER,
|
||||
path_json TEXT,
|
||||
timestamp INTEGER NOT NULL
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func TestNewDBHasIncrementalAutoVacuum(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.db")
|
||||
|
||||
db := createFreshDBWithAutoVacuum(t, path)
|
||||
defer db.Close()
|
||||
|
||||
var autoVacuum int
|
||||
if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if autoVacuum != 2 {
|
||||
t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExistingDBHasAutoVacuumNone(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create DB WITHOUT setting auto_vacuum (simulates old DB)
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
_, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var autoVacuum int
|
||||
if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
if autoVacuum != 0 {
|
||||
t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumOnStartupMigratesDB(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create DB without auto_vacuum (old DB)
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
_, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var before int
|
||||
db.QueryRow("PRAGMA auto_vacuum").Scan(&before)
|
||||
if before != 0 {
|
||||
t.Fatalf("precondition: expected auto_vacuum=0, got %d", before)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
// Simulate vacuumOnStartup migration using openRW
|
||||
rw, err := openRW(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rw.Exec("VACUUM"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw.Close()
|
||||
|
||||
// Verify migration
|
||||
db2, err := sql.Open("sqlite", path+"?mode=ro")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db2.Close()
|
||||
|
||||
var after int
|
||||
if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after != 2 {
|
||||
t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncrementalVacuumReducesFreelist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.db")
|
||||
|
||||
db := createFreshDBWithAutoVacuum(t, path)
|
||||
|
||||
// Insert a bunch of data
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
for i := 0; i < 500; i++ {
|
||||
_, err := db.Exec(
|
||||
"INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)",
|
||||
strings.Repeat("AA", 200), // ~400 bytes each
|
||||
"hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)),
|
||||
now,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get file size before delete
|
||||
db.Close()
|
||||
infoBefore, _ := os.Stat(path)
|
||||
sizeBefore := infoBefore.Size()
|
||||
|
||||
// Reopen and delete all
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
defer db.Close()
|
||||
|
||||
_, err = db.Exec("DELETE FROM transmissions")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check freelist before vacuum
|
||||
var freelistBefore int64
|
||||
db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore)
|
||||
if freelistBefore == 0 {
|
||||
t.Fatal("expected non-zero freelist after DELETE")
|
||||
}
|
||||
|
||||
// Run incremental vacuum
|
||||
_, err = db.Exec("PRAGMA incremental_vacuum(10000)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check freelist after vacuum
|
||||
var freelistAfter int64
|
||||
db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter)
|
||||
if freelistAfter >= freelistBefore {
|
||||
t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter)
|
||||
}
|
||||
|
||||
// Checkpoint WAL and check file size shrunk
|
||||
db.Exec("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
db.Close()
|
||||
infoAfter, _ := os.Stat(path)
|
||||
sizeAfter := infoAfter.Size()
|
||||
if sizeAfter >= sizeBefore {
|
||||
t.Logf("warning: file did not shrink (before=%d after=%d) — may depend on page reuse", sizeBefore, sizeAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckAutoVacuumLogs(t *testing.T) {
|
||||
// This test verifies checkAutoVacuum doesn't panic on various configs
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create a fresh DB with auto_vacuum=INCREMENTAL
|
||||
dbConn := createFreshDBWithAutoVacuum(t, path)
|
||||
db := &DB{conn: dbConn, path: path}
|
||||
cfg := &Config{}
|
||||
|
||||
// Should not panic
|
||||
checkAutoVacuum(db, cfg, path)
|
||||
dbConn.Close()
|
||||
|
||||
// Create a DB without auto_vacuum
|
||||
path2 := filepath.Join(dir, "test2.db")
|
||||
dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)")
|
||||
dbConn2.SetMaxOpenConns(1)
|
||||
dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)")
|
||||
db2 := &DB{conn: dbConn2, path: path2}
|
||||
|
||||
// Should log warning but not panic
|
||||
checkAutoVacuum(db2, cfg, path2)
|
||||
dbConn2.Close()
|
||||
}
|
||||
|
||||
func TestConfigIncrementalVacuumPages(t *testing.T) {
|
||||
// Default
|
||||
cfg := &Config{}
|
||||
if cfg.IncrementalVacuumPages() != 1024 {
|
||||
t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages())
|
||||
}
|
||||
|
||||
// Custom
|
||||
cfg.DB = &DBConfig{IncrementalVacuumPages: 512}
|
||||
if cfg.IncrementalVacuumPages() != 512 {
|
||||
t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages())
|
||||
}
|
||||
|
||||
// Zero should return default
|
||||
cfg.DB.IncrementalVacuumPages = 0
|
||||
if cfg.IncrementalVacuumPages() != 1024 {
|
||||
t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages())
|
||||
}
|
||||
}
|
||||
+101
-17
@@ -10,7 +10,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/sigvalidate"
|
||||
)
|
||||
|
||||
@@ -106,7 +105,6 @@ type Payload struct {
|
||||
Tag uint32 `json:"tag,omitempty"`
|
||||
AuthCode uint32 `json:"authCode,omitempty"`
|
||||
TraceFlags *int `json:"traceFlags,omitempty"`
|
||||
SNRValues []float64 `json:"snrValues,omitempty"`
|
||||
RawHex string `json:"raw,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
@@ -166,9 +164,8 @@ func decodePath(pathByte byte, buf []byte, offset int) (Path, int) {
|
||||
}, totalBytes
|
||||
}
|
||||
|
||||
// isTransportRoute delegates to packetpath.IsTransportRoute.
|
||||
func isTransportRoute(routeType int) bool {
|
||||
return packetpath.IsTransportRoute(routeType)
|
||||
return routeType == RouteTransportFlood || routeType == RouteTransportDirect
|
||||
}
|
||||
|
||||
func decodeEncryptedPayload(typeName string, buf []byte) Payload {
|
||||
@@ -408,19 +405,6 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
|
||||
}
|
||||
// The header path hops count represents SNR entries = completed hops
|
||||
hopsCompleted := path.HashCount
|
||||
// Extract per-hop SNR from header path bytes (int8, quarter-dB encoding)
|
||||
if hopsCompleted > 0 && len(path.Hops) >= hopsCompleted {
|
||||
snrVals := make([]float64, 0, hopsCompleted)
|
||||
for i := 0; i < hopsCompleted; i++ {
|
||||
b, err := hex.DecodeString(path.Hops[i])
|
||||
if err == nil && len(b) == 1 {
|
||||
snrVals = append(snrVals, float64(int8(b[0]))/4.0)
|
||||
}
|
||||
}
|
||||
if len(snrVals) > 0 {
|
||||
payload.SNRValues = snrVals
|
||||
}
|
||||
}
|
||||
pathBytes, err := hex.DecodeString(payload.PathData)
|
||||
if err == nil && payload.TraceFlags != nil {
|
||||
// path_sz from flags byte is a power-of-two exponent per firmware:
|
||||
@@ -457,6 +441,106 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
|
||||
}, nil
|
||||
}
|
||||
|
||||
// HexRange represents a labeled byte range for the hex breakdown visualization.
|
||||
type HexRange struct {
|
||||
Start int `json:"start"`
|
||||
End int `json:"end"`
|
||||
Label string `json:"label"`
|
||||
}
|
||||
|
||||
// Breakdown holds colored byte ranges returned by the packet detail endpoint.
|
||||
type Breakdown struct {
|
||||
Ranges []HexRange `json:"ranges"`
|
||||
}
|
||||
|
||||
// BuildBreakdown computes labeled byte ranges for each section of a MeshCore packet.
|
||||
// The returned ranges are consumed by createColoredHexDump() and buildHexLegend()
|
||||
// in the frontend (public/app.js).
|
||||
func BuildBreakdown(hexString string) *Breakdown {
|
||||
hexString = strings.ReplaceAll(hexString, " ", "")
|
||||
hexString = strings.ReplaceAll(hexString, "\n", "")
|
||||
hexString = strings.ReplaceAll(hexString, "\r", "")
|
||||
buf, err := hex.DecodeString(hexString)
|
||||
if err != nil || len(buf) < 2 {
|
||||
return &Breakdown{Ranges: []HexRange{}}
|
||||
}
|
||||
|
||||
var ranges []HexRange
|
||||
offset := 0
|
||||
|
||||
// Byte 0: Header
|
||||
ranges = append(ranges, HexRange{Start: 0, End: 0, Label: "Header"})
|
||||
offset = 1
|
||||
|
||||
header := decodeHeader(buf[0])
|
||||
|
||||
// Bytes 1-4: Transport Codes (TRANSPORT_FLOOD / TRANSPORT_DIRECT only)
|
||||
if isTransportRoute(header.RouteType) {
|
||||
if len(buf) < offset+4 {
|
||||
return &Breakdown{Ranges: ranges}
|
||||
}
|
||||
ranges = append(ranges, HexRange{Start: offset, End: offset + 3, Label: "Transport Codes"})
|
||||
offset += 4
|
||||
}
|
||||
|
||||
if offset >= len(buf) {
|
||||
return &Breakdown{Ranges: ranges}
|
||||
}
|
||||
|
||||
// Next byte: Path Length (bits 7-6 = hashSize-1, bits 5-0 = hashCount)
|
||||
ranges = append(ranges, HexRange{Start: offset, End: offset, Label: "Path Length"})
|
||||
pathByte := buf[offset]
|
||||
offset++
|
||||
|
||||
hashSize := int(pathByte>>6) + 1
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
pathBytes := hashSize * hashCount
|
||||
|
||||
// Path hops
|
||||
if hashCount > 0 && offset+pathBytes <= len(buf) {
|
||||
ranges = append(ranges, HexRange{Start: offset, End: offset + pathBytes - 1, Label: "Path"})
|
||||
}
|
||||
offset += pathBytes
|
||||
|
||||
if offset >= len(buf) {
|
||||
return &Breakdown{Ranges: ranges}
|
||||
}
|
||||
|
||||
payloadStart := offset
|
||||
|
||||
// Payload — break ADVERT into named sub-fields; everything else is one Payload range
|
||||
if header.PayloadType == PayloadADVERT && len(buf)-payloadStart >= 100 {
|
||||
ranges = append(ranges, HexRange{Start: payloadStart, End: payloadStart + 31, Label: "PubKey"})
|
||||
ranges = append(ranges, HexRange{Start: payloadStart + 32, End: payloadStart + 35, Label: "Timestamp"})
|
||||
ranges = append(ranges, HexRange{Start: payloadStart + 36, End: payloadStart + 99, Label: "Signature"})
|
||||
|
||||
appStart := payloadStart + 100
|
||||
if appStart < len(buf) {
|
||||
ranges = append(ranges, HexRange{Start: appStart, End: appStart, Label: "Flags"})
|
||||
appFlags := buf[appStart]
|
||||
fOff := appStart + 1
|
||||
if appFlags&0x10 != 0 && fOff+8 <= len(buf) {
|
||||
ranges = append(ranges, HexRange{Start: fOff, End: fOff + 3, Label: "Latitude"})
|
||||
ranges = append(ranges, HexRange{Start: fOff + 4, End: fOff + 7, Label: "Longitude"})
|
||||
fOff += 8
|
||||
}
|
||||
if appFlags&0x20 != 0 && fOff+2 <= len(buf) {
|
||||
fOff += 2
|
||||
}
|
||||
if appFlags&0x40 != 0 && fOff+2 <= len(buf) {
|
||||
fOff += 2
|
||||
}
|
||||
if appFlags&0x80 != 0 && fOff < len(buf) {
|
||||
ranges = append(ranges, HexRange{Start: fOff, End: len(buf) - 1, Label: "Name"})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ranges = append(ranges, HexRange{Start: payloadStart, End: len(buf) - 1, Label: "Payload"})
|
||||
}
|
||||
|
||||
return &Breakdown{Ranges: ranges}
|
||||
}
|
||||
|
||||
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
|
||||
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
|
||||
// route-independent identifier for the same logical packet. For TRACE packets,
|
||||
|
||||
+140
-48
@@ -97,6 +97,146 @@ func TestDecodePacket_FloodHasNoCodes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_InvalidHex(t *testing.T) {
|
||||
b := BuildBreakdown("not-hex!")
|
||||
if len(b.Ranges) != 0 {
|
||||
t.Errorf("expected empty ranges for invalid hex, got %d", len(b.Ranges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_TooShort(t *testing.T) {
|
||||
b := BuildBreakdown("11") // 1 byte — no path byte
|
||||
if len(b.Ranges) != 0 {
|
||||
t.Errorf("expected empty ranges for too-short packet, got %d", len(b.Ranges))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_FloodNonAdvert(t *testing.T) {
|
||||
// Header 0x15: route=1/FLOOD, payload=5/GRP_TXT
|
||||
// PathByte 0x01: 1 hop, 1-byte hash
|
||||
// PathHop: AA
|
||||
// Payload: FF0011
|
||||
b := BuildBreakdown("1501AAFFFF00")
|
||||
labels := rangeLabels(b.Ranges)
|
||||
expect := []string{"Header", "Path Length", "Path", "Payload"}
|
||||
if !equalLabels(labels, expect) {
|
||||
t.Errorf("expected labels %v, got %v", expect, labels)
|
||||
}
|
||||
// Verify byte positions
|
||||
assertRange(t, b.Ranges, "Header", 0, 0)
|
||||
assertRange(t, b.Ranges, "Path Length", 1, 1)
|
||||
assertRange(t, b.Ranges, "Path", 2, 2)
|
||||
assertRange(t, b.Ranges, "Payload", 3, 5)
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_TransportFlood(t *testing.T) {
|
||||
// Header 0x14: route=0/TRANSPORT_FLOOD, payload=5/GRP_TXT
|
||||
// TransportCodes: AABBCCDD (4 bytes)
|
||||
// PathByte 0x01: 1 hop, 1-byte hash
|
||||
// PathHop: EE
|
||||
// Payload: FF00
|
||||
b := BuildBreakdown("14AABBCCDD01EEFF00")
|
||||
assertRange(t, b.Ranges, "Header", 0, 0)
|
||||
assertRange(t, b.Ranges, "Transport Codes", 1, 4)
|
||||
assertRange(t, b.Ranges, "Path Length", 5, 5)
|
||||
assertRange(t, b.Ranges, "Path", 6, 6)
|
||||
assertRange(t, b.Ranges, "Payload", 7, 8)
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_FloodNoHops(t *testing.T) {
|
||||
// Header 0x15: FLOOD/GRP_TXT; PathByte 0x00: 0 hops; Payload: AABB
|
||||
b := BuildBreakdown("150000AABB")
|
||||
assertRange(t, b.Ranges, "Header", 0, 0)
|
||||
assertRange(t, b.Ranges, "Path Length", 1, 1)
|
||||
// No Path range since hashCount=0
|
||||
for _, r := range b.Ranges {
|
||||
if r.Label == "Path" {
|
||||
t.Error("expected no Path range for zero-hop packet")
|
||||
}
|
||||
}
|
||||
assertRange(t, b.Ranges, "Payload", 2, 4)
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_AdvertBasic(t *testing.T) {
|
||||
// Header 0x11: FLOOD/ADVERT
|
||||
// PathByte 0x01: 1 hop, 1-byte hash
|
||||
// PathHop: AA
|
||||
// Payload: 100 bytes (PubKey32 + Timestamp4 + Signature64) + Flags=0x02 (repeater, no extras)
|
||||
pubkey := repeatHex("AB", 32)
|
||||
ts := "00000000" // 4 bytes
|
||||
sig := repeatHex("CD", 64)
|
||||
flags := "02"
|
||||
hex := "1101AA" + pubkey + ts + sig + flags
|
||||
b := BuildBreakdown(hex)
|
||||
assertRange(t, b.Ranges, "Header", 0, 0)
|
||||
assertRange(t, b.Ranges, "Path Length", 1, 1)
|
||||
assertRange(t, b.Ranges, "Path", 2, 2)
|
||||
assertRange(t, b.Ranges, "PubKey", 3, 34)
|
||||
assertRange(t, b.Ranges, "Timestamp", 35, 38)
|
||||
assertRange(t, b.Ranges, "Signature", 39, 102)
|
||||
assertRange(t, b.Ranges, "Flags", 103, 103)
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_AdvertWithLocation(t *testing.T) {
|
||||
// flags=0x12: hasLocation bit set
|
||||
pubkey := repeatHex("00", 32)
|
||||
ts := "00000000"
|
||||
sig := repeatHex("00", 64)
|
||||
flags := "12" // 0x10 = hasLocation
|
||||
latBytes := "00000000"
|
||||
lonBytes := "00000000"
|
||||
hex := "1101AA" + pubkey + ts + sig + flags + latBytes + lonBytes
|
||||
b := BuildBreakdown(hex)
|
||||
assertRange(t, b.Ranges, "Latitude", 104, 107)
|
||||
assertRange(t, b.Ranges, "Longitude", 108, 111)
|
||||
}
|
||||
|
||||
func TestBuildBreakdown_AdvertWithName(t *testing.T) {
|
||||
// flags=0x82: hasName bit set
|
||||
pubkey := repeatHex("00", 32)
|
||||
ts := "00000000"
|
||||
sig := repeatHex("00", 64)
|
||||
flags := "82" // 0x80 = hasName
|
||||
name := "4E6F6465" // "Node" in hex
|
||||
hex := "1101AA" + pubkey + ts + sig + flags + name
|
||||
b := BuildBreakdown(hex)
|
||||
assertRange(t, b.Ranges, "Name", 104, 107)
|
||||
}
|
||||
|
||||
// helpers
|
||||
|
||||
func rangeLabels(ranges []HexRange) []string {
|
||||
out := make([]string, len(ranges))
|
||||
for i, r := range ranges {
|
||||
out[i] = r.Label
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func equalLabels(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func assertRange(t *testing.T, ranges []HexRange, label string, wantStart, wantEnd int) {
|
||||
t.Helper()
|
||||
for _, r := range ranges {
|
||||
if r.Label == label {
|
||||
if r.Start != wantStart || r.End != wantEnd {
|
||||
t.Errorf("range %q: want [%d,%d], got [%d,%d]", label, wantStart, wantEnd, r.Start, r.End)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("range %q not found in %v", label, rangeLabels(ranges))
|
||||
}
|
||||
|
||||
func TestZeroHopDirectHashSize(t *testing.T) {
|
||||
// DIRECT (RouteType=2) + REQ (PayloadType=0) → header byte = 0x02
|
||||
@@ -440,51 +580,3 @@ func TestDecodeAdvertSignatureValidation(t *testing.T) {
|
||||
t.Error("expected SignatureValid to be nil when validation disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceSNRValues(t *testing.T) {
|
||||
// TRACE packet with 3 SNR bytes in header path:
|
||||
// SNR byte 0: 0x14 = int8(20) → 20/4.0 = 5.0 dB
|
||||
// SNR byte 1: 0xF4 = int8(-12) → -12/4.0 = -3.0 dB
|
||||
// SNR byte 2: 0x08 = int8(8) → 8/4.0 = 2.0 dB
|
||||
// header: DIRECT+TRACE = (0<<6)|(9<<2)|2 = 0x26
|
||||
// path_length: hash_size=0b00 (1-byte), hash_count=3 → 0x03
|
||||
hex := "2603" + "14F408" + // header + path_byte + 3 SNR bytes
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"00" + // flags=0 → path_sz=1
|
||||
"AABBCCDD" // 4 route hops (1-byte each)
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.Payload.SNRValues == nil {
|
||||
t.Fatal("expected SNRValues to be populated")
|
||||
}
|
||||
if len(pkt.Payload.SNRValues) != 3 {
|
||||
t.Fatalf("expected 3 SNR values, got %d", len(pkt.Payload.SNRValues))
|
||||
}
|
||||
expected := []float64{5.0, -3.0, 2.0}
|
||||
for i, want := range expected {
|
||||
if pkt.Payload.SNRValues[i] != want {
|
||||
t.Errorf("SNRValues[%d] = %v, want %v", i, pkt.Payload.SNRValues[i], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacket_TraceNoSNRValues(t *testing.T) {
|
||||
// TRACE with 0 SNR bytes → SNRValues should be nil/empty
|
||||
hex := "2600" + // header + path_byte (0 hops)
|
||||
"01000000" + // tag
|
||||
"02000000" + // authCode
|
||||
"00" + // flags
|
||||
"AABB" // 2 route hops
|
||||
|
||||
pkt, err := DecodePacket(hex, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if len(pkt.Payload.SNRValues) != 0 {
|
||||
t.Errorf("expected empty SNRValues, got %v", pkt.Payload.SNRValues)
|
||||
}
|
||||
}
|
||||
|
||||
+10
-32
@@ -247,11 +247,6 @@ func TestEvictStale_CleansNodeIndexes(t *testing.T) {
|
||||
|
||||
func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Create a temp DB for on-demand SQL fetch during eviction
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
store := &PacketStore{
|
||||
packets: make([]*StoreTx, 0),
|
||||
byHash: make(map[string]*StoreTx),
|
||||
@@ -272,33 +267,25 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
rfCacheTTL: 15 * time.Second,
|
||||
retentionHours: 24,
|
||||
db: db,
|
||||
useResolvedPathIndex: true,
|
||||
}
|
||||
store.initResolvedPathIndex()
|
||||
|
||||
// Create a packet indexed via resolved_path pubkeys
|
||||
// Create a packet indexed only via resolved_path (no decoded JSON pubkeys)
|
||||
relayPK := "relay0001abcdef"
|
||||
txID := 1
|
||||
obsID := 100
|
||||
tx := &StoreTx{
|
||||
ID: txID,
|
||||
ID: 1,
|
||||
Hash: "hash_rp_001",
|
||||
FirstSeen: now.Add(-48 * time.Hour).UTC().Format(time.RFC3339),
|
||||
}
|
||||
rpPtr := &relayPK
|
||||
obs := &StoreObs{
|
||||
ID: obsID,
|
||||
TransmissionID: txID,
|
||||
ID: 100,
|
||||
TransmissionID: 1,
|
||||
ObserverID: "obs0",
|
||||
Timestamp: tx.FirstSeen,
|
||||
ResolvedPath: []*string{rpPtr},
|
||||
}
|
||||
tx.Observations = append(tx.Observations, obs)
|
||||
|
||||
// Insert into DB so on-demand SQL fetch works during eviction
|
||||
db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)",
|
||||
txID, tx.Hash, tx.FirstSeen)
|
||||
db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)",
|
||||
obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`)
|
||||
tx.ResolvedPath = []*string{rpPtr}
|
||||
|
||||
store.packets = append(store.packets, tx)
|
||||
store.byHash[tx.Hash] = tx
|
||||
@@ -306,9 +293,8 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
store.byObsID[obs.ID] = obs
|
||||
store.byObserver["obs0"] = append(store.byObserver["obs0"], obs)
|
||||
|
||||
// Index relay via decode-window simulation
|
||||
store.addToByNode(tx, relayPK)
|
||||
store.addToResolvedPubkeyIndex(txID, []string{relayPK})
|
||||
// Index via resolved_path
|
||||
store.indexByNode(tx)
|
||||
|
||||
// Verify indexed
|
||||
if len(store.byNode[relayPK]) != 1 {
|
||||
@@ -318,7 +304,7 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
t.Fatalf("expected nodeHashes[%s] to contain %s", relayPK, tx.Hash)
|
||||
}
|
||||
|
||||
evicted := store.RunEviction()
|
||||
evicted := store.EvictStale()
|
||||
if evicted != 1 {
|
||||
t.Fatalf("expected 1 evicted, got %d", evicted)
|
||||
}
|
||||
@@ -330,14 +316,6 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
|
||||
if _, exists := store.nodeHashes[relayPK]; exists {
|
||||
t.Fatalf("expected nodeHashes[%s] to be deleted after eviction", relayPK)
|
||||
}
|
||||
// Verify resolved pubkey index is cleaned up
|
||||
h := resolvedPubkeyHash(relayPK)
|
||||
if len(store.resolvedPubkeyIndex[h]) != 0 {
|
||||
t.Fatalf("expected resolvedPubkeyIndex to be empty after eviction")
|
||||
}
|
||||
if _, exists := store.resolvedPubkeyReverse[txID]; exists {
|
||||
t.Fatalf("expected resolvedPubkeyReverse to be empty after eviction")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvictStale_RunEvictionThreadSafe(t *testing.T) {
|
||||
|
||||
@@ -14,14 +14,6 @@ replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter
|
||||
|
||||
replace github.com/meshcore-analyzer/sigvalidate => ../../internal/sigvalidate
|
||||
|
||||
require github.com/meshcore-analyzer/packetpath v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath
|
||||
|
||||
require github.com/meshcore-analyzer/dbconfig v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// readiness tracks whether background init goroutines have completed.
|
||||
// Set to 1 once store.Load, pickBestObservation, and neighbor graph build are done.
|
||||
var readiness atomic.Int32
|
||||
|
||||
// handleHealthz returns 200 when the server is ready to serve queries,
|
||||
// or 503 while background initialization is still running.
|
||||
func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
if readiness.Load() == 0 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"ready": false,
|
||||
"reason": "loading",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
var loadedTx, loadedObs int
|
||||
if s.store != nil {
|
||||
s.store.mu.RLock()
|
||||
loadedTx = len(s.store.packets)
|
||||
for _, p := range s.store.packets {
|
||||
loadedObs += len(p.Observations)
|
||||
}
|
||||
s.store.mu.RUnlock()
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"ready": true,
|
||||
"loadedTx": loadedTx,
|
||||
"loadedObs": loadedObs,
|
||||
})
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHealthzNotReady(t *testing.T) {
|
||||
// Ensure readiness is 0 (not ready)
|
||||
readiness.Store(0)
|
||||
defer readiness.Store(0)
|
||||
|
||||
srv := &Server{store: &PacketStore{}}
|
||||
req := httptest.NewRequest("GET", "/api/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleHealthz(w, req)
|
||||
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected 503, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON: %v", err)
|
||||
}
|
||||
if resp["ready"] != false {
|
||||
t.Fatalf("expected ready=false, got %v", resp["ready"])
|
||||
}
|
||||
if resp["reason"] != "loading" {
|
||||
t.Fatalf("expected reason=loading, got %v", resp["reason"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthzReady(t *testing.T) {
|
||||
readiness.Store(1)
|
||||
defer readiness.Store(0)
|
||||
|
||||
srv := &Server{store: &PacketStore{}}
|
||||
req := httptest.NewRequest("GET", "/api/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleHealthz(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON: %v", err)
|
||||
}
|
||||
if resp["ready"] != true {
|
||||
t.Fatalf("expected ready=true, got %v", resp["ready"])
|
||||
}
|
||||
if _, ok := resp["loadedTx"]; !ok {
|
||||
t.Fatal("missing loadedTx field")
|
||||
}
|
||||
if _, ok := resp["loadedObs"]; !ok {
|
||||
t.Fatal("missing loadedObs field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthzAntiTautology(t *testing.T) {
|
||||
// When readiness is 0, must NOT return 200
|
||||
readiness.Store(0)
|
||||
defer readiness.Store(0)
|
||||
|
||||
srv := &Server{store: &PacketStore{}}
|
||||
req := httptest.NewRequest("GET", "/api/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleHealthz(w, req)
|
||||
|
||||
if w.Code == http.StatusOK {
|
||||
t.Fatal("anti-tautology: handler returned 200 when readiness=0; gating is broken")
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9"
|
||||
|
||||
// setupIssue673Store builds an in-memory store with one repeater node having:
|
||||
// - one ADVERT packet (legitimately indexed in byNode)
|
||||
// - one GRP_TXT packet whose decoded text contains the node's pubkey (false-positive candidate)
|
||||
func setupIssue673Store(t *testing.T) (*PacketStore, *DB) {
|
||||
t.Helper()
|
||||
db := setupTestDB(t)
|
||||
|
||||
_, err := db.conn.Exec(
|
||||
"INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)",
|
||||
issue673NodePK, "Quail Hollow Park", "repeater",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ps := NewPacketStore(db, nil)
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
pt4 := 4 // ADVERT
|
||||
pt5 := 5 // GRP_TXT
|
||||
|
||||
advertDecoded, _ := json.Marshal(map[string]interface{}{"pubKey": issue673NodePK})
|
||||
advert := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "advert_hash_673",
|
||||
PayloadType: &pt4,
|
||||
DecodedJSON: string(advertDecoded),
|
||||
FirstSeen: now,
|
||||
}
|
||||
|
||||
otherPK := "aabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccdd"
|
||||
chatDecoded, _ := json.Marshal(map[string]interface{}{
|
||||
"srcPubKey": otherPK,
|
||||
"text": "Check out node " + issue673NodePK + " on the analyzer",
|
||||
})
|
||||
chat := &StoreTx{
|
||||
ID: 2,
|
||||
Hash: "chat_hash_673",
|
||||
PayloadType: &pt5,
|
||||
DecodedJSON: string(chatDecoded),
|
||||
FirstSeen: now,
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.packets = append(ps.packets, advert, chat)
|
||||
ps.byHash[advert.Hash] = advert
|
||||
ps.byHash[chat.Hash] = chat
|
||||
ps.byTxID[advert.ID] = advert
|
||||
ps.byTxID[chat.ID] = chat
|
||||
ps.byNode[issue673NodePK] = []*StoreTx{advert}
|
||||
ps.mu.Unlock()
|
||||
|
||||
return ps, db
|
||||
}
|
||||
|
||||
// TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText verifies that a GRP_TXT packet
|
||||
// whose message text contains a node's pubkey is not counted in that node's analytics.
|
||||
func TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText(t *testing.T) {
|
||||
ps, db := setupIssue673Store(t)
|
||||
defer db.Close()
|
||||
|
||||
analytics, err := ps.GetNodeAnalytics(issue673NodePK, 30)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if analytics == nil {
|
||||
t.Fatal("expected analytics, got nil")
|
||||
}
|
||||
|
||||
for _, ptc := range analytics.PacketTypeBreakdown {
|
||||
if ptc.PayloadType == 5 {
|
||||
t.Errorf("GRP_TXT (type 5) should not appear in analytics for repeater node, got count=%d", ptc.Count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilterPackets_NodeQueryDoesNotMatchChatText verifies that the slow path of
|
||||
// filterPackets (node filter combined with Since) does not return a GRP_TXT packet
|
||||
// whose pubkey appears only in message text, not in a structured pubkey field.
|
||||
func TestFilterPackets_NodeQueryDoesNotMatchChatText(t *testing.T) {
|
||||
ps, db := setupIssue673Store(t)
|
||||
defer db.Close()
|
||||
|
||||
yesterday := time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
result := ps.QueryPackets(PacketQuery{Node: issue673NodePK, Since: yesterday, Limit: 50})
|
||||
|
||||
if result.Total != 1 {
|
||||
t.Errorf("expected 1 packet for node (ADVERT only), got %d", result.Total)
|
||||
}
|
||||
for _, pkt := range result.Packets {
|
||||
if pkt["hash"] == "chat_hash_673" {
|
||||
t.Errorf("GRP_TXT with pubkey in message text was incorrectly returned for node query")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestIssue804_AnalyticsAttributesByRepeaterRegion verifies that analytics
|
||||
// (specifically GetAnalyticsHashSizes) attribute multi-byte nodes to the
|
||||
// REPEATER's home region, not the observer that happened to hear the relay.
|
||||
//
|
||||
// Scenario from #804:
|
||||
// - PDX-Repeater is a multi-byte (hashSize=2) repeater whose ZERO-HOP direct
|
||||
// adverts are only heard by obs-PDX (a PDX observer). That zero-hop direct
|
||||
// advert is the most reliable home-region signal — it cannot have been
|
||||
// relayed.
|
||||
// - A flood advert from PDX-Repeater (hashSize=2) propagates and is heard by
|
||||
// obs-SJC (a SJC observer) via a multi-hop relay path.
|
||||
// - When the user asks for region=SJC analytics, the PDX-Repeater MUST NOT
|
||||
// pollute SJC's multiByteNodes — it lives in PDX.
|
||||
// - The result should also expose attributionMethod="repeater" so the API
|
||||
// consumer knows which method was used.
|
||||
//
|
||||
// Pre-fix behavior: PDX-Repeater appears in SJC's multiByteNodes because the
|
||||
// filter is observer-based. This test fails on the pre-fix code at the
|
||||
// "want PDX-Repeater EXCLUDED" assertion.
|
||||
func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Observers: one in PDX, one in SJC
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obs-pdx', 'Obs PDX', 'PDX', ?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
|
||||
// PDX-Repeater node (lives in Portland)
|
||||
pdxPK := "pdx0000000000001"
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role)
|
||||
VALUES (?, 'PDX-Repeater', 'repeater')`, pdxPK)
|
||||
|
||||
// SJC-Repeater node (lives in San Jose) — sanity baseline
|
||||
sjcPK := "sjc0000000000001"
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role)
|
||||
VALUES (?, 'SJC-Repeater', 'repeater')`, sjcPK)
|
||||
|
||||
pdxDecoded := `{"pubKey":"` + pdxPK + `","name":"PDX-Repeater","type":"ADVERT","flags":{"isRepeater":true}}`
|
||||
sjcDecoded := `{"pubKey":"` + sjcPK + `","name":"SJC-Repeater","type":"ADVERT","flags":{"isRepeater":true}}`
|
||||
|
||||
// 1) PDX-Repeater zero-hop DIRECT advert heard only by obs-PDX.
|
||||
// Establishes PDX as the repeater's home region.
|
||||
// raw_hex header 0x12 = route_type 2 (direct), payload_type 4
|
||||
// pathByte 0x40 (hashSize bits=01 → 2, hop_count=0)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1240aabbccdd', 'pdx_zh_direct', ?, 2, 4, ?)`, recent, pdxDecoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch)
|
||||
|
||||
// 2) PDX-Repeater FLOOD advert with hashSize=2 (reliable).
|
||||
// Heard ONLY by obs-SJC via a relay path (this is the polluting case).
|
||||
// raw_hex header 0x11 = route_type 1 (flood), payload_type 4
|
||||
// pathByte 0x41 (hashSize bits=01 → 2, hop_count=1)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1141aabbccdd', 'pdx_flood', ?, 1, 4, ?)`, recent, pdxDecoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 2, 8.0, -95, '["aa11"]', ?)`, recentEpoch)
|
||||
|
||||
// 3) SJC-Repeater zero-hop DIRECT advert heard only by obs-SJC.
|
||||
// Establishes SJC as the repeater's home region.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1240ccddeeff', 'sjc_zh_direct', ?, 2, 4, ?)`, recent, sjcDecoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (3, 2, 14.0, -82, '[]', ?)`, recentEpoch)
|
||||
|
||||
// 4) SJC-Repeater FLOOD advert with hashSize=2, heard by obs-SJC.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('1141ccddeeff', 'sjc_flood', ?, 1, 4, ?)`, recent, sjcDecoded)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (4, 2, 11.0, -88, '["cc22"]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
t.Run("region=SJC excludes PDX-Repeater (heard but not home)", func(t *testing.T) {
|
||||
result := store.GetAnalyticsHashSizes("SJC")
|
||||
|
||||
mb, ok := result["multiByteNodes"].([]map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected multiByteNodes slice")
|
||||
}
|
||||
|
||||
var foundPDX, foundSJC bool
|
||||
for _, n := range mb {
|
||||
pk, _ := n["pubkey"].(string)
|
||||
if pk == pdxPK {
|
||||
foundPDX = true
|
||||
}
|
||||
if pk == sjcPK {
|
||||
foundSJC = true
|
||||
}
|
||||
}
|
||||
|
||||
if foundPDX {
|
||||
t.Errorf("PDX-Repeater leaked into SJC analytics — region attribution still observer-based (#804 not fixed)")
|
||||
}
|
||||
if !foundSJC {
|
||||
t.Errorf("SJC-Repeater missing from SJC analytics — fix over-filtered")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("API exposes attributionMethod", func(t *testing.T) {
|
||||
result := store.GetAnalyticsHashSizes("SJC")
|
||||
method, ok := result["attributionMethod"].(string)
|
||||
if !ok {
|
||||
t.Fatal("expected attributionMethod string field on result")
|
||||
}
|
||||
if method != "repeater" {
|
||||
t.Errorf("attributionMethod = %q, want %q", method, "repeater")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("region=PDX excludes SJC-Repeater", func(t *testing.T) {
|
||||
result := store.GetAnalyticsHashSizes("PDX")
|
||||
mb, _ := result["multiByteNodes"].([]map[string]interface{})
|
||||
|
||||
var foundPDX, foundSJC bool
|
||||
for _, n := range mb {
|
||||
pk, _ := n["pubkey"].(string)
|
||||
if pk == pdxPK {
|
||||
foundPDX = true
|
||||
}
|
||||
if pk == sjcPK {
|
||||
foundSJC = true
|
||||
}
|
||||
}
|
||||
if !foundPDX {
|
||||
t.Errorf("PDX-Repeater missing from PDX analytics")
|
||||
}
|
||||
if foundSJC {
|
||||
t.Errorf("SJC-Repeater leaked into PDX analytics")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestRepro810 reproduces #810: when the longest-path observation has NULL
|
||||
// resolved_path but a shorter-path observation has one, fetchResolvedPathForTxBest
|
||||
// returns nil → /api/nodes/{pk}/health.recentPackets[].resolved_path is missing
|
||||
// while /api/packets shows it.
|
||||
func TestRepro810(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent)
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent)
|
||||
// Longest-path obs WITHOUT resolved_path
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch)
|
||||
// Shorter-path obs WITH resolved_path
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
// Sanity: /api/packets should show resolved_path for this tx.
|
||||
reqP := httptest.NewRequest("GET", "/api/packets?limit=10", nil)
|
||||
wP := httptest.NewRecorder()
|
||||
router.ServeHTTP(wP, reqP)
|
||||
var pktsBody map[string]interface{}
|
||||
json.Unmarshal(wP.Body.Bytes(), &pktsBody)
|
||||
pkts, _ := pktsBody["packets"].([]interface{})
|
||||
hasOnPackets := false
|
||||
for _, p := range pkts {
|
||||
pm := p.(map[string]interface{})
|
||||
if pm["hash"] == "testhash00000001" && pm["resolved_path"] != nil {
|
||||
hasOnPackets = true
|
||||
}
|
||||
}
|
||||
if !hasOnPackets {
|
||||
t.Fatal("precondition: /api/packets must report resolved_path for tx")
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
rp, _ := body["recentPackets"].([]interface{})
|
||||
if len(rp) == 0 {
|
||||
t.Fatal("no recentPackets")
|
||||
}
|
||||
for _, p := range rp {
|
||||
pm := p.(map[string]interface{})
|
||||
if pm["hash"] == "testhash00000001" {
|
||||
if pm["resolved_path"] == nil {
|
||||
t.Fatal("BUG #810: /health.recentPackets resolved_path is nil despite /api/packets reporting it")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("tx not found in recentPackets")
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestIssue871_NoNullHashOrTimestamp verifies that /api/packets never returns
|
||||
// packets with null/empty hash or null timestamp (issue #871).
|
||||
func TestIssue871_NoNullHashOrTimestamp(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
seedTestData(t, db)
|
||||
|
||||
// Insert bad legacy data: packet with empty hash
|
||||
now := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('DEAD', '', ?, 1, 4, '{}')`, now)
|
||||
// Insert bad legacy data: packet with NULL first_seen (timestamp)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('BEEF', 'aa11bb22cc33dd44', NULL, 1, 4, '{}')`)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load failed: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/packets?limit=200", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Packets []map[string]interface{} `json:"packets"`
|
||||
}
|
||||
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode error: %v", err)
|
||||
}
|
||||
|
||||
for i, p := range resp.Packets {
|
||||
hash, _ := p["hash"]
|
||||
ts, _ := p["timestamp"]
|
||||
if hash == nil || hash == "" {
|
||||
t.Errorf("packet[%d] has null/empty hash: %v", i, p)
|
||||
}
|
||||
if ts == nil || ts == "" {
|
||||
t.Errorf("packet[%d] has null/empty timestamp: %v", i, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-49
@@ -148,9 +148,6 @@ func main() {
|
||||
stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers)
|
||||
}
|
||||
|
||||
// Check auto_vacuum mode and optionally migrate (#919)
|
||||
checkAutoVacuum(database, cfg, resolvedDB)
|
||||
|
||||
// In-memory packet store
|
||||
store := NewPacketStore(database, cfg.PacketStore, cfg.CacheTTL)
|
||||
if err := store.Load(); err != nil {
|
||||
@@ -174,27 +171,6 @@ func main() {
|
||||
database.hasResolvedPath = true // detectSchema ran before column was added; fix the flag
|
||||
}
|
||||
|
||||
// Ensure observers.inactive column exists (PR #954 filters on it; ingestor migration
|
||||
// adds it but server may run against DBs ingestor never touched, e.g. e2e fixture).
|
||||
if err := ensureObserverInactiveColumn(dbPath); err != nil {
|
||||
log.Printf("[store] warning: could not add observers.inactive column: %v", err)
|
||||
}
|
||||
|
||||
// Ensure observers.last_packet_at column exists (PR #905 reads it; ingestor migration
|
||||
// adds it but server may run against DBs ingestor never touched, e.g. e2e fixture).
|
||||
if err := ensureLastPacketAtColumn(dbPath); err != nil {
|
||||
log.Printf("[store] warning: could not add observers.last_packet_at column: %v", err)
|
||||
}
|
||||
|
||||
// Soft-delete observers that are in the blacklist (mark inactive=1) so
|
||||
// historical data from a prior unblocked window is hidden too.
|
||||
if len(cfg.ObserverBlacklist) > 0 {
|
||||
softDeleteBlacklistedObservers(dbPath, cfg.ObserverBlacklist)
|
||||
}
|
||||
|
||||
// WaitGroup for background init steps that gate /api/healthz readiness.
|
||||
var initWg sync.WaitGroup
|
||||
|
||||
// Load or build neighbor graph
|
||||
if neighborEdgesTableExists(database.conn) {
|
||||
store.graph = loadNeighborEdgesFromDB(database.conn)
|
||||
@@ -202,17 +178,16 @@ func main() {
|
||||
} else {
|
||||
log.Printf("[neighbor] no persisted edges found, will build in background...")
|
||||
store.graph = NewNeighborGraph() // empty graph — gets populated by background goroutine
|
||||
initWg.Add(1)
|
||||
go func() {
|
||||
defer initWg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[neighbor] graph build panic recovered: %v", r)
|
||||
}
|
||||
}()
|
||||
rw, rwErr := cachedRW(dbPath)
|
||||
rw, rwErr := openRW(dbPath)
|
||||
if rwErr == nil {
|
||||
edgeCount := buildAndPersistEdges(store, rw)
|
||||
rw.Close()
|
||||
log.Printf("[neighbor] persisted %d edges", edgeCount)
|
||||
}
|
||||
built := BuildFromStore(store)
|
||||
@@ -227,9 +202,7 @@ func main() {
|
||||
// API serves best-effort data until this completes (~10s for 100K txs).
|
||||
// Processes in chunks of 5000, releasing the lock between chunks so API
|
||||
// handlers remain responsive.
|
||||
initWg.Add(1)
|
||||
go func() {
|
||||
defer initWg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[store] pickBestObservation panic recovered: %v", r)
|
||||
@@ -257,13 +230,6 @@ func main() {
|
||||
log.Printf("[store] initial pickBestObservation complete (%d transmissions)", totalPackets)
|
||||
}()
|
||||
|
||||
// Mark server ready once all background init completes.
|
||||
go func() {
|
||||
initWg.Wait()
|
||||
readiness.Store(1)
|
||||
log.Printf("[server] readiness: ready=true (background init complete)")
|
||||
}()
|
||||
|
||||
// WebSocket hub
|
||||
hub := NewHub()
|
||||
|
||||
@@ -300,7 +266,6 @@ func main() {
|
||||
defer stopEviction()
|
||||
|
||||
// Auto-prune old packets if retention.packetDays is configured
|
||||
vacuumPages := cfg.IncrementalVacuumPages()
|
||||
var stopPrune func()
|
||||
if cfg.Retention != nil && cfg.Retention.PacketDays > 0 {
|
||||
days := cfg.Retention.PacketDays
|
||||
@@ -321,9 +286,6 @@ func main() {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
if n > 0 {
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
@@ -332,9 +294,6 @@ func main() {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
if n > 0 {
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
}
|
||||
}
|
||||
case <-pruneDone:
|
||||
return
|
||||
@@ -362,12 +321,10 @@ func main() {
|
||||
}()
|
||||
time.Sleep(2 * time.Minute) // stagger after packet prune
|
||||
database.PruneOldMetrics(metricsDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
for {
|
||||
select {
|
||||
case <-metricsPruneTicker.C:
|
||||
database.PruneOldMetrics(metricsDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
case <-metricsPruneDone:
|
||||
return
|
||||
}
|
||||
@@ -397,12 +354,10 @@ func main() {
|
||||
}()
|
||||
time.Sleep(3 * time.Minute) // stagger after metrics prune
|
||||
database.RemoveStaleObservers(observerDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
for {
|
||||
select {
|
||||
case <-observerPruneTicker.C:
|
||||
database.RemoveStaleObservers(observerDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
case <-observerPruneDone:
|
||||
return
|
||||
}
|
||||
@@ -433,7 +388,6 @@ func main() {
|
||||
g := store.graph
|
||||
store.mu.RUnlock()
|
||||
PruneNeighborEdges(dbPath, g, maxAgeDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
for {
|
||||
select {
|
||||
case <-edgePruneTicker.C:
|
||||
@@ -441,7 +395,6 @@ func main() {
|
||||
g := store.graph
|
||||
store.mu.RUnlock()
|
||||
PruneNeighborEdges(dbPath, g, maxAgeDays)
|
||||
runIncrementalVacuum(resolvedDB, vacuumPages)
|
||||
case <-edgePruneDone:
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemorySnapshot is a point-in-time view of process memory across several
|
||||
// vantage points. Values are in MB (1024*1024 bytes), rounded to one decimal.
|
||||
//
|
||||
// Field invariants (typical, not guaranteed under exotic conditions):
|
||||
//
|
||||
// processRSSMB >= goSysMB >= goHeapInuseMB >= storeDataMB
|
||||
//
|
||||
// - processRSSMB is what the kernel charges the process (resident set).
|
||||
// Read from /proc/self/status `VmRSS:` on Linux; falls back to goSysMB
|
||||
// on other platforms or when /proc is unavailable.
|
||||
// - goSysMB is the total memory obtained from the OS by the Go runtime
|
||||
// (heap, stacks, GC metadata, mspans, mcache, etc.). Includes
|
||||
// fragmentation and unused-but-mapped span overhead.
|
||||
// - goHeapInuseMB is the live, in-use Go heap (HeapInuse). Excludes
|
||||
// idle spans and runtime overhead.
|
||||
// - storeDataMB is the in-store packet byte estimate (transmissions +
|
||||
// observations). Subset of HeapInuse. Does not include index maps,
|
||||
// analytics caches, broadcast queues, or runtime overhead. Used as
|
||||
// the input to the eviction watermark.
|
||||
//
|
||||
// processRSSMB and storeDataMB are monotonic only relative to ingest +
|
||||
// eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB
|
||||
// fluctuate with GC.
|
||||
//
|
||||
// cgoBytesMB intentionally absent: this build uses the pure-Go
|
||||
// modernc.org/sqlite driver, so there is no cgo allocator to measure.
|
||||
// Reintroduce only if we ever switch back to mattn/go-sqlite3.
|
||||
type MemorySnapshot struct {
|
||||
ProcessRSSMB float64 `json:"processRSSMB"`
|
||||
GoHeapInuseMB float64 `json:"goHeapInuseMB"`
|
||||
GoSysMB float64 `json:"goSysMB"`
|
||||
StoreDataMB float64 `json:"storeDataMB"`
|
||||
}
|
||||
|
||||
// rssCache rate-limits the /proc/self/status read. Go memory stats are
|
||||
// already cached by Server.getMemStats (5s TTL). We use a tighter 1s TTL
|
||||
// here so processRSSMB stays reasonably fresh during ops debugging
|
||||
// without paying the syscall cost on every /api/stats hit.
|
||||
var (
|
||||
rssCacheMu sync.Mutex
|
||||
rssCacheValueMB float64
|
||||
rssCacheCachedAt time.Time
|
||||
)
|
||||
|
||||
const rssCacheTTL = 1 * time.Second
|
||||
|
||||
// getMemorySnapshot composes a MemorySnapshot using the Server's existing
|
||||
// runtime.MemStats cache (5s TTL, used by /api/health and /api/perf too)
|
||||
// plus a rate-limited /proc RSS read. storeDataMB is supplied by the
|
||||
// caller because the packet store is the source of truth.
|
||||
func (s *Server) getMemorySnapshot(storeDataMB float64) MemorySnapshot {
|
||||
ms := s.getMemStats()
|
||||
|
||||
rssCacheMu.Lock()
|
||||
if time.Since(rssCacheCachedAt) > rssCacheTTL {
|
||||
rssCacheValueMB = readProcRSSMB()
|
||||
rssCacheCachedAt = time.Now()
|
||||
}
|
||||
rssMB := rssCacheValueMB
|
||||
rssCacheMu.Unlock()
|
||||
|
||||
if rssMB <= 0 {
|
||||
// Fallback when /proc is unavailable (non-Linux, sandboxes, etc.).
|
||||
// runtime.Sys is an upper bound on Go-attributable memory and a
|
||||
// reasonable proxy for pure-Go builds.
|
||||
rssMB = float64(ms.Sys) / 1048576.0
|
||||
}
|
||||
|
||||
return MemorySnapshot{
|
||||
ProcessRSSMB: roundMB(rssMB),
|
||||
GoHeapInuseMB: roundMB(float64(ms.HeapInuse) / 1048576.0),
|
||||
GoSysMB: roundMB(float64(ms.Sys) / 1048576.0),
|
||||
StoreDataMB: roundMB(storeDataMB),
|
||||
}
|
||||
}
|
||||
|
||||
// readProcRSSMB parses /proc/self/status for the VmRSS line. Returns 0 on
|
||||
// any failure (file missing, malformed line, parse error) — the caller
|
||||
// then uses a runtime fallback. Linux only; macOS/Windows return 0.
|
||||
//
|
||||
// Safety notes (djb): the file path is hard-coded, no untrusted input is
|
||||
// concatenated. We bound the read at 8 KiB (the whole status file is
|
||||
// well under 4 KiB on modern kernels) so a corrupt /proc can't OOM us.
|
||||
// We only parse digits with strconv; no shell, no exec, no format strings.
|
||||
func readProcRSSMB() float64 {
|
||||
const maxStatusBytes = 8 * 1024
|
||||
f, err := os.Open("/proc/self/status")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
buf := make([]byte, maxStatusBytes)
|
||||
n, err := f.Read(buf)
|
||||
if err != nil && n == 0 {
|
||||
return 0
|
||||
}
|
||||
for _, line := range strings.Split(string(buf[:n]), "\n") {
|
||||
if !strings.HasPrefix(line, "VmRSS:") {
|
||||
continue
|
||||
}
|
||||
// Format: "VmRSS:\t 123456 kB"
|
||||
fields := strings.Fields(line[len("VmRSS:"):])
|
||||
if len(fields) < 2 {
|
||||
return 0
|
||||
}
|
||||
kb, err := strconv.ParseFloat(fields[0], 64)
|
||||
if err != nil || kb < 0 {
|
||||
return 0
|
||||
}
|
||||
// Unit is kB per kernel convention; convert to MB.
|
||||
return kb / 1024.0
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func roundMB(v float64) float64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(int64(v*10+0.5)) / 10.0
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEnrichNodeWithMultiByte(t *testing.T) {
|
||||
t.Run("nil entry leaves no fields", func(t *testing.T) {
|
||||
node := map[string]interface{}{"public_key": "abc123"}
|
||||
EnrichNodeWithMultiByte(node, nil)
|
||||
if _, ok := node["multi_byte_status"]; ok {
|
||||
t.Error("expected no multi_byte_status with nil entry")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("confirmed entry sets fields", func(t *testing.T) {
|
||||
node := map[string]interface{}{"public_key": "abc123"}
|
||||
entry := &MultiByteCapEntry{
|
||||
Status: "confirmed",
|
||||
Evidence: "advert",
|
||||
MaxHashSize: 2,
|
||||
}
|
||||
EnrichNodeWithMultiByte(node, entry)
|
||||
if node["multi_byte_status"] != "confirmed" {
|
||||
t.Errorf("expected confirmed, got %v", node["multi_byte_status"])
|
||||
}
|
||||
if node["multi_byte_evidence"] != "advert" {
|
||||
t.Errorf("expected advert, got %v", node["multi_byte_evidence"])
|
||||
}
|
||||
if node["multi_byte_max_hash_size"] != 2 {
|
||||
t.Errorf("expected 2, got %v", node["multi_byte_max_hash_size"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("suspected entry sets fields", func(t *testing.T) {
|
||||
node := map[string]interface{}{"public_key": "abc123"}
|
||||
entry := &MultiByteCapEntry{
|
||||
Status: "suspected",
|
||||
Evidence: "path",
|
||||
MaxHashSize: 2,
|
||||
}
|
||||
EnrichNodeWithMultiByte(node, entry)
|
||||
if node["multi_byte_status"] != "suspected" {
|
||||
t.Errorf("expected suspected, got %v", node["multi_byte_status"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown entry sets status unknown", func(t *testing.T) {
|
||||
node := map[string]interface{}{"public_key": "abc123"}
|
||||
entry := &MultiByteCapEntry{
|
||||
Status: "unknown",
|
||||
MaxHashSize: 1,
|
||||
}
|
||||
EnrichNodeWithMultiByte(node, entry)
|
||||
if node["multi_byte_status"] != "unknown" {
|
||||
t.Errorf("expected unknown, got %v", node["multi_byte_status"])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) {
|
||||
// Node A at lat=45, lon=-122. Candidate B1 at lat=45.1, lon=-122.1 (close).
|
||||
// Candidate B2 at lat=10, lon=10 (far away). Prefix "b0" matches both.
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0}
|
||||
|
||||
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2})
|
||||
|
||||
@@ -62,8 +62,8 @@ func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) {
|
||||
|
||||
// Test 2: Ambiguous edge merged with existing resolved edge (count accumulation).
|
||||
func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) {
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
|
||||
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB})
|
||||
|
||||
@@ -133,9 +133,9 @@ func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) {
|
||||
// Test 3: Ambiguous edge left as-is when resolution fails.
|
||||
func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) {
|
||||
// Two candidates, neither has GPS, no affinity data — resolution falls through.
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"}
|
||||
nodeB1 := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "B1"}
|
||||
nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffff", Name: "B2"}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"}
|
||||
nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "B1"}
|
||||
nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "B2"}
|
||||
|
||||
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2})
|
||||
|
||||
@@ -175,7 +175,7 @@ func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) {
|
||||
|
||||
// Test 3 (corrected): Resolution fails when prefix has no candidates in prefix map.
|
||||
func TestResolveAmbiguousEdges_NoMatch(t *testing.T) {
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"}
|
||||
// pm has no entries matching prefix "zz"
|
||||
pm := buildPrefixMap([]nodeInfo{nodeA})
|
||||
|
||||
@@ -215,8 +215,8 @@ func TestResolveAmbiguousEdges_NoMatch(t *testing.T) {
|
||||
// Test 6: Phase 1 edge collection unchanged (no regression).
|
||||
func TestPhase1EdgeCollection_Unchanged(t *testing.T) {
|
||||
// Build a simple graph and verify non-ambiguous edges are not touched.
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
|
||||
ts := time.Now().UTC().Format(time.RFC3339)
|
||||
payloadType := 4
|
||||
@@ -232,7 +232,7 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) {
|
||||
Observations: obs,
|
||||
}
|
||||
|
||||
store := ngTestStore([]nodeInfo{nodeA, nodeB, {Role: "repeater", PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx})
|
||||
store := ngTestStore([]nodeInfo{nodeA, nodeB, {PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx})
|
||||
graph := BuildFromStore(store)
|
||||
|
||||
edges := graph.Neighbors("aaaa1111")
|
||||
@@ -255,8 +255,8 @@ func TestPhase1EdgeCollection_Unchanged(t *testing.T) {
|
||||
|
||||
// Test 7: Merge preserves higher LastSeen timestamp.
|
||||
func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) {
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB})
|
||||
|
||||
graph := NewNeighborGraph()
|
||||
@@ -307,10 +307,10 @@ func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) {
|
||||
|
||||
// Test 5: Integration — node with both 1-byte and 2-byte prefix observations shows single entry.
|
||||
func TestIntegration_DualPrefixSingleNeighbor(t *testing.T) {
|
||||
nodeA := nodeInfo{Role: "repeater", PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{Role: "repeater", PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeB2 := nodeInfo{Role: "repeater", PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0}
|
||||
observer := nodeInfo{Role: "repeater", PublicKey: "cccc3333cccc3333", Name: "Observer"}
|
||||
nodeA := nodeInfo{PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
|
||||
nodeB := nodeInfo{PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
|
||||
nodeB2 := nodeInfo{PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0}
|
||||
observer := nodeInfo{PublicKey: "cccc3333cccc3333", Name: "Observer"}
|
||||
|
||||
ts := time.Now().UTC().Format(time.RFC3339)
|
||||
pt := 4
|
||||
|
||||
@@ -86,9 +86,9 @@ func TestBuildNeighborGraph_EmptyStore(t *testing.T) {
|
||||
func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) {
|
||||
// ADVERT from X, path=["R1_prefix"] → edges: X↔R1 and Observer↔R1
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["r1aa"]`, nowStr, ngFloatPtr(-10)),
|
||||
@@ -132,10 +132,10 @@ func TestBuildNeighborGraph_AdvertSingleHopPath(t *testing.T) {
|
||||
func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) {
|
||||
// ADVERT from X, path=["R1","R2"] → X↔R1 and Observer↔R2
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil),
|
||||
@@ -170,8 +170,8 @@ func TestBuildNeighborGraph_AdvertMultiHopPath(t *testing.T) {
|
||||
func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) {
|
||||
// ADVERT from X, path=[] → X↔Observer direct edge
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `[]`, nowStr, nil),
|
||||
@@ -195,8 +195,8 @@ func TestBuildNeighborGraph_AdvertZeroHop(t *testing.T) {
|
||||
func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) {
|
||||
// Non-ADVERT, path=[] → no edges
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `[]`, nowStr, nil),
|
||||
@@ -212,10 +212,10 @@ func TestBuildNeighborGraph_NonAdvertEmptyPath(t *testing.T) {
|
||||
func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) {
|
||||
// Non-ADVERT with path=["R1","R2"] → only Observer↔R2, NO originator edge
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil),
|
||||
@@ -236,9 +236,9 @@ func TestBuildNeighborGraph_NonAdvertOnlyObserverEdge(t *testing.T) {
|
||||
func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) {
|
||||
// Non-ADVERT with path=["R1"] → Observer↔R1 only
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["r1aa"]`, nowStr, nil),
|
||||
@@ -259,10 +259,10 @@ func TestBuildNeighborGraph_NonAdvertSingleHop(t *testing.T) {
|
||||
func TestBuildNeighborGraph_HashCollision(t *testing.T) {
|
||||
// Two nodes share prefix "a3" → ambiguous edge
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "a3bb1111", Name: "CandidateA"},
|
||||
{Role: "repeater", PublicKey: "a3bb2222", Name: "CandidateB"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "a3bb1111", Name: "CandidateA"},
|
||||
{PublicKey: "a3bb2222", Name: "CandidateB"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["a3bb"]`, nowStr, nil),
|
||||
@@ -308,13 +308,13 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) {
|
||||
// CandidateB has no known neighbors (Jaccard = 0).
|
||||
// An ambiguous edge X↔prefix "a3" with candidates [A, B] should auto-resolve to A.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "n1111111", Name: "N1"},
|
||||
{Role: "repeater", PublicKey: "n2222222", Name: "N2"},
|
||||
{Role: "repeater", PublicKey: "n3333333", Name: "N3"},
|
||||
{Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"},
|
||||
{Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "n1111111", Name: "N1"},
|
||||
{PublicKey: "n2222222", Name: "N2"},
|
||||
{PublicKey: "n3333333", Name: "N3"},
|
||||
{PublicKey: "a3001111", Name: "CandidateA"},
|
||||
{PublicKey: "a3002222", Name: "CandidateB"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
|
||||
// Create resolved edges: X↔N1, X↔N2, X↔N3, A↔N1, A↔N2, A↔N3
|
||||
@@ -373,11 +373,11 @@ func TestBuildNeighborGraph_ConfidenceAutoResolve(t *testing.T) {
|
||||
func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) {
|
||||
// Two candidates with identical neighbor sets → should NOT auto-resolve.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "n1111111", Name: "N1"},
|
||||
{Role: "repeater", PublicKey: "a3001111", Name: "CandidateA"},
|
||||
{Role: "repeater", PublicKey: "a3002222", Name: "CandidateB"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "n1111111", Name: "N1"},
|
||||
{PublicKey: "a3001111", Name: "CandidateA"},
|
||||
{PublicKey: "a3002222", Name: "CandidateB"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
|
||||
var txs []*StoreTx
|
||||
@@ -425,8 +425,8 @@ func TestBuildNeighborGraph_EqualScoresAmbiguous(t *testing.T) {
|
||||
func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) {
|
||||
// Observer's own prefix in path → should NOT create self-edge.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["obs0"]`, nowStr, nil),
|
||||
@@ -445,8 +445,8 @@ func TestBuildNeighborGraph_ObserverSelfEdgeGuard(t *testing.T) {
|
||||
func TestBuildNeighborGraph_OrphanPrefix(t *testing.T) {
|
||||
// Path contains prefix matching zero nodes → edge recorded as unresolved.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["ff99"]`, nowStr, nil),
|
||||
@@ -506,9 +506,9 @@ func TestAffinityScore_StaleAndLow(t *testing.T) {
|
||||
|
||||
func TestBuildNeighborGraph_CountAccumulation(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
|
||||
var txs []*StoreTx
|
||||
@@ -535,10 +535,10 @@ func TestBuildNeighborGraph_CountAccumulation(t *testing.T) {
|
||||
|
||||
func TestBuildNeighborGraph_MultipleObservers(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Obs1"},
|
||||
{Role: "repeater", PublicKey: "obs00002", Name: "Obs2"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "obs00001", Name: "Obs1"},
|
||||
{PublicKey: "obs00002", Name: "Obs2"},
|
||||
}
|
||||
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
@@ -565,9 +565,9 @@ func TestBuildNeighborGraph_MultipleObservers(t *testing.T) {
|
||||
|
||||
func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
|
||||
tx := ngMakeTx(1, 4, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
@@ -592,10 +592,10 @@ func TestBuildNeighborGraph_TimeDecayOldObservations(t *testing.T) {
|
||||
func TestBuildNeighborGraph_ADVERTOnlyConstraint(t *testing.T) {
|
||||
// Non-ADVERT: should NOT create originator↔path[0] edge, only observer↔path[last].
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{Role: "repeater", PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{Role: "repeater", PublicKey: "obs00001", Name: "Observer"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeX"},
|
||||
{PublicKey: "r1aabbcc", Name: "R1"},
|
||||
{PublicKey: "r2ddeeff", Name: "R2"},
|
||||
{PublicKey: "obs00001", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 2, ngFromNodeJSON("aaaa1111"), []*StoreObs{
|
||||
ngMakeObs("obs00001", `["r1aa","r2dd"]`, nowStr, nil),
|
||||
@@ -631,9 +631,9 @@ func ngPubKeyJSON(pubkey string) string {
|
||||
func TestBuildNeighborGraph_AdvertPubKeyField(t *testing.T) {
|
||||
// Real ADVERTs use "pubKey", not "from_node". Verify the builder handles it.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"},
|
||||
{Role: "repeater", PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"},
|
||||
{Role: "repeater", PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"},
|
||||
{PublicKey: "99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234", Name: "Originator"},
|
||||
{PublicKey: "r1aabbccdd001122334455667788990011223344556677889900112233445566", Name: "R1"},
|
||||
{PublicKey: "obs0000100112233445566778899001122334455667788990011223344556677", Name: "Observer"},
|
||||
}
|
||||
tx := ngMakeTx(1, 4, ngPubKeyJSON("99bf37abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234"), []*StoreObs{
|
||||
ngMakeObs("obs0000100112233445566778899001122334455667788990011223344556677", `["r1"]`, nowStr, ngFloatPtr(-8.5)),
|
||||
@@ -666,10 +666,10 @@ func TestBuildNeighborGraph_OneByteHashPrefixes(t *testing.T) {
|
||||
// Real-world scenario: 1-byte hash prefixes with multiple candidates.
|
||||
// Should create edges (possibly ambiguous) rather than empty graph.
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"},
|
||||
{Role: "repeater", PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"},
|
||||
{Role: "repeater", PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"},
|
||||
{Role: "repeater", PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"},
|
||||
{PublicKey: "c0dedad400000000000000000000000000000000000000000000000000000001", Name: "NodeC0-1"},
|
||||
{PublicKey: "c0dedad900000000000000000000000000000000000000000000000000000002", Name: "NodeC0-2"},
|
||||
{PublicKey: "a3bbccdd00000000000000000000000000000000000000000000000000000003", Name: "Originator"},
|
||||
{PublicKey: "obs1234500000000000000000000000000000000000000000000000000000004", Name: "Observer"},
|
||||
}
|
||||
// ADVERT from Originator with 1-byte path hop "c0"
|
||||
tx := ngMakeTx(1, 4, ngPubKeyJSON("a3bbccdd00000000000000000000000000000000000000000000000000000003"), []*StoreObs{
|
||||
@@ -809,10 +809,10 @@ func TestExtractFromNode_UsesCachedParse(t *testing.T) {
|
||||
func BenchmarkBuildFromStore(b *testing.B) {
|
||||
// Simulate a dataset with many packets and repeated pubkeys
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aaaa1111", Name: "NodeA"},
|
||||
{Role: "repeater", PublicKey: "bbbb2222", Name: "NodeB"},
|
||||
{Role: "repeater", PublicKey: "cccc3333", Name: "NodeC"},
|
||||
{Role: "repeater", PublicKey: "dddd4444", Name: "NodeD"},
|
||||
{PublicKey: "aaaa1111", Name: "NodeA"},
|
||||
{PublicKey: "bbbb2222", Name: "NodeB"},
|
||||
{PublicKey: "cccc3333", Name: "NodeC"},
|
||||
{PublicKey: "dddd4444", Name: "NodeD"},
|
||||
}
|
||||
const numPackets = 1000
|
||||
packets := make([]*StoreTx, 0, numPackets)
|
||||
|
||||
+23
-167
@@ -20,10 +20,11 @@ var persistSem = make(chan struct{}, 1)
|
||||
// ensureNeighborEdgesTable creates the neighbor_edges table if it doesn't exist.
|
||||
// Uses a separate read-write connection since the main DB is read-only.
|
||||
func ensureNeighborEdgesTable(dbPath string) error {
|
||||
rw, err := cachedRW(dbPath)
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open rw for neighbor_edges: %w", err)
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
_, err = rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges (
|
||||
node_a TEXT NOT NULL,
|
||||
@@ -128,11 +129,12 @@ func asyncPersistResolvedPathsAndEdges(dbPath string, obsUpdates []persistObsUpd
|
||||
go func() {
|
||||
defer func() { <-persistSem }()
|
||||
|
||||
rw, err := cachedRW(dbPath)
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[store] %s rw open error: %v", logPrefix, err)
|
||||
return
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
if len(obsUpdates) > 0 {
|
||||
sqlTx, err := rw.Begin()
|
||||
@@ -247,10 +249,11 @@ func buildAndPersistEdges(store *PacketStore, rw *sql.DB) int {
|
||||
|
||||
// ensureResolvedPathColumn adds the resolved_path column to observations if missing.
|
||||
func ensureResolvedPathColumn(dbPath string) error {
|
||||
rw, err := cachedRW(dbPath)
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
// Check if column already exists
|
||||
rows, err := rw.Query("PRAGMA table_info(observations)")
|
||||
@@ -278,115 +281,6 @@ func ensureResolvedPathColumn(dbPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureObserverInactiveColumn adds the inactive column to observers if missing.
|
||||
// The column was originally added by ingestor migration (cmd/ingestor/db.go:344) to
|
||||
// support soft-delete via RemoveStaleObservers + filtered reads (PR #954). When the
|
||||
// server starts against a DB that was never touched by the ingestor (e.g. the e2e
|
||||
// fixture), the column is missing and read queries that filter on it (GetObservers,
|
||||
// GetStats) silently fail with "no such column: inactive" — leaving /api/observers
|
||||
// returning empty.
|
||||
func ensureObserverInactiveColumn(dbPath string) error {
|
||||
rw, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := rw.Query("PRAGMA table_info(observers)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil && colName == "inactive" {
|
||||
return nil // already exists
|
||||
}
|
||||
}
|
||||
|
||||
_, err = rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0")
|
||||
if err != nil {
|
||||
return fmt.Errorf("add inactive column: %w", err)
|
||||
}
|
||||
log.Println("[store] Added inactive column to observers")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureLastPacketAtColumn adds the last_packet_at column to observers if missing.
|
||||
// The column was originally added by ingestor migration (observers_last_packet_at_v1)
|
||||
// to track the most recent packet observation time separately from status updates.
|
||||
// When the server starts against a DB that was never touched by the ingestor (e.g.
|
||||
// the e2e fixture), the column is missing and read queries that reference it
|
||||
// (GetObservers, GetObserverByID) fail with "no such column: last_packet_at".
|
||||
func ensureLastPacketAtColumn(dbPath string) error {
|
||||
rw, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := rw.Query("PRAGMA table_info(observers)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil && colName == "last_packet_at" {
|
||||
return nil // already exists
|
||||
}
|
||||
}
|
||||
|
||||
_, err = rw.Exec("ALTER TABLE observers ADD COLUMN last_packet_at TEXT")
|
||||
if err != nil {
|
||||
return fmt.Errorf("add last_packet_at column: %w", err)
|
||||
}
|
||||
log.Println("[store] Added last_packet_at column to observers")
|
||||
return nil
|
||||
}
|
||||
|
||||
// softDeleteBlacklistedObservers marks observers matching the blacklist as
|
||||
// inactive=1 so they are hidden from API responses. Runs once at startup.
|
||||
func softDeleteBlacklistedObservers(dbPath string, blacklist []string) {
|
||||
rw, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[observer-blacklist] warning: could not open DB for soft-delete: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
placeholders := make([]string, 0, len(blacklist))
|
||||
args := make([]interface{}, 0, len(blacklist))
|
||||
for _, pk := range blacklist {
|
||||
trimmed := strings.TrimSpace(pk)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
placeholders = append(placeholders, "LOWER(?)")
|
||||
args = append(args, trimmed)
|
||||
}
|
||||
if len(placeholders) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
query := "UPDATE observers SET inactive = 1 WHERE LOWER(id) IN (" + strings.Join(placeholders, ",") + ") AND (inactive IS NULL OR inactive = 0)"
|
||||
result, err := rw.Exec(query, args...)
|
||||
if err != nil {
|
||||
log.Printf("[observer-blacklist] warning: soft-delete failed: %v", err)
|
||||
return
|
||||
}
|
||||
if n, _ := result.RowsAffected(); n > 0 {
|
||||
log.Printf("[observer-blacklist] soft-deleted %d blacklisted observer(s)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// resolvePathForObs resolves hop prefixes to full pubkeys for an observation.
|
||||
// Returns nil if path is empty.
|
||||
func resolvePathForObs(pathJSON, observerID string, tx *StoreTx, pm *prefixMap, graph *NeighborGraph) []*string {
|
||||
@@ -487,13 +381,7 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
|
||||
}
|
||||
}
|
||||
for _, obs := range tx.Observations {
|
||||
// Check if this observation has been resolved: look up in the index.
|
||||
// If the tx has no reverse-map entries AND path is non-empty, it needs backfill.
|
||||
hasRP := false
|
||||
if _, ok := store.resolvedPubkeyReverse[tx.ID]; ok {
|
||||
hasRP = true
|
||||
}
|
||||
if !hasRP && obs.PathJSON != "" && obs.PathJSON != "[]" {
|
||||
if obs.ResolvedPath == nil && obs.PathJSON != "" && obs.PathJSON != "[]" {
|
||||
allPending = append(allPending, obsRef{
|
||||
obsID: obs.ID,
|
||||
pathJSON: obs.PathJSON,
|
||||
@@ -522,12 +410,16 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
|
||||
var rw *sql.DB
|
||||
if dbPath != "" {
|
||||
var err error
|
||||
rw, err = cachedRW(dbPath)
|
||||
rw, err = openRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[store] async backfill: open rw error: %v", err)
|
||||
}
|
||||
}
|
||||
// rw is cached process-wide; do not close
|
||||
defer func() {
|
||||
if rw != nil {
|
||||
rw.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
totalProcessed := 0
|
||||
for totalProcessed < totalPending {
|
||||
@@ -590,61 +482,24 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
|
||||
}
|
||||
}
|
||||
|
||||
// Update in-memory state: update resolved pubkey index, re-pick best observation,
|
||||
// and invalidate LRU cache entries for backfilled observations (#800).
|
||||
//
|
||||
// Lock ordering: always take s.mu BEFORE lruMu. The read path
|
||||
// (fetchResolvedPathForObs) takes lruMu independently of s.mu,
|
||||
// so we must NOT hold s.mu while taking lruMu. Instead, collect
|
||||
// obsIDs to invalidate under s.mu, release it, then take lruMu.
|
||||
// Update in-memory state and re-pick best observation under a single
|
||||
// write lock. The per-tx pickBestObservation is O(observations) which is
|
||||
// typically <10 per tx — negligible cost vs. the race risk of splitting
|
||||
// the lock (pollAndMerge can append to tx.Observations concurrently).
|
||||
store.mu.Lock()
|
||||
affectedSet := make(map[string]bool)
|
||||
lruInvalidate := make([]int, 0, len(results))
|
||||
for _, r := range results {
|
||||
// Remove old index entries for this tx, then re-add with new pubkeys
|
||||
if obs, ok := store.byObsID[r.obsID]; ok {
|
||||
obs.ResolvedPath = r.rp
|
||||
}
|
||||
if !affectedSet[r.txHash] {
|
||||
affectedSet[r.txHash] = true
|
||||
if tx, ok := store.byHash[r.txHash]; ok {
|
||||
store.removeFromResolvedPubkeyIndex(tx.ID)
|
||||
pickBestObservation(tx)
|
||||
}
|
||||
}
|
||||
// Add new resolved pubkeys to index
|
||||
if tx, ok := store.byHash[r.txHash]; ok {
|
||||
pks := extractResolvedPubkeys(r.rp)
|
||||
store.addToResolvedPubkeyIndex(tx.ID, pks)
|
||||
// Update byNode for relay nodes
|
||||
for _, pk := range pks {
|
||||
store.addToByNode(tx, pk)
|
||||
}
|
||||
// Update byPathHop resolved-key entries
|
||||
hopsSeen := make(map[string]bool)
|
||||
for _, hop := range txGetParsedPath(tx) {
|
||||
hopsSeen[strings.ToLower(hop)] = true
|
||||
}
|
||||
for _, pk := range pks {
|
||||
if !hopsSeen[pk] {
|
||||
hopsSeen[pk] = true
|
||||
store.byPathHop[pk] = append(store.byPathHop[pk], tx)
|
||||
}
|
||||
}
|
||||
}
|
||||
lruInvalidate = append(lruInvalidate, r.obsID)
|
||||
}
|
||||
// Re-pick best observation for affected transmissions
|
||||
for txHash := range affectedSet {
|
||||
if tx, ok := store.byHash[txHash]; ok {
|
||||
pickBestObservation(tx)
|
||||
}
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
// Invalidate LRU entries AFTER releasing s.mu to maintain lock
|
||||
// ordering (lruMu must never be taken while s.mu is held).
|
||||
store.lruMu.Lock()
|
||||
for _, obsID := range lruInvalidate {
|
||||
store.lruDelete(obsID)
|
||||
}
|
||||
store.lruMu.Unlock()
|
||||
}
|
||||
|
||||
totalProcessed += len(chunk)
|
||||
@@ -752,10 +607,11 @@ func PruneNeighborEdges(dbPath string, graph *NeighborGraph, maxAgeDays int) (in
|
||||
|
||||
// 1. Prune from SQLite using a read-write connection
|
||||
var dbPruned int64
|
||||
rw, err := cachedRW(dbPath)
|
||||
rw, err := openRW(dbPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune neighbor_edges: open rw: %w", err)
|
||||
}
|
||||
defer rw.Close()
|
||||
res, err := rw.Exec("DELETE FROM neighbor_edges WHERE last_seen < ?", cutoff.Format(time.RFC3339))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune neighbor_edges: %w", err)
|
||||
|
||||
@@ -38,7 +38,7 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) {
|
||||
observer_id TEXT, observer_name TEXT, direction TEXT,
|
||||
snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT,
|
||||
resolved_path TEXT, raw_hex TEXT
|
||||
resolved_path TEXT
|
||||
)`)
|
||||
conn.Exec(`CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
|
||||
@@ -58,8 +58,8 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) {
|
||||
func TestResolvePathForObs(t *testing.T) {
|
||||
// Build a prefix map with known nodes
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{Role: "repeater", PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"},
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{PublicKey: "bbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-BB"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
graph := NewNeighborGraph()
|
||||
@@ -97,7 +97,7 @@ func TestResolvePathForObs_EmptyPath(t *testing.T) {
|
||||
|
||||
func TestResolvePathForObs_Unresolvable(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
@@ -203,14 +203,14 @@ func TestLoadNeighborEdgesFromDB(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
|
||||
// After #800 refactor, resolved_path is no longer stored on StoreTx/StoreObs structs.
|
||||
// Broadcast maps carry resolved_path from the decode-window, not from struct fields.
|
||||
// This test verifies pickBestObservation no longer sets ResolvedPath on tx.
|
||||
// Verify resolved_path appears in broadcast maps
|
||||
pk := "aabbccdd"
|
||||
obs := &StoreObs{
|
||||
ID: 1,
|
||||
ObserverID: "obs1",
|
||||
ObserverName: "Observer 1",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
}
|
||||
|
||||
@@ -221,26 +221,32 @@ func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
|
||||
}
|
||||
pickBestObservation(tx)
|
||||
|
||||
// tx should NOT have a ResolvedPath field anymore (compile-time guard)
|
||||
// Verify the best observation's fields are propagated correctly
|
||||
if tx.ObserverID != "obs1" {
|
||||
t.Errorf("expected ObserverID=obs1, got %s", tx.ObserverID)
|
||||
if tx.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be set on tx after pickBestObservation")
|
||||
}
|
||||
if *tx.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("expected resolved path to be aabbccdd, got %s", *tx.ResolvedPath[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInTxToMap(t *testing.T) {
|
||||
// After #800, txToMap no longer includes resolved_path from the struct.
|
||||
// resolved_path is only available via on-demand SQL fetch (txToMapWithRP).
|
||||
pk := "aabbccdd"
|
||||
tx := &StoreTx{
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
PathJSON: `["aa"]`,
|
||||
obsKeys: make(map[string]bool),
|
||||
ID: 1,
|
||||
Hash: "abc123",
|
||||
PathJSON: `["aa"]`,
|
||||
ResolvedPath: []*string{&pk},
|
||||
obsKeys: make(map[string]bool),
|
||||
}
|
||||
|
||||
m := txToMap(tx)
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should not be in txToMap output (removed in #800)")
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path not in txToMap output")
|
||||
}
|
||||
rpSlice, ok := rp.([]*string)
|
||||
if !ok || len(rpSlice) != 1 || *rpSlice[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected resolved_path: %v", rp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,7 +270,7 @@ func TestEnsureResolvedPathColumn(t *testing.T) {
|
||||
conn, _ := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
|
||||
conn.Exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER,
|
||||
observer_id TEXT, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
observer_id TEXT, path_json TEXT, timestamp TEXT
|
||||
)`)
|
||||
conn.Close()
|
||||
|
||||
@@ -359,21 +365,27 @@ func TestLoadWithResolvedPath(t *testing.T) {
|
||||
t.Fatalf("expected 1 observation, got %d", len(tx.Observations))
|
||||
}
|
||||
|
||||
// After #800, ResolvedPath is not stored on StoreObs struct.
|
||||
// Instead, resolved pubkeys are in the membership index.
|
||||
_ = tx.Observations[0] // obs exists
|
||||
h := resolvedPubkeyHash("aabbccdd")
|
||||
if len(store.resolvedPubkeyIndex[h]) != 1 {
|
||||
t.Fatal("expected resolved pubkey to be indexed")
|
||||
obs := tx.Observations[0]
|
||||
if obs.ResolvedPath == nil {
|
||||
t.Fatal("expected ResolvedPath to be loaded")
|
||||
}
|
||||
if len(obs.ResolvedPath) != 1 || *obs.ResolvedPath[0] != "aabbccdd" {
|
||||
t.Errorf("unexpected ResolvedPath: %v", obs.ResolvedPath)
|
||||
}
|
||||
|
||||
// Check that pickBestObservation propagated resolved_path to tx
|
||||
if tx.ResolvedPath == nil || len(tx.ResolvedPath) != 1 {
|
||||
t.Error("expected ResolvedPath to be propagated to tx")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedPathInAPIResponse(t *testing.T) {
|
||||
// After #800, TransmissionResp no longer has ResolvedPath field.
|
||||
// resolved_path is included dynamically in map-based API responses.
|
||||
// Test that TransmissionResp properly marshals resolved_path
|
||||
pk := "aabbccddee"
|
||||
resp := TransmissionResp{
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
ID: 1,
|
||||
Hash: "test",
|
||||
ResolvedPath: []*string{&pk, nil},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(resp)
|
||||
@@ -384,9 +396,19 @@ func TestResolvedPathInAPIResponse(t *testing.T) {
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(data, &m)
|
||||
|
||||
// resolved_path should NOT be in the marshaled JSON
|
||||
if _, ok := m["resolved_path"]; ok {
|
||||
t.Error("resolved_path should not be in TransmissionResp JSON (#800)")
|
||||
rp, ok := m["resolved_path"]
|
||||
if !ok {
|
||||
t.Fatal("resolved_path missing from JSON")
|
||||
}
|
||||
rpArr, ok := rp.([]interface{})
|
||||
if !ok || len(rpArr) != 2 {
|
||||
t.Fatalf("unexpected resolved_path shape: %v", rp)
|
||||
}
|
||||
if rpArr[0] != "aabbccddee" {
|
||||
t.Errorf("first element wrong: %v", rpArr[0])
|
||||
}
|
||||
if rpArr[1] != nil {
|
||||
t.Errorf("second element should be null: %v", rpArr[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,8 +459,8 @@ func TestExtractEdgesFromObs_NonAdvertNoPath(t *testing.T) {
|
||||
|
||||
func TestExtractEdgesFromObs_WithPath(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{Role: "repeater", PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{Role: "repeater", PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"},
|
||||
{PublicKey: "aabbccddee1234567890aabbccddee1234567890aabbccddee1234567890aabb", Name: "Node-AA"},
|
||||
{PublicKey: "ffgghhii1234567890aabbccddee1234567890aabbccddee1234567890aabb11", Name: "Node-FF"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
@@ -538,62 +560,3 @@ func TestOpenRW_BusyTimeout(t *testing.T) {
|
||||
t.Errorf("expected busy_timeout=5000, got %d", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureLastPacketAtColumn(t *testing.T) {
|
||||
// Create a temp DB with observers table missing last_packet_at
|
||||
dir := t.TempDir()
|
||||
dbPath := dir + "/test.db"
|
||||
db, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = db.Exec(`CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
last_seen TEXT,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
inactive INTEGER DEFAULT 0
|
||||
)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
// First call: should add the column
|
||||
if err := ensureLastPacketAtColumn(dbPath); err != nil {
|
||||
t.Fatalf("first call failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify column exists
|
||||
db2, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db2.Close()
|
||||
|
||||
var found bool
|
||||
rows, err := db2.Query("PRAGMA table_info(observers)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var colName string
|
||||
var colType sql.NullString
|
||||
var notNull, pk int
|
||||
var dflt sql.NullString
|
||||
if rows.Scan(&cid, &colName, &colType, ¬Null, &dflt, &pk) == nil && colName == "last_packet_at" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("last_packet_at column not found after migration")
|
||||
}
|
||||
|
||||
// Idempotency: second call should succeed without error
|
||||
if err := ensureLastPacketAtColumn(dbPath); err != nil {
|
||||
t.Fatalf("idempotent call failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConfigIsObserverBlacklisted(t *testing.T) {
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"OBS1", "obs2", " Obs3 "},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{"OBS1", true},
|
||||
{"obs1", true}, // case-insensitive
|
||||
{"OBS2", true},
|
||||
{"Obs3", true}, // whitespace trimmed
|
||||
{"obs4", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := cfg.IsObserverBlacklisted(tt.id)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsObserverBlacklisted(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigIsObserverBlacklistedEmpty(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("empty blacklist should not match anything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigIsObserverBlacklistedNil(t *testing.T) {
|
||||
var cfg *Config
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("nil config should not match anything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverBlacklistFiltersHandleObservers(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', datetime('now'))")
|
||||
db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"badobs"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp ObserverListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
for _, obs := range resp.Observers {
|
||||
if obs.ID == "badobs" {
|
||||
t.Error("blacklisted observer should not appear in observers list")
|
||||
}
|
||||
}
|
||||
|
||||
foundGood := false
|
||||
for _, obs := range resp.Observers {
|
||||
if obs.ID == "goodobs" {
|
||||
foundGood = true
|
||||
}
|
||||
}
|
||||
if !foundGood {
|
||||
t.Error("non-blacklisted observer should appear in observers list")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverBlacklistFiltersObserverDetail(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))")
|
||||
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"badobs"},
|
||||
}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/observers/badobs", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("expected 404 for blacklisted observer detail, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoObserverBlacklistPassesAll(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', datetime('now'))")
|
||||
|
||||
cfg := &Config{}
|
||||
srv := NewServer(db, cfg, NewHub())
|
||||
srv.RegisterRoutes(setupTestRouter(srv))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
w := httptest.NewRecorder()
|
||||
srv.router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
|
||||
var resp ObserverListResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("failed to parse response: %v", err)
|
||||
}
|
||||
|
||||
foundSome := false
|
||||
for _, obs := range resp.Observers {
|
||||
if obs.ID == "someobs" {
|
||||
foundSome = true
|
||||
}
|
||||
}
|
||||
if !foundSome {
|
||||
t.Error("without blacklist, observer should appear")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverBlacklistConcurrent(t *testing.T) {
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"AA", "BB", "CC"},
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
for i := 0; i < 50; i++ {
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for j := 0; j < 100; j++ {
|
||||
cfg.IsObserverBlacklisted("AA")
|
||||
cfg.IsObserverBlacklisted("DD")
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,6 @@ func routeDescriptions() map[string]routeMeta {
|
||||
"POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true},
|
||||
"POST /api/admin/prune": {Summary: "Prune old data", Description: "Deletes packets and nodes older than the configured retention period.", Tag: "admin", Auth: true},
|
||||
"GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true},
|
||||
"GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-<unix>.db.", Tag: "admin", Auth: true},
|
||||
|
||||
// Packets
|
||||
"GET /api/packets": {Summary: "List packets", Description: "Returns decoded packets with filtering, sorting, and pagination.", Tag: "packets",
|
||||
|
||||
@@ -1,427 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── Path Inspector ────────────────────────────────────────────────────────────
|
||||
// POST /api/paths/inspect — beam-search scorer for prefix path candidates.
|
||||
// Spec: issue #944 §2.1–2.5.
|
||||
|
||||
// pathInspectRequest is the JSON body for the inspect endpoint.
|
||||
type pathInspectRequest struct {
|
||||
Prefixes []string `json:"prefixes"`
|
||||
Context *pathInspectContext `json:"context,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
}
|
||||
|
||||
type pathInspectContext struct {
|
||||
ObserverID string `json:"observerId,omitempty"`
|
||||
Since string `json:"since,omitempty"`
|
||||
Until string `json:"until,omitempty"`
|
||||
}
|
||||
|
||||
// pathCandidate is one scored candidate path in the response.
|
||||
type pathCandidate struct {
|
||||
Path []string `json:"path"`
|
||||
Names []string `json:"names"`
|
||||
Score float64 `json:"score"`
|
||||
Speculative bool `json:"speculative"`
|
||||
Evidence pathEvidence `json:"evidence"`
|
||||
}
|
||||
|
||||
type pathEvidence struct {
|
||||
PerHop []hopEvidence `json:"perHop"`
|
||||
}
|
||||
|
||||
type hopEvidence struct {
|
||||
Prefix string `json:"prefix"`
|
||||
CandidatesConsidered int `json:"candidatesConsidered"`
|
||||
Chosen string `json:"chosen"`
|
||||
EdgeWeight float64 `json:"edgeWeight"`
|
||||
Alternatives []hopAlternative `json:"alternatives,omitempty"`
|
||||
}
|
||||
|
||||
// hopAlternative shows a candidate that was considered but not chosen for this hop.
|
||||
type hopAlternative struct {
|
||||
PublicKey string `json:"publicKey"`
|
||||
Name string `json:"name"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
type pathInspectResponse struct {
|
||||
Candidates []pathCandidate `json:"candidates"`
|
||||
Input map[string]interface{} `json:"input"`
|
||||
Stats map[string]interface{} `json:"stats"`
|
||||
}
|
||||
|
||||
// beamEntry represents a partial path being extended during beam search.
|
||||
type beamEntry struct {
|
||||
pubkeys []string
|
||||
names []string
|
||||
evidence []hopEvidence
|
||||
score float64 // product of per-hop scores (pre-geometric-mean)
|
||||
}
|
||||
|
||||
const (
|
||||
beamWidth = 20
|
||||
maxInputHops = 64
|
||||
maxPrefixBytes = 3
|
||||
maxRequestItems = 64
|
||||
geoMaxKm = 50.0
|
||||
hopScoreFloor = 0.05
|
||||
speculativeThreshold = 0.7
|
||||
inspectCacheTTL = 30 * time.Second
|
||||
inspectBodyLimit = 4096
|
||||
)
|
||||
|
||||
// Weights per spec §2.3.
|
||||
const (
|
||||
wEdge = 0.35
|
||||
wGeo = 0.20
|
||||
wRecency = 0.15
|
||||
wSelectivity = 0.30
|
||||
)
|
||||
|
||||
func (s *Server) handlePathInspect(w http.ResponseWriter, r *http.Request) {
|
||||
// Body limit per spec §2.1.
|
||||
r.Body = http.MaxBytesReader(w, r.Body, inspectBodyLimit)
|
||||
|
||||
var req pathInspectRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, `{"error":"invalid JSON"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate prefixes.
|
||||
if len(req.Prefixes) == 0 {
|
||||
http.Error(w, `{"error":"prefixes required"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(req.Prefixes) > maxRequestItems {
|
||||
http.Error(w, `{"error":"too many prefixes (max 64)"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Normalize + validate each prefix.
|
||||
prefixByteLen := -1
|
||||
for i, p := range req.Prefixes {
|
||||
p = strings.ToLower(strings.TrimSpace(p))
|
||||
req.Prefixes[i] = p
|
||||
if len(p) == 0 || len(p)%2 != 0 {
|
||||
http.Error(w, `{"error":"prefixes must be even-length hex"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if _, err := hex.DecodeString(p); err != nil {
|
||||
http.Error(w, `{"error":"prefixes must be valid hex"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
byteLen := len(p) / 2
|
||||
if byteLen > maxPrefixBytes {
|
||||
http.Error(w, `{"error":"prefix exceeds 3 bytes"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if prefixByteLen == -1 {
|
||||
prefixByteLen = byteLen
|
||||
} else if byteLen != prefixByteLen {
|
||||
http.Error(w, `{"error":"mixed prefix lengths not allowed"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
limit := req.Limit
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
if limit > 50 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
// Check cache.
|
||||
cacheKey := s.store.inspectCacheKey(req)
|
||||
s.store.inspectMu.RLock()
|
||||
if cached, ok := s.store.inspectCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
|
||||
s.store.inspectMu.RUnlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(cached.data)
|
||||
return
|
||||
}
|
||||
s.store.inspectMu.RUnlock()
|
||||
|
||||
// Snapshot data under read lock.
|
||||
nodes, pm := s.store.getCachedNodesAndPM()
|
||||
|
||||
// Build pubkey→nodeInfo map for O(1) geo lookup in scorer.
|
||||
nodeByPK := make(map[string]*nodeInfo, len(nodes))
|
||||
for i := range nodes {
|
||||
nodeByPK[strings.ToLower(nodes[i].PublicKey)] = &nodes[i]
|
||||
}
|
||||
|
||||
// Get neighbor graph; handle cold start.
|
||||
graph := s.store.graph
|
||||
if graph == nil || graph.IsStale() {
|
||||
rebuilt := make(chan struct{})
|
||||
go func() {
|
||||
s.store.ensureNeighborGraph()
|
||||
close(rebuilt)
|
||||
}()
|
||||
select {
|
||||
case <-rebuilt:
|
||||
graph = s.store.graph
|
||||
case <-time.After(2 * time.Second):
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"retry": true})
|
||||
return
|
||||
}
|
||||
if graph == nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"retry": true})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
start := now
|
||||
|
||||
// Beam search.
|
||||
beam := s.store.beamSearch(req.Prefixes, pm, graph, nodeByPK, now)
|
||||
|
||||
// Sort by score descending, take top limit.
|
||||
sortBeam(beam)
|
||||
if len(beam) > limit {
|
||||
beam = beam[:limit]
|
||||
}
|
||||
|
||||
// Build response with per-hop alternatives (spec §2.7, M2 fix).
|
||||
candidates := make([]pathCandidate, 0, len(beam))
|
||||
for _, entry := range beam {
|
||||
nHops := len(entry.pubkeys)
|
||||
var score float64
|
||||
if nHops > 0 {
|
||||
score = math.Pow(entry.score, 1.0/float64(nHops))
|
||||
}
|
||||
|
||||
// Populate per-hop alternatives: other candidates at each hop that weren't chosen.
|
||||
evidence := make([]hopEvidence, len(entry.evidence))
|
||||
copy(evidence, entry.evidence)
|
||||
for hi, ev := range evidence {
|
||||
if hi >= len(req.Prefixes) {
|
||||
break
|
||||
}
|
||||
prefix := req.Prefixes[hi]
|
||||
allCands := pm.m[prefix]
|
||||
var alts []hopAlternative
|
||||
for _, c := range allCands {
|
||||
if !canAppearInPath(c.Role) || c.PublicKey == ev.Chosen {
|
||||
continue
|
||||
}
|
||||
// Score this alternative in context of the partial path up to this hop.
|
||||
var partialEntry beamEntry
|
||||
if hi > 0 {
|
||||
partialEntry = beamEntry{pubkeys: entry.pubkeys[:hi], names: entry.names[:hi], score: 1.0}
|
||||
}
|
||||
altScore := s.store.scoreHop(partialEntry, c, ev.CandidatesConsidered, graph, nodeByPK, now, hi)
|
||||
alts = append(alts, hopAlternative{PublicKey: c.PublicKey, Name: c.Name, Score: math.Round(altScore*1000) / 1000})
|
||||
}
|
||||
// Sort alts by score desc, cap at 5.
|
||||
sort.Slice(alts, func(i, j int) bool { return alts[i].Score > alts[j].Score })
|
||||
if len(alts) > 5 {
|
||||
alts = alts[:5]
|
||||
}
|
||||
evidence[hi] = hopEvidence{
|
||||
Prefix: ev.Prefix,
|
||||
CandidatesConsidered: ev.CandidatesConsidered,
|
||||
Chosen: ev.Chosen,
|
||||
EdgeWeight: ev.EdgeWeight,
|
||||
Alternatives: alts,
|
||||
}
|
||||
}
|
||||
|
||||
candidates = append(candidates, pathCandidate{
|
||||
Path: entry.pubkeys,
|
||||
Names: entry.names,
|
||||
Score: math.Round(score*1000) / 1000,
|
||||
Speculative: score < speculativeThreshold,
|
||||
Evidence: pathEvidence{PerHop: evidence},
|
||||
})
|
||||
}
|
||||
|
||||
elapsed := time.Since(start).Milliseconds()
|
||||
resp := pathInspectResponse{
|
||||
Candidates: candidates,
|
||||
Input: map[string]interface{}{
|
||||
"prefixes": req.Prefixes,
|
||||
"hops": len(req.Prefixes),
|
||||
},
|
||||
Stats: map[string]interface{}{
|
||||
"beamWidth": beamWidth,
|
||||
"expansionsRun": len(req.Prefixes) * beamWidth,
|
||||
"elapsedMs": elapsed,
|
||||
},
|
||||
}
|
||||
|
||||
// Cache result (and evict stale entries).
|
||||
s.store.inspectMu.Lock()
|
||||
if s.store.inspectCache == nil {
|
||||
s.store.inspectCache = make(map[string]*inspectCachedResult)
|
||||
}
|
||||
now2 := time.Now()
|
||||
for k, v := range s.store.inspectCache {
|
||||
if now2.After(v.expiresAt) {
|
||||
delete(s.store.inspectCache, k)
|
||||
}
|
||||
}
|
||||
s.store.inspectCache[cacheKey] = &inspectCachedResult{
|
||||
data: resp,
|
||||
expiresAt: now2.Add(inspectCacheTTL),
|
||||
}
|
||||
s.store.inspectMu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
type inspectCachedResult struct {
|
||||
data pathInspectResponse
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
func (s *PacketStore) inspectCacheKey(req pathInspectRequest) string {
|
||||
key := strings.Join(req.Prefixes, ",")
|
||||
if req.Context != nil {
|
||||
key += "|" + req.Context.ObserverID + "|" + req.Context.Since + "|" + req.Context.Until
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func (s *PacketStore) beamSearch(prefixes []string, pm *prefixMap, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time) []beamEntry {
|
||||
// Start with empty beam.
|
||||
beam := []beamEntry{{pubkeys: nil, names: nil, evidence: nil, score: 1.0}}
|
||||
|
||||
for hopIdx, prefix := range prefixes {
|
||||
candidates := pm.m[prefix]
|
||||
// Filter by role at lookup time (spec §2.2 step 2).
|
||||
var filtered []nodeInfo
|
||||
for _, c := range candidates {
|
||||
if canAppearInPath(c.Role) {
|
||||
filtered = append(filtered, c)
|
||||
}
|
||||
}
|
||||
|
||||
candidateCount := len(filtered)
|
||||
if candidateCount == 0 {
|
||||
// No candidates for this hop — beam dies.
|
||||
return nil
|
||||
}
|
||||
|
||||
var nextBeam []beamEntry
|
||||
for _, entry := range beam {
|
||||
for _, cand := range filtered {
|
||||
hopScore := s.scoreHop(entry, cand, candidateCount, graph, nodeByPK, now, hopIdx)
|
||||
if hopScore < hopScoreFloor {
|
||||
hopScore = hopScoreFloor
|
||||
}
|
||||
|
||||
newEntry := beamEntry{
|
||||
pubkeys: append(append([]string{}, entry.pubkeys...), cand.PublicKey),
|
||||
names: append(append([]string{}, entry.names...), cand.Name),
|
||||
evidence: append(append([]hopEvidence{}, entry.evidence...), hopEvidence{
|
||||
Prefix: prefix,
|
||||
CandidatesConsidered: candidateCount,
|
||||
Chosen: cand.PublicKey,
|
||||
EdgeWeight: hopScore,
|
||||
}),
|
||||
score: entry.score * hopScore,
|
||||
}
|
||||
nextBeam = append(nextBeam, newEntry)
|
||||
}
|
||||
}
|
||||
|
||||
// Prune to beam width.
|
||||
sortBeam(nextBeam)
|
||||
if len(nextBeam) > beamWidth {
|
||||
nextBeam = nextBeam[:beamWidth]
|
||||
}
|
||||
beam = nextBeam
|
||||
}
|
||||
|
||||
return beam
|
||||
}
|
||||
|
||||
func (s *PacketStore) scoreHop(entry beamEntry, cand nodeInfo, candidateCount int, graph *NeighborGraph, nodeByPK map[string]*nodeInfo, now time.Time, hopIdx int) float64 {
|
||||
var edgeScore float64
|
||||
var geoScore float64 = 1.0
|
||||
var recencyScore float64 = 1.0
|
||||
|
||||
if hopIdx == 0 || len(entry.pubkeys) == 0 {
|
||||
// First hop: no prior node to compare against.
|
||||
edgeScore = 1.0
|
||||
} else {
|
||||
lastPK := entry.pubkeys[len(entry.pubkeys)-1]
|
||||
|
||||
// Single scan over neighbors for both edge weight and recency.
|
||||
edges := graph.Neighbors(lastPK)
|
||||
var foundEdge *NeighborEdge
|
||||
for _, e := range edges {
|
||||
peer := e.NodeA
|
||||
if strings.EqualFold(peer, lastPK) {
|
||||
peer = e.NodeB
|
||||
}
|
||||
if strings.EqualFold(peer, cand.PublicKey) {
|
||||
foundEdge = e
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if foundEdge != nil {
|
||||
edgeScore = foundEdge.Score(now)
|
||||
hoursSince := now.Sub(foundEdge.LastSeen).Hours()
|
||||
if hoursSince <= 24 {
|
||||
recencyScore = 1.0
|
||||
} else {
|
||||
recencyScore = math.Max(0.1, 24.0/hoursSince)
|
||||
}
|
||||
} else {
|
||||
edgeScore = 0
|
||||
recencyScore = 0
|
||||
}
|
||||
|
||||
// Geographic plausibility.
|
||||
prevNode := nodeByPK[strings.ToLower(lastPK)]
|
||||
if prevNode != nil && prevNode.HasGPS && cand.HasGPS {
|
||||
dist := haversineKm(prevNode.Lat, prevNode.Lon, cand.Lat, cand.Lon)
|
||||
if dist > geoMaxKm {
|
||||
geoScore = math.Max(0.1, geoMaxKm/dist)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prefix selectivity.
|
||||
selectivityScore := 1.0 / float64(candidateCount)
|
||||
|
||||
return wEdge*edgeScore + wGeo*geoScore + wRecency*recencyScore + wSelectivity*selectivityScore
|
||||
}
|
||||
|
||||
|
||||
func sortBeam(beam []beamEntry) {
|
||||
sort.Slice(beam, func(i, j int) bool {
|
||||
return beam[i].score > beam[j].score
|
||||
})
|
||||
}
|
||||
|
||||
// ensureNeighborGraph triggers a graph rebuild if nil or stale.
|
||||
func (s *PacketStore) ensureNeighborGraph() {
|
||||
if s.graph != nil && !s.graph.IsStale() {
|
||||
return
|
||||
}
|
||||
g := BuildFromStore(s)
|
||||
s.graph = g
|
||||
}
|
||||
@@ -1,308 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ─── Unit tests for path inspector (issue #944) ────────────────────────────────
|
||||
|
||||
func TestScoreHop_EdgeWeight(t *testing.T) {
|
||||
store := &PacketStore{}
|
||||
graph := NewNeighborGraph()
|
||||
now := time.Now()
|
||||
|
||||
// Add an edge between A and B.
|
||||
graph.mu.Lock()
|
||||
edge := &NeighborEdge{
|
||||
NodeA: "aaaa", NodeB: "bbbb",
|
||||
Count: 50, LastSeen: now.Add(-1 * time.Hour),
|
||||
Observers: map[string]bool{"obs1": true},
|
||||
}
|
||||
key := edgeKey{"aaaa", "bbbb"}
|
||||
graph.edges[key] = edge
|
||||
graph.byNode["aaaa"] = append(graph.byNode["aaaa"], edge)
|
||||
graph.byNode["bbbb"] = append(graph.byNode["bbbb"], edge)
|
||||
graph.mu.Unlock()
|
||||
|
||||
entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"NodeA"}}
|
||||
cand := nodeInfo{PublicKey: "bbbb", Name: "NodeB", Role: "repeater"}
|
||||
|
||||
score := store.scoreHop(entry, cand, 2, graph, nil, now, 1)
|
||||
|
||||
// With edge present, edgeScore > 0. With 2 candidates, selectivity = 0.5.
|
||||
// Anti-tautology: if we zero out edge weight constant, score would change.
|
||||
if score <= 0.05 {
|
||||
t.Errorf("expected score > floor, got %f", score)
|
||||
}
|
||||
|
||||
// No edge: score should be lower.
|
||||
candNoEdge := nodeInfo{PublicKey: "cccc", Name: "NodeC", Role: "repeater"}
|
||||
scoreNoEdge := store.scoreHop(entry, candNoEdge, 2, graph, nil, now, 1)
|
||||
if scoreNoEdge >= score {
|
||||
t.Errorf("expected no-edge score (%f) < edge score (%f)", scoreNoEdge, score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoreHop_FirstHop(t *testing.T) {
|
||||
store := &PacketStore{}
|
||||
graph := NewNeighborGraph()
|
||||
now := time.Now()
|
||||
|
||||
entry := beamEntry{pubkeys: nil, names: nil}
|
||||
cand := nodeInfo{PublicKey: "aaaa", Name: "NodeA", Role: "repeater"}
|
||||
|
||||
score := store.scoreHop(entry, cand, 3, graph, nil, now, 0)
|
||||
// First hop: edgeScore=1.0, geoScore=1.0, recencyScore=1.0, selectivity=1/3
|
||||
// = 0.35*1 + 0.20*1 + 0.15*1 + 0.30*(1/3) = 0.35+0.20+0.15+0.10 = 0.80
|
||||
expected := 0.35 + 0.20 + 0.15 + 0.30/3.0
|
||||
if score < expected-0.01 || score > expected+0.01 {
|
||||
t.Errorf("expected ~%f, got %f", expected, score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoreHop_GeoPlausibility(t *testing.T) {
|
||||
store := &PacketStore{}
|
||||
store.nodeCache = []nodeInfo{
|
||||
{PublicKey: "aaaa", Name: "A", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true},
|
||||
{PublicKey: "bbbb", Name: "B", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true}, // ~1.4km
|
||||
{PublicKey: "cccc", Name: "C", Role: "repeater", Lat: 40.0, Lon: -120.0, HasGPS: true}, // ~400km
|
||||
}
|
||||
store.nodePM = buildPrefixMap(store.nodeCache)
|
||||
store.nodeCacheTime = time.Now()
|
||||
|
||||
graph := NewNeighborGraph()
|
||||
now := time.Now()
|
||||
|
||||
nodeByPK := map[string]*nodeInfo{
|
||||
"aaaa": &store.nodeCache[0],
|
||||
"bbbb": &store.nodeCache[1],
|
||||
"cccc": &store.nodeCache[2],
|
||||
}
|
||||
|
||||
entry := beamEntry{pubkeys: []string{"aaaa"}, names: []string{"A"}}
|
||||
|
||||
// Close node should score higher than far node (geo component).
|
||||
scoreClose := store.scoreHop(entry, store.nodeCache[1], 2, graph, nodeByPK, now, 1)
|
||||
scoreFar := store.scoreHop(entry, store.nodeCache[2], 2, graph, nodeByPK, now, 1)
|
||||
if scoreFar >= scoreClose {
|
||||
t.Errorf("expected far node score (%f) < close node score (%f)", scoreFar, scoreClose)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBeamSearch_WidthCap(t *testing.T) {
|
||||
store := &PacketStore{}
|
||||
graph := NewNeighborGraph()
|
||||
graph.builtAt = time.Now()
|
||||
now := time.Now()
|
||||
|
||||
// Create 25 nodes that all match prefix "aa".
|
||||
var nodes []nodeInfo
|
||||
for i := 0; i < 25; i++ {
|
||||
// Each node has pubkey starting with "aa" followed by unique hex.
|
||||
pk := "aa" + strings.Repeat("0", 4) + fmt.Sprintf("%02x", i)
|
||||
nodes = append(nodes, nodeInfo{PublicKey: pk, Name: pk, Role: "repeater"})
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
// Two hops of "aa" — should produce 25*25=625 combos, pruned to 20.
|
||||
beam := store.beamSearch([]string{"aa", "aa"}, pm, graph, nil, now)
|
||||
if len(beam) > beamWidth {
|
||||
t.Errorf("beam exceeded width: got %d, want <= %d", len(beam), beamWidth)
|
||||
}
|
||||
// Anti-tautology: without beam pruning, we'd have up to 25*min(25,beamWidth)=500 entries.
|
||||
// The test verifies pruning is effective.
|
||||
}
|
||||
|
||||
func TestBeamSearch_Speculative(t *testing.T) {
|
||||
store := &PacketStore{}
|
||||
graph := NewNeighborGraph()
|
||||
graph.builtAt = time.Now()
|
||||
now := time.Now()
|
||||
|
||||
// Create nodes with no edges and multiple candidates — should result in low scores (speculative).
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "aabb", Name: "N1", Role: "repeater"},
|
||||
{PublicKey: "aabb22", Name: "N1b", Role: "repeater"},
|
||||
{PublicKey: "ccdd", Name: "N2", Role: "repeater"},
|
||||
{PublicKey: "ccdd22", Name: "N2b", Role: "repeater"},
|
||||
{PublicKey: "ccdd33", Name: "N2c", Role: "repeater"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
beam := store.beamSearch([]string{"aa", "cc"}, pm, graph, nil, now)
|
||||
if len(beam) == 0 {
|
||||
t.Fatal("expected at least one result")
|
||||
}
|
||||
|
||||
// Score should be < 0.7 since there's no edge and multiple candidates (speculative).
|
||||
nHops := len(beam[0].pubkeys)
|
||||
score := 1.0
|
||||
if nHops > 0 {
|
||||
product := beam[0].score
|
||||
score = pow(product, 1.0/float64(nHops))
|
||||
}
|
||||
if score >= speculativeThreshold {
|
||||
t.Errorf("expected speculative score (< %f), got %f", speculativeThreshold, score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_EmptyPrefixes(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
body := `{"prefixes":[]}`
|
||||
rr := doInspectRequest(srv, body)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_OddLengthPrefix(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
body := `{"prefixes":["abc"]}`
|
||||
rr := doInspectRequest(srv, body)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for odd-length prefix, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_MixedLengths(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
body := `{"prefixes":["aa","bbcc"]}`
|
||||
rr := doInspectRequest(srv, body)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for mixed lengths, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_TooLongPrefix(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
body := `{"prefixes":["aabbccdd"]}`
|
||||
rr := doInspectRequest(srv, body)
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for >3-byte prefix, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_TooManyPrefixes(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
prefixes := make([]string, 65)
|
||||
for i := range prefixes {
|
||||
prefixes[i] = "aa"
|
||||
}
|
||||
b, _ := json.Marshal(map[string]interface{}{"prefixes": prefixes})
|
||||
rr := doInspectRequest(srv, string(b))
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("expected 400 for >64 prefixes, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlePathInspect_ValidRequest(t *testing.T) {
|
||||
srv := newTestServerForInspect(t)
|
||||
|
||||
// Seed nodes in the store — multiple candidates per prefix to lower selectivity.
|
||||
srv.store.nodeCache = []nodeInfo{
|
||||
{PublicKey: "aabb1234", Name: "NodeA", Role: "repeater", Lat: 37.0, Lon: -122.0, HasGPS: true},
|
||||
{PublicKey: "aabb5678", Name: "NodeA2", Role: "repeater"},
|
||||
{PublicKey: "ccdd5678", Name: "NodeB", Role: "repeater", Lat: 37.01, Lon: -122.01, HasGPS: true},
|
||||
{PublicKey: "ccdd9999", Name: "NodeB2", Role: "repeater"},
|
||||
{PublicKey: "ccdd1111", Name: "NodeB3", Role: "repeater"},
|
||||
}
|
||||
srv.store.nodePM = buildPrefixMap(srv.store.nodeCache)
|
||||
srv.store.nodeCacheTime = time.Now()
|
||||
srv.store.graph = NewNeighborGraph()
|
||||
srv.store.graph.builtAt = time.Now()
|
||||
|
||||
body := `{"prefixes":["aa","cc"]}`
|
||||
rr := doInspectRequest(srv, body)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var resp pathInspectResponse
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON response: %v", err)
|
||||
}
|
||||
if len(resp.Candidates) == 0 {
|
||||
t.Error("expected at least one candidate")
|
||||
}
|
||||
if resp.Candidates[0].Speculative != true {
|
||||
// No edge between nodes, so score should be < 0.7.
|
||||
t.Error("expected speculative=true for no-edge path")
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func newTestServerForInspect(t *testing.T) *Server {
|
||||
t.Helper()
|
||||
store := &PacketStore{
|
||||
inspectCache: make(map[string]*inspectCachedResult),
|
||||
}
|
||||
store.graph = NewNeighborGraph()
|
||||
store.graph.builtAt = time.Now()
|
||||
return &Server{store: store}
|
||||
}
|
||||
|
||||
func doInspectRequest(srv *Server, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("POST", "/api/paths/inspect", bytes.NewBufferString(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
srv.handlePathInspect(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
func pow(base, exp float64) float64 {
|
||||
return math.Pow(base, exp)
|
||||
}
|
||||
|
||||
// BenchmarkBeamSearch — performance proof for spec §2.5 (<100ms p99 for ≤64 hops).
|
||||
// Anti-tautology: removing beam pruning makes this ~625x slower; timing assertion catches it.
|
||||
func BenchmarkBeamSearch(b *testing.B) {
|
||||
// Setup: 100 nodes, 10-hop prefix input, realistic neighbor graph.
|
||||
// Anti-tautology: removing beam pruning makes this ~625x slower.
|
||||
store := &PacketStore{}
|
||||
pm := &prefixMap{m: make(map[string][]nodeInfo)}
|
||||
graph := NewNeighborGraph()
|
||||
nodes := make([]nodeInfo, 100)
|
||||
|
||||
now := time.Now()
|
||||
for i := 0; i < 100; i++ {
|
||||
pk := fmt.Sprintf("%064x", i)
|
||||
prefix := fmt.Sprintf("%02x", i%256)
|
||||
node := nodeInfo{PublicKey: pk, Name: fmt.Sprintf("Node%d", i), Role: "repeater", Lat: 37.0 + float64(i)*0.01, Lon: -122.0 + float64(i)*0.01}
|
||||
nodes[i] = node
|
||||
pm.m[prefix] = append(pm.m[prefix], node)
|
||||
// Add neighbor edges to create a connected graph.
|
||||
if i > 0 {
|
||||
prevPK := fmt.Sprintf("%064x", i-1)
|
||||
key := makeEdgeKey(prevPK, pk)
|
||||
edge := &NeighborEdge{NodeA: prevPK, NodeB: pk, LastSeen: now, Count: 10}
|
||||
graph.edges[key] = edge
|
||||
graph.byNode[prevPK] = append(graph.byNode[prevPK], edge)
|
||||
graph.byNode[pk] = append(graph.byNode[pk], edge)
|
||||
}
|
||||
}
|
||||
|
||||
// 10-hop input using prefixes that map to multiple candidates.
|
||||
prefixes := make([]string, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
prefixes[i] = fmt.Sprintf("%02x", (i*3)%256)
|
||||
}
|
||||
|
||||
nodeByPK := make(map[string]*nodeInfo)
|
||||
for idx := range nodes {
|
||||
nodeByPK[nodes[idx].PublicKey] = &nodes[idx]
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
store.beamSearch(prefixes, pm, graph, nodeByPK, now)
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestHandleNodePaths_PrefixCollisionExclusion verifies that paths through a node
|
||||
// sharing a 2-char prefix with another node are not returned as false positives
|
||||
// when they have no resolved_path data (issue #929).
|
||||
//
|
||||
// Setup:
|
||||
// - nodeA (target): pubkey starts with "7a", no GPS
|
||||
// - nodeB (other): pubkey starts with "7a", has GPS → "7a" resolves to nodeB
|
||||
// - tx1: path ["7a"], resolved_path NULL → false positive candidate, must be excluded
|
||||
// - tx2: path ["7a"], resolved_path contains nodeA pubkey → SQL-confirmed, must be included
|
||||
func TestHandleNodePaths_PrefixCollisionExclusion(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := time.Now().Add(-1 * time.Hour).Unix()
|
||||
|
||||
nodeAPK := "7acb1111aaaabbbb"
|
||||
nodeBPK := "7aff2222ccccdddd" // same "7a" prefix, has GPS so resolveHop("7a") picks B
|
||||
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, 'NodeA', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeAPK, recent)
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, 'NodeB', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeBPK, recent)
|
||||
|
||||
// tx1: no resolved_path — should be excluded by hop-level check
|
||||
db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (10, 'AA', 'hash_fp', ?)`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path)
|
||||
VALUES (10, NULL, '["7a"]', ?, NULL)`, recentEpoch)
|
||||
|
||||
// tx2: resolved_path confirms nodeA — must be included
|
||||
db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (11, 'BB', 'hash_tp', ?)`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path)
|
||||
VALUES (11, NULL, '["7a"]', ?, ?)`, recentEpoch, `["`+nodeAPK+`"]`)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/"+nodeAPK+"/paths", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp NodePathsResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
// Only the SQL-confirmed path (tx2) should be present; tx1 (false positive) must be excluded.
|
||||
// tx1 and tx2 share the same raw path ["7a"] so they collapse into 1 unique path group.
|
||||
// If tx1 were included, TotalTransmissions would be 2.
|
||||
if resp.TotalPaths != 1 {
|
||||
t.Errorf("expected 1 path group, got %d", resp.TotalPaths)
|
||||
}
|
||||
if resp.TotalTransmissions != 1 {
|
||||
t.Errorf("expected 1 transmission (false positive tx1 excluded), got %d", resp.TotalTransmissions)
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCanAppearInPath(t *testing.T) {
|
||||
cases := []struct {
|
||||
role string
|
||||
want bool
|
||||
}{
|
||||
{"repeater", true},
|
||||
{"Repeater", true},
|
||||
{"REPEATER", true},
|
||||
{"room_server", true},
|
||||
{"Room_Server", true},
|
||||
{"room", true},
|
||||
{"companion", false},
|
||||
{"sensor", false},
|
||||
{"", false},
|
||||
{"unknown", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := canAppearInPath(tc.role); got != tc.want {
|
||||
t.Errorf("canAppearInPath(%q) = %v, want %v", tc.role, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPrefixMap_ExcludesCompanions(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
if len(pm.m) != 0 {
|
||||
t.Fatalf("expected empty prefix map, got %d entries", len(pm.m))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPrefixMap_ExcludesSensors(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
if len(pm.m) != 0 {
|
||||
t.Fatalf("expected empty prefix map, got %d entries", len(pm.m))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWithContext_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r, _, _ := pm.resolveWithContext("7a", nil, nil)
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWithContext_NilWhenOnlySensorMatchesPrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r, _, _ := pm.resolveWithContext("7a", nil, nil)
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil for sensor-only prefix, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWithContext_PrefersRepeaterOverCompanionAtSamePrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"},
|
||||
{PublicKey: "7a5678901234", Role: "repeater", Name: "MyRepeater"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r, _, _ := pm.resolveWithContext("7a", nil, nil)
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil result")
|
||||
}
|
||||
if r.Name != "MyRepeater" {
|
||||
t.Fatalf("expected MyRepeater, got %s", r.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWithContext_PrefersRoomServerOverCompanionAtSamePrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "ab1234abcdef", Role: "companion", Name: "MyCompanion"},
|
||||
{PublicKey: "ab5678901234", Role: "room_server", Name: "MyRoom"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r, _, _ := pm.resolveWithContext("ab", nil, nil)
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil result")
|
||||
}
|
||||
if r.Name != "MyRoom" {
|
||||
t.Fatalf("expected MyRoom, got %s", r.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_NilWhenOnlyCompanionMatchesPrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "MyCompanion"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r := pm.resolve("7a")
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil from resolve() for companion-only prefix, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolve_NilWhenOnlySensorMatchesPrefix(t *testing.T) {
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "sensor", Name: "MySensor"},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r := pm.resolve("7a")
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil from resolve() for sensor-only prefix, got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWithContext_PicksRepeaterEvenWhenCompanionHasGPS(t *testing.T) {
|
||||
// Adversarial: companion has GPS, repeater doesn't. Role filter should
|
||||
// exclude companion entirely, so repeater wins despite lacking GPS.
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "GPSCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true},
|
||||
{PublicKey: "7a5678901234", Role: "repeater", Name: "NoGPSRepeater", Lat: 0, Lon: 0, HasGPS: false},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
r, _, _ := pm.resolveWithContext("7a", nil, nil)
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil result")
|
||||
}
|
||||
if r.Name != "NoGPSRepeater" {
|
||||
t.Fatalf("expected NoGPSRepeater (role filter excludes companion), got %s", r.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDistancesForTx_CompanionNeverInResolvedChain(t *testing.T) {
|
||||
// Integration test: a path with a prefix matching both a companion and a
|
||||
// repeater. The resolveHop function (using buildPrefixMap) should only
|
||||
// return the repeater.
|
||||
nodes := []nodeInfo{
|
||||
{PublicKey: "7a1234abcdef", Role: "companion", Name: "BadCompanion", Lat: 37.0, Lon: -122.0, HasGPS: true},
|
||||
{PublicKey: "7a5678901234", Role: "repeater", Name: "GoodRepeater", Lat: 38.0, Lon: -123.0, HasGPS: true},
|
||||
{PublicKey: "bb1111111111", Role: "repeater", Name: "OtherRepeater", Lat: 39.0, Lon: -124.0, HasGPS: true},
|
||||
}
|
||||
pm := buildPrefixMap(nodes)
|
||||
|
||||
nodeByPk := make(map[string]*nodeInfo)
|
||||
for i := range nodes {
|
||||
nodeByPk[nodes[i].PublicKey] = &nodes[i]
|
||||
}
|
||||
repeaterSet := map[string]bool{
|
||||
"7a5678901234": true,
|
||||
"bb1111111111": true,
|
||||
}
|
||||
|
||||
// Build a synthetic StoreTx with a path ["7a", "bb"] and a sender with GPS
|
||||
senderPK := "cc0000000000"
|
||||
sender := nodeInfo{PublicKey: senderPK, Role: "repeater", Name: "Sender", Lat: 36.0, Lon: -121.0, HasGPS: true}
|
||||
nodeByPk[senderPK] = &sender
|
||||
|
||||
pathJSON, _ := json.Marshal([]string{"7a", "bb"})
|
||||
decoded, _ := json.Marshal(map[string]interface{}{"pubKey": senderPK})
|
||||
|
||||
tx := &StoreTx{
|
||||
PathJSON: string(pathJSON),
|
||||
DecodedJSON: string(decoded),
|
||||
FirstSeen: "2026-04-30T12:00",
|
||||
}
|
||||
|
||||
resolveHop := func(hop string) *nodeInfo {
|
||||
return pm.resolve(hop)
|
||||
}
|
||||
|
||||
hops, pathRec := computeDistancesForTx(tx, nodeByPk, repeaterSet, resolveHop)
|
||||
|
||||
// Verify BadCompanion's pubkey never appears in hops
|
||||
badPK := "7a1234abcdef"
|
||||
for i, h := range hops {
|
||||
if h.FromPk == badPK || h.ToPk == badPK {
|
||||
t.Fatalf("hop[%d] contains BadCompanion pubkey: from=%s to=%s", i, h.FromPk, h.ToPk)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify BadCompanion's pubkey never appears in pathRec
|
||||
if pathRec == nil {
|
||||
t.Fatal("expected non-nil path record (3 GPS nodes in chain)")
|
||||
}
|
||||
for i, hop := range pathRec.Hops {
|
||||
if hop.FromPk == badPK || hop.ToPk == badPK {
|
||||
t.Fatalf("pathRec.Hops[%d] contains BadCompanion pubkey: from=%s to=%s", i, hop.FromPk, hop.ToPk)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify GoodRepeater IS in the chain (proves the prefix was resolved to the right node)
|
||||
goodPK := "7a5678901234"
|
||||
foundGood := false
|
||||
for _, hop := range pathRec.Hops {
|
||||
if hop.FromPk == goodPK || hop.ToPk == goodPK {
|
||||
foundGood = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundGood {
|
||||
t.Fatal("expected GoodRepeater (7a5678901234) in pathRec.Hops but not found")
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue #770: the region filter dropdown's "All" option was being sent to the
|
||||
// backend as ?region=All. The backend then tried to match observers with IATA
|
||||
// code "ALL", which never exists, producing an empty channel/packet list.
|
||||
//
|
||||
// "All" / "ALL" / "all" / "" must all be treated as "no region filter".
|
||||
func TestNormalizeRegionCodes_AllIsNoFilter(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
}{
|
||||
{"empty", ""},
|
||||
{"literal All (frontend dropdown label)", "All"},
|
||||
{"upper ALL", "ALL"},
|
||||
{"lower all", "all"},
|
||||
{"All with whitespace", " All "},
|
||||
{"All in csv with empty siblings", "All,"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := normalizeRegionCodes(tc.in)
|
||||
if got != nil {
|
||||
t.Errorf("normalizeRegionCodes(%q) = %v, want nil (no filter)", tc.in, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Real region codes must still pass through unchanged (case-folded to upper).
|
||||
// This locks in that the "All" handling does not regress legitimate filters.
|
||||
func TestNormalizeRegionCodes_RealCodesPreserved(t *testing.T) {
|
||||
got := normalizeRegionCodes("sjc,PDX")
|
||||
if len(got) != 2 || got[0] != "SJC" || got[1] != "PDX" {
|
||||
t.Errorf("normalizeRegionCodes(\"sjc,PDX\") = %v, want [SJC PDX]", got)
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
func TestResolveWithContext_UniquePrefix(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2},
|
||||
{PublicKey: "a1b2c3d4", Name: "Node-A", HasGPS: true, Lat: 1, Lon: 2},
|
||||
})
|
||||
ni, confidence, _ := pm.resolveWithContext("a1b2c3d4", nil, nil)
|
||||
if ni == nil || ni.Name != "Node-A" {
|
||||
@@ -24,7 +24,7 @@ func TestResolveWithContext_UniquePrefix(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_NoMatch(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1b2c3d4", Name: "Node-A"},
|
||||
{PublicKey: "a1b2c3d4", Name: "Node-A"},
|
||||
})
|
||||
ni, confidence, _ := pm.resolveWithContext("ff", nil, nil)
|
||||
if ni != nil {
|
||||
@@ -37,8 +37,8 @@ func TestResolveWithContext_NoMatch(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_AffinityWins(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1"},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2"},
|
||||
{PublicKey: "a1aaaaaa", Name: "Node-A1"},
|
||||
{PublicKey: "a1bbbbbb", Name: "Node-A2"},
|
||||
})
|
||||
|
||||
graph := NewNeighborGraph()
|
||||
@@ -60,9 +60,9 @@ func TestResolveWithContext_AffinityWins(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21},
|
||||
{Role: "repeater", PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1},
|
||||
{PublicKey: "a1aaaaaa", Name: "Node-A1", HasGPS: true, Lat: 10, Lon: 20},
|
||||
{PublicKey: "a1bbbbbb", Name: "Node-A2", HasGPS: true, Lat: 11, Lon: 21},
|
||||
{PublicKey: "c0c0c0c0", Name: "Ctx", HasGPS: true, Lat: 10.1, Lon: 20.1},
|
||||
})
|
||||
|
||||
graph := NewNeighborGraph()
|
||||
@@ -85,8 +85,8 @@ func TestResolveWithContext_AffinityTooClose_FallsToGeo(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_GPSPreference(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
{PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
})
|
||||
|
||||
ni, confidence, _ := pm.resolveWithContext("a1", nil, nil)
|
||||
@@ -100,8 +100,8 @@ func TestResolveWithContext_GPSPreference(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_FirstMatchFallback(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "First"},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "Second"},
|
||||
{PublicKey: "a1aaaaaa", Name: "First"},
|
||||
{PublicKey: "a1bbbbbb", Name: "Second"},
|
||||
})
|
||||
|
||||
ni, confidence, _ := pm.resolveWithContext("a1", nil, nil)
|
||||
@@ -115,8 +115,8 @@ func TestResolveWithContext_FirstMatchFallback(t *testing.T) {
|
||||
|
||||
func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) {
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
{PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
})
|
||||
|
||||
ni, confidence, _ := pm.resolveWithContext("a1", []string{"someone"}, nil)
|
||||
@@ -131,8 +131,8 @@ func TestResolveWithContext_NilGraphFallsToGPS(t *testing.T) {
|
||||
func TestResolveWithContext_BackwardCompatResolve(t *testing.T) {
|
||||
// Verify original resolve() still works unchanged
|
||||
pm := buildPrefixMap([]nodeInfo{
|
||||
{Role: "repeater", PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{Role: "repeater", PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
{PublicKey: "a1aaaaaa", Name: "NoGPS"},
|
||||
{PublicKey: "a1bbbbbb", Name: "HasGPS", HasGPS: true, Lat: 1, Lon: 2},
|
||||
})
|
||||
ni := pm.resolve("a1")
|
||||
if ni == nil || ni.Name != "HasGPS" {
|
||||
@@ -164,8 +164,8 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) {
|
||||
_ = srv
|
||||
|
||||
// Insert a unique node
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"ff11223344", "UniqueNode", 37.0, -122.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"ff11223344", "UniqueNode", 37.0, -122.0)
|
||||
srv.store.InvalidateNodeCache()
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ff11223344", nil)
|
||||
@@ -189,10 +189,10 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) {
|
||||
func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"ee1aaaaaaa", "Node-E1", 37.0, -122.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"ee1bbbbbbb", "Node-E2", 38.0, -121.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"ee1aaaaaaa", "Node-E1", 37.0, -122.0)
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"ee1bbbbbbb", "Node-E2", 38.0, -121.0)
|
||||
srv.store.InvalidateNodeCache()
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=ee1", nil)
|
||||
@@ -224,12 +224,12 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) {
|
||||
func TestResolveHopsAPI_WithAffinityContext(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"dd1aaaaaaa", "Node-D1", 37.0, -122.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"dd1bbbbbbb", "Node-D2", 38.0, -121.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"c0c0c0c0c0", "Context", 37.1, -122.1, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"dd1aaaaaaa", "Node-D1", 37.0, -122.0)
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"dd1bbbbbbb", "Node-D2", 38.0, -121.0)
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"c0c0c0c0c0", "Context", 37.1, -122.1)
|
||||
|
||||
// Invalidate node cache so the PM includes newly inserted nodes.
|
||||
srv.store.cacheMu.Lock()
|
||||
@@ -279,8 +279,8 @@ func TestResolveHopsAPI_WithAffinityContext(t *testing.T) {
|
||||
func TestResolveHopsAPI_ResponseShape(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)",
|
||||
"bb1aaaaaaa", "Node-B1", 37.0, -122.0, "repeater")
|
||||
srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon) VALUES (?, ?, ?, ?)",
|
||||
"bb1aaaaaaa", "Node-B1", 37.0, -122.0)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=bb1a", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
@@ -1,475 +0,0 @@
|
||||
package main
|
||||
|
||||
// Lock ordering contract (MUST be followed everywhere):
|
||||
//
|
||||
// s.mu → s.lruMu (s.mu is the outer lock, lruMu is the inner lock)
|
||||
//
|
||||
// • Never acquire s.lruMu while holding s.mu.
|
||||
// • fetchResolvedPathForObs takes lruMu independently — callers under s.mu
|
||||
// must NOT call it directly; instead collect IDs under s.mu, release, then
|
||||
// do LRU ops under lruMu separately.
|
||||
// • The backfill path (backfillResolvedPathsAsync) follows this by collecting
|
||||
// obsIDs to invalidate under s.mu, releasing it, then taking lruMu.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"hash/fnv"
|
||||
"log"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// resolvedPubkeyHash computes a fast 64-bit hash for membership index keying.
|
||||
// Uses FNV-1a from stdlib — good distribution, no external dependency.
|
||||
func resolvedPubkeyHash(pk string) uint64 {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(strings.ToLower(pk)))
|
||||
return h.Sum64()
|
||||
}
|
||||
|
||||
// addToResolvedPubkeyIndex adds a txID under each resolved pubkey hash.
|
||||
// Deduplicates both within a single call AND across calls — won't add the
|
||||
// same (hash, txID) pair twice even when called multiple times for the same tx.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) addToResolvedPubkeyIndex(txID int, resolvedPubkeys []string) {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
seen := make(map[uint64]bool, len(resolvedPubkeys))
|
||||
for _, pk := range resolvedPubkeys {
|
||||
if pk == "" {
|
||||
continue
|
||||
}
|
||||
h := resolvedPubkeyHash(pk)
|
||||
if seen[h] {
|
||||
continue
|
||||
}
|
||||
seen[h] = true
|
||||
|
||||
// Cross-call dedup: check if (h, txID) already exists in forward index.
|
||||
existing := s.resolvedPubkeyIndex[h]
|
||||
alreadyPresent := false
|
||||
for _, id := range existing {
|
||||
if id == txID {
|
||||
alreadyPresent = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if alreadyPresent {
|
||||
continue
|
||||
}
|
||||
|
||||
s.resolvedPubkeyIndex[h] = append(existing, txID)
|
||||
s.resolvedPubkeyReverse[txID] = append(s.resolvedPubkeyReverse[txID], h)
|
||||
}
|
||||
}
|
||||
|
||||
// removeFromResolvedPubkeyIndex removes all index entries for a txID using the reverse map.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) removeFromResolvedPubkeyIndex(txID int) {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
hashes := s.resolvedPubkeyReverse[txID]
|
||||
for _, h := range hashes {
|
||||
list := s.resolvedPubkeyIndex[h]
|
||||
// Remove ALL occurrences of txID (not just the first) to prevent orphans.
|
||||
filtered := list[:0]
|
||||
for _, id := range list {
|
||||
if id != txID {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
delete(s.resolvedPubkeyIndex, h)
|
||||
} else {
|
||||
s.resolvedPubkeyIndex[h] = filtered
|
||||
}
|
||||
}
|
||||
delete(s.resolvedPubkeyReverse, txID)
|
||||
}
|
||||
|
||||
// extractResolvedPubkeys extracts all non-nil, non-empty pubkeys from a resolved path.
|
||||
func extractResolvedPubkeys(rp []*string) []string {
|
||||
if len(rp) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, 0, len(rp))
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" {
|
||||
result = append(result, *p)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// mergeResolvedPubkeys collects unique non-empty pubkeys from multiple resolved paths.
|
||||
func mergeResolvedPubkeys(paths ...[]*string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var result []string
|
||||
for _, rp := range paths {
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" && !seen[*p] {
|
||||
seen[*p] = true
|
||||
result = append(result, *p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// nodeInResolvedPathViaIndex checks whether a transmission is associated with
|
||||
// a target pubkey using the membership index + collision-safety SQL check.
|
||||
// Must be called under s.mu RLock at minimum.
|
||||
func (s *PacketStore) nodeInResolvedPathViaIndex(tx *StoreTx, targetPK string) bool {
|
||||
if !s.useResolvedPathIndex {
|
||||
// Flag off: can't disambiguate, keep candidate (conservative)
|
||||
return true
|
||||
}
|
||||
|
||||
// If this tx has no indexed pubkeys at all, we can't disambiguate —
|
||||
// keep the candidate (same as old behavior for NULL resolved_path).
|
||||
if _, hasReverse := s.resolvedPubkeyReverse[tx.ID]; !hasReverse {
|
||||
return true
|
||||
}
|
||||
|
||||
h := resolvedPubkeyHash(targetPK)
|
||||
txIDs := s.resolvedPubkeyIndex[h]
|
||||
|
||||
// Check if this tx's ID is in the candidate list
|
||||
for _, id := range txIDs {
|
||||
if id == tx.ID {
|
||||
// Found in index. Collision-safety: verify with SQL.
|
||||
if s.db != nil && s.db.conn != nil {
|
||||
return s.confirmResolvedPathContains(tx.ID, targetPK)
|
||||
}
|
||||
return true // no DB, trust the index
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// confirmResolvedPathContains verifies an exact pubkey match in resolved_path
|
||||
// via SQL. This is the collision-safety fallback for the membership index.
|
||||
func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return true
|
||||
}
|
||||
// Use INSTR with surrounding quotes for exact match — avoids LIKE escape issues.
|
||||
// resolved_path format: ["pubkey1","pubkey2",...]
|
||||
needle := `"` + strings.ToLower(pubkey) + `"`
|
||||
var count int
|
||||
err := s.db.conn.QueryRow(
|
||||
`SELECT COUNT(*) FROM observations WHERE transmission_id = ? AND INSTR(LOWER(resolved_path), ?) > 0`,
|
||||
txID, needle,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
return true // on error, keep the candidate
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
// fetchResolvedPathsForTx fetches resolved_path from SQLite for all observations
|
||||
// of a transmission. Used for on-demand API responses and eviction cleanup.
|
||||
func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.db.conn.Query(
|
||||
`SELECT id, resolved_path FROM observations WHERE transmission_id = ? AND resolved_path IS NOT NULL`,
|
||||
txID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := make(map[int][]*string)
|
||||
for rows.Next() {
|
||||
var obsID int
|
||||
var rpJSON sql.NullString
|
||||
if err := rows.Scan(&obsID, &rpJSON); err != nil {
|
||||
continue
|
||||
}
|
||||
if rpJSON.Valid && rpJSON.String != "" {
|
||||
result[obsID] = unmarshalResolvedPath(rpJSON.String)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// fetchResolvedPathForObs fetches resolved_path for a single observation,
|
||||
// using the LRU cache.
|
||||
func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check LRU cache first
|
||||
s.lruMu.RLock()
|
||||
if s.apiResolvedPathLRU != nil {
|
||||
if entry, ok := s.apiResolvedPathLRU[obsID]; ok {
|
||||
s.lruMu.RUnlock()
|
||||
return entry
|
||||
}
|
||||
}
|
||||
s.lruMu.RUnlock()
|
||||
|
||||
var rpJSON sql.NullString
|
||||
err := s.db.conn.QueryRow(
|
||||
`SELECT resolved_path FROM observations WHERE id = ?`, obsID,
|
||||
).Scan(&rpJSON)
|
||||
if err != nil || !rpJSON.Valid {
|
||||
return nil
|
||||
}
|
||||
rp := unmarshalResolvedPath(rpJSON.String)
|
||||
|
||||
// Store in LRU
|
||||
s.lruMu.Lock()
|
||||
s.lruPut(obsID, rp)
|
||||
s.lruMu.Unlock()
|
||||
|
||||
return rp
|
||||
}
|
||||
|
||||
// fetchResolvedPathForTxBest returns the best observation's resolved_path for a tx.
|
||||
//
|
||||
// "Best" = the longest path_json among observations that actually have a stored
|
||||
// resolved_path. Earlier versions picked the longest-path obs unconditionally
|
||||
// and queried SQL for that single ID — if the longest-path obs had NULL
|
||||
// resolved_path while a shorter sibling had one, the call returned nil and
|
||||
// callers (e.g. /api/nodes/{pk}/health.recentPackets) lost the field. Fixes
|
||||
// #810 by checking all observations and falling back to the longest sibling
|
||||
// that has a stored path.
|
||||
func (s *PacketStore) fetchResolvedPathForTxBest(tx *StoreTx) []*string {
|
||||
if tx == nil || len(tx.Observations) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Fast path: try the longest-path obs first via the LRU/SQL helper.
|
||||
longest := tx.Observations[0]
|
||||
longestLen := pathLen(longest.PathJSON)
|
||||
for _, obs := range tx.Observations[1:] {
|
||||
if l := pathLen(obs.PathJSON); l > longestLen {
|
||||
longest = obs
|
||||
longestLen = l
|
||||
}
|
||||
}
|
||||
if rp := s.fetchResolvedPathForObs(longest.ID); rp != nil {
|
||||
return rp
|
||||
}
|
||||
// Fallback: longest-path obs has no stored resolved_path. Query all
|
||||
// observations for this tx and pick the one with the longest path_json
|
||||
// that actually has a stored resolved_path.
|
||||
rpMap := s.fetchResolvedPathsForTx(tx.ID)
|
||||
if len(rpMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
var bestRP []*string
|
||||
bestObsID := 0
|
||||
bestLen := -1
|
||||
for _, obs := range tx.Observations {
|
||||
rp, ok := rpMap[obs.ID]
|
||||
if !ok || rp == nil {
|
||||
continue
|
||||
}
|
||||
if l := pathLen(obs.PathJSON); l > bestLen {
|
||||
bestLen = l
|
||||
bestRP = rp
|
||||
bestObsID = obs.ID
|
||||
}
|
||||
}
|
||||
// Populate LRU so repeat lookups for this tx don't re-issue the multi-row
|
||||
// SQL fallback (e.g. dashboard polling /api/nodes/{pk}/health).
|
||||
if bestRP != nil && bestObsID != 0 {
|
||||
s.lruMu.Lock()
|
||||
s.lruPut(bestObsID, bestRP)
|
||||
s.lruMu.Unlock()
|
||||
}
|
||||
return bestRP
|
||||
}
|
||||
|
||||
// --- Simple LRU cache for resolved paths ---
|
||||
|
||||
const lruMaxSize = 10000
|
||||
|
||||
// lruPut adds an entry. Must be called under s.lruMu write lock.
|
||||
func (s *PacketStore) lruPut(obsID int, rp []*string) {
|
||||
if s.apiResolvedPathLRU == nil {
|
||||
return
|
||||
}
|
||||
if _, exists := s.apiResolvedPathLRU[obsID]; exists {
|
||||
return
|
||||
}
|
||||
// Compact lruOrder if stale entries exceed 50% of capacity.
|
||||
// This prevents effective capacity degradation after bulk deletions.
|
||||
if len(s.lruOrder) >= lruMaxSize && len(s.apiResolvedPathLRU) < lruMaxSize/2 {
|
||||
compacted := make([]int, 0, len(s.apiResolvedPathLRU))
|
||||
for _, id := range s.lruOrder {
|
||||
if _, ok := s.apiResolvedPathLRU[id]; ok {
|
||||
compacted = append(compacted, id)
|
||||
}
|
||||
}
|
||||
s.lruOrder = compacted
|
||||
}
|
||||
if len(s.lruOrder) >= lruMaxSize {
|
||||
// Evict oldest, skipping stale entries
|
||||
for len(s.lruOrder) > 0 {
|
||||
evictID := s.lruOrder[0]
|
||||
s.lruOrder = s.lruOrder[1:]
|
||||
if _, ok := s.apiResolvedPathLRU[evictID]; ok {
|
||||
delete(s.apiResolvedPathLRU, evictID)
|
||||
break
|
||||
}
|
||||
// stale entry — skip and continue
|
||||
}
|
||||
}
|
||||
s.apiResolvedPathLRU[obsID] = rp
|
||||
s.lruOrder = append(s.lruOrder, obsID)
|
||||
}
|
||||
|
||||
// lruDelete removes an entry. Must be called under s.lruMu write lock.
|
||||
func (s *PacketStore) lruDelete(obsID int) {
|
||||
if s.apiResolvedPathLRU == nil {
|
||||
return
|
||||
}
|
||||
delete(s.apiResolvedPathLRU, obsID)
|
||||
// Don't scan lruOrder — eviction handles stale entries naturally.
|
||||
}
|
||||
|
||||
// resolvedPubkeysForEvictionBatch fetches resolved pubkeys for multiple txIDs
|
||||
// from SQL in a single batched query. Returns a map from txID to unique pubkeys.
|
||||
// MUST be called WITHOUT holding s.mu — this is the whole point of the batch approach.
|
||||
// Chunks queries to stay under SQLite's 500-parameter limit.
|
||||
func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]string {
|
||||
result := make(map[int][]string, len(txIDs))
|
||||
if len(txIDs) == 0 || s.db == nil || s.db.conn == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
const chunkSize = 499 // SQLite SQLITE_MAX_VARIABLE_NUMBER default is 999; stay well under
|
||||
for start := 0; start < len(txIDs); start += chunkSize {
|
||||
end := start + chunkSize
|
||||
if end > len(txIDs) {
|
||||
end = len(txIDs)
|
||||
}
|
||||
chunk := txIDs[start:end]
|
||||
|
||||
// Build query with placeholders
|
||||
placeholders := make([]byte, 0, len(chunk)*2)
|
||||
args := make([]interface{}, len(chunk))
|
||||
for i, id := range chunk {
|
||||
if i > 0 {
|
||||
placeholders = append(placeholders, ',')
|
||||
}
|
||||
placeholders = append(placeholders, '?')
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" +
|
||||
string(placeholders) + ") AND resolved_path IS NOT NULL"
|
||||
|
||||
rows, err := s.db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var txID int
|
||||
var rpJSON sql.NullString
|
||||
if err := rows.Scan(&txID, &rpJSON); err != nil {
|
||||
continue
|
||||
}
|
||||
if !rpJSON.Valid || rpJSON.String == "" {
|
||||
continue
|
||||
}
|
||||
rp := unmarshalResolvedPath(rpJSON.String)
|
||||
for _, p := range rp {
|
||||
if p != nil && *p != "" {
|
||||
result[txID] = append(result[txID], *p)
|
||||
}
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
|
||||
// Deduplicate per-txID
|
||||
for txID, pks := range result {
|
||||
seen := make(map[string]bool, len(pks))
|
||||
deduped := pks[:0]
|
||||
for _, pk := range pks {
|
||||
if !seen[pk] {
|
||||
seen[pk] = true
|
||||
deduped = append(deduped, pk)
|
||||
}
|
||||
}
|
||||
result[txID] = deduped
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// initResolvedPathIndex initializes the resolved path index data structures.
|
||||
func (s *PacketStore) initResolvedPathIndex() {
|
||||
s.resolvedPubkeyIndex = make(map[uint64][]int, 4096)
|
||||
s.resolvedPubkeyReverse = make(map[int][]uint64, 4096)
|
||||
s.apiResolvedPathLRU = make(map[int][]*string, lruMaxSize)
|
||||
s.lruOrder = make([]int, 0, lruMaxSize)
|
||||
}
|
||||
|
||||
// CompactResolvedPubkeyIndex reclaims memory from the resolved pubkey index maps
|
||||
// after eviction. It removes empty forward-index entries (shouldn't exist if
|
||||
// removeFromResolvedPubkeyIndex is correct, but defense in depth) and clips
|
||||
// oversized slice backing arrays where cap > 2*len.
|
||||
// Must be called under s.mu write lock.
|
||||
func (s *PacketStore) CompactResolvedPubkeyIndex() {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
for h, ids := range s.resolvedPubkeyIndex {
|
||||
if len(ids) == 0 {
|
||||
delete(s.resolvedPubkeyIndex, h)
|
||||
continue
|
||||
}
|
||||
// Clip oversized backing arrays: if cap > 2*len, reallocate.
|
||||
if cap(ids) > 2*len(ids)+8 {
|
||||
clipped := make([]int, len(ids))
|
||||
copy(clipped, ids)
|
||||
s.resolvedPubkeyIndex[h] = clipped
|
||||
}
|
||||
}
|
||||
for txID, hashes := range s.resolvedPubkeyReverse {
|
||||
if len(hashes) == 0 {
|
||||
delete(s.resolvedPubkeyReverse, txID)
|
||||
continue
|
||||
}
|
||||
if cap(hashes) > 2*len(hashes)+8 {
|
||||
clipped := make([]uint64, len(hashes))
|
||||
copy(clipped, hashes)
|
||||
s.resolvedPubkeyReverse[txID] = clipped
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// defaultMaxResolvedPubkeyIndexEntries is the default hard cap for the forward
|
||||
// index. When exceeded, a warning is logged. No auto-eviction — that's the
|
||||
// eviction ticker's job.
|
||||
const defaultMaxResolvedPubkeyIndexEntries = 5_000_000
|
||||
|
||||
// CheckResolvedPubkeyIndexSize logs a warning if the resolved pubkey forward
|
||||
// index exceeds the configured maximum entries. Must be called under s.mu
|
||||
// read lock at minimum.
|
||||
func (s *PacketStore) CheckResolvedPubkeyIndexSize() {
|
||||
if !s.useResolvedPathIndex {
|
||||
return
|
||||
}
|
||||
maxEntries := s.maxResolvedPubkeyIndexEntries
|
||||
if maxEntries <= 0 {
|
||||
maxEntries = defaultMaxResolvedPubkeyIndexEntries
|
||||
}
|
||||
fwdLen := len(s.resolvedPubkeyIndex)
|
||||
revLen := len(s.resolvedPubkeyReverse)
|
||||
if fwdLen > maxEntries || revLen > maxEntries {
|
||||
log.Printf("[store] WARNING: resolvedPubkeyIndex size exceeds limit — forward=%d reverse=%d limit=%d",
|
||||
fwdLen, revLen, maxEntries)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,133 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RoleStats summarises one role's population and clock-skew posture.
|
||||
type RoleStats struct {
|
||||
Role string `json:"role"`
|
||||
NodeCount int `json:"nodeCount"`
|
||||
WithSkew int `json:"withSkew"`
|
||||
MeanAbsSkewSec float64 `json:"meanAbsSkewSec"`
|
||||
MedianAbsSkewSec float64 `json:"medianAbsSkewSec"`
|
||||
OkCount int `json:"okCount"`
|
||||
WarningCount int `json:"warningCount"`
|
||||
CriticalCount int `json:"criticalCount"`
|
||||
AbsurdCount int `json:"absurdCount"`
|
||||
NoClockCount int `json:"noClockCount"`
|
||||
}
|
||||
|
||||
// RoleAnalyticsResponse is the payload returned by /api/analytics/roles.
|
||||
type RoleAnalyticsResponse struct {
|
||||
TotalNodes int `json:"totalNodes"`
|
||||
Roles []RoleStats `json:"roles"`
|
||||
}
|
||||
|
||||
// normalizeRole canonicalises a role string so empty/unknown roles bucket
|
||||
// together and case differences don't fragment the distribution.
|
||||
func normalizeRole(r string) string {
|
||||
r = strings.ToLower(strings.TrimSpace(r))
|
||||
if r == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// computeRoleAnalytics groups nodes by role and aggregates clock-skew per
|
||||
// role. Pure function: takes the node roster and the per-pubkey skew map and
|
||||
// returns the response — no store / lock dependencies, easy to unit test.
|
||||
//
|
||||
// `nodesByPubkey` lists every known node (pubkey → role). `skewByPubkey`
|
||||
// is the subset of pubkeys that have clock-skew data with their severity and
|
||||
// most-recent corrected skew (in seconds, signed — we take |x| for averages).
|
||||
func computeRoleAnalytics(nodesByPubkey map[string]string, skewByPubkey map[string]*NodeClockSkew) RoleAnalyticsResponse {
|
||||
type bucket struct {
|
||||
stats RoleStats
|
||||
absSkews []float64
|
||||
}
|
||||
buckets := make(map[string]*bucket)
|
||||
for pk, rawRole := range nodesByPubkey {
|
||||
role := normalizeRole(rawRole)
|
||||
b, ok := buckets[role]
|
||||
if !ok {
|
||||
b = &bucket{stats: RoleStats{Role: role}}
|
||||
buckets[role] = b
|
||||
}
|
||||
b.stats.NodeCount++
|
||||
cs, has := skewByPubkey[pk]
|
||||
if !has || cs == nil {
|
||||
continue
|
||||
}
|
||||
b.stats.WithSkew++
|
||||
abs := math.Abs(cs.RecentMedianSkewSec)
|
||||
if abs == 0 {
|
||||
abs = math.Abs(cs.LastSkewSec)
|
||||
}
|
||||
b.absSkews = append(b.absSkews, abs)
|
||||
switch cs.Severity {
|
||||
case SkewOK:
|
||||
b.stats.OkCount++
|
||||
case SkewWarning:
|
||||
b.stats.WarningCount++
|
||||
case SkewCritical:
|
||||
b.stats.CriticalCount++
|
||||
case SkewAbsurd:
|
||||
b.stats.AbsurdCount++
|
||||
case SkewNoClock:
|
||||
b.stats.NoClockCount++
|
||||
}
|
||||
}
|
||||
resp := RoleAnalyticsResponse{Roles: make([]RoleStats, 0, len(buckets))}
|
||||
for _, b := range buckets {
|
||||
if n := len(b.absSkews); n > 0 {
|
||||
sum := 0.0
|
||||
for _, v := range b.absSkews {
|
||||
sum += v
|
||||
}
|
||||
b.stats.MeanAbsSkewSec = round(sum/float64(n), 2)
|
||||
sorted := make([]float64, n)
|
||||
copy(sorted, b.absSkews)
|
||||
sort.Float64s(sorted)
|
||||
if n%2 == 1 {
|
||||
b.stats.MedianAbsSkewSec = round(sorted[n/2], 2)
|
||||
} else {
|
||||
b.stats.MedianAbsSkewSec = round((sorted[n/2-1]+sorted[n/2])/2, 2)
|
||||
}
|
||||
}
|
||||
resp.TotalNodes += b.stats.NodeCount
|
||||
resp.Roles = append(resp.Roles, b.stats)
|
||||
}
|
||||
// Sort: largest population first, then role name for stable output.
|
||||
sort.Slice(resp.Roles, func(i, j int) bool {
|
||||
if resp.Roles[i].NodeCount != resp.Roles[j].NodeCount {
|
||||
return resp.Roles[i].NodeCount > resp.Roles[j].NodeCount
|
||||
}
|
||||
return resp.Roles[i].Role < resp.Roles[j].Role
|
||||
})
|
||||
return resp
|
||||
}
|
||||
|
||||
// handleAnalyticsRoles serves /api/analytics/roles.
|
||||
func (s *Server) handleAnalyticsRoles(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store == nil {
|
||||
writeJSON(w, RoleAnalyticsResponse{Roles: []RoleStats{}})
|
||||
return
|
||||
}
|
||||
nodes, _ := s.store.getCachedNodesAndPM()
|
||||
roles := make(map[string]string, len(nodes))
|
||||
for _, n := range nodes {
|
||||
roles[n.PublicKey] = n.Role
|
||||
}
|
||||
skewMap := make(map[string]*NodeClockSkew)
|
||||
for _, cs := range s.store.GetFleetClockSkew() {
|
||||
if cs == nil {
|
||||
continue
|
||||
}
|
||||
skewMap[cs.Pubkey] = cs
|
||||
}
|
||||
writeJSON(w, computeRoleAnalytics(roles, skewMap))
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestComputeRoleAnalytics_Distribution verifies that computeRoleAnalytics
|
||||
// groups nodes by role, normalises empty/case-different roles, and sorts the
|
||||
// output largest-population first. Asserts on the public RoleAnalyticsResponse
|
||||
// shape so the bar is "behaviour", not "compiles".
|
||||
func TestComputeRoleAnalytics_Distribution(t *testing.T) {
|
||||
nodes := map[string]string{
|
||||
"pk_a": "Repeater",
|
||||
"pk_b": "repeater",
|
||||
"pk_c": "companion",
|
||||
"pk_d": "",
|
||||
"pk_e": "ROOM_SERVER",
|
||||
}
|
||||
got := computeRoleAnalytics(nodes, nil)
|
||||
|
||||
if got.TotalNodes != 5 {
|
||||
t.Fatalf("TotalNodes = %d, want 5", got.TotalNodes)
|
||||
}
|
||||
if len(got.Roles) != 4 {
|
||||
t.Fatalf("len(Roles) = %d, want 4 (repeater, companion, room_server, unknown), got %+v", len(got.Roles), got.Roles)
|
||||
}
|
||||
if got.Roles[0].Role != "repeater" || got.Roles[0].NodeCount != 2 {
|
||||
t.Errorf("Roles[0] = %+v, want {repeater,2}", got.Roles[0])
|
||||
}
|
||||
// Empty roles should bucket as "unknown".
|
||||
foundUnknown := false
|
||||
for _, r := range got.Roles {
|
||||
if r.Role == "unknown" {
|
||||
foundUnknown = true
|
||||
if r.NodeCount != 1 {
|
||||
t.Errorf("unknown bucket NodeCount = %d, want 1", r.NodeCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundUnknown {
|
||||
t.Errorf("no 'unknown' bucket for empty roles in %+v", got.Roles)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeRoleAnalytics_SkewAggregation verifies per-role clock-skew
|
||||
// aggregation: counts by severity, mean and median absolute skew.
|
||||
func TestComputeRoleAnalytics_SkewAggregation(t *testing.T) {
|
||||
nodes := map[string]string{
|
||||
"pk_1": "repeater",
|
||||
"pk_2": "repeater",
|
||||
"pk_3": "repeater",
|
||||
}
|
||||
skews := map[string]*NodeClockSkew{
|
||||
"pk_1": {Pubkey: "pk_1", RecentMedianSkewSec: 10, Severity: SkewOK},
|
||||
"pk_2": {Pubkey: "pk_2", RecentMedianSkewSec: -400, Severity: SkewWarning},
|
||||
"pk_3": {Pubkey: "pk_3", RecentMedianSkewSec: 7200, Severity: SkewCritical},
|
||||
}
|
||||
got := computeRoleAnalytics(nodes, skews)
|
||||
if len(got.Roles) != 1 {
|
||||
t.Fatalf("len(Roles) = %d, want 1; got %+v", len(got.Roles), got.Roles)
|
||||
}
|
||||
r := got.Roles[0]
|
||||
if r.WithSkew != 3 {
|
||||
t.Errorf("WithSkew = %d, want 3", r.WithSkew)
|
||||
}
|
||||
if r.OkCount != 1 || r.WarningCount != 1 || r.CriticalCount != 1 {
|
||||
t.Errorf("severity counts = ok %d, warn %d, crit %d; want 1/1/1", r.OkCount, r.WarningCount, r.CriticalCount)
|
||||
}
|
||||
// mean(|10|, |−400|, |7200|) = 7610/3 ≈ 2536.67
|
||||
if r.MeanAbsSkewSec < 2536 || r.MeanAbsSkewSec > 2537 {
|
||||
t.Errorf("MeanAbsSkewSec = %v, want ~2536.67", r.MeanAbsSkewSec)
|
||||
}
|
||||
// median(10, 400, 7200) = 400
|
||||
if r.MedianAbsSkewSec != 400 {
|
||||
t.Errorf("MedianAbsSkewSec = %v, want 400", r.MedianAbsSkewSec)
|
||||
}
|
||||
}
|
||||
+25
-189
@@ -16,7 +16,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
)
|
||||
|
||||
// Server holds shared state for route handlers.
|
||||
@@ -104,9 +103,6 @@ func (s *Server) getMemStats() runtime.MemStats {
|
||||
// RegisterRoutes sets up all HTTP routes on the given router.
|
||||
func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
s.router = r
|
||||
// CORS middleware (must run before route handlers)
|
||||
r.Use(s.corsMiddleware)
|
||||
|
||||
// Performance instrumentation middleware
|
||||
r.Use(s.perfMiddleware)
|
||||
|
||||
@@ -121,9 +117,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/config/map", s.handleConfigMap).Methods("GET")
|
||||
r.HandleFunc("/api/config/geo-filter", s.handleConfigGeoFilter).Methods("GET")
|
||||
|
||||
// Readiness endpoint (gated on background init completion)
|
||||
r.HandleFunc("/api/healthz", s.handleHealthz).Methods("GET")
|
||||
|
||||
// System endpoints
|
||||
r.HandleFunc("/api/health", s.handleHealth).Methods("GET")
|
||||
r.HandleFunc("/api/stats", s.handleStats).Methods("GET")
|
||||
@@ -132,7 +125,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST")
|
||||
r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET")
|
||||
r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET")
|
||||
r.Handle("/api/backup", s.requireAPIKey(http.HandlerFunc(s.handleBackup))).Methods("GET")
|
||||
|
||||
// Packet endpoints
|
||||
r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")
|
||||
@@ -159,7 +151,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/nodes", s.handleNodes).Methods("GET")
|
||||
|
||||
// Analytics endpoints
|
||||
r.HandleFunc("/api/analytics/roles", s.handleAnalyticsRoles).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/rf", s.handleAnalyticsRF).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/topology", s.handleAnalyticsTopology).Methods("GET")
|
||||
r.HandleFunc("/api/analytics/channels", s.handleAnalyticsChannels).Methods("GET")
|
||||
@@ -181,7 +172,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/observers/{id}", s.handleObserverDetail).Methods("GET")
|
||||
r.HandleFunc("/api/observers", s.handleObservers).Methods("GET")
|
||||
r.HandleFunc("/api/traces/{hash}", s.handleTraces).Methods("GET")
|
||||
r.HandleFunc("/api/paths/inspect", s.handlePathInspect).Methods("POST")
|
||||
r.HandleFunc("/api/iata-coords", s.handleIATACoords).Methods("GET")
|
||||
r.HandleFunc("/api/audio-lab/buckets", s.handleAudioLabBuckets).Methods("GET")
|
||||
|
||||
@@ -579,16 +569,6 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
backfillProgress = 1
|
||||
}
|
||||
|
||||
// Memory accounting (#832). storeDataMB is the in-store packet byte
|
||||
// estimate (the old "trackedMB"); processRSSMB / goHeapInuseMB / goSysMB
|
||||
// give ops the breakdown needed to reason about real RSS. All values
|
||||
// share a single 1s-cached snapshot to amortize ReadMemStats cost.
|
||||
var storeDataMB float64
|
||||
if s.store != nil {
|
||||
storeDataMB = s.store.trackedMemoryMB()
|
||||
}
|
||||
mem := s.getMemorySnapshot(storeDataMB)
|
||||
|
||||
resp := &StatsResponse{
|
||||
TotalPackets: stats.TotalPackets,
|
||||
TotalTransmissions: &stats.TotalTransmissions,
|
||||
@@ -612,12 +592,6 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
BackfillProgress: backfillProgress,
|
||||
SignatureDrops: s.db.GetSignatureDropCount(),
|
||||
HashMigrationComplete: s.store != nil && s.store.hashMigrationComplete.Load(),
|
||||
|
||||
TrackedMB: mem.StoreDataMB, // deprecated alias
|
||||
StoreDataMB: mem.StoreDataMB,
|
||||
ProcessRSSMB: mem.ProcessRSSMB,
|
||||
GoHeapInuseMB: mem.GoHeapInuseMB,
|
||||
GoSysMB: mem.GoSysMB,
|
||||
}
|
||||
|
||||
s.statsMu.Lock()
|
||||
@@ -800,7 +774,6 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
|
||||
Until: r.URL.Query().Get("until"),
|
||||
Region: r.URL.Query().Get("region"),
|
||||
Node: r.URL.Query().Get("node"),
|
||||
Channel: r.URL.Query().Get("channel"),
|
||||
Order: "DESC",
|
||||
ExpandObservations: r.URL.Query().Get("expand") == "observations",
|
||||
}
|
||||
@@ -903,11 +876,9 @@ func (s *Server) handleBatchObservations(w http.ResponseWriter, r *http.Request)
|
||||
func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
param := mux.Vars(r)["id"]
|
||||
var packet map[string]interface{}
|
||||
fromDB := false
|
||||
|
||||
isHash := hashPattern.MatchString(strings.ToLower(param))
|
||||
if s.store != nil {
|
||||
if isHash {
|
||||
if hashPattern.MatchString(strings.ToLower(param)) {
|
||||
packet = s.store.GetPacketByHash(param)
|
||||
}
|
||||
if packet == nil {
|
||||
@@ -920,25 +891,6 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// DB fallback: in-memory PacketStore prunes old entries, but the SQLite
|
||||
// DB retains them and is the source for /api/nodes recentAdverts. Without
|
||||
// this fallback, links from node-detail pages 404 once the packet ages out.
|
||||
if packet == nil && s.db != nil {
|
||||
if isHash {
|
||||
if dbPkt, err := s.db.GetPacketByHash(param); err == nil && dbPkt != nil {
|
||||
packet = dbPkt
|
||||
fromDB = true
|
||||
}
|
||||
}
|
||||
if packet == nil {
|
||||
if id, parseErr := strconv.Atoi(param); parseErr == nil {
|
||||
if dbPkt, err := s.db.GetTransmissionByID(id); err == nil && dbPkt != nil {
|
||||
packet = dbPkt
|
||||
fromDB = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if packet == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
@@ -949,9 +901,6 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store != nil {
|
||||
observations = s.store.GetObservationsForHash(hash)
|
||||
}
|
||||
if len(observations) == 0 && fromDB && s.db != nil && hash != "" {
|
||||
observations = s.db.GetObservationsForHash(hash)
|
||||
}
|
||||
observationCount := len(observations)
|
||||
if observationCount == 0 {
|
||||
observationCount = 1
|
||||
@@ -967,9 +916,11 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
|
||||
pathHops = []interface{}{}
|
||||
}
|
||||
|
||||
rawHex, _ := packet["raw_hex"].(string)
|
||||
writeJSON(w, PacketDetailResponse{
|
||||
Packet: packet,
|
||||
Path: pathHops,
|
||||
Breakdown: BuildBreakdown(rawHex),
|
||||
ObservationCount: observationCount,
|
||||
Observations: mapSliceToObservations(observations),
|
||||
})
|
||||
@@ -1028,17 +979,8 @@ func (s *Server) handlePostPacket(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
contentHash := ComputeContentHash(hexStr)
|
||||
pathJSON := "[]"
|
||||
// For TRACE packets, path_json must be the payload-decoded route hops
|
||||
// (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values.
|
||||
// For all other packet types, derive path from raw_hex (#886).
|
||||
if !packetpath.PathBytesAreHops(byte(decoded.Header.PayloadType)) {
|
||||
if len(decoded.Path.Hops) > 0 {
|
||||
if pj, e := json.Marshal(decoded.Path.Hops); e == nil {
|
||||
pathJSON = string(pj)
|
||||
}
|
||||
}
|
||||
} else if hops, err := packetpath.DecodePathFromRawHex(hexStr); err == nil && len(hops) > 0 {
|
||||
if pj, e := json.Marshal(hops); e == nil {
|
||||
if len(decoded.Path.Hops) > 0 {
|
||||
if pj, e := json.Marshal(decoded.Path.Hops); e == nil {
|
||||
pathJSON = string(pj)
|
||||
}
|
||||
}
|
||||
@@ -1096,11 +1038,9 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if s.store != nil {
|
||||
hashInfo := s.store.GetNodeHashSizeInfo()
|
||||
mbCap := s.store.GetMultiByteCapMap()
|
||||
for _, node := range nodes {
|
||||
if pk, ok := node["public_key"].(string); ok {
|
||||
EnrichNodeWithHashSize(node, hashInfo[pk])
|
||||
EnrichNodeWithMultiByte(node, mbCap[pk])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1159,44 +1099,14 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
node, err := s.db.GetNodeByPubkey(pubkey)
|
||||
if err != nil {
|
||||
writeError(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
// Issue #772: short-URL fallback. If exact pubkey lookup misses and the
|
||||
// path looks like a hex prefix (>=8 chars, <64), try prefix resolution.
|
||||
if node == nil && len(pubkey) >= 8 && len(pubkey) < 64 {
|
||||
resolved, ambiguous, perr := s.db.GetNodeByPrefix(pubkey)
|
||||
if perr != nil {
|
||||
writeError(w, 500, perr.Error())
|
||||
return
|
||||
}
|
||||
if ambiguous {
|
||||
writeError(w, http.StatusConflict, "Ambiguous prefix: multiple nodes match. Use a longer prefix.")
|
||||
return
|
||||
}
|
||||
if resolved != nil {
|
||||
if pk, _ := resolved["public_key"].(string); pk != "" && s.cfg.IsBlacklisted(pk) {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
node = resolved
|
||||
}
|
||||
}
|
||||
if node == nil {
|
||||
if err != nil || node == nil {
|
||||
writeError(w, 404, "Not found")
|
||||
return
|
||||
}
|
||||
// From here on use the canonical pubkey for downstream lookups.
|
||||
if pk, _ := node["public_key"].(string); pk != "" {
|
||||
pubkey = pk
|
||||
}
|
||||
|
||||
if s.store != nil {
|
||||
hashInfo := s.store.GetNodeHashSizeInfo()
|
||||
EnrichNodeWithHashSize(node, hashInfo[pubkey])
|
||||
mbCap := s.store.GetMultiByteCapMap()
|
||||
EnrichNodeWithMultiByte(node, mbCap[pubkey])
|
||||
}
|
||||
|
||||
name := ""
|
||||
@@ -1298,85 +1208,39 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
_, pm := s.store.getCachedNodesAndPM()
|
||||
|
||||
// Collect candidate transmissions from the index, deduplicating by tx ID.
|
||||
// confirmedByFullKey tracks TXs found via the full-pubkey index key — these are
|
||||
// already resolved_path-confirmed and bypass the hop-level check below.
|
||||
confirmedByFullKey := make(map[int]bool)
|
||||
seen := make(map[int]bool)
|
||||
var candidates []*StoreTx
|
||||
addCandidates := func(key string, confirmed bool) {
|
||||
addCandidates := func(key string) {
|
||||
for _, tx := range s.store.byPathHop[key] {
|
||||
if !seen[tx.ID] {
|
||||
seen[tx.ID] = true
|
||||
if confirmed {
|
||||
confirmedByFullKey[tx.ID] = true
|
||||
}
|
||||
candidates = append(candidates, tx)
|
||||
}
|
||||
}
|
||||
}
|
||||
addCandidates(lowerPK, true) // full pubkey match (from resolved_path) → confirmed
|
||||
addCandidates(prefix1, false) // 2-char raw hop match
|
||||
addCandidates(prefix2, false) // 4-char raw hop match
|
||||
addCandidates(lowerPK) // full pubkey match (from resolved_path)
|
||||
addCandidates(prefix1) // 2-char raw hop match
|
||||
addCandidates(prefix2) // 4-char raw hop match
|
||||
// Also check any raw hops that start with prefix2 (longer prefixes).
|
||||
// Raw hops are typically 2 chars, so iterate only keys with HasPrefix
|
||||
// on the small set of index keys rather than all packets.
|
||||
for key := range s.store.byPathHop {
|
||||
if len(key) > 4 && len(key) < len(lowerPK) && strings.HasPrefix(key, prefix2) {
|
||||
addCandidates(key, false)
|
||||
addCandidates(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Post-filter: verify target node actually appears in each candidate's resolved_path.
|
||||
// The byPathHop index uses short prefixes which can collide (e.g. "c0" matches multiple nodes).
|
||||
// We lean on resolved_path (from neighbor affinity graph) to disambiguate.
|
||||
//
|
||||
// Collect candidate IDs and index membership under the read lock, then release
|
||||
// the lock before running SQL queries (confirmResolvedPathContains does disk I/O).
|
||||
type candidateCheck struct {
|
||||
tx *StoreTx
|
||||
hasReverse bool
|
||||
inIndex bool
|
||||
}
|
||||
checks := make([]candidateCheck, len(candidates))
|
||||
for i, tx := range candidates {
|
||||
cc := candidateCheck{tx: tx}
|
||||
if !s.store.useResolvedPathIndex {
|
||||
cc.inIndex = true // flag off — keep all
|
||||
} else if _, hasRev := s.store.resolvedPubkeyReverse[tx.ID]; !hasRev {
|
||||
cc.inIndex = true // no indexed pubkeys — keep (conservative)
|
||||
} else {
|
||||
h := resolvedPubkeyHash(lowerPK)
|
||||
for _, id := range s.store.resolvedPubkeyIndex[h] {
|
||||
if id == tx.ID {
|
||||
cc.hasReverse = true // needs SQL confirmation
|
||||
break
|
||||
}
|
||||
}
|
||||
// If not in index at all, it's a definite no
|
||||
filtered := candidates[:0] // reuse backing array
|
||||
for _, tx := range candidates {
|
||||
if nodeInResolvedPath(tx, lowerPK) {
|
||||
filtered = append(filtered, tx)
|
||||
}
|
||||
checks[i] = cc
|
||||
}
|
||||
s.store.mu.RUnlock()
|
||||
|
||||
// Now run SQL checks outside the lock for candidates that need confirmation.
|
||||
confirmedBySQL := make(map[int]bool)
|
||||
filtered := candidates[:0]
|
||||
for _, cc := range checks {
|
||||
if cc.inIndex {
|
||||
filtered = append(filtered, cc.tx)
|
||||
} else if cc.hasReverse {
|
||||
if s.store.confirmResolvedPathContains(cc.tx.ID, lowerPK) {
|
||||
filtered = append(filtered, cc.tx)
|
||||
confirmedBySQL[cc.tx.ID] = true
|
||||
}
|
||||
}
|
||||
// else: not in index → exclude
|
||||
}
|
||||
candidates = filtered
|
||||
|
||||
// Re-acquire read lock for the aggregation phase that reads store data.
|
||||
s.store.mu.RLock()
|
||||
|
||||
type pathAgg struct {
|
||||
Hops []PathHopResp
|
||||
Count int
|
||||
@@ -1395,14 +1259,10 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
return r
|
||||
}
|
||||
for _, tx := range candidates {
|
||||
totalTransmissions++
|
||||
hops := txGetParsedPath(tx)
|
||||
resolvedHops := make([]PathHopResp, len(hops))
|
||||
sigParts := make([]string, len(hops))
|
||||
// For candidates not confirmed via full-pubkey index or SQL, verify that at
|
||||
// least one hop actually resolves to the target. This catches prefix collisions
|
||||
// (e.g. two nodes sharing a "7a" 1-byte prefix) that slipped through the
|
||||
// conservative resolved_path fallback.
|
||||
containsTarget := confirmedByFullKey[tx.ID] || confirmedBySQL[tx.ID]
|
||||
for i, hop := range hops {
|
||||
resolved := resolveHop(hop)
|
||||
entry := PathHopResp{Prefix: hop, Name: hop}
|
||||
@@ -1414,22 +1274,11 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
|
||||
entry.Lon = resolved.Lon
|
||||
}
|
||||
sigParts[i] = resolved.PublicKey
|
||||
if strings.ToLower(resolved.PublicKey) == lowerPK {
|
||||
containsTarget = true
|
||||
}
|
||||
} else {
|
||||
sigParts[i] = hop
|
||||
// Unresolvable hop: keep conservative if prefix could be the target.
|
||||
if strings.HasPrefix(lowerPK, strings.ToLower(hop)) {
|
||||
containsTarget = true
|
||||
}
|
||||
}
|
||||
resolvedHops[i] = entry
|
||||
}
|
||||
if !containsTarget {
|
||||
continue
|
||||
}
|
||||
totalTransmissions++
|
||||
|
||||
sig := strings.Join(sigParts, "→")
|
||||
agg := pathGroups[sig]
|
||||
@@ -1555,9 +1404,8 @@ func (s *Server) handleFleetClockSkew(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
|
||||
region := r.URL.Query().Get("region")
|
||||
window := ParseTimeWindow(r)
|
||||
if s.store != nil {
|
||||
writeJSON(w, s.store.GetAnalyticsRFWithWindow(region, window))
|
||||
writeJSON(w, s.store.GetAnalyticsRF(region))
|
||||
return
|
||||
}
|
||||
writeJSON(w, RFAnalyticsResponse{
|
||||
@@ -1576,9 +1424,8 @@ func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleAnalyticsTopology(w http.ResponseWriter, r *http.Request) {
|
||||
region := r.URL.Query().Get("region")
|
||||
window := ParseTimeWindow(r)
|
||||
if s.store != nil {
|
||||
data := s.store.GetAnalyticsTopologyWithWindow(region, window)
|
||||
data := s.store.GetAnalyticsTopology(region)
|
||||
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
|
||||
data = s.filterBlacklistedFromTopology(data)
|
||||
}
|
||||
@@ -1600,8 +1447,7 @@ func (s *Server) handleAnalyticsTopology(w http.ResponseWriter, r *http.Request)
|
||||
func (s *Server) handleAnalyticsChannels(w http.ResponseWriter, r *http.Request) {
|
||||
if s.store != nil {
|
||||
region := r.URL.Query().Get("region")
|
||||
window := ParseTimeWindow(r)
|
||||
writeJSON(w, s.store.GetAnalyticsChannelsWithWindow(region, window))
|
||||
writeJSON(w, s.store.GetAnalyticsChannels(region))
|
||||
return
|
||||
}
|
||||
channels, _ := s.db.GetChannels()
|
||||
@@ -1995,10 +1841,6 @@ func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
result := make([]ObserverResp, 0, len(observers))
|
||||
for _, o := range observers {
|
||||
// Defense in depth: skip observers that are in the blacklist
|
||||
if s.cfg != nil && s.cfg.IsObserverBlacklisted(o.ID) {
|
||||
continue
|
||||
}
|
||||
plh := 0
|
||||
if c, ok := pktCounts[o.ID]; ok {
|
||||
plh = c
|
||||
@@ -2018,7 +1860,6 @@ func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
|
||||
ClientVersion: o.ClientVersion, Radio: o.Radio,
|
||||
BatteryMv: o.BatteryMv, UptimeSecs: o.UptimeSecs,
|
||||
NoiseFloor: o.NoiseFloor,
|
||||
LastPacketAt: o.LastPacketAt,
|
||||
PacketsLastHour: plh,
|
||||
Lat: lat, Lon: lon, NodeRole: nodeRole,
|
||||
})
|
||||
@@ -2031,13 +1872,6 @@ func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (s *Server) handleObserverDetail(w http.ResponseWriter, r *http.Request) {
|
||||
id := mux.Vars(r)["id"]
|
||||
|
||||
// Defense in depth: reject blacklisted observer
|
||||
if s.cfg != nil && s.cfg.IsObserverBlacklisted(id) {
|
||||
writeError(w, 404, "Observer not found")
|
||||
return
|
||||
}
|
||||
|
||||
obs, err := s.db.GetObserverByID(id)
|
||||
if err != nil || obs == nil {
|
||||
writeError(w, 404, "Observer not found")
|
||||
@@ -2060,7 +1894,6 @@ func (s *Server) handleObserverDetail(w http.ResponseWriter, r *http.Request) {
|
||||
ClientVersion: obs.ClientVersion, Radio: obs.Radio,
|
||||
BatteryMv: obs.BatteryMv, UptimeSecs: obs.UptimeSecs,
|
||||
NoiseFloor: obs.NoiseFloor,
|
||||
LastPacketAt: obs.LastPacketAt,
|
||||
PacketsLastHour: plh,
|
||||
})
|
||||
}
|
||||
@@ -2454,6 +2287,9 @@ func mapSliceToTransmissions(maps []map[string]interface{}) []TransmissionResp {
|
||||
tx.PathJSON = m["path_json"]
|
||||
tx.Direction = m["direction"]
|
||||
tx.Score = m["score"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
tx.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, tx)
|
||||
}
|
||||
return result
|
||||
@@ -2474,10 +2310,10 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
|
||||
obs.SNR = m["snr"]
|
||||
obs.RSSI = m["rssi"]
|
||||
obs.PathJSON = m["path_json"]
|
||||
obs.ResolvedPath = m["resolved_path"]
|
||||
obs.Direction = m["direction"]
|
||||
obs.RawHex = m["raw_hex"]
|
||||
obs.Timestamp = m["timestamp"]
|
||||
if rp, ok := m["resolved_path"].([]*string); ok {
|
||||
obs.ResolvedPath = rp
|
||||
}
|
||||
result = append(result, obs)
|
||||
}
|
||||
return result
|
||||
|
||||
+41
-205
@@ -3681,55 +3681,67 @@ func TestNodePathsPrefixCollisionFilter(t *testing.T) {
|
||||
func TestNodeInResolvedPath(t *testing.T) {
|
||||
target := "aabbccdd11223344"
|
||||
|
||||
// After #800, nodeInResolvedPath is replaced by nodeInResolvedPathViaIndex
|
||||
// which uses the membership index. Test the index-based approach.
|
||||
store := &PacketStore{
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
useResolvedPathIndex: true,
|
||||
}
|
||||
store.initResolvedPathIndex()
|
||||
|
||||
// Case 1: tx indexed with target pubkey
|
||||
tx1 := &StoreTx{ID: 1}
|
||||
store.addToResolvedPubkeyIndex(1, []string{target})
|
||||
if !store.nodeInResolvedPathViaIndex(tx1, target) {
|
||||
t.Error("should match when index contains target")
|
||||
// Case 1: tx.ResolvedPath contains target
|
||||
pk := "aabbccdd11223344"
|
||||
tx1 := &StoreTx{ResolvedPath: []*string{&pk}}
|
||||
if !nodeInResolvedPath(tx1, target) {
|
||||
t.Error("should match when ResolvedPath contains target")
|
||||
}
|
||||
|
||||
// Case 2: tx indexed with different pubkey
|
||||
tx2 := &StoreTx{ID: 2}
|
||||
store.addToResolvedPubkeyIndex(2, []string{"aacafe0000000000"})
|
||||
if store.nodeInResolvedPathViaIndex(tx2, target) {
|
||||
t.Error("should not match when index contains different node")
|
||||
// Case 2: tx.ResolvedPath contains different node
|
||||
other := "aacafe0000000000"
|
||||
tx2 := &StoreTx{ResolvedPath: []*string{&other}}
|
||||
if nodeInResolvedPath(tx2, target) {
|
||||
t.Error("should not match when ResolvedPath contains different node")
|
||||
}
|
||||
|
||||
// Case 3: tx not in index at all — should match (no data to disambiguate)
|
||||
tx3 := &StoreTx{ID: 3}
|
||||
if !store.nodeInResolvedPathViaIndex(tx3, target) {
|
||||
t.Error("should match when tx has no index entries (no data to disambiguate)")
|
||||
// Case 3: nil ResolvedPath — should match (no data to disambiguate, keep it)
|
||||
tx3 := &StoreTx{}
|
||||
if !nodeInResolvedPath(tx3, target) {
|
||||
t.Error("should match when ResolvedPath is nil (no data to disambiguate)")
|
||||
}
|
||||
|
||||
// Case 4: ResolvedPath with nil elements only — has data but no match
|
||||
tx4 := &StoreTx{ResolvedPath: []*string{nil, nil}}
|
||||
if nodeInResolvedPath(tx4, target) {
|
||||
t.Error("should not match when all ResolvedPath elements are nil")
|
||||
}
|
||||
|
||||
// Case 5: target in observation but not in tx.ResolvedPath
|
||||
tx5 := &StoreTx{
|
||||
ResolvedPath: []*string{&other},
|
||||
Observations: []*StoreObs{
|
||||
{ResolvedPath: []*string{&pk}},
|
||||
},
|
||||
}
|
||||
if !nodeInResolvedPath(tx5, target) {
|
||||
t.Error("should match when observation's ResolvedPath contains target")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathHopIndexIncrementalUpdate(t *testing.T) {
|
||||
// After #800, addTxToPathHopIndex only indexes raw hops (not resolved pubkeys).
|
||||
// Resolved pubkeys are handled by the resolved pubkey membership index.
|
||||
// Test that addTxToPathHopIndex and removeTxFromPathHopIndex work correctly
|
||||
idx := make(map[string][]*StoreTx)
|
||||
|
||||
pk1 := "fullpubkey1"
|
||||
tx1 := &StoreTx{
|
||||
ID: 1,
|
||||
PathJSON: `["ab","cd"]`,
|
||||
ResolvedPath: []*string{&pk1, nil},
|
||||
}
|
||||
|
||||
addTxToPathHopIndex(idx, tx1)
|
||||
|
||||
// Should be indexed under "ab" and "cd" only (no resolved pubkey)
|
||||
// Should be indexed under "ab", "cd", and "fullpubkey1"
|
||||
if len(idx["ab"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'ab', got %d", len(idx["ab"]))
|
||||
}
|
||||
if len(idx["cd"]) != 1 {
|
||||
t.Errorf("expected 1 entry for 'cd', got %d", len(idx["cd"]))
|
||||
}
|
||||
if len(idx["fullpubkey1"]) != 1 {
|
||||
t.Errorf("expected 1 entry for resolved pubkey, got %d", len(idx["fullpubkey1"]))
|
||||
}
|
||||
|
||||
// Add another tx with overlapping hop
|
||||
tx2 := &StoreTx{
|
||||
@@ -3754,6 +3766,9 @@ func TestPathHopIndexIncrementalUpdate(t *testing.T) {
|
||||
if _, ok := idx["cd"]; ok {
|
||||
t.Error("expected 'cd' key to be deleted after removal")
|
||||
}
|
||||
if _, ok := idx["fullpubkey1"]; ok {
|
||||
t.Error("expected resolved pubkey key to be deleted after removal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAPIEndpoints(t *testing.T) {
|
||||
@@ -3793,182 +3808,3 @@ func TestMetricsAPIEndpoints(t *testing.T) {
|
||||
t.Errorf("expected 1 observer in summary, got %v", resp2["observers"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeHealth_RecentPackets_ResolvedPath verifies that recentPackets in the
|
||||
// node health endpoint include resolved_path (regression for Codex review item #2).
|
||||
func TestNodeHealth_RecentPackets_ResolvedPath(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
rp, ok := body["recentPackets"].([]interface{})
|
||||
if !ok || len(rp) == 0 {
|
||||
t.Fatal("expected non-empty recentPackets")
|
||||
}
|
||||
// At least one packet should have resolved_path (tx 1 has observations with resolved_path)
|
||||
found := false
|
||||
for _, p := range rp {
|
||||
pm, ok := p.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if pm["resolved_path"] != nil {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("expected at least one recentPacket with resolved_path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketsExpand_ResolvedPath verifies that expandObservations=true includes
|
||||
// resolved_path on expanded observations (regression for Codex review item #3).
|
||||
func TestPacketsExpand_ResolvedPath(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/packets?expand=observations&limit=10", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
packets, ok := body["packets"].([]interface{})
|
||||
if !ok || len(packets) == 0 {
|
||||
t.Fatal("expected non-empty packets")
|
||||
}
|
||||
// Find a packet with observations that should have resolved_path
|
||||
found := false
|
||||
for _, p := range packets {
|
||||
pm, ok := p.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
obs, ok := pm["observations"].([]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, o := range obs {
|
||||
om, ok := o.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if om["resolved_path"] != nil {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("expected at least one expanded observation with resolved_path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetailFallsBackToDBWhenStoreMisses verifies that handlePacketDetail
|
||||
// serves transmissions present in the DB but absent from the in-memory store.
|
||||
// This is the recentAdverts → "Not found" bug (#827).
|
||||
func TestPacketDetailFallsBackToDBWhenStoreMisses(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
// Insert a transmission directly into the DB AFTER store.Load(), so the
|
||||
// in-memory PacketStore won't see it. Mirrors the production case where
|
||||
// the store has pruned an entry but the DB still has it.
|
||||
const dbOnlyHash = "deadbeef00112233"
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('FFEE', ?, ?, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
var txID int
|
||||
if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", dbOnlyHash).Scan(&txID); err != nil {
|
||||
t.Fatalf("lookup tx id: %v", err)
|
||||
}
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO observations
|
||||
(transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (?, 1, 7.5, -99, '[]', ?)`, txID, time.Now().Unix()); err != nil {
|
||||
t.Fatalf("insert obs: %v", err)
|
||||
}
|
||||
|
||||
// Confirm the store really doesn't have it (precondition for the fix).
|
||||
if got := srv.store.GetPacketByHash(dbOnlyHash); got != nil {
|
||||
t.Fatalf("test precondition failed: store unexpectedly has %s", dbOnlyHash)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/packets/"+dbOnlyHash, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkt, ok := body["packet"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected packet object")
|
||||
}
|
||||
if pkt["hash"] != dbOnlyHash {
|
||||
t.Errorf("expected hash %s, got %v", dbOnlyHash, pkt["hash"])
|
||||
}
|
||||
// Observations fallback should populate from DB too.
|
||||
obs, _ := body["observations"].([]interface{})
|
||||
if len(obs) == 0 {
|
||||
t.Errorf("expected DB observations to be returned, got 0")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetail404WhenAbsentFromBoth verifies that a hash present in
|
||||
// neither store nor DB still returns 404 (no false positives from the fallback).
|
||||
func TestPacketDetail404WhenAbsentFromBoth(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/packets/0011223344556677", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 404 {
|
||||
t.Errorf("expected 404, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestPacketDetailPrefersStoreOverDB verifies the store result wins when the
|
||||
// hash exists in both — the DB fallback must not double-fetch / overwrite.
|
||||
func TestPacketDetailPrefersStoreOverDB(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
// abc123def4567890 is seeded in both DB and (after Load) the store.
|
||||
const hash = "abc123def4567890"
|
||||
if got := srv.store.GetPacketByHash(hash); got == nil {
|
||||
t.Fatalf("test precondition failed: store should have %s", hash)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/packets/"+hash, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
json.Unmarshal(w.Body.Bytes(), &body)
|
||||
pkt, _ := body["packet"].(map[string]interface{})
|
||||
if pkt == nil || pkt["hash"] != hash {
|
||||
t.Fatalf("expected packet with hash %s, got %v", hash, pkt)
|
||||
}
|
||||
// observation_count comes from store observations (2 seeded for tx 1).
|
||||
if cnt, _ := body["observation_count"].(float64); cnt != 2 {
|
||||
t.Errorf("expected observation_count=2 (from store), got %v", body["observation_count"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// rwCache holds a process-wide cached RW connection per database path.
|
||||
// Instead of opening and closing a new RW connection on every call to openRW,
|
||||
// we cache a single *sql.DB (which internally manages one connection due to
|
||||
// SetMaxOpenConns(1)). This eliminates repeated open/close overhead for
|
||||
// vacuum, prune, persist operations that run frequently (#921).
|
||||
var rwCache = struct {
|
||||
mu sync.Mutex
|
||||
conns map[string]*sql.DB
|
||||
}{conns: make(map[string]*sql.DB)}
|
||||
|
||||
// cachedRW returns a cached read-write connection for the given dbPath.
|
||||
// The connection is created on first call and reused thereafter.
|
||||
// Callers MUST NOT call Close() on the returned *sql.DB.
|
||||
func cachedRW(dbPath string) (*sql.DB, error) {
|
||||
rwCache.mu.Lock()
|
||||
defer rwCache.mu.Unlock()
|
||||
|
||||
if db, ok := rwCache.conns[dbPath]; ok {
|
||||
return db, nil
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL", dbPath)
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.Exec("PRAGMA busy_timeout = 5000"); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("set busy_timeout: %w", err)
|
||||
}
|
||||
rwCache.conns[dbPath] = db
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// closeRWCache closes all cached RW connections (for tests/shutdown).
|
||||
func closeRWCache() {
|
||||
rwCache.mu.Lock()
|
||||
defer rwCache.mu.Unlock()
|
||||
for k, db := range rwCache.conns {
|
||||
db.Close()
|
||||
delete(rwCache.conns, k)
|
||||
}
|
||||
}
|
||||
|
||||
// rwCacheLen returns the number of cached connections (for testing).
|
||||
func rwCacheLen() int {
|
||||
rwCache.mu.Lock()
|
||||
defer rwCache.mu.Unlock()
|
||||
return len(rwCache.conns)
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCachedRW_ReturnsSameHandle(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
// Create the DB file
|
||||
f, _ := os.Create(dbPath)
|
||||
f.Close()
|
||||
|
||||
defer closeRWCache()
|
||||
|
||||
db1, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("first cachedRW: %v", err)
|
||||
}
|
||||
db2, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("second cachedRW: %v", err)
|
||||
}
|
||||
if db1 != db2 {
|
||||
t.Fatalf("cachedRW returned different handles: %p vs %p", db1, db2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedRW_100Calls_SingleConnection(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
f, _ := os.Create(dbPath)
|
||||
f.Close()
|
||||
|
||||
defer closeRWCache()
|
||||
|
||||
var first interface{}
|
||||
for i := 0; i < 100; i++ {
|
||||
db, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("call %d: %v", i, err)
|
||||
}
|
||||
if i == 0 {
|
||||
first = db
|
||||
} else if db != first {
|
||||
t.Fatalf("call %d returned different handle", i)
|
||||
}
|
||||
}
|
||||
if rwCacheLen() != 1 {
|
||||
t.Fatalf("expected 1 cached connection, got %d", rwCacheLen())
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue #772 — shortened URL for easier sending over the mesh.
|
||||
//
|
||||
// Public keys are 64 hex chars. Operators want to share node URLs over a
|
||||
// mesh radio link where every byte counts. We allow truncating the pubkey
|
||||
// in the URL down to a minimum 8-hex-char prefix; the server resolves the
|
||||
// prefix back to the full pubkey when (and only when) it is unambiguous.
|
||||
|
||||
func TestResolveNodePrefix_Unique(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// "aabbccdd" uniquely identifies the seeded TestRepeater (pubkey aabbccdd11223344).
|
||||
node, ambiguous, err := db.GetNodeByPrefix("aabbccdd")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if ambiguous {
|
||||
t.Fatalf("expected unambiguous match, got ambiguous=true")
|
||||
}
|
||||
if node == nil {
|
||||
t.Fatalf("expected node, got nil")
|
||||
}
|
||||
if got, _ := node["public_key"].(string); got != "aabbccdd11223344" {
|
||||
t.Errorf("expected public_key aabbccdd11223344, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveNodePrefix_Ambiguous(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// Insert a second node sharing the 8-char prefix "aabbccdd".
|
||||
if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, advert_count)
|
||||
VALUES ('aabbccdd99887766', 'OtherNode', 'companion', 1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
node, ambiguous, err := db.GetNodeByPrefix("aabbccdd")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected err: %v", err)
|
||||
}
|
||||
if !ambiguous {
|
||||
t.Fatalf("expected ambiguous=true for shared prefix, got false (node=%v)", node)
|
||||
}
|
||||
if node != nil {
|
||||
t.Errorf("expected nil node when ambiguous, got %v", node["public_key"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveNodePrefix_TooShort(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// <8 hex chars must NOT resolve, even if it would be unique.
|
||||
node, _, err := db.GetNodeByPrefix("aabbccd")
|
||||
if err == nil && node != nil {
|
||||
t.Errorf("expected nil/error for 7-char prefix, got node %v", node["public_key"])
|
||||
}
|
||||
}
|
||||
|
||||
// Route-level: GET /api/nodes/<8-char-prefix> resolves to the full node.
|
||||
func TestNodeDetailRoute_PrefixResolves(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200 for unique 8-char prefix, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var body NodeDetailResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
pk, _ := body.Node["public_key"].(string)
|
||||
if pk != "aabbccdd11223344" {
|
||||
t.Errorf("expected resolved pubkey aabbccdd11223344, got %q", pk)
|
||||
}
|
||||
}
|
||||
|
||||
// Route-level: GET /api/nodes/<ambiguous-prefix> returns 409 with a hint.
|
||||
func TestNodeDetailRoute_PrefixAmbiguous(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, advert_count)
|
||||
VALUES ('aabbccdd99887766', 'OtherNode', 'companion', 1)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusConflict {
|
||||
t.Fatalf("expected 409 for ambiguous prefix, got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestStatsMemoryFields verifies that /api/stats exposes the new memory
|
||||
// breakdown introduced for issue #832: storeDataMB, processRSSMB,
|
||||
// goHeapInuseMB, goSysMB, plus the deprecated trackedMB alias.
|
||||
//
|
||||
// We assert presence, type, sign, and ordering invariants — but NOT
|
||||
// "RSS within X% of true RSS" because that is flaky in CI under cgo,
|
||||
// containerization, and shared-runner load.
|
||||
func TestStatsMemoryFields(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/stats", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("json decode: %v", err)
|
||||
}
|
||||
|
||||
required := []string{"trackedMB", "storeDataMB", "processRSSMB", "goHeapInuseMB", "goSysMB"}
|
||||
values := make(map[string]float64, len(required))
|
||||
for _, k := range required {
|
||||
v, ok := body[k]
|
||||
if !ok {
|
||||
t.Fatalf("missing field %q in /api/stats response", k)
|
||||
}
|
||||
f, ok := v.(float64)
|
||||
if !ok {
|
||||
t.Fatalf("field %q is %T, expected float64", k, v)
|
||||
}
|
||||
if f < 0 {
|
||||
t.Errorf("field %q is negative: %v", k, f)
|
||||
}
|
||||
values[k] = f
|
||||
}
|
||||
|
||||
// trackedMB is a deprecated alias for storeDataMB; they must match.
|
||||
if values["trackedMB"] != values["storeDataMB"] {
|
||||
t.Errorf("trackedMB (%v) != storeDataMB (%v); they must remain aliased",
|
||||
values["trackedMB"], values["storeDataMB"])
|
||||
}
|
||||
|
||||
// Ordering invariants. goSys is the runtime's view of total OS memory;
|
||||
// HeapInuse is a subset of it. storeData is a subset of HeapInuse.
|
||||
// processRSS may be 0 in environments without /proc — treat 0 as
|
||||
// "unknown" rather than a failure.
|
||||
if values["goHeapInuseMB"] > values["goSysMB"]+0.5 {
|
||||
t.Errorf("invariant violated: goHeapInuseMB (%v) > goSysMB (%v)",
|
||||
values["goHeapInuseMB"], values["goSysMB"])
|
||||
}
|
||||
if values["storeDataMB"] > values["goHeapInuseMB"]+0.5 && values["storeDataMB"] > 0 {
|
||||
// In the test fixture storeDataMB is typically 0 (no packets in
|
||||
// store); only enforce the bound when both are nonzero.
|
||||
t.Errorf("invariant violated: storeDataMB (%v) > goHeapInuseMB (%v)",
|
||||
values["storeDataMB"], values["goHeapInuseMB"])
|
||||
}
|
||||
if values["processRSSMB"] > 0 && values["goSysMB"] > 0 {
|
||||
// goSys can briefly exceed RSS if pages are reserved-but-not-touched,
|
||||
// so allow some slack.
|
||||
if values["goSysMB"] > values["processRSSMB"]*4 {
|
||||
t.Errorf("suspicious: goSysMB (%v) >> processRSSMB (%v)",
|
||||
values["goSysMB"], values["processRSSMB"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsMemoryFieldsRawJSON spot-checks that the JSON wire format uses
|
||||
// the documented camelCase names (no accidental rename through struct tags).
|
||||
func TestStatsMemoryFieldsRawJSON(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/stats", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
body := w.Body.String()
|
||||
for _, key := range []string{
|
||||
`"trackedMB":`, `"storeDataMB":`,
|
||||
`"processRSSMB":`, `"goHeapInuseMB":`, `"goSysMB":`,
|
||||
} {
|
||||
if !strings.Contains(body, key) {
|
||||
t.Errorf("missing %s in raw response: %s", key, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
+380
-1059
File diff suppressed because it is too large
Load Diff
@@ -1,116 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func f64(v float64) *float64 { return &v }
|
||||
|
||||
func TestDedupeTopHopsByPair(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: f64(5.0), Hash: "h1", Timestamp: "t1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: f64(8.0), Hash: "h2", Timestamp: "t2"},
|
||||
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 80, Type: "R↔R", SNR: f64(3.0), Hash: "h3", Timestamp: "t3"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 70, Type: "R↔R", SNR: f64(6.0), Hash: "h4", Timestamp: "t4"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 60, Type: "R↔R", SNR: f64(4.0), Hash: "h5", Timestamp: "t5"},
|
||||
{FromPk: "CCC", ToPk: "DDD", FromName: "C", ToName: "D", Dist: 50, Type: "C↔R", SNR: f64(7.0), Hash: "h6", Timestamp: "t6"},
|
||||
}
|
||||
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
|
||||
if len(result) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(result))
|
||||
}
|
||||
|
||||
// First entry: A↔B pair, max distance = 100, obsCount = 5
|
||||
ab := result[0]
|
||||
if ab["dist"].(float64) != 100 {
|
||||
t.Errorf("expected dist 100, got %v", ab["dist"])
|
||||
}
|
||||
if ab["obsCount"].(int) != 5 {
|
||||
t.Errorf("expected obsCount 5, got %v", ab["obsCount"])
|
||||
}
|
||||
if ab["hash"].(string) != "h1" {
|
||||
t.Errorf("expected hash h1 (from max-dist record), got %v", ab["hash"])
|
||||
}
|
||||
if ab["bestSnr"].(float64) != 8.0 {
|
||||
t.Errorf("expected bestSnr 8.0, got %v", ab["bestSnr"])
|
||||
}
|
||||
// medianSnr of [3,4,5,6,8] = 5.0
|
||||
if ab["medianSnr"].(float64) != 5.0 {
|
||||
t.Errorf("expected medianSnr 5.0, got %v", ab["medianSnr"])
|
||||
}
|
||||
|
||||
// Second entry: C↔D pair
|
||||
cd := result[1]
|
||||
if cd["dist"].(float64) != 50 {
|
||||
t.Errorf("expected dist 50, got %v", cd["dist"])
|
||||
}
|
||||
if cd["obsCount"].(int) != 1 {
|
||||
t.Errorf("expected obsCount 1, got %v", cd["obsCount"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsReversePairMerges(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 50, Type: "R↔R", Hash: "h1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 80, Type: "R↔R", Hash: "h2"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(result))
|
||||
}
|
||||
if result[0]["obsCount"].(int) != 2 {
|
||||
t.Errorf("expected obsCount 2, got %v", result[0]["obsCount"])
|
||||
}
|
||||
if result[0]["dist"].(float64) != 80 {
|
||||
t.Errorf("expected dist 80, got %v", result[0]["dist"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsNilSNR(t *testing.T) {
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: nil, Hash: "h1"},
|
||||
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: nil, Hash: "h2"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(result))
|
||||
}
|
||||
if result[0]["bestSnr"] != nil {
|
||||
t.Errorf("expected bestSnr nil, got %v", result[0]["bestSnr"])
|
||||
}
|
||||
if result[0]["medianSnr"] != nil {
|
||||
t.Errorf("expected medianSnr nil, got %v", result[0]["medianSnr"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsLimit(t *testing.T) {
|
||||
// Generate 25 unique pairs, verify limit=20 caps output
|
||||
hops := make([]distHopRecord, 25)
|
||||
for i := range hops {
|
||||
hops[i] = distHopRecord{
|
||||
FromPk: "A", ToPk: string(rune('a' + i)),
|
||||
Dist: float64(i), Type: "R↔R", Hash: "h",
|
||||
}
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
if len(result) != 20 {
|
||||
t.Errorf("expected 20 entries, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeTopHopsEvenMedian(t *testing.T) {
|
||||
// Even count: median = avg of two middle values
|
||||
hops := []distHopRecord{
|
||||
{FromPk: "A", ToPk: "B", Dist: 10, Type: "R↔R", SNR: f64(2.0), Hash: "h1"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 20, Type: "R↔R", SNR: f64(4.0), Hash: "h2"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 30, Type: "R↔R", SNR: f64(6.0), Hash: "h3"},
|
||||
{FromPk: "A", ToPk: "B", Dist: 40, Type: "R↔R", SNR: f64(8.0), Hash: "h4"},
|
||||
}
|
||||
result := dedupeHopsByPair(hops, 20)
|
||||
// sorted SNR: [2,4,6,8], median = (4+6)/2 = 5.0
|
||||
if result[0]["medianSnr"].(float64) != 5.0 {
|
||||
t.Errorf("expected medianSnr 5.0, got %v", result[0]["medianSnr"])
|
||||
}
|
||||
}
|
||||
+4
-10
@@ -42,20 +42,14 @@
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"snr": {
|
||||
"type": "number"
|
||||
},
|
||||
"hash": {
|
||||
"type": "string"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string"
|
||||
},
|
||||
"bestSnr": {
|
||||
"type": "number"
|
||||
},
|
||||
"medianSnr": {
|
||||
"type": "number"
|
||||
},
|
||||
"obsCount": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1586,4 +1580,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TimeWindow is a half-open time range used to bound analytics queries.
|
||||
// Empty Since/Until means unbounded on that end (backwards compatible).
|
||||
type TimeWindow struct {
|
||||
Since string // RFC3339, empty = unbounded
|
||||
Until string // RFC3339, empty = unbounded
|
||||
// Label is a stable identifier for the user-requested window
|
||||
// (e.g. "24h"). For relative windows it is the original alias; for
|
||||
// absolute ranges it is empty (Since/Until are already stable).
|
||||
// Used only for cache keying so that "?window=24h" produces a single
|
||||
// cache entry instead of one per second.
|
||||
Label string
|
||||
}
|
||||
|
||||
// IsZero reports whether the window imposes no bounds at all.
|
||||
func (w TimeWindow) IsZero() bool {
|
||||
return w.Since == "" && w.Until == ""
|
||||
}
|
||||
|
||||
// CacheKey returns a deterministic key suitable for analytics caches.
|
||||
// For relative windows the key is the alias label so that the cache
|
||||
// remains stable across the wall-clock advancing.
|
||||
func (w TimeWindow) CacheKey() string {
|
||||
if w.IsZero() {
|
||||
return ""
|
||||
}
|
||||
if w.Label != "" {
|
||||
return "rel:" + w.Label
|
||||
}
|
||||
return w.Since + "|" + w.Until
|
||||
}
|
||||
|
||||
// Includes reports whether ts (an RFC3339-style string) falls within the
|
||||
// window. Empty ts is treated as included (for callers that don't have a
|
||||
// timestamp on every observation).
|
||||
//
|
||||
// Comparison is done by parsing both sides into time.Time. Lex compare is
|
||||
// unsafe here because stored timestamps carry millisecond precision
|
||||
// ("...HH:MM:SS.000Z") while bounds emitted by ParseTimeWindow do not
|
||||
// ("...HH:MM:SSZ"), and '.' (0x2e) sorts before 'Z' (0x5a). If a timestamp
|
||||
// fails to parse we fall back to lex compare to preserve old behavior.
|
||||
func (w TimeWindow) Includes(ts string) bool {
|
||||
if ts == "" {
|
||||
return true
|
||||
}
|
||||
tt, terr := parseAnyRFC3339(ts)
|
||||
if w.Since != "" {
|
||||
if s, err := parseAnyRFC3339(w.Since); err == nil && terr == nil {
|
||||
if tt.Before(s) {
|
||||
return false
|
||||
}
|
||||
} else if ts < w.Since {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w.Until != "" {
|
||||
if u, err := parseAnyRFC3339(w.Until); err == nil && terr == nil {
|
||||
if tt.After(u) {
|
||||
return false
|
||||
}
|
||||
} else if ts > w.Until {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseAnyRFC3339 accepts both fractional-second ("...000Z") and second-
|
||||
// precision ("...Z") RFC3339 timestamps. time.RFC3339Nano handles both.
|
||||
func parseAnyRFC3339(s string) (time.Time, error) {
|
||||
return time.Parse(time.RFC3339Nano, s)
|
||||
}
|
||||
|
||||
// ParseTimeWindow extracts a TimeWindow from query params.
|
||||
//
|
||||
// Supported parameters:
|
||||
//
|
||||
// ?window=1h | 24h | 7d | 30d — relative window ending "now"
|
||||
// ?from=<RFC3339>&to=<RFC3339> — absolute custom range (either bound optional)
|
||||
//
|
||||
// When neither is set, returns the zero TimeWindow (unbounded; original behavior).
|
||||
// Invalid values are silently ignored to preserve backwards compatibility.
|
||||
func ParseTimeWindow(r *http.Request) TimeWindow {
|
||||
q := r.URL.Query()
|
||||
|
||||
// Absolute range takes precedence if either bound is set.
|
||||
from := q.Get("from")
|
||||
to := q.Get("to")
|
||||
if from != "" || to != "" {
|
||||
w := TimeWindow{}
|
||||
if from != "" {
|
||||
if t, err := time.Parse(time.RFC3339, from); err == nil {
|
||||
w.Since = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
if to != "" {
|
||||
if t, err := time.Parse(time.RFC3339, to); err == nil {
|
||||
w.Until = t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Relative window.
|
||||
if win := q.Get("window"); win != "" {
|
||||
var d time.Duration
|
||||
switch win {
|
||||
case "1h":
|
||||
d = 1 * time.Hour
|
||||
case "24h", "1d":
|
||||
d = 24 * time.Hour
|
||||
case "3d":
|
||||
d = 3 * 24 * time.Hour
|
||||
case "7d", "1w":
|
||||
d = 7 * 24 * time.Hour
|
||||
case "30d":
|
||||
d = 30 * 24 * time.Hour
|
||||
default:
|
||||
// Unknown values are silently ignored — backwards compatible.
|
||||
return TimeWindow{}
|
||||
}
|
||||
since := time.Now().UTC().Add(-d).Format(time.RFC3339)
|
||||
return TimeWindow{Since: since, Label: win}
|
||||
}
|
||||
|
||||
return TimeWindow{}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue #842 — selectable analytics timeframes.
|
||||
// Backend must accept ?window=1h|24h|7d|30d and ?from=/?to= and yield a
|
||||
// TimeWindow that correctly bounds analytics queries.
|
||||
|
||||
func TestParseTimeWindow_Window24h(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "/api/analytics/rf?window=24h", nil)
|
||||
w := ParseTimeWindow(r)
|
||||
if w.Since == "" {
|
||||
t.Fatalf("window=24h: expected non-empty Since, got %q", w.Since)
|
||||
}
|
||||
since, err := time.Parse(time.RFC3339, w.Since)
|
||||
if err != nil {
|
||||
t.Fatalf("window=24h: Since %q is not RFC3339: %v", w.Since, err)
|
||||
}
|
||||
delta := time.Since(since)
|
||||
if delta < 23*time.Hour || delta > 25*time.Hour {
|
||||
t.Fatalf("window=24h: Since should be ~24h ago, got delta=%v", delta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimeWindow_WindowAliases(t *testing.T) {
|
||||
cases := map[string]time.Duration{
|
||||
"1h": 1 * time.Hour,
|
||||
"24h": 24 * time.Hour,
|
||||
"7d": 7 * 24 * time.Hour,
|
||||
"30d": 30 * 24 * time.Hour,
|
||||
}
|
||||
for q, want := range cases {
|
||||
r := httptest.NewRequest("GET", "/api/analytics/rf?window="+q, nil)
|
||||
got := ParseTimeWindow(r)
|
||||
if got.Since == "" {
|
||||
t.Errorf("window=%s: empty Since", q)
|
||||
continue
|
||||
}
|
||||
since, err := time.Parse(time.RFC3339, got.Since)
|
||||
if err != nil {
|
||||
t.Errorf("window=%s: bad RFC3339 %q", q, got.Since)
|
||||
continue
|
||||
}
|
||||
delta := time.Since(since)
|
||||
// allow 5 minutes of slack
|
||||
if delta < want-5*time.Minute || delta > want+5*time.Minute {
|
||||
t.Errorf("window=%s: expected ~%v, got %v", q, want, delta)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimeWindow_FromTo(t *testing.T) {
|
||||
from := "2026-04-01T00:00:00Z"
|
||||
to := "2026-04-08T00:00:00Z"
|
||||
r := httptest.NewRequest("GET", "/api/analytics/rf?from="+from+"&to="+to, nil)
|
||||
w := ParseTimeWindow(r)
|
||||
if w.Since != from {
|
||||
t.Errorf("expected Since=%q, got %q", from, w.Since)
|
||||
}
|
||||
if w.Until != to {
|
||||
t.Errorf("expected Until=%q, got %q", to, w.Until)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTimeWindow_NoParams_BackwardsCompatible(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "/api/analytics/rf", nil)
|
||||
w := ParseTimeWindow(r)
|
||||
if !w.IsZero() {
|
||||
t.Errorf("no params should yield zero window, got %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeWindow_Includes(t *testing.T) {
|
||||
w := TimeWindow{Since: "2026-04-01T00:00:00Z", Until: "2026-04-08T00:00:00Z"}
|
||||
if !w.Includes("2026-04-05T12:00:00Z") {
|
||||
t.Error("mid-range ts should be included")
|
||||
}
|
||||
if w.Includes("2026-03-31T23:59:59Z") {
|
||||
t.Error("ts before Since should be excluded")
|
||||
}
|
||||
if w.Includes("2026-04-08T00:00:01Z") {
|
||||
t.Error("ts after Until should be excluded")
|
||||
}
|
||||
// Empty ts always included (some observations lack timestamps)
|
||||
if !w.Includes("") {
|
||||
t.Error("empty ts should be included")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeWindow_CacheKey_DistinctPerWindow(t *testing.T) {
|
||||
a := TimeWindow{Since: "2026-04-01T00:00:00Z"}
|
||||
b := TimeWindow{Since: "2026-04-02T00:00:00Z"}
|
||||
z := TimeWindow{}
|
||||
if a.CacheKey() == b.CacheKey() {
|
||||
t.Error("different windows must produce different cache keys")
|
||||
}
|
||||
if z.CacheKey() != "" {
|
||||
t.Errorf("zero window cache key must be empty, got %q", z.CacheKey())
|
||||
}
|
||||
if !strings.Contains(a.CacheKey(), "2026-04-01") {
|
||||
t.Errorf("cache key should encode Since, got %q", a.CacheKey())
|
||||
}
|
||||
}
|
||||
|
||||
// Self-review fixes (#1018 polish).
|
||||
|
||||
// B1: a relative window must produce a STABLE cache key across calls,
|
||||
// otherwise the analytics cache thrashes (one entry per second).
|
||||
func TestTimeWindow_RelativeWindow_StableCacheKey(t *testing.T) {
|
||||
r1 := httptest.NewRequest("GET", "/api/analytics/rf?window=24h", nil)
|
||||
w1 := ParseTimeWindow(r1)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
r2 := httptest.NewRequest("GET", "/api/analytics/rf?window=24h", nil)
|
||||
w2 := ParseTimeWindow(r2)
|
||||
if w1.CacheKey() != w2.CacheKey() {
|
||||
t.Fatalf("relative window cache key must be stable across calls, got %q vs %q", w1.CacheKey(), w2.CacheKey())
|
||||
}
|
||||
}
|
||||
|
||||
// B2: stored timestamps use millisecond precision (".000Z") while RFC3339
|
||||
// bounds have none. Includes() must use time-based compare, not lex compare,
|
||||
// so tx past Until are correctly excluded regardless of fractional digits.
|
||||
func TestTimeWindow_Includes_FractionalSecondsBoundary(t *testing.T) {
|
||||
w := TimeWindow{Until: "2026-04-08T00:00:00Z"}
|
||||
// A tx 1ms past Until should NOT be included.
|
||||
if w.Includes("2026-04-08T00:00:00.001Z") {
|
||||
t.Error("ts 1ms past Until must be excluded; lex compare against fractional ts is wrong")
|
||||
}
|
||||
// A tx well inside the window must be included.
|
||||
if !w.Includes("2026-04-07T23:59:59.999Z") {
|
||||
t.Error("ts just before Until must be included")
|
||||
}
|
||||
|
||||
w2 := TimeWindow{Since: "2026-04-01T00:00:00Z"}
|
||||
// A tx at exactly Since should be included.
|
||||
if !w2.Includes("2026-04-01T00:00:00.000Z") {
|
||||
t.Error("ts exactly at Since must be included; lex compare excludes it because '.' < 'Z'")
|
||||
}
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// TestTopologyDedup_RepeatersMergeByPubkey verifies that topRepeaters
|
||||
// merges entries whose hop prefixes resolve unambiguously to the same node.
|
||||
func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
exec := func(s string) {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
exec(`CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, frequency REAL
|
||||
)`)
|
||||
exec(`CREATE TABLE schema_version (version INTEGER)`)
|
||||
exec(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
// Insert two repeater nodes with distinct pubkeys.
|
||||
// AQUA: pubkey starts with 0735bc...
|
||||
// BETA: pubkey starts with 99aabb...
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('0735bc6dda4d1122aabbccdd', 'AQUA', 'Repeater')`)
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('99aabb001122334455667788', 'BETA', 'Repeater')`)
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Create packets:
|
||||
// - 10 packets with path ["07", "99aa"] (short prefix for AQUA, medium for BETA)
|
||||
// - 5 packets with path ["0735bc", "99"] (medium prefix for AQUA, short for BETA)
|
||||
// - 3 packets with path ["0735bc6dda4d", "99aabb"] (long prefix for both)
|
||||
txID := 1
|
||||
obsID := 1
|
||||
insertTx := func(path string, count int) {
|
||||
for i := 0; i < count; i++ {
|
||||
ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", txID)
|
||||
conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)",
|
||||
txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID))
|
||||
conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts)
|
||||
txID++
|
||||
obsID++
|
||||
}
|
||||
}
|
||||
|
||||
insertTx(`["07","99aa"]`, 10)
|
||||
insertTx(`["0735bc","99"]`, 5)
|
||||
insertTx(`["0735bc6d","99aabb"]`, 3)
|
||||
|
||||
// Total: AQUA appears as "07" (10×), "0735bc" (5×), "0735bc6d" (3×) = 18 total
|
||||
// Total: BETA appears as "99aa" (10×), "99" (5×), "99aabb" (3×) = 18 total
|
||||
// After dedup, each should appear ONCE with count=18.
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.conn.Close()
|
||||
|
||||
store := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 100})
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := store.computeAnalyticsTopology("", TimeWindow{})
|
||||
topRepeaters := result["topRepeaters"].([]map[string]interface{})
|
||||
|
||||
// Build a map of pubkey → total count from topRepeaters
|
||||
pubkeyCounts := map[string]int{}
|
||||
for _, entry := range topRepeaters {
|
||||
pk, _ := entry["pubkey"].(string)
|
||||
if pk == "" {
|
||||
continue
|
||||
}
|
||||
pubkeyCounts[pk] += entry["count"].(int)
|
||||
}
|
||||
|
||||
// Each pubkey should appear exactly once in topRepeaters
|
||||
aquaEntries := 0
|
||||
betaEntries := 0
|
||||
for _, entry := range topRepeaters {
|
||||
pk, _ := entry["pubkey"].(string)
|
||||
if pk == "0735bc6dda4d1122aabbccdd" {
|
||||
aquaEntries++
|
||||
}
|
||||
if pk == "99aabb001122334455667788" {
|
||||
betaEntries++
|
||||
}
|
||||
}
|
||||
|
||||
if aquaEntries != 1 {
|
||||
t.Errorf("AQUA should appear exactly once in topRepeaters after dedup, got %d entries", aquaEntries)
|
||||
for _, e := range topRepeaters {
|
||||
t.Logf(" entry: hop=%v name=%v pubkey=%v count=%v", e["hop"], e["name"], e["pubkey"], e["count"])
|
||||
}
|
||||
}
|
||||
if betaEntries != 1 {
|
||||
t.Errorf("BETA should appear exactly once in topRepeaters after dedup, got %d entries", betaEntries)
|
||||
}
|
||||
|
||||
// Check that the merged count is correct (18 each)
|
||||
if c := pubkeyCounts["0735bc6dda4d1122aabbccdd"]; c != 18 {
|
||||
t.Errorf("AQUA total count should be 18, got %d", c)
|
||||
}
|
||||
if c := pubkeyCounts["99aabb001122334455667788"]; c != 18 {
|
||||
t.Errorf("BETA total count should be 18, got %d", c)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTopologyDedup_AmbiguousPrefixNotMerged verifies that ambiguous short
|
||||
// prefixes (matching multiple nodes) are NOT merged — they stay separate.
|
||||
func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
exec := func(s string) {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
exec(`CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, frequency REAL
|
||||
)`)
|
||||
exec(`CREATE TABLE schema_version (version INTEGER)`)
|
||||
exec(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
// Two nodes whose pubkeys share the prefix "ab" — collision!
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('ab11223344556677aabbccdd', 'NODE_A', 'Repeater')`)
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('ab99887766554433aabbccdd', 'NODE_B', 'Repeater')`)
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
txID := 1
|
||||
obsID := 1
|
||||
|
||||
// 10 packets with hop "ab" — ambiguous (matches both NODE_A and NODE_B)
|
||||
for i := 0; i < 10; i++ {
|
||||
ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", txID)
|
||||
conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)",
|
||||
txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID))
|
||||
conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab"]`, ts)
|
||||
txID++
|
||||
obsID++
|
||||
}
|
||||
// 5 packets with hop "ab1122" — unambiguous (only NODE_A)
|
||||
for i := 0; i < 5; i++ {
|
||||
ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", txID)
|
||||
conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)",
|
||||
txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID))
|
||||
conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab1122"]`, ts)
|
||||
txID++
|
||||
obsID++
|
||||
}
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.conn.Close()
|
||||
|
||||
store := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 100})
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := store.computeAnalyticsTopology("", TimeWindow{})
|
||||
topRepeaters := result["topRepeaters"].([]map[string]interface{})
|
||||
|
||||
// "ab" is ambiguous — should NOT be merged with "ab1122"
|
||||
// We expect two separate entries: one for "ab" (count=10) and one for "ab1122" (count=5)
|
||||
foundAb := false
|
||||
foundAb1122 := false
|
||||
for _, entry := range topRepeaters {
|
||||
hop := entry["hop"].(string)
|
||||
count := entry["count"].(int)
|
||||
if hop == "ab" {
|
||||
foundAb = true
|
||||
if count != 10 {
|
||||
t.Errorf("ambiguous hop 'ab' should have count=10, got %d", count)
|
||||
}
|
||||
}
|
||||
if hop == "ab1122" {
|
||||
foundAb1122 = true
|
||||
if count != 5 {
|
||||
t.Errorf("unambiguous hop 'ab1122' should have count=5, got %d", count)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundAb {
|
||||
t.Error("ambiguous hop 'ab' should remain as separate entry")
|
||||
}
|
||||
if !foundAb1122 {
|
||||
t.Error("unambiguous hop 'ab1122' should remain as separate entry (not merged with ambiguous 'ab')")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTopologyDedup_PairsMergeByPubkey verifies that topPairs merges
|
||||
// pair entries whose hops resolve unambiguously to the same node pair.
|
||||
func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
exec := func(s string) {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
exec(`CREATE TABLE transmissions (
|
||||
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`)
|
||||
exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
|
||||
exec(`CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, frequency REAL
|
||||
)`)
|
||||
exec(`CREATE TABLE schema_version (version INTEGER)`)
|
||||
exec(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('0735bc6dda4d1122aabbccdd', 'AQUA', 'Repeater')`)
|
||||
exec(`INSERT INTO nodes (public_key, name, role) VALUES ('99aabb001122334455667788', 'BETA', 'Repeater')`)
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
txID := 1
|
||||
obsID := 1
|
||||
insertTx := func(path string, count int) {
|
||||
for i := 0; i < count; i++ {
|
||||
ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", txID)
|
||||
conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)",
|
||||
txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID))
|
||||
conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts)
|
||||
txID++
|
||||
obsID++
|
||||
}
|
||||
}
|
||||
|
||||
// Path ["07","99aa"] → pair "07|99aa", 10 times
|
||||
// Path ["0735bc","99"] → pair "0735bc|99" but sorted = "0735bc|99", 5 times
|
||||
// Wait: pair sorting is by string comparison: "07" < "99aa", "0735bc" < "99"
|
||||
// After dedup both should merge to AQUA|BETA pair with count=15
|
||||
insertTx(`["07","99aa"]`, 10)
|
||||
insertTx(`["0735bc","99"]`, 5)
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.conn.Close()
|
||||
|
||||
store := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 100})
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
result := store.computeAnalyticsTopology("", TimeWindow{})
|
||||
topPairs := result["topPairs"].([]map[string]interface{})
|
||||
|
||||
// Should have exactly 1 pair entry for AQUA-BETA with count=15
|
||||
aquaBetaPairs := 0
|
||||
totalCount := 0
|
||||
for _, entry := range topPairs {
|
||||
pkA, _ := entry["pubkeyA"].(string)
|
||||
pkB, _ := entry["pubkeyB"].(string)
|
||||
if (pkA == "0735bc6dda4d1122aabbccdd" && pkB == "99aabb001122334455667788") ||
|
||||
(pkA == "99aabb001122334455667788" && pkB == "0735bc6dda4d1122aabbccdd") {
|
||||
aquaBetaPairs++
|
||||
totalCount += entry["count"].(int)
|
||||
}
|
||||
}
|
||||
|
||||
if aquaBetaPairs != 1 {
|
||||
t.Errorf("AQUA-BETA pair should appear exactly once after dedup, got %d entries", aquaBetaPairs)
|
||||
for _, e := range topPairs {
|
||||
t.Logf(" pair: hopA=%v hopB=%v count=%v pkA=%v pkB=%v", e["hopA"], e["hopB"], e["count"], e["pubkeyA"], e["pubkeyB"])
|
||||
}
|
||||
}
|
||||
if totalCount != 15 {
|
||||
t.Errorf("AQUA-BETA pair total count should be 15, got %d", totalCount)
|
||||
}
|
||||
}
|
||||
@@ -69,11 +69,13 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// After #800, touchRelayLastSeen takes a []string of pubkeys (from decode-window)
|
||||
pks := []string{"relay1"}
|
||||
pk := "relay1"
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{&pk},
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
s.touchRelayLastSeen(pks, now)
|
||||
s.touchRelayLastSeen(tx, now)
|
||||
|
||||
// Verify it was written
|
||||
var lastSeen sql.NullString
|
||||
@@ -86,7 +88,7 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
db.conn.Exec("UPDATE nodes SET last_seen = NULL WHERE public_key = ?", "relay1")
|
||||
|
||||
// Call again within 5 minutes — should be debounced (no write)
|
||||
s.touchRelayLastSeen(pks, now.Add(2*time.Minute))
|
||||
s.touchRelayLastSeen(tx, now.Add(2*time.Minute))
|
||||
|
||||
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
|
||||
if lastSeen.Valid {
|
||||
@@ -94,14 +96,14 @@ func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
|
||||
}
|
||||
|
||||
// Call after 5 minutes — should write again
|
||||
s.touchRelayLastSeen(pks, now.Add(6*time.Minute))
|
||||
s.touchRelayLastSeen(tx, now.Add(6*time.Minute))
|
||||
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
|
||||
if !lastSeen.Valid {
|
||||
t.Fatal("expected write after debounce interval expired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchRelayLastSeen_SkipsEmptyPubkeys(t *testing.T) {
|
||||
func TestTouchRelayLastSeen_SkipsNilResolvedPath(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
@@ -110,9 +112,13 @@ func TestTouchRelayLastSeen_SkipsEmptyPubkeys(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// Empty pubkeys — should not panic or error
|
||||
s.touchRelayLastSeen([]string{}, time.Now())
|
||||
s.touchRelayLastSeen(nil, time.Now())
|
||||
// tx with nil entries and empty resolved_path
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{nil, nil},
|
||||
}
|
||||
|
||||
// Should not panic or error
|
||||
s.touchRelayLastSeen(tx, time.Now())
|
||||
}
|
||||
|
||||
func TestTouchRelayLastSeen_NilDB(t *testing.T) {
|
||||
@@ -121,6 +127,11 @@ func TestTouchRelayLastSeen_NilDB(t *testing.T) {
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
pk := "abc"
|
||||
tx := &StoreTx{
|
||||
ResolvedPath: []*string{&pk},
|
||||
}
|
||||
|
||||
// Should not panic with nil db
|
||||
s.touchRelayLastSeen([]string{"abc"}, time.Now())
|
||||
s.touchRelayLastSeen(tx, time.Now())
|
||||
}
|
||||
|
||||
@@ -52,26 +52,26 @@ func TestEstimateStoreTxBytes_ManyHopsSubpaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEstimateStoreObsBytes_AfterRefactor verifies that after #800 refactor,
|
||||
// observations no longer have ResolvedPath overhead in their estimate.
|
||||
func TestEstimateStoreObsBytes_AfterRefactor(t *testing.T) {
|
||||
obs := &StoreObs{
|
||||
// TestEstimateStoreObsBytes_WithResolvedPath verifies that observations with
|
||||
// ResolvedPath estimate more than those without.
|
||||
func TestEstimateStoreObsBytes_WithResolvedPath(t *testing.T) {
|
||||
s1, s2, s3 := "node1", "node2", "node3"
|
||||
|
||||
obsNoRP := &StoreObs{
|
||||
ObserverID: "obs1",
|
||||
PathJSON: `["a","b"]`,
|
||||
}
|
||||
obsWithRP := &StoreObs{
|
||||
ObserverID: "obs1",
|
||||
PathJSON: `["a","b"]`,
|
||||
ResolvedPath: []*string{&s1, &s2, &s3},
|
||||
}
|
||||
|
||||
est := estimateStoreObsBytes(obs)
|
||||
if est <= 0 {
|
||||
t.Errorf("estimate should be positive, got %d", est)
|
||||
}
|
||||
// After #800, all obs estimates should be the same (no RP field variation)
|
||||
obs2 := &StoreObs{
|
||||
ObserverID: "obs1",
|
||||
PathJSON: `["a","b"]`,
|
||||
}
|
||||
est2 := estimateStoreObsBytes(obs2)
|
||||
if est != est2 {
|
||||
t.Errorf("estimates should be equal after #800 (no RP field), got %d vs %d", est, est2)
|
||||
estNo := estimateStoreObsBytes(obsNoRP)
|
||||
estWith := estimateStoreObsBytes(obsWithRP)
|
||||
|
||||
if estWith <= estNo {
|
||||
t.Errorf("obs with ResolvedPath (%d) should estimate more than without (%d)", estWith, estNo)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,9 +155,11 @@ func BenchmarkEstimateStoreTxBytes(b *testing.B) {
|
||||
|
||||
// BenchmarkEstimateStoreObsBytes verifies the obs estimate function is fast.
|
||||
func BenchmarkEstimateStoreObsBytes(b *testing.B) {
|
||||
s := "resolvedNodePubkey123456"
|
||||
obs := &StoreObs{
|
||||
ObserverID: "observer1234",
|
||||
PathJSON: `["a","b","c"]`,
|
||||
ObserverID: "observer1234",
|
||||
PathJSON: `["a","b","c"]`,
|
||||
ResolvedPath: []*string{&s, &s, &s},
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
|
||||
+4
-23
@@ -72,22 +72,6 @@ type StatsResponse struct {
|
||||
BackfillProgress float64 `json:"backfillProgress"`
|
||||
SignatureDrops int64 `json:"signatureDrops,omitempty"`
|
||||
HashMigrationComplete bool `json:"hashMigrationComplete"`
|
||||
|
||||
// Memory accounting (issue #832). All values in MB.
|
||||
//
|
||||
// StoreDataMB ("trackedMB" historically) is the in-store packet byte
|
||||
// estimate — useful packet bytes only. Subset of HeapInuse. Used as
|
||||
// the eviction watermark input. NOT a proxy for RSS; ops dashboards
|
||||
// should prefer ProcessRSSMB for capacity decisions.
|
||||
//
|
||||
// Old field name TrackedMB is retained for backward compatibility
|
||||
// with pre-v3.6 consumers; it carries the same value as StoreDataMB
|
||||
// and is deprecated.
|
||||
TrackedMB float64 `json:"trackedMB"` // deprecated alias for storeDataMB
|
||||
StoreDataMB float64 `json:"storeDataMB"` // in-store packet bytes (subset of heap)
|
||||
ProcessRSSMB float64 `json:"processRSSMB"` // process RSS from /proc (Linux) or runtime.Sys fallback
|
||||
GoHeapInuseMB float64 `json:"goHeapInuseMB"` // runtime.MemStats.HeapInuse
|
||||
GoSysMB float64 `json:"goSysMB"` // runtime.MemStats.Sys (total Go-managed)
|
||||
}
|
||||
|
||||
// ─── Health ────────────────────────────────────────────────────────────────────
|
||||
@@ -263,6 +247,7 @@ type TransmissionResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
Direction interface{} `json:"direction"`
|
||||
Score interface{} `json:"score,omitempty"`
|
||||
Observations []ObservationResp `json:"observations,omitempty"`
|
||||
@@ -277,9 +262,7 @@ type ObservationResp struct {
|
||||
SNR interface{} `json:"snr"`
|
||||
RSSI interface{} `json:"rssi"`
|
||||
PathJSON interface{} `json:"path_json"`
|
||||
ResolvedPath interface{} `json:"resolved_path,omitempty"`
|
||||
Direction interface{} `json:"direction,omitempty"`
|
||||
RawHex interface{} `json:"raw_hex,omitempty"`
|
||||
ResolvedPath []*string `json:"resolved_path,omitempty"`
|
||||
Timestamp interface{} `json:"timestamp"`
|
||||
}
|
||||
|
||||
@@ -315,6 +298,7 @@ type PacketTimestampsResponse struct {
|
||||
type PacketDetailResponse struct {
|
||||
Packet interface{} `json:"packet"`
|
||||
Path []interface{} `json:"path"`
|
||||
Breakdown *Breakdown `json:"breakdown"`
|
||||
ObservationCount int `json:"observation_count"`
|
||||
Observations []ObservationResp `json:"observations,omitempty"`
|
||||
}
|
||||
@@ -680,9 +664,7 @@ type DistanceHop struct {
|
||||
ToPk string `json:"toPk"`
|
||||
Dist float64 `json:"dist"`
|
||||
Type string `json:"type"`
|
||||
BestSnr interface{} `json:"bestSnr"`
|
||||
MedianSnr interface{} `json:"medianSnr"`
|
||||
ObsCount int `json:"obsCount"`
|
||||
SNR interface{} `json:"snr"`
|
||||
Hash string `json:"hash"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
@@ -859,7 +841,6 @@ type ObserverResp struct {
|
||||
BatteryMv interface{} `json:"battery_mv"`
|
||||
UptimeSecs interface{} `json:"uptime_secs"`
|
||||
NoiseFloor interface{} `json:"noise_floor"`
|
||||
LastPacketAt interface{} `json:"last_packet_at"`
|
||||
PacketsLastHour int `json:"packetsLastHour"`
|
||||
Lat interface{} `json:"lat"`
|
||||
Lon interface{} `json:"lon"`
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning
|
||||
// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if
|
||||
// the operator has set db.vacuumOnStartup: true in config (#919).
|
||||
func checkAutoVacuum(db *DB, cfg *Config, dbPath string) {
|
||||
var autoVacuum int
|
||||
if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
|
||||
log.Printf("[db] warning: could not read auto_vacuum: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if autoVacuum == 2 {
|
||||
log.Printf("[db] auto_vacuum=INCREMENTAL")
|
||||
return
|
||||
}
|
||||
|
||||
modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"}
|
||||
mode := modes[autoVacuum]
|
||||
if mode == "" {
|
||||
mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum)
|
||||
}
|
||||
|
||||
log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+
|
||||
"Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+
|
||||
"See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode)
|
||||
|
||||
if cfg.DB != nil && cfg.DB.VacuumOnStartup {
|
||||
// WARNING: Full VACUUM creates a temporary copy of the entire DB file.
|
||||
// Requires ~2× the DB file size in free disk space or it will fail.
|
||||
log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...")
|
||||
start := time.Now()
|
||||
|
||||
rw, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[db] VACUUM failed: could not open RW connection: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err)
|
||||
return
|
||||
}
|
||||
if _, err := rw.Exec("VACUUM"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond))
|
||||
|
||||
// Re-check
|
||||
var newMode int
|
||||
if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil {
|
||||
if newMode == 2 {
|
||||
log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)")
|
||||
} else {
|
||||
log.Printf("[db] warning: auto_vacuum=%d after VACUUM — expected 2", newMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write
|
||||
// connection. Safe to call on auto_vacuum=NONE databases (noop).
|
||||
func runIncrementalVacuum(dbPath string, pages int) {
|
||||
rw, err := cachedRW(dbPath)
|
||||
if err != nil {
|
||||
log.Printf("[vacuum] could not open RW connection: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
|
||||
log.Printf("[vacuum] incremental_vacuum error: %v", err)
|
||||
}
|
||||
}
|
||||
+5
-15
@@ -3,19 +3,12 @@
|
||||
"apiKey": "your-secret-api-key-here",
|
||||
"nodeBlacklist": [],
|
||||
"_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.",
|
||||
"observerIATAWhitelist": [],
|
||||
"_comment_observerIATAWhitelist": "Global IATA region whitelist. When non-empty, only observers whose IATA code (from MQTT topic) matches are processed. Case-insensitive. Empty = allow all. Unlike per-source iataFilter, this applies across all MQTT sources.",
|
||||
"retention": {
|
||||
"nodeDays": 7,
|
||||
"observerDays": 14,
|
||||
"packetDays": 30,
|
||||
"_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)."
|
||||
},
|
||||
"db": {
|
||||
"vacuumOnStartup": false,
|
||||
"incrementalVacuumPages": 1024,
|
||||
"_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919."
|
||||
},
|
||||
"https": {
|
||||
"cert": "/path/to/cert.pem",
|
||||
"key": "/path/to/key.pem",
|
||||
@@ -131,9 +124,7 @@
|
||||
"SFO",
|
||||
"OAK",
|
||||
"MRY"
|
||||
],
|
||||
"region": "SJC",
|
||||
"connectTimeoutSec": 45
|
||||
]
|
||||
}
|
||||
],
|
||||
"channelKeys": {
|
||||
@@ -173,7 +164,7 @@
|
||||
[37.20, -122.52]
|
||||
],
|
||||
"bufferKm": 20,
|
||||
"_comment": "Optional. Restricts ingestion and API responses to nodes within the polygon + bufferKm. Polygon is an array of [lat, lon] pairs (minimum 3). Use the GeoFilter Builder (`/geofilter-builder.html`) to draw a polygon, save drafts to localStorage with Save Draft, and export a config snippet with Download — paste the snippet here as the `geo_filter` block. Remove this section to disable filtering. Nodes with no GPS fix are always allowed through."
|
||||
"_comment": "Optional. Restricts ingestion and API responses to nodes within the polygon + bufferKm. Polygon is an array of [lat, lon] pairs (minimum 3). Use tools/geofilter-builder.html to draw a polygon visually. Remove this section to disable filtering. Nodes with no GPS fix are always allowed through."
|
||||
},
|
||||
"regions": {
|
||||
"SJC": "San Jose, US",
|
||||
@@ -217,8 +208,7 @@
|
||||
"packetStore": {
|
||||
"maxMemoryMB": 1024,
|
||||
"estimatedPacketBytes": 450,
|
||||
"retentionHours": 168,
|
||||
"_comment": "In-memory packet store. maxMemoryMB caps RAM usage. retentionHours: only packets younger than this are loaded on startup and kept in memory (0 = unlimited, not recommended for large DBs — causes OOM on cold start). 168 = 7 days. Must be ≤ retention.packetDays * 24."
|
||||
"_comment": "In-memory packet store. maxMemoryMB caps RAM usage. All packets loaded on startup, served from RAM."
|
||||
},
|
||||
"resolvedPath": {
|
||||
"backfillHours": 24,
|
||||
@@ -228,10 +218,10 @@
|
||||
"maxAgeDays": 5,
|
||||
"_comment": "Neighbor edges older than this many days are pruned on startup and daily. Default: 5."
|
||||
},
|
||||
"_comment_mqttSources": "Each source connects to an MQTT broker. topics: what to subscribe to. iataFilter: only ingest packets from these regions (optional). region: default IATA region for this source — used when packet/topic doesn't specify one (optional, priority: payload > topic > this field).",
|
||||
"_comment_mqttSources": "Each source connects to an MQTT broker. topics: what to subscribe to. iataFilter: only ingest packets from these regions (optional).",
|
||||
"_comment_channelKeys": "Hex keys for decrypting channel messages. Key name = channel display name. public channel key is well-known.",
|
||||
"_comment_hashChannels": "Channel names whose keys are derived via SHA256. Key = SHA256(name)[:16]. Listed here so the ingestor can auto-derive keys.",
|
||||
"_comment_defaultRegion": "IATA code shown by default in region filters.",
|
||||
"_comment_mapDefaults": "Initial map center [lat, lon] and zoom level.",
|
||||
"_comment_regions": "IATA code → display name mapping for the region filter UI. Each key is a 3-letter IATA code that an observer is tagged with (resolved priority: MQTT payload `region` field > topic-derived region > mqttSources.region). Observers without an IATA tag will not appear under any region filter — only under 'All Regions'. The region filter dropdown shows one entry per code listed here PLUS any extra IATA codes the server discovers from observers at runtime (so you can omit codes here and they will still be selectable, just labelled with the bare IATA code instead of a friendly name). Selecting 'All Regions' (or no region) returns results from every observer including those with no IATA tag; selecting one or more codes restricts results to packets observed by observers tagged with those codes. The reserved value 'All' (case-insensitive) is treated as 'no filter' on the server, so the URL ?region=All behaves identically to omitting the param. Issue #770."
|
||||
"_comment_regions": "IATA code to display name mapping. Packets are tagged with region codes by MQTT topic structure."
|
||||
}
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
# Scope Stats Page — Design Spec
|
||||
|
||||
**Issue**: Kpa-clawbot/CoreScope#899
|
||||
**Date**: 2026-04-23
|
||||
**Branch target**: `master`
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Add a dedicated **Scopes** page showing scope/region statistics for MeshCore transport-route packets. Scope filtering in MeshCore uses `TRANSPORT_FLOOD` (route_type 0) and `TRANSPORT_DIRECT` (route_type 3) packets that carry two 16-bit transport codes. Code1 ≠ `0000` means the packet is region-scoped.
|
||||
|
||||
Feature 3 from the issue (default scope per client via advert) is **not implemented** — the advert format has no scope field in the current firmware.
|
||||
|
||||
---
|
||||
|
||||
## How Scopes Work (Firmware)
|
||||
|
||||
Transport code derivation (authoritative source: `meshcore-dev/MeshCore`):
|
||||
|
||||
```
|
||||
key = SHA256("#regionname")[:16] // TransportKeyStore::getAutoKeyFor
|
||||
Code1 = HMAC-SHA256(key, type || payload) // TransportKey::calcTransportCode, 2-byte output
|
||||
```
|
||||
|
||||
Code1 is a **per-message** HMAC — the same region produces a different Code1 for every message. Identifying a region from Code1 requires knowing the region name in advance and recomputing the HMAC.
|
||||
|
||||
`Code1 = 0000` is the "no scope" sentinel (also `FFFF` is reserved). Packets with route_type 1 or 2 (plain FLOOD/DIRECT) carry no transport codes.
|
||||
|
||||
---
|
||||
|
||||
## Config
|
||||
|
||||
Add `hashRegions` to the ingestor `Config` struct in `cmd/ingestor/config.go`, mirroring `hashChannels`:
|
||||
|
||||
```json
|
||||
"hashRegions": ["#belgium", "#eu", "#brussels"]
|
||||
```
|
||||
|
||||
Normalization (same rules as `hashChannels`):
|
||||
- Trim whitespace
|
||||
- Prepend `#` if missing
|
||||
- Skip empty entries
|
||||
|
||||
---
|
||||
|
||||
## Ingestor Changes
|
||||
|
||||
### Key derivation (`loadRegionKeys`)
|
||||
|
||||
```go
|
||||
func loadRegionKeys(cfg *Config) map[string][]byte {
|
||||
// key = first 16 bytes of SHA256("#regionname")
|
||||
}
|
||||
```
|
||||
|
||||
Returns `map[string][]byte` (region name → 16-byte HMAC key). Called once at startup, stored on the `Store`.
|
||||
|
||||
### Decoder: expose raw payload bytes
|
||||
|
||||
Add `PayloadRaw []byte` to `DecodedPacket` in `cmd/ingestor/decoder.go`. Populated from the raw `buf` slice at the payload offset — zero-copy slice, no allocation. This is the **encrypted** payload bytes, matching what the firmware feeds into `calcTransportCode`.
|
||||
|
||||
### At-ingest region matching
|
||||
|
||||
In `BuildPacketData`:
|
||||
- Skip if `route_type` not in `{0, 3}` → `scope_name` stays `nil`
|
||||
- If `Code1 == "0000"` → `scope_name = nil` (unscoped transport, no scope involvement)
|
||||
- If `Code1 != "0000"` → try each region key:
|
||||
```
|
||||
HMAC-SHA256(key, payloadType_byte || PayloadRaw) → first 2 bytes as uint16
|
||||
```
|
||||
First match → `scope_name = "#regionname"`. No match → `scope_name = ""` (unknown scope).
|
||||
|
||||
Add `ScopeName *string` to `PacketData`.
|
||||
|
||||
### MQTT-sourced packets (DM / CHAN paths in main.go)
|
||||
|
||||
These are injected directly without going through `BuildPacketData`. They use `route_type = 1` (FLOOD), so they are never transport-route packets. No scope matching needed for these paths.
|
||||
|
||||
---
|
||||
|
||||
## Database
|
||||
|
||||
### Migration
|
||||
|
||||
```sql
|
||||
ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL;
|
||||
CREATE INDEX idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL;
|
||||
```
|
||||
|
||||
### Column semantics
|
||||
|
||||
| Value | Meaning |
|
||||
|-------|---------|
|
||||
| `NULL` | Either: non-transport-route packet (route_type 1/2), or transport-route with Code1=0000 |
|
||||
| `""` (empty string) | Transport-route, Code1 ≠ 0000, but no configured region matched |
|
||||
| `"#belgium"` | Matched named region |
|
||||
|
||||
The API stats queries resolve the NULL ambiguity by always filtering `route_type IN (0, 3)` first:
|
||||
- `unscoped` count = `route_type IN (0,3) AND scope_name IS NULL`
|
||||
- `scoped` count = `route_type IN (0,3) AND scope_name IS NOT NULL`
|
||||
|
||||
### Backfill
|
||||
|
||||
On migration, re-decode `raw_hex` for all rows where `route_type IN (0, 3)` and `scope_name IS NULL`. Run the same HMAC matching logic. Rows with `Code1 = 0000` remain `NULL`.
|
||||
|
||||
The backfill runs in the existing migration framework in `cmd/ingestor/db.go`. If no regions are configured, backfill is skipped.
|
||||
|
||||
---
|
||||
|
||||
## API
|
||||
|
||||
### `GET /api/scope-stats`
|
||||
|
||||
**Query param**: `window` — one of `1h`, `24h` (default), `7d`
|
||||
|
||||
**Time-series bucket sizes**:
|
||||
| Window | Bucket |
|
||||
|--------|--------|
|
||||
| `1h` | 5 min |
|
||||
| `24h` | 1 hour |
|
||||
| `7d` | 6 hours|
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"window": "24h",
|
||||
"summary": {
|
||||
"transportTotal": 1240,
|
||||
"scoped": 890,
|
||||
"unscoped": 350,
|
||||
"unknownScope": 42
|
||||
},
|
||||
"byRegion": [
|
||||
{ "name": "#belgium", "count": 612 },
|
||||
{ "name": "#eu", "count": 236 }
|
||||
],
|
||||
"timeSeries": [
|
||||
{ "t": "2026-04-23T10:00:00Z", "scoped": 45, "unscoped": 18 },
|
||||
{ "t": "2026-04-23T11:00:00Z", "scoped": 51, "unscoped": 22 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `transportTotal` = `scoped + unscoped` (transport-route packets only)
|
||||
- `scoped` = Code1 ≠ 0000 (named + unknown)
|
||||
- `unscoped` = transport-route with Code1 = 0000
|
||||
- `unknownScope` = scoped but no region name matched (subset of `scoped`)
|
||||
- `byRegion` sorted by count descending, excludes unknown
|
||||
- `timeSeries` covers the full window at the bucket granularity
|
||||
|
||||
Route: `GET /api/scope-stats` registered in `cmd/server/routes.go`.
|
||||
No auth required (same as other read endpoints).
|
||||
TTL cache: 30 seconds (heavier query than `/api/stats`).
|
||||
|
||||
---
|
||||
|
||||
## Frontend
|
||||
|
||||
### Navigation
|
||||
|
||||
Add nav link between Channels and Nodes in `public/index.html`:
|
||||
```html
|
||||
<a href="#/scopes" class="nav-link" data-route="scopes">Scopes</a>
|
||||
```
|
||||
|
||||
### `public/scopes.js`
|
||||
|
||||
Three sections on the page:
|
||||
|
||||
**1. Summary cards** (reuse existing card CSS pattern from home/analytics pages)
|
||||
- Transport total, Scoped, Unscoped, Unknown scope
|
||||
- Each card shows count + percentage of transport total
|
||||
|
||||
**2. Per-region table**
|
||||
Columns: Region, Messages, % of Scoped
|
||||
Sorted by count descending. Last row: "Unknown scope" (italic) if unknownScope > 0.
|
||||
Shows "No regions configured" message if `byRegion` is empty and `unknownScope = 0`.
|
||||
|
||||
**3. Time-series chart**
|
||||
- Window selector: `1h / 24h / 7d` (default 24h)
|
||||
- Two lines: **Scoped** (blue) and **Unscoped** (grey)
|
||||
- Uses the same lightweight canvas chart pattern as other pages (no external chart lib)
|
||||
|
||||
### Cache buster
|
||||
|
||||
`scopes.js` added to the `__BUST__` entries in `index.html` in the same commit.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit tests for `loadRegionKeys`: normalization, key bytes match firmware SHA256 derivation
|
||||
- Unit tests for HMAC matching: known Code1 value computed from firmware logic, verified against Go implementation
|
||||
- Integration test: ingest a synthetic transport-route packet with a known region, assert `scope_name` column is set correctly
|
||||
- API test: `GET /api/scope-stats` returns correct summary counts against fixture DB
|
||||
|
||||
---
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Feature 3 (default scope per client via advert) — firmware has no advert scope field
|
||||
- Drill-down from region row to filtered packet list (deferred)
|
||||
- Private regions (`$`-prefixed) — use secret keys not publicly derivable
|
||||
@@ -98,22 +98,6 @@ How long (in hours) before a node is marked degraded or silent:
|
||||
| `retention.nodeDays` | `7` | Nodes not seen in N days move to inactive |
|
||||
| `retention.packetDays` | `30` | Packets older than N days are deleted daily |
|
||||
|
||||
> **Note:** Lowering retention does **not** immediately shrink the database file.
|
||||
> SQLite marks deleted pages as free but does not return them to the filesystem
|
||||
> unless [incremental auto-vacuum](database.md) is enabled. New databases created
|
||||
> after v0.x.x have auto-vacuum enabled automatically. Existing databases require
|
||||
> a one-time migration — see the [Database](database.md) guide.
|
||||
|
||||
## Database
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `db.vacuumOnStartup` | `false` | Run a one-time full `VACUUM` on startup to enable incremental auto-vacuum (blocks for minutes on large DBs) |
|
||||
| `db.incrementalVacuumPages` | `1024` | Free pages returned to the OS after each retention reaper cycle |
|
||||
|
||||
See [Database](database.md) for details on SQLite auto-vacuum, WAL, and manual maintenance.
|
||||
See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background.
|
||||
|
||||
## Channel decryption
|
||||
|
||||
| Field | Description |
|
||||
@@ -166,9 +150,6 @@ Lower values = fresher data but more server load.
|
||||
|-------|---------|-------------|
|
||||
| `packetStore.maxMemoryMB` | `1024` | Maximum RAM for in-memory packet store |
|
||||
| `packetStore.estimatedPacketBytes` | `450` | Estimated bytes per packet (for memory budgeting) |
|
||||
| `packetStore.retentionHours` | `0` | Only load packets younger than N hours on startup and keep them in memory. **Set this on any instance with a large DB.** `0` = unlimited (loads full DB history — causes OOM on cold start when the DB has hundreds of thousands of paths). Recommended: same as `retention.packetDays × 24` (e.g. `168` for 7 days). |
|
||||
|
||||
> **Warning:** Leaving `retentionHours` at `0` on a large database will cause the server to OOM-kill itself on every cold start. The full packet history is loaded into the subpath index at startup; a DB with ~280K paths produces ~13M index entries before the process is killed.
|
||||
|
||||
## Timestamps
|
||||
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
# Database
|
||||
|
||||
CoreScope uses SQLite in WAL (Write-Ahead Log) mode for both the server
|
||||
(read-only) and ingestor (read-write).
|
||||
|
||||
## WAL mode
|
||||
|
||||
WAL mode allows concurrent reads while writes happen. It is set automatically
|
||||
at connection time via `PRAGMA journal_mode=WAL`. No operator action needed.
|
||||
|
||||
The WAL file (`meshcore.db-wal`) grows during writes and is checkpointed
|
||||
(merged back into the main DB) periodically and at clean shutdown.
|
||||
|
||||
## Auto-vacuum
|
||||
|
||||
By default, SQLite does not shrink the database file after `DELETE` operations.
|
||||
Deleted pages are marked free and reused by future writes, but the file size
|
||||
on disk stays the same. This is surprising when lowering retention settings.
|
||||
|
||||
### New databases
|
||||
|
||||
Databases created after this feature was added automatically have
|
||||
`PRAGMA auto_vacuum = INCREMENTAL`. After each retention reaper cycle,
|
||||
CoreScope runs `PRAGMA incremental_vacuum(N)` to return free pages to the OS.
|
||||
|
||||
### Existing databases
|
||||
|
||||
The `auto_vacuum` mode is stored in the database header and can only be changed
|
||||
by rewriting the entire file with `VACUUM`. CoreScope will **not** do this
|
||||
automatically — on large databases (5+ GB seen in the wild) it takes minutes
|
||||
and holds an exclusive lock.
|
||||
|
||||
**To migrate an existing database:**
|
||||
|
||||
1. At startup, CoreScope logs a warning:
|
||||
```
|
||||
[db] auto_vacuum=NONE — DB needs one-time VACUUM to enable incremental auto-vacuum.
|
||||
```
|
||||
2. **Ensure at least 2× the database file size in free disk space.** Full VACUUM
|
||||
creates a temporary copy of the entire file — on a near-full disk it will fail.
|
||||
3. Set `db.vacuumOnStartup: true` in your `config.json`:
|
||||
```json
|
||||
{
|
||||
"db": {
|
||||
"vacuumOnStartup": true
|
||||
}
|
||||
}
|
||||
```
|
||||
4. Restart CoreScope. The one-time `VACUUM` will run and block startup.
|
||||
5. After migration, remove or set `vacuumOnStartup: false` — it's not needed again.
|
||||
|
||||
### Configuration
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `db.vacuumOnStartup` | `false` | One-time full VACUUM to enable incremental auto-vacuum |
|
||||
| `db.incrementalVacuumPages` | `1024` | Pages returned to OS per reaper cycle |
|
||||
|
||||
## Manual VACUUM
|
||||
|
||||
You can also run a manual vacuum from the SQLite CLI:
|
||||
|
||||
```bash
|
||||
sqlite3 data/meshcore.db "PRAGMA auto_vacuum = INCREMENTAL; VACUUM;"
|
||||
```
|
||||
|
||||
This is equivalent to `vacuumOnStartup: true` but can be done offline.
|
||||
|
||||
> ⚠️ Full VACUUM requires **2× the database file size** in free disk space (it
|
||||
> creates a temporary copy). Check with `ls -lh data/meshcore.db` before running.
|
||||
|
||||
## Checking current mode
|
||||
|
||||
```bash
|
||||
sqlite3 data/meshcore.db "PRAGMA auto_vacuum;"
|
||||
```
|
||||
|
||||
- `0` = NONE (default for old databases)
|
||||
- `1` = FULL (automatic, but slower writes)
|
||||
- `2` = INCREMENTAL (recommended — CoreScope triggers vacuum after deletes)
|
||||
|
||||
See [#919](https://github.com/Kpa-clawbot/CoreScope/issues/919) for background on this feature.
|
||||
@@ -1,17 +0,0 @@
|
||||
// Package dbconfig provides the shared DBConfig struct used by both the server
|
||||
// and ingestor binaries for SQLite vacuum and maintenance settings (#919, #921).
|
||||
package dbconfig
|
||||
|
||||
// DBConfig controls SQLite vacuum and maintenance behavior (#919).
|
||||
type DBConfig struct {
|
||||
VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL
|
||||
IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024)
|
||||
}
|
||||
|
||||
// GetIncrementalVacuumPages returns the configured pages or 1024 default.
|
||||
func (c *DBConfig) GetIncrementalVacuumPages() int {
|
||||
if c != nil && c.IncrementalVacuumPages > 0 {
|
||||
return c.IncrementalVacuumPages
|
||||
}
|
||||
return 1024
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package dbconfig
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetIncrementalVacuumPages_Default(t *testing.T) {
|
||||
var c *DBConfig
|
||||
if got := c.GetIncrementalVacuumPages(); got != 1024 {
|
||||
t.Fatalf("nil DBConfig: got %d, want 1024", got)
|
||||
}
|
||||
c = &DBConfig{}
|
||||
if got := c.GetIncrementalVacuumPages(); got != 1024 {
|
||||
t.Fatalf("zero DBConfig: got %d, want 1024", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIncrementalVacuumPages_Configured(t *testing.T) {
|
||||
c := &DBConfig{IncrementalVacuumPages: 512}
|
||||
if got := c.GetIncrementalVacuumPages(); got != 512 {
|
||||
t.Fatalf("got %d, want 512", got)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
module github.com/meshcore-analyzer/dbconfig
|
||||
|
||||
go 1.22
|
||||
@@ -1,3 +0,0 @@
|
||||
module github.com/meshcore-analyzer/packetpath
|
||||
|
||||
go 1.22
|
||||
@@ -1,76 +0,0 @@
|
||||
// Package packetpath provides shared helpers for extracting path hops from
|
||||
// raw MeshCore packet hex bytes.
|
||||
package packetpath
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DecodePathFromRawHex extracts the header path hops directly from raw hex bytes.
|
||||
// This is the authoritative path that matches what's in raw_hex, as opposed to
|
||||
// decoded.Path.Hops which may be overwritten for TRACE packets (issue #886).
|
||||
//
|
||||
// WARNING: This function returns the literal header path bytes regardless of
|
||||
// payload type. For TRACE packets these bytes are SNR values, NOT hop hashes.
|
||||
// Callers that may receive TRACE packets MUST check PathBytesAreHops(payloadType)
|
||||
// first, or use the safer DecodeHopsForPayload wrapper.
|
||||
func DecodePathFromRawHex(rawHex string) ([]string, error) {
|
||||
buf, err := hex.DecodeString(rawHex)
|
||||
if err != nil || len(buf) < 2 {
|
||||
return nil, fmt.Errorf("invalid or too-short hex")
|
||||
}
|
||||
|
||||
headerByte := buf[0]
|
||||
offset := 1
|
||||
if IsTransportRoute(int(headerByte & 0x03)) {
|
||||
if len(buf) < offset+4 {
|
||||
return nil, fmt.Errorf("too short for transport codes")
|
||||
}
|
||||
offset += 4
|
||||
}
|
||||
if offset >= len(buf) {
|
||||
return nil, fmt.Errorf("too short for path byte")
|
||||
}
|
||||
|
||||
pathByte := buf[offset]
|
||||
offset++
|
||||
|
||||
hashSize := int(pathByte>>6) + 1
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
|
||||
hops := make([]string, 0, hashCount)
|
||||
for i := 0; i < hashCount; i++ {
|
||||
start := offset + i*hashSize
|
||||
end := start + hashSize
|
||||
if end > len(buf) {
|
||||
break
|
||||
}
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(buf[start:end])))
|
||||
}
|
||||
return hops, nil
|
||||
}
|
||||
|
||||
// DecodeHopsForPayload returns the header path hops only when the payload type's
|
||||
// header bytes are actually route hops (i.e. PathBytesAreHops(payloadType) is true).
|
||||
// For TRACE packets it returns (nil, ErrPayloadHasNoHeaderHops) so the caller is
|
||||
// forced to source hops from the decoded payload instead.
|
||||
//
|
||||
// Prefer this over DecodePathFromRawHex when the payload type is known.
|
||||
func DecodeHopsForPayload(rawHex string, payloadType byte) ([]string, error) {
|
||||
if !PathBytesAreHops(payloadType) {
|
||||
return nil, ErrPayloadHasNoHeaderHops
|
||||
}
|
||||
return DecodePathFromRawHex(rawHex)
|
||||
}
|
||||
|
||||
// ErrPayloadHasNoHeaderHops is returned by DecodeHopsForPayload when the
|
||||
// payload type repurposes the raw_hex header path bytes (e.g. TRACE → SNR values).
|
||||
var ErrPayloadHasNoHeaderHops = errPayloadHasNoHeaderHops{}
|
||||
|
||||
type errPayloadHasNoHeaderHops struct{}
|
||||
|
||||
func (errPayloadHasNoHeaderHops) Error() string {
|
||||
return "payload type repurposes header path bytes; source hops from decoded payload"
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package packetpath
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDecodePathFromRawHex_Basic(t *testing.T) {
|
||||
// Build a simple FLOOD packet (route_type=1) with 2 hops of hashSize=1
|
||||
// header: route_type=1, payload_type=2 (TXT_MSG), version=0 → 0b00_0010_01 = 0x09
|
||||
// path byte: hashSize=1 (bits 7-6 = 0), hashCount=2 (bits 5-0 = 2) → 0x02
|
||||
// hops: AB, CD
|
||||
// payload: some bytes
|
||||
raw := "0902ABCD" + "DEADBEEF"
|
||||
hops, err := DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(hops) != 2 || hops[0] != "AB" || hops[1] != "CD" {
|
||||
t.Fatalf("expected [AB, CD], got %v", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_ZeroHops(t *testing.T) {
|
||||
// DIRECT route (type=2), no hops → 0b00_0010_10 = 0x0A
|
||||
// path byte: 0x00 (0 hops)
|
||||
raw := "0A00" + "DEADBEEF"
|
||||
hops, err := DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(hops) != 0 {
|
||||
t.Fatalf("expected 0 hops, got %v", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_TransportRoute(t *testing.T) {
|
||||
// TRANSPORT_FLOOD (route_type=0), payload_type=5 (GRP_TXT), version=0
|
||||
// header: 0b00_0101_00 = 0x14
|
||||
// transport codes: 4 bytes
|
||||
// path byte: hashSize=1, hashCount=1 → 0x01
|
||||
// hop: FF
|
||||
raw := "14" + "00112233" + "01" + "FF" + "DEAD"
|
||||
hops, err := DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(hops) != 1 || hops[0] != "FF" {
|
||||
t.Fatalf("expected [FF], got %v", hops)
|
||||
}
|
||||
}
|
||||
|
||||
// buildTracePacket creates a TRACE packet hex string where header path bytes are
|
||||
// SNR values, and payload contains the actual route hops.
|
||||
func buildTracePacket() (rawHex string, headerPathHops []string, payloadHops []string) {
|
||||
// DIRECT route (type=2), TRACE payload (type=9), version=0
|
||||
// header byte: 0b00_1001_10 = 0x26
|
||||
headerByte := byte(0x26)
|
||||
|
||||
// Header path: 2 SNR bytes (hashSize=1, hashCount=2) → path byte = 0x02
|
||||
// SNR values: 0x1A (26 dB), 0x0F (15 dB)
|
||||
pathByte := byte(0x02)
|
||||
snrBytes := []byte{0x1A, 0x0F}
|
||||
|
||||
// TRACE payload: tag(4) + authCode(4) + flags(1) + path hops
|
||||
tag := []byte{0x01, 0x00, 0x00, 0x00}
|
||||
authCode := []byte{0x02, 0x00, 0x00, 0x00}
|
||||
// flags: path_sz=0 (1 byte hops), other bits=0 → 0x00
|
||||
flags := byte(0x00)
|
||||
// Payload hops: AA, BB, CC (the actual route)
|
||||
payloadPathBytes := []byte{0xAA, 0xBB, 0xCC}
|
||||
|
||||
var buf []byte
|
||||
buf = append(buf, headerByte, pathByte)
|
||||
buf = append(buf, snrBytes...)
|
||||
buf = append(buf, tag...)
|
||||
buf = append(buf, authCode...)
|
||||
buf = append(buf, flags)
|
||||
buf = append(buf, payloadPathBytes...)
|
||||
|
||||
rawHex = strings.ToUpper(hex.EncodeToString(buf))
|
||||
headerPathHops = []string{"1A", "0F"} // SNR values — NOT route hops
|
||||
payloadHops = []string{"AA", "BB", "CC"} // actual route hops from payload
|
||||
return
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_TraceReturnsSNR(t *testing.T) {
|
||||
rawHex, expectedSNR, _ := buildTracePacket()
|
||||
hops, err := DecodePathFromRawHex(rawHex)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
// DecodePathFromRawHex always returns header path bytes — for TRACE these are SNR values
|
||||
if len(hops) != len(expectedSNR) {
|
||||
t.Fatalf("expected %d hops (SNR), got %d: %v", len(expectedSNR), len(hops), hops)
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expectedSNR[i] {
|
||||
t.Errorf("hop[%d]: expected %s, got %s", i, expectedSNR[i], h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTracePathJSON_UsesPayloadHops(t *testing.T) {
|
||||
// This test validates the TRACE vs non-TRACE logic that callers should implement:
|
||||
// For TRACE: path_json = decoded.Path.Hops (payload-decoded route hops)
|
||||
// For non-TRACE: path_json = DecodePathFromRawHex(raw_hex)
|
||||
rawHex, snrHops, payloadHops := buildTracePacket()
|
||||
|
||||
// DecodePathFromRawHex returns SNR bytes for TRACE
|
||||
headerHops, _ := DecodePathFromRawHex(rawHex)
|
||||
headerJSON, _ := json.Marshal(headerHops)
|
||||
|
||||
// payload hops (what decoded.Path.Hops would return after TRACE decoding)
|
||||
payloadJSON, _ := json.Marshal(payloadHops)
|
||||
|
||||
// They must differ — SNR != route hops
|
||||
if string(headerJSON) == string(payloadJSON) {
|
||||
t.Fatalf("SNR hops and payload hops should differ for TRACE; both are %s", headerJSON)
|
||||
}
|
||||
|
||||
// For TRACE, path_json should be payloadHops, not headerHops
|
||||
_ = snrHops // snrHops == headerHops — used for documentation
|
||||
t.Logf("TRACE: header path (SNR) = %s, payload path (route) = %s", headerJSON, payloadJSON)
|
||||
}
|
||||
|
||||
func TestDecodeHopsForPayload_NonTrace(t *testing.T) {
|
||||
// header 0x01, path_len 0x02, hops 0xAA 0xBB, then payload bytes
|
||||
raw := "0102AABB00"
|
||||
hops, err := DecodeHopsForPayload(raw, 0x05) // GRP_TXT — header path bytes ARE hops
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(hops) != 2 || hops[0] != "AA" || hops[1] != "BB" {
|
||||
t.Errorf("expected [AA BB], got %v", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeHopsForPayload_TraceReturnsError(t *testing.T) {
|
||||
raw := "010205F00100"
|
||||
hops, err := DecodeHopsForPayload(raw, PayloadTRACE)
|
||||
if err != ErrPayloadHasNoHeaderHops {
|
||||
t.Errorf("expected ErrPayloadHasNoHeaderHops, got %v", err)
|
||||
}
|
||||
if hops != nil {
|
||||
t.Errorf("expected nil hops for TRACE, got %v", hops)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package packetpath
|
||||
|
||||
// Route type constants (header bits 1-0).
|
||||
const (
|
||||
RouteTransportFlood = 0
|
||||
RouteFlood = 1
|
||||
RouteDirect = 2
|
||||
RouteTransportDirect = 3
|
||||
)
|
||||
|
||||
// PayloadTRACE is the payload type constant for TRACE packets.
|
||||
const PayloadTRACE = 0x09
|
||||
|
||||
// IsTransportRoute returns true for TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3).
|
||||
func IsTransportRoute(routeType int) bool {
|
||||
return routeType == RouteTransportFlood || routeType == RouteTransportDirect
|
||||
}
|
||||
|
||||
// PathBytesAreHops returns true when the raw_hex header path bytes represent
|
||||
// route hop hashes (the normal case). Returns false for packet types where
|
||||
// header path bytes are repurposed (e.g. TRACE uses them for SNR values).
|
||||
func PathBytesAreHops(payloadType byte) bool {
|
||||
return payloadType != PayloadTRACE
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package packetpath
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsTransportRoute(t *testing.T) {
|
||||
if !IsTransportRoute(RouteTransportFlood) {
|
||||
t.Error("RouteTransportFlood should be transport")
|
||||
}
|
||||
if !IsTransportRoute(RouteTransportDirect) {
|
||||
t.Error("RouteTransportDirect should be transport")
|
||||
}
|
||||
if IsTransportRoute(RouteFlood) {
|
||||
t.Error("RouteFlood should not be transport")
|
||||
}
|
||||
if IsTransportRoute(RouteDirect) {
|
||||
t.Error("RouteDirect should not be transport")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathBytesAreHops(t *testing.T) {
|
||||
if PathBytesAreHops(PayloadTRACE) {
|
||||
t.Error("PathBytesAreHops(PayloadTRACE) should be false")
|
||||
}
|
||||
// All other known payload types should return true.
|
||||
otherTypes := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}
|
||||
for _, pt := range otherTypes {
|
||||
if !PathBytesAreHops(pt) {
|
||||
t.Errorf("PathBytesAreHops(0x%02X) should be true", pt)
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
-94
@@ -28,7 +28,7 @@
|
||||
|
||||
function barChart(data, labels, colors, w = 800, h = 220, pad = 40) {
|
||||
const max = Math.max(...data, 1);
|
||||
const barW = Math.max(1, Math.min((w - pad * 2) / data.length - 2, 30));
|
||||
const barW = Math.min((w - pad * 2) / data.length - 2, 30);
|
||||
let svg = `<svg viewBox="0 0 ${w} ${h}" style="width:100%;max-height:${h}px" role="img" aria-label="Bar chart showing data distribution"><title>Bar chart showing data distribution</title>`;
|
||||
// Grid
|
||||
for (let i = 0; i <= 4; i++) {
|
||||
@@ -75,16 +75,6 @@
|
||||
<h2>📊 Mesh Analytics</h2>
|
||||
<p class="text-muted">Deep dive into your mesh network data</p>
|
||||
<div id="analyticsRegionFilter" class="region-filter-container"></div>
|
||||
<div class="time-window-filter" style="margin:8px 0">
|
||||
<label for="analyticsTimeWindow" style="font-size:0.9em;color:var(--text-muted);margin-right:6px">Time window:</label>
|
||||
<select id="analyticsTimeWindow" data-testid="analytics-time-window" aria-label="Time window">
|
||||
<option value="">All data</option>
|
||||
<option value="1h">Last 1 hour</option>
|
||||
<option value="24h">Last 24 hours</option>
|
||||
<option value="7d">Last 7 days</option>
|
||||
<option value="30d">Last 30 days</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="analytics-tabs" id="analyticsTabs" role="tablist" aria-label="Analytics tabs">
|
||||
<button class="tab-btn active" data-tab="overview">Overview</button>
|
||||
<button class="tab-btn" data-tab="rf">RF / Signal</button>
|
||||
@@ -133,12 +123,6 @@
|
||||
RegionFilter.init(document.getElementById('analyticsRegionFilter'));
|
||||
RegionFilter.onChange(function () { loadAnalytics(); });
|
||||
|
||||
// Time-window picker (#842) — refresh analytics on change.
|
||||
const tw = document.getElementById('analyticsTimeWindow');
|
||||
if (tw) {
|
||||
tw.addEventListener('change', function () { loadAnalytics(); });
|
||||
}
|
||||
|
||||
// Delegated click/keyboard handler for clickable table rows
|
||||
const analyticsContent = document.getElementById('analyticsContent');
|
||||
if (analyticsContent) {
|
||||
@@ -166,24 +150,14 @@
|
||||
async function loadAnalytics() {
|
||||
try {
|
||||
_analyticsData = {};
|
||||
const rqs = RegionFilter.regionQueryString(); // "®ion=..." or ""
|
||||
// Time window picker (#842) — append &window=… when set.
|
||||
// NOTE: only the three window-aware endpoints (rf/topology/channels)
|
||||
// receive ?window=…; hash-sizes and hash-collisions are about node
|
||||
// identity / hash-byte distribution and intentionally span all data.
|
||||
const twEl = document.getElementById('analyticsTimeWindow');
|
||||
const twVal = twEl ? twEl.value : '';
|
||||
const tws = twVal ? '&window=' + encodeURIComponent(twVal) : '';
|
||||
const baseQS = rqs.slice(1); // drop leading '&', "" or "region=…"
|
||||
const sepBase = baseQS ? '?' + baseQS : '';
|
||||
const windowedQS = (rqs + tws).slice(1);
|
||||
const sepWin = windowedQS ? '?' + windowedQS : '';
|
||||
const rqs = RegionFilter.regionQueryString();
|
||||
const sep = rqs ? '?' + rqs.slice(1) : '';
|
||||
const [hashData, rfData, topoData, chanData, collisionData] = await Promise.all([
|
||||
api('/analytics/hash-sizes' + sepBase, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/rf' + sepWin, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/topology' + sepWin, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/channels' + sepWin, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/hash-collisions' + sepBase, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/hash-sizes' + sep, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/rf' + sep, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/topology' + sep, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/channels' + sep, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
api('/analytics/hash-collisions' + sep, { ttl: CLIENT_TTL.analyticsRF }),
|
||||
]);
|
||||
_analyticsData = { hashData, rfData, topoData, chanData, collisionData };
|
||||
renderTab(_currentTab);
|
||||
@@ -289,25 +263,7 @@
|
||||
<div class="analytics-row">
|
||||
<div class="analytics-card flex-1">
|
||||
<h3>📈 Packets / Hour</h3>
|
||||
${(() => {
|
||||
const pph = rf.packetsPerHour;
|
||||
const counts = pph.map(h => h.count);
|
||||
// Decimate x-axis labels to avoid overlap
|
||||
const totalHours = pph.length;
|
||||
// Pick label interval: <=24h show every 6h, <=72h every 12h, else every 24h
|
||||
const labelInterval = totalHours <= 24 ? 6 : totalHours <= 72 ? 12 : 24;
|
||||
const labels = pph.map((h, i) => {
|
||||
const hh = h.hour.slice(11, 13); // "HH"
|
||||
const hourNum = parseInt(hh, 10);
|
||||
if (hourNum % labelInterval === 0) {
|
||||
// For multi-day ranges, show date on 00h boundaries
|
||||
if (totalHours > 48 && hourNum === 0) return h.hour.slice(5, 10);
|
||||
return hh + 'h';
|
||||
}
|
||||
return ''; // skip label
|
||||
});
|
||||
return barChart(counts, labels, 'var(--accent)');
|
||||
})()}
|
||||
${barChart(rf.packetsPerHour.map(h=>h.count), rf.packetsPerHour.map(h=>h.hour.slice(11)+'h'), 'var(--accent)')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -668,13 +624,14 @@
|
||||
if (!data || !data.rings.length) return '<div class="text-muted">No path data for this observer</div>';
|
||||
let html = `<div class="reach-rings">`;
|
||||
data.rings.forEach(ring => {
|
||||
const opacity = Math.max(0.3, 1 - ring.hops * 0.06);
|
||||
const nodeLinks = ring.nodes.slice(0, 8).map(n => {
|
||||
const label = n.name ? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>` : `<span class="mono">${n.hop}</span>`;
|
||||
const detail = n.distRange ? ` <span class="text-muted">(${n.distRange})</span>` : '';
|
||||
return label + detail;
|
||||
}).join(', ');
|
||||
const extra = ring.nodes.length > 8 ? ` <span class="text-muted">+${ring.nodes.length - 8} more</span>` : '';
|
||||
html += `<div class="reach-ring">
|
||||
html += `<div class="reach-ring" style="opacity:${opacity}">
|
||||
<div class="reach-hop">${ring.hops} hop${ring.hops > 1 ? 's' : ''}</div>
|
||||
<div class="reach-nodes">${nodeLinks}${extra}</div>
|
||||
<div class="reach-count">${ring.nodes.length} node${ring.nodes.length > 1 ? 's' : ''}</div>
|
||||
@@ -718,6 +675,7 @@
|
||||
});
|
||||
let html = '<div class="reach-rings">';
|
||||
Object.entries(byDist).sort((a, b) => +a[0] - +b[0]).forEach(([dist, nodes]) => {
|
||||
const opacity = Math.max(0.3, 1 - (+dist) * 0.06);
|
||||
const nodeLinks = nodes.slice(0, 10).map(n => {
|
||||
const label = n.name
|
||||
? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>`
|
||||
@@ -725,7 +683,7 @@
|
||||
return label + ` <span class="text-muted">via ${esc(n.observer_name)}</span>`;
|
||||
}).join(', ');
|
||||
const extra = nodes.length > 10 ? ` <span class="text-muted">+${nodes.length - 10} more</span>` : '';
|
||||
html += `<div class="reach-ring">
|
||||
html += `<div class="reach-ring" style="opacity:${opacity}">
|
||||
<div class="reach-hop">${dist} hop${+dist > 1 ? 's' : ''}</div>
|
||||
<div class="reach-nodes">${nodeLinks}${extra}</div>
|
||||
<div class="reach-count">${nodes.length} node${nodes.length > 1 ? 's' : ''}</div>
|
||||
@@ -882,44 +840,29 @@
|
||||
}
|
||||
}
|
||||
|
||||
var CHANNEL_TIMELINE_MAX_SERIES = 8;
|
||||
|
||||
function renderChannelTimeline(data) {
|
||||
if (!data.length) return '<div class="text-muted">No data</div>';
|
||||
var hours = []; var hourSet = {};
|
||||
var channelList = []; var channelSet = {};
|
||||
var lookup = {};
|
||||
var channelVolume = {};
|
||||
var maxCount = 1;
|
||||
for (var i = 0; i < data.length; i++) {
|
||||
var d = data[i];
|
||||
if (!hourSet[d.hour]) { hourSet[d.hour] = 1; hours.push(d.hour); }
|
||||
if (!channelSet[d.channel]) { channelSet[d.channel] = 1; channelList.push(d.channel); }
|
||||
lookup[d.hour + '|' + d.channel] = d.count;
|
||||
channelVolume[d.channel] = (channelVolume[d.channel] || 0) + d.count;
|
||||
if (d.count > maxCount) maxCount = d.count;
|
||||
}
|
||||
hours.sort();
|
||||
// Sort channels by total volume descending, cap to top N
|
||||
channelList.sort(function(a, b) { return channelVolume[b] - channelVolume[a]; });
|
||||
var hiddenCount = Math.max(0, channelList.length - CHANNEL_TIMELINE_MAX_SERIES);
|
||||
var visibleChannels = channelList.slice(0, CHANNEL_TIMELINE_MAX_SERIES);
|
||||
|
||||
var maxCount = 1;
|
||||
for (var vi = 0; vi < visibleChannels.length; vi++) {
|
||||
for (var hi2 = 0; hi2 < hours.length; hi2++) {
|
||||
var c = lookup[hours[hi2] + '|' + visibleChannels[vi]] || 0;
|
||||
if (c > maxCount) maxCount = c;
|
||||
}
|
||||
}
|
||||
|
||||
var colors = ['#ef4444','#22c55e','#3b82f6','#f59e0b','#8b5cf6','#ec4899','#14b8a6','#64748b'];
|
||||
var w = 600, h = 180, pad = 35;
|
||||
var xScale = (w - pad * 2) / Math.max(hours.length - 1, 1);
|
||||
var yScale = (h - pad * 2) / maxCount;
|
||||
var svg = '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:100%;max-height:180px" role="img" aria-label="Channel message activity over time"><title>Channel message activity over time</title>';
|
||||
for (var ci = 0; ci < visibleChannels.length; ci++) {
|
||||
for (var ci = 0; ci < channelList.length; ci++) {
|
||||
var pts = [];
|
||||
for (var hi = 0; hi < hours.length; hi++) {
|
||||
var count = lookup[hours[hi] + '|' + visibleChannels[ci]] || 0;
|
||||
var count = lookup[hours[hi] + '|' + channelList[ci]] || 0;
|
||||
var x = pad + hi * xScale;
|
||||
var y = h - pad - count * yScale;
|
||||
pts.push(x + ',' + y);
|
||||
@@ -933,11 +876,8 @@
|
||||
}
|
||||
svg += '</svg>';
|
||||
var legendParts = [];
|
||||
for (var lci = 0; lci < visibleChannels.length; lci++) {
|
||||
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(visibleChannels[lci]) + '</span>');
|
||||
}
|
||||
if (hiddenCount > 0) {
|
||||
legendParts.push('<span class="text-muted">+' + hiddenCount + ' more</span>');
|
||||
for (var lci = 0; lci < channelList.length; lci++) {
|
||||
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(channelList[lci]) + '</span>');
|
||||
}
|
||||
svg += '<div class="timeline-legend">' + legendParts.join('') + '</div>';
|
||||
return svg;
|
||||
@@ -1758,8 +1698,8 @@
|
||||
|
||||
<div class="subpath-section">
|
||||
<h5>⏱️ Timeline</h5>
|
||||
<div>First seen: ${data.firstSeen ? (typeof formatAbsoluteTimestamp === 'function' ? formatAbsoluteTimestamp(data.firstSeen) : new Date(data.firstSeen).toLocaleString()) : '—'}</div>
|
||||
<div>Last seen: ${data.lastSeen ? (typeof formatAbsoluteTimestamp === 'function' ? formatAbsoluteTimestamp(data.lastSeen) : new Date(data.lastSeen).toLocaleString()) : '—'}</div>
|
||||
<div>First seen: ${data.firstSeen ? new Date(data.firstSeen).toLocaleString() : '—'}</div>
|
||||
<div>Last seen: ${data.lastSeen ? new Date(data.lastSeen).toLocaleString() : '—'}</div>
|
||||
</div>
|
||||
|
||||
${data.observers.length ? `
|
||||
@@ -1997,18 +1937,15 @@
|
||||
}
|
||||
|
||||
// Top hops leaderboard
|
||||
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">Obs</th><th scope="col">Best SNR</th><th scope="col">Median SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
|
||||
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
|
||||
const top20 = data.topHops.slice(0, 20);
|
||||
top20.forEach((h, i) => {
|
||||
const fromLink = h.fromPk ? `<a href="#/nodes/${encodeURIComponent(h.fromPk)}" class="analytics-link">${esc(h.fromName)}</a>` : esc(h.fromName || '?');
|
||||
const toLink = h.toPk ? `<a href="#/nodes/${encodeURIComponent(h.toPk)}" class="analytics-link">${esc(h.toName)}</a>` : esc(h.toName || '?');
|
||||
const bestSnr = h.bestSnr != null ? Number(h.bestSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
|
||||
const medianSnr = h.medianSnr != null ? Number(h.medianSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
|
||||
const obs = h.obsCount != null ? h.obsCount : 1;
|
||||
const snr = h.snr != null ? h.snr + ' dB' : '<span class="text-muted">—</span>';
|
||||
const pktLink = h.hash ? `<a href="#/packet/${encodeURIComponent(h.hash)}" class="analytics-link mono" style="font-size:0.85em">${esc(h.hash.slice(0, 12))}…</a>` : '—';
|
||||
const mapBtn = h.fromPk && h.toPk ? `<button class="btn-icon dist-map-hop" data-from="${esc(h.fromPk)}" data-to="${esc(h.toPk)}" title="View on map">🗺️</button>` : '';
|
||||
const tsTitle = h.timestamp ? `Best observation: ${h.timestamp}` : '';
|
||||
html += `<tr title="${esc(tsTitle)}"><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${obs}</td><td>${bestSnr}</td><td>${medianSnr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
|
||||
html += `<tr><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${snr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
|
||||
});
|
||||
html += `</tbody></table></div>`;
|
||||
|
||||
@@ -2686,7 +2623,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
const name = esc(n.name || n.public_key.slice(0, 12));
|
||||
const role = n.role ? `<span class="text-muted" style="font-size:0.82em">${esc(n.role)}</span>` : '';
|
||||
const hs = n.hash_size ? ` <span class="text-muted" style="font-size:0.78em;opacity:0.7">${n.hash_size}B hash</span>` : '';
|
||||
const when = n.last_seen ? ` <span class="text-muted" style="font-size:0.8em">${(typeof formatAbsoluteTimestamp === 'function') ? formatAbsoluteTimestamp(n.last_seen) : new Date(n.last_seen).toLocaleDateString()}</span>` : '';
|
||||
const when = n.last_seen ? ` <span class="text-muted" style="font-size:0.8em">${new Date(n.last_seen).toLocaleDateString()}</span>` : '';
|
||||
return `<div style="padding:3px 0"><a href="#/nodes/${encodeURIComponent(n.public_key)}" class="analytics-link">${name}</a> ${role}${hs}${when}</div>`;
|
||||
}
|
||||
|
||||
@@ -3184,7 +3121,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
const t = new Date(d.t);
|
||||
const x = sx(t.getTime());
|
||||
const y = sy(d.v);
|
||||
const ts = (typeof formatAbsoluteTimestamp === 'function') ? formatAbsoluteTimestamp(d.t) : t.toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC');
|
||||
const ts = t.toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC');
|
||||
const tip = `${label}: ${formatV(d.v)}${unit}\n${ts}`;
|
||||
svg += `<circle cx="${x.toFixed(1)}" cy="${y.toFixed(1)}" r="8" fill="transparent" stroke="none" pointer-events="all"><title>${tip}</title></circle>`;
|
||||
});
|
||||
@@ -3198,7 +3135,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
const idx = Math.floor(i * (data.length - 1) / Math.max(xTicks - 1, 1));
|
||||
const t = new Date(data[idx].t);
|
||||
const x = sx(t.getTime());
|
||||
const label = (typeof formatChartAxisLabel === 'function') ? formatChartAxisLabel(t, true) : t.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
|
||||
const label = t.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
|
||||
svg += `<text x="${x.toFixed(1)}" y="${h - 5}" text-anchor="middle" font-size="9" fill="var(--text-muted)">${label}</text>`;
|
||||
}
|
||||
return svg;
|
||||
@@ -3511,7 +3448,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
if (sortKey === 'severity') {
|
||||
v = (SKEW_SEVERITY_ORDER[a.severity] || 9) - (SKEW_SEVERITY_ORDER[b.severity] || 9);
|
||||
} else if (sortKey === 'skew') {
|
||||
v = Math.abs(window.currentSkewValue(b) || 0) - Math.abs(window.currentSkewValue(a) || 0);
|
||||
v = Math.abs(b.medianSkewSec || 0) - Math.abs(a.medianSkewSec || 0);
|
||||
} else if (sortKey === 'name') {
|
||||
v = (a.nodeName || '').localeCompare(b.nodeName || '');
|
||||
} else if (sortKey === 'drift') {
|
||||
@@ -3538,13 +3475,12 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
|
||||
var rowsHtml = filtered.map(function(n) {
|
||||
var rowClass = 'clock-fleet-row--' + (n.severity || 'ok');
|
||||
var lastAdv = n.lastObservedTS ? new Date(n.lastObservedTS * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC') : '—';
|
||||
var skewVal = window.currentSkewValue(n);
|
||||
var skewText = n.severity === 'no_clock' ? 'No Clock' : formatSkew(skewVal);
|
||||
var skewText = n.severity === 'no_clock' ? 'No Clock' : formatSkew(n.medianSkewSec);
|
||||
var driftText = n.severity === 'no_clock' || !n.driftPerDaySec ? '–' : formatDrift(n.driftPerDaySec);
|
||||
return '<tr class="' + rowClass + '" data-pubkey="' + esc(n.pubkey) + '" style="cursor:pointer">' +
|
||||
'<td><strong>' + esc(n.nodeName || n.pubkey.slice(0, 12)) + '</strong></td>' +
|
||||
'<td style="font-family:var(--mono,monospace)">' + skewText + '</td>' +
|
||||
'<td>' + renderSkewBadge(n.severity, skewVal, n) + '</td>' +
|
||||
'<td>' + renderSkewBadge(n.severity, n.medianSkewSec) + '</td>' +
|
||||
'<td style="font-family:var(--mono,monospace)">' + driftText + '</td>' +
|
||||
'<td style="font-size:11px">' + lastAdv + '</td>' +
|
||||
'</tr>';
|
||||
|
||||
+6
-145
@@ -4,81 +4,14 @@
|
||||
// --- Route/Payload name maps ---
|
||||
const ROUTE_TYPES = { 0: 'TRANSPORT_FLOOD', 1: 'FLOOD', 2: 'DIRECT', 3: 'TRANSPORT_DIRECT' };
|
||||
const PAYLOAD_TYPES = { 0: 'Request', 1: 'Response', 2: 'Direct Msg', 3: 'ACK', 4: 'Advert', 5: 'Channel Msg', 6: 'Group Data', 7: 'Anon Req', 8: 'Path', 9: 'Trace', 10: 'Multipart', 11: 'Control', 15: 'Raw Custom' };
|
||||
const PAYLOAD_COLORS = { 0: 'req', 1: 'response', 2: 'txt-msg', 3: 'ack', 4: 'advert', 5: 'grp-txt', 6: 'grp-data', 7: 'anon-req', 8: 'path', 9: 'trace', 10: 'multipart', 11: 'control', 15: 'raw-custom' };
|
||||
const PAYLOAD_COLORS = { 0: 'req', 1: 'response', 2: 'txt-msg', 3: 'ack', 4: 'advert', 5: 'grp-txt', 7: 'anon-req', 8: 'path', 9: 'trace' };
|
||||
|
||||
function routeTypeName(n) { return ROUTE_TYPES[n] || 'UNKNOWN'; }
|
||||
function payloadTypeName(n) { return PAYLOAD_TYPES[n] || 'UNKNOWN'; }
|
||||
function payloadTypeColor(n) { return PAYLOAD_COLORS[n] || 'unknown'; }
|
||||
function isTransportRoute(rt) { return rt === 0 || rt === 3; }
|
||||
/** Byte offset of path_len in raw_hex: 5 for transport routes (4 bytes of next/last hop codes precede it), 1 otherwise. */
|
||||
function getPathLenOffset(routeType) { return isTransportRoute(routeType) ? 5 : 1; }
|
||||
function transportBadge(rt) { return isTransportRoute(rt) ? ' <span class="badge badge-transport" title="' + routeTypeName(rt) + '">T</span>' : ''; }
|
||||
|
||||
/**
|
||||
* Compute breakdown byte ranges from raw_hex on the client.
|
||||
* Mirrors cmd/server/decoder.go BuildBreakdown(). Used so per-observation raw_hex
|
||||
* (which can differ in path length from the top-level packet) gets accurate
|
||||
* highlighted byte ranges, instead of using the server-supplied breakdown
|
||||
* computed once from the top-level raw_hex.
|
||||
*/
|
||||
function computeBreakdownRanges(hexString, routeType, payloadType) {
|
||||
if (!hexString) return [];
|
||||
const clean = hexString.replace(/\s+/g, '');
|
||||
const bytes = clean.length / 2;
|
||||
if (bytes < 2) return [];
|
||||
const ranges = [];
|
||||
// Header
|
||||
ranges.push({ start: 0, end: 0, label: 'Header' });
|
||||
let offset = 1;
|
||||
if (isTransportRoute(routeType)) {
|
||||
if (bytes < offset + 4) return ranges;
|
||||
ranges.push({ start: offset, end: offset + 3, label: 'Transport Codes' });
|
||||
offset += 4;
|
||||
}
|
||||
if (offset >= bytes) return ranges;
|
||||
// Path Length byte
|
||||
ranges.push({ start: offset, end: offset, label: 'Path Length' });
|
||||
const pathByte = parseInt(clean.slice(offset * 2, offset * 2 + 2), 16);
|
||||
offset += 1;
|
||||
if (isNaN(pathByte)) return ranges;
|
||||
const hashSize = (pathByte >> 6) + 1;
|
||||
const hashCount = pathByte & 0x3F;
|
||||
const pathBytes = hashSize * hashCount;
|
||||
if (hashCount > 0 && offset + pathBytes <= bytes) {
|
||||
ranges.push({ start: offset, end: offset + pathBytes - 1, label: 'Path' });
|
||||
}
|
||||
offset += pathBytes;
|
||||
if (offset >= bytes) return ranges;
|
||||
const payloadStart = offset;
|
||||
// ADVERT (payload_type 4) gets sub-fields when full record present
|
||||
if (payloadType === 4 && bytes - payloadStart >= 100) {
|
||||
ranges.push({ start: payloadStart, end: payloadStart + 31, label: 'PubKey' });
|
||||
ranges.push({ start: payloadStart + 32, end: payloadStart + 35, label: 'Timestamp' });
|
||||
ranges.push({ start: payloadStart + 36, end: payloadStart + 99, label: 'Signature' });
|
||||
const appStart = payloadStart + 100;
|
||||
if (appStart < bytes) {
|
||||
ranges.push({ start: appStart, end: appStart, label: 'Flags' });
|
||||
const appFlags = parseInt(clean.slice(appStart * 2, appStart * 2 + 2), 16);
|
||||
let fOff = appStart + 1;
|
||||
if (!isNaN(appFlags)) {
|
||||
if ((appFlags & 0x10) && fOff + 8 <= bytes) {
|
||||
ranges.push({ start: fOff, end: fOff + 3, label: 'Latitude' });
|
||||
ranges.push({ start: fOff + 4, end: fOff + 7, label: 'Longitude' });
|
||||
fOff += 8;
|
||||
}
|
||||
if ((appFlags & 0x20) && fOff + 2 <= bytes) fOff += 2;
|
||||
if ((appFlags & 0x40) && fOff + 2 <= bytes) fOff += 2;
|
||||
if ((appFlags & 0x80) && fOff < bytes) {
|
||||
ranges.push({ start: fOff, end: bytes - 1, label: 'Name' });
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ranges.push({ start: payloadStart, end: bytes - 1, label: 'Payload' });
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
// --- Utilities ---
|
||||
const _apiPerf = { calls: 0, totalMs: 0, log: [], cacheHits: 0 };
|
||||
const _apiCache = new Map();
|
||||
@@ -309,39 +242,6 @@ function formatTimestampWithTooltip(isoString, mode) {
|
||||
return { text, tooltip, isFuture };
|
||||
}
|
||||
|
||||
// Format a Date for chart axis labels, respecting customizer timestamp settings.
|
||||
// shortForm: true = time only (for intra-day), false = date+time (multi-day).
|
||||
function formatChartAxisLabel(d, shortForm) {
|
||||
if (!(d instanceof Date) || !isFinite(d.getTime())) return '—';
|
||||
var timezone = (typeof getTimestampTimezone === 'function') ? getTimestampTimezone() : 'local';
|
||||
var preset = (typeof getTimestampFormatPreset === 'function') ? getTimestampFormatPreset() : 'iso';
|
||||
var useUtc = timezone === 'utc';
|
||||
|
||||
if (preset === 'locale') {
|
||||
if (shortForm) {
|
||||
var opts = { hour: '2-digit', minute: '2-digit' };
|
||||
if (useUtc) opts.timeZone = 'UTC';
|
||||
return d.toLocaleTimeString([], opts);
|
||||
}
|
||||
var opts2 = { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' };
|
||||
if (useUtc) opts2.timeZone = 'UTC';
|
||||
return d.toLocaleString([], opts2);
|
||||
}
|
||||
|
||||
// ISO-style (iso or iso-seconds)
|
||||
var hour = useUtc ? d.getUTCHours() : d.getHours();
|
||||
var minute = useUtc ? d.getUTCMinutes() : d.getMinutes();
|
||||
var timeStr = pad2(hour) + ':' + pad2(minute);
|
||||
if (preset === 'iso-seconds') {
|
||||
var sec = useUtc ? d.getUTCSeconds() : d.getSeconds();
|
||||
timeStr += ':' + pad2(sec);
|
||||
}
|
||||
if (shortForm) return timeStr;
|
||||
var month = useUtc ? d.getUTCMonth() + 1 : d.getMonth() + 1;
|
||||
var day = useUtc ? d.getUTCDate() : d.getDate();
|
||||
return pad2(month) + '-' + pad2(day) + ' ' + timeStr;
|
||||
}
|
||||
|
||||
function truncate(str, len) {
|
||||
if (!str) return '';
|
||||
return str.length > len ? str.slice(0, len) + '…' : str;
|
||||
@@ -538,21 +438,6 @@ const pages = {};
|
||||
|
||||
function registerPage(name, mod) { pages[name] = mod; }
|
||||
|
||||
// Tools landing page — shows sub-menu with Trace and Path Inspector (spec §2.8, M1 fix).
|
||||
registerPage('tools-landing', {
|
||||
init: function (container) {
|
||||
container.innerHTML =
|
||||
'<div class="tools-landing">' +
|
||||
'<h2>Tools</h2>' +
|
||||
'<div class="tools-menu">' +
|
||||
'<a href="#/tools/path-inspector" class="tools-card"><h3>🔍 Path Inspector</h3><p>Resolve prefix paths to candidate full-pubkey routes with confidence scoring.</p></a>' +
|
||||
'<a href="#/tools/trace/" class="tools-card"><h3>📡 Trace Viewer</h3><p>View detailed packet traces by hash.</p></a>' +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
},
|
||||
destroy: function () {}
|
||||
});
|
||||
|
||||
let currentPage = null;
|
||||
|
||||
function closeNav() {
|
||||
@@ -573,12 +458,6 @@ function closeMoreMenu() {
|
||||
function navigate() {
|
||||
closeNav();
|
||||
|
||||
// Backward-compat redirect: #/traces/<hash> → #/tools/trace/<hash> (issue #944).
|
||||
if (location.hash.startsWith('#/traces/')) {
|
||||
location.hash = location.hash.replace('#/traces/', '#/tools/trace/');
|
||||
return;
|
||||
}
|
||||
|
||||
const hash = location.hash.replace('#/', '') || 'packets';
|
||||
const route = hash.split('?')[0];
|
||||
|
||||
@@ -606,27 +485,9 @@ function navigate() {
|
||||
basePage = 'observer-detail';
|
||||
}
|
||||
|
||||
// Tools sub-routing (issue #944): tools/trace/<hash>, tools/path-inspector
|
||||
if (basePage === 'tools') {
|
||||
if (routeParam && routeParam.startsWith('trace/')) {
|
||||
basePage = 'traces';
|
||||
routeParam = routeParam.substring(6); // strip "trace/"
|
||||
} else if (routeParam === 'path-inspector' || (routeParam && routeParam.startsWith('path-inspector'))) {
|
||||
basePage = 'path-inspector';
|
||||
routeParam = null;
|
||||
} else if (!routeParam) {
|
||||
// Default tools landing shows menu with both entries.
|
||||
basePage = 'tools-landing';
|
||||
}
|
||||
}
|
||||
// Also support old #/traces (no sub-path) → traces page.
|
||||
if (basePage === 'traces' && !routeParam) {
|
||||
basePage = 'traces';
|
||||
}
|
||||
|
||||
// Update nav active state
|
||||
document.querySelectorAll('.nav-link[data-route]').forEach(el => {
|
||||
el.classList.toggle('active', el.dataset.route === basePage || (el.dataset.route === 'tools' && (basePage === 'traces' || basePage === 'path-inspector' || basePage === 'tools-landing')));
|
||||
el.classList.toggle('active', el.dataset.route === basePage);
|
||||
});
|
||||
// Update "More" button to show active state if a low-priority page is selected
|
||||
var moreBtn = document.getElementById('navMoreBtn');
|
||||
@@ -998,11 +859,10 @@ window.addEventListener('DOMContentLoaded', () => {
|
||||
}).catch(() => {
|
||||
window.SITE_CONFIG = { timestamps: { defaultMode: 'ago', timezone: 'local', formatPreset: 'iso', customFormat: '', allowCustomFormat: false } };
|
||||
if (window._customizerV2) window._customizerV2.init(window.SITE_CONFIG);
|
||||
}).finally(() => {
|
||||
if (!location.hash || location.hash === '#/') location.hash = '#/home';
|
||||
else navigate();
|
||||
});
|
||||
|
||||
// Navigate immediately — don't gate data-fetching pages on cosmetic theme fetch
|
||||
if (!location.hash || location.hash === '#/') location.hash = '#/home';
|
||||
else navigate();
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -1167,6 +1027,7 @@ function makeColumnsResizable(tableSelector, storageKey) {
|
||||
// Add resize handles
|
||||
ths.forEach((th, i) => {
|
||||
if (i === ths.length - 1) return;
|
||||
th.style.position = 'relative';
|
||||
const handle = document.createElement('div');
|
||||
handle.className = 'col-resize-handle';
|
||||
handle.addEventListener('mousedown', (e) => {
|
||||
|
||||
@@ -120,8 +120,8 @@
|
||||
var ph = rect.height;
|
||||
var vw = window.innerWidth;
|
||||
var vh = window.innerHeight;
|
||||
var finalX = x + pw > vw ? Math.max(0, vw - pw - 14) : x;
|
||||
var finalY = y + ph > vh ? Math.max(0, vh - ph - 14) : y;
|
||||
var finalX = x + pw > vw ? Math.max(0, vw - pw - 8) : x;
|
||||
var finalY = y + ph > vh ? Math.max(0, vh - ph - 8) : y;
|
||||
el.style.left = finalX + 'px';
|
||||
el.style.top = finalY + 'px';
|
||||
}
|
||||
@@ -228,6 +228,12 @@
|
||||
if (ch) showPopover(ch, e.clientX, e.clientY);
|
||||
});
|
||||
|
||||
feed.addEventListener('contextmenu', function(e) {
|
||||
var item = e.target.closest('.live-feed-item');
|
||||
if (!item || !item._ccChannel) return;
|
||||
e.preventDefault();
|
||||
showPopover(item._ccChannel, e.clientX, e.clientY);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+41
-185
@@ -15,7 +15,6 @@ window.ChannelDecrypt = (function () {
|
||||
'use strict';
|
||||
|
||||
var STORAGE_KEY = 'corescope_channel_keys';
|
||||
var LABELS_KEY = 'corescope_channel_labels';
|
||||
var CACHE_KEY = 'corescope_channel_cache';
|
||||
|
||||
// ---- Hex utilities ----
|
||||
@@ -38,25 +37,6 @@ window.ChannelDecrypt = (function () {
|
||||
|
||||
// ---- Key derivation ----
|
||||
|
||||
// Detect whether SubtleCrypto is available. SubtleCrypto is only exposed
|
||||
// in **secure contexts** (HTTPS or localhost) — when CoreScope is served
|
||||
// over plain HTTP, `crypto.subtle` is undefined and any digest/HMAC call
|
||||
// throws. We fall back to the vendored pure-JS implementation in
|
||||
// public/vendor/sha256-hmac.js. PR #1021 did the same for AES-ECB.
|
||||
function hasSubtle() {
|
||||
return typeof crypto !== 'undefined' && crypto && crypto.subtle && typeof crypto.subtle.digest === 'function';
|
||||
}
|
||||
|
||||
function pureCryptoOrThrow() {
|
||||
var host = (typeof window !== 'undefined') ? window
|
||||
: (typeof self !== 'undefined') ? self : null;
|
||||
if (!host || !host.PureCrypto || !host.PureCrypto.sha256 || !host.PureCrypto.hmacSha256) {
|
||||
throw new Error('PureCrypto vendor module not loaded (public/vendor/sha256-hmac.js). ' +
|
||||
'crypto.subtle is unavailable (HTTP context) and no fallback present.');
|
||||
}
|
||||
return host.PureCrypto;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive AES-128 key from channel name: SHA-256("#channelname")[:16].
|
||||
* @param {string} channelName - e.g. "#LongFast"
|
||||
@@ -64,12 +44,8 @@ window.ChannelDecrypt = (function () {
|
||||
*/
|
||||
async function deriveKey(channelName) {
|
||||
var enc = new TextEncoder();
|
||||
var data = enc.encode(channelName);
|
||||
if (hasSubtle()) {
|
||||
var hash = await crypto.subtle.digest('SHA-256', data);
|
||||
return new Uint8Array(hash).slice(0, 16);
|
||||
}
|
||||
return pureCryptoOrThrow().sha256(data).slice(0, 16);
|
||||
var hash = await crypto.subtle.digest('SHA-256', enc.encode(channelName));
|
||||
return new Uint8Array(hash).slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -78,41 +54,46 @@ window.ChannelDecrypt = (function () {
|
||||
* @returns {Promise<number>} single byte (0-255)
|
||||
*/
|
||||
async function computeChannelHash(key) {
|
||||
if (hasSubtle()) {
|
||||
var hash = await crypto.subtle.digest('SHA-256', key);
|
||||
return new Uint8Array(hash)[0];
|
||||
}
|
||||
return pureCryptoOrThrow().sha256(key)[0];
|
||||
var hash = await crypto.subtle.digest('SHA-256', key);
|
||||
return new Uint8Array(hash)[0];
|
||||
}
|
||||
|
||||
// ---- AES-128-ECB via vendored pure-JS implementation ----
|
||||
//
|
||||
// Web Crypto exposes AES-CBC/CTR/GCM but NOT raw AES-ECB. The previous
|
||||
// implementation simulated ECB with AES-CBC + zero IV + a dummy PKCS7
|
||||
// padding block; that hack throws OperationError on real ciphertext
|
||||
// because Web Crypto validates PKCS7 padding on the decrypted output
|
||||
// and the dummy padding bytes rarely form a valid PKCS7 sequence
|
||||
// after decryption. We use a pure-JS AES-128 ECB core
|
||||
// (public/vendor/aes-ecb.js, MIT, derived from aes-js by Richard
|
||||
// Moore) so decryption is deterministic across browsers and works in
|
||||
// HTTP contexts.
|
||||
// ---- AES-128-ECB via Web Crypto (CBC with zero IV, block-by-block) ----
|
||||
|
||||
/**
|
||||
* Decrypt AES-128-ECB.
|
||||
* Decrypt AES-128-ECB by decrypting each 16-byte block independently
|
||||
* using AES-CBC with a zero IV (equivalent to ECB for single blocks).
|
||||
* @param {Uint8Array} key - 16-byte AES key
|
||||
* @param {Uint8Array} ciphertext - must be a non-zero multiple of 16 bytes
|
||||
* @returns {Promise<Uint8Array|null>} plaintext, or null on invalid input
|
||||
* @param {Uint8Array} ciphertext - must be multiple of 16 bytes
|
||||
* @returns {Promise<Uint8Array>} plaintext
|
||||
*/
|
||||
async function decryptECB(key, ciphertext) {
|
||||
if (!ciphertext || ciphertext.length === 0 || ciphertext.length % 16 !== 0) {
|
||||
if (ciphertext.length === 0 || ciphertext.length % 16 !== 0) {
|
||||
return null;
|
||||
}
|
||||
var host = (typeof window !== 'undefined') ? window
|
||||
: (typeof self !== 'undefined') ? self : null;
|
||||
if (!host || !host.AES_ECB || !host.AES_ECB.decrypt) {
|
||||
throw new Error('AES_ECB vendor module not loaded (public/vendor/aes-ecb.js)');
|
||||
var cryptoKey = await crypto.subtle.importKey(
|
||||
'raw', key, { name: 'AES-CBC' }, false, ['decrypt']
|
||||
);
|
||||
var zeroIV = new Uint8Array(16);
|
||||
var plaintext = new Uint8Array(ciphertext.length);
|
||||
|
||||
for (var i = 0; i < ciphertext.length; i += 16) {
|
||||
var block = ciphertext.slice(i, i + 16);
|
||||
// Append a dummy block (16 bytes of 0x10 = PKCS7 padding for empty next block)
|
||||
// so Web Crypto doesn't complain about padding
|
||||
var padded = new Uint8Array(32);
|
||||
padded.set(block, 0);
|
||||
// Second block is PKCS7 padding: 16 bytes of 0x10
|
||||
for (var j = 16; j < 32; j++) padded[j] = 16;
|
||||
|
||||
var decrypted = await crypto.subtle.decrypt(
|
||||
{ name: 'AES-CBC', iv: zeroIV }, cryptoKey, padded
|
||||
);
|
||||
var decBytes = new Uint8Array(decrypted);
|
||||
plaintext.set(decBytes.slice(0, 16), i);
|
||||
}
|
||||
return host.AES_ECB.decrypt(key, ciphertext);
|
||||
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
// ---- MAC verification ----
|
||||
@@ -130,17 +111,13 @@ window.ChannelDecrypt = (function () {
|
||||
secret.set(key, 0);
|
||||
// remaining 16 bytes are already 0
|
||||
|
||||
var cryptoKey = await crypto.subtle.importKey(
|
||||
'raw', secret, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']
|
||||
);
|
||||
var sig = await crypto.subtle.sign('HMAC', cryptoKey, ciphertext);
|
||||
var sigBytes = new Uint8Array(sig);
|
||||
|
||||
var macBytes = hexToBytes(macHex);
|
||||
var sigBytes;
|
||||
if (hasSubtle() && typeof crypto.subtle.importKey === 'function' && typeof crypto.subtle.sign === 'function') {
|
||||
var cryptoKey = await crypto.subtle.importKey(
|
||||
'raw', secret, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']
|
||||
);
|
||||
var sig = await crypto.subtle.sign('HMAC', cryptoKey, ciphertext);
|
||||
sigBytes = new Uint8Array(sig);
|
||||
} else {
|
||||
sigBytes = pureCryptoOrThrow().hmacSha256(secret, ciphertext);
|
||||
}
|
||||
return sigBytes[0] === macBytes[0] && sigBytes[1] === macBytes[1];
|
||||
}
|
||||
|
||||
@@ -210,96 +187,12 @@ window.ChannelDecrypt = (function () {
|
||||
// Alias used by channels.js
|
||||
var decryptPacket = decrypt;
|
||||
|
||||
// ---- Live PSK decrypt (WS path) ----
|
||||
//
|
||||
// Build a Map<channelHashByte, { channelName, keyBytes, keyHex }> from all
|
||||
// stored PSK keys so the WebSocket handler can do an O(1) lookup on each
|
||||
// incoming GRP_TXT packet. Hash byte derivation is async, so we cache the
|
||||
// map between calls and only rebuild when the stored-keys set changes.
|
||||
var _keyMapCache = null;
|
||||
var _keyMapSig = '';
|
||||
|
||||
function _keysSignature(keys) {
|
||||
var names = Object.keys(keys).sort();
|
||||
var sig = '';
|
||||
for (var i = 0; i < names.length; i++) {
|
||||
sig += names[i] + '=' + keys[names[i]] + ';';
|
||||
}
|
||||
return sig;
|
||||
}
|
||||
|
||||
async function buildKeyMap() {
|
||||
var keys = getKeys();
|
||||
var sig = _keysSignature(keys);
|
||||
if (_keyMapCache && _keyMapSig === sig) return _keyMapCache;
|
||||
var map = new Map();
|
||||
var names = Object.keys(keys);
|
||||
for (var i = 0; i < names.length; i++) {
|
||||
var channelName = names[i];
|
||||
var keyHex = keys[channelName];
|
||||
if (!keyHex || typeof keyHex !== 'string') continue;
|
||||
var keyBytes;
|
||||
try { keyBytes = hexToBytes(keyHex); } catch (e) { continue; }
|
||||
if (keyBytes.length !== 16) continue;
|
||||
var hashByte;
|
||||
try { hashByte = await computeChannelHash(keyBytes); } catch (e) { continue; }
|
||||
// First-write-wins on collision (rare): different channel names can
|
||||
// hash to the same byte. The downstream MAC check still gates rendering.
|
||||
if (!map.has(hashByte)) {
|
||||
map.set(hashByte, { channelName: channelName, keyBytes: keyBytes, keyHex: keyHex });
|
||||
}
|
||||
}
|
||||
_keyMapCache = map;
|
||||
_keyMapSig = sig;
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to decrypt a live GRP_TXT payload using a prebuilt key map.
|
||||
* Returns { sender, text, channelName, channelHashByte } on success,
|
||||
* or null when no key matches, MAC verification fails, or the payload
|
||||
* is not an encrypted GRP_TXT.
|
||||
*/
|
||||
async function tryDecryptLive(payload, keyMap) {
|
||||
if (!payload || payload.type !== 'GRP_TXT') return null;
|
||||
if (!payload.encryptedData || !payload.mac) return null;
|
||||
if (!keyMap || typeof keyMap.get !== 'function') return null;
|
||||
var hashByte = payload.channelHash;
|
||||
// channelHash arrives as either a number or a hex string in some paths;
|
||||
// normalize to number so Map.get hits.
|
||||
if (typeof hashByte === 'string') {
|
||||
var n = parseInt(hashByte, 16);
|
||||
if (!isFinite(n)) return null;
|
||||
hashByte = n;
|
||||
}
|
||||
if (typeof hashByte !== 'number') return null;
|
||||
var entry = keyMap.get(hashByte);
|
||||
if (!entry) return null;
|
||||
var result;
|
||||
try {
|
||||
result = await decrypt(entry.keyBytes, payload.mac, payload.encryptedData);
|
||||
} catch (e) { return null; }
|
||||
if (!result) return null;
|
||||
return {
|
||||
sender: result.sender || 'Unknown',
|
||||
text: result.message || '',
|
||||
channelName: entry.channelName,
|
||||
channelHashByte: hashByte,
|
||||
timestamp: result.timestamp || null
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// ---- Key storage (localStorage) ----
|
||||
|
||||
function saveKey(channelName, keyHex, label) {
|
||||
function saveKey(channelName, keyHex) {
|
||||
var keys = getKeys();
|
||||
keys[channelName] = keyHex;
|
||||
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
|
||||
_keyMapCache = null; // invalidate live-decrypt index
|
||||
if (typeof label === 'string' && label.trim()) {
|
||||
saveLabel(channelName, label.trim());
|
||||
}
|
||||
}
|
||||
|
||||
// Alias used by channels.js
|
||||
@@ -319,39 +212,8 @@ window.ChannelDecrypt = (function () {
|
||||
var keys = getKeys();
|
||||
delete keys[channelName];
|
||||
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
|
||||
_keyMapCache = null; // invalidate live-decrypt index
|
||||
// Also clear cached messages and any label for this channel (#1020)
|
||||
// Also clear cached messages for this channel
|
||||
clearChannelCache(channelName);
|
||||
var labels = getLabels();
|
||||
if (labels[channelName]) {
|
||||
delete labels[channelName];
|
||||
try { localStorage.setItem(LABELS_KEY, JSON.stringify(labels)); } catch (e) { /* quota */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ---- User-supplied display labels (#1020) ----
|
||||
// Stored separately from keys so we can display friendly names instead of
|
||||
// psk:<hex8> for user-added PSK channels.
|
||||
function getLabels() {
|
||||
try {
|
||||
var raw = localStorage.getItem(LABELS_KEY);
|
||||
return raw ? JSON.parse(raw) : {};
|
||||
} catch (e) { return {}; }
|
||||
}
|
||||
|
||||
function getLabel(channelName) {
|
||||
var labels = getLabels();
|
||||
return labels[channelName] || '';
|
||||
}
|
||||
|
||||
function saveLabel(channelName, label) {
|
||||
var labels = getLabels();
|
||||
if (typeof label === 'string' && label.trim()) {
|
||||
labels[channelName] = label.trim();
|
||||
} else {
|
||||
delete labels[channelName];
|
||||
}
|
||||
try { localStorage.setItem(LABELS_KEY, JSON.stringify(labels)); } catch (e) { /* quota */ }
|
||||
}
|
||||
|
||||
/** Remove cached messages for a specific channel (by name or hash). */
|
||||
@@ -424,16 +286,10 @@ window.ChannelDecrypt = (function () {
|
||||
getKeys: getKeys,
|
||||
getStoredKeys: getStoredKeys,
|
||||
removeKey: removeKey,
|
||||
// #1020: optional user-friendly display labels for stored keys
|
||||
saveLabel: saveLabel,
|
||||
getLabel: getLabel,
|
||||
getLabels: getLabels,
|
||||
clearChannelCache: clearChannelCache,
|
||||
cacheMessages: cacheMessages,
|
||||
getCachedMessages: getCachedMessages,
|
||||
setCache: setCache,
|
||||
getCache: getCache,
|
||||
buildKeyMap: buildKeyMap,
|
||||
tryDecryptLive: tryDecryptLive
|
||||
getCache: getCache
|
||||
};
|
||||
})();
|
||||
|
||||
+54
-246
@@ -339,10 +339,8 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Add a user channel by name (#channelname) or hex key.
|
||||
// `label` (#1020) is an optional friendly name shown in the sidebar instead
|
||||
// of "psk:<hex8>" — stored alongside the key in localStorage.
|
||||
async function addUserChannel(val, label) {
|
||||
// Add a user channel by name (#channelname) or hex key
|
||||
async function addUserChannel(val) {
|
||||
var displayName = val.startsWith('#') ? val : (isHexKey(val) ? val.substring(0, 8) + '…' : '#' + val);
|
||||
showAddStatus('Decrypting ' + displayName + ' messages…', 'loading');
|
||||
var channelName, keyHex;
|
||||
@@ -361,8 +359,7 @@
|
||||
keyHex = ChannelDecrypt.bytesToHex(keyBytes2);
|
||||
}
|
||||
|
||||
// #1020: persist optional user-supplied label alongside the key
|
||||
ChannelDecrypt.storeKey(channelName, keyHex, label);
|
||||
ChannelDecrypt.storeKey(channelName, keyHex);
|
||||
|
||||
// Compute channel hash byte to find matching encrypted channels
|
||||
var keyBytes3 = ChannelDecrypt.hexToBytes(keyHex);
|
||||
@@ -381,53 +378,35 @@
|
||||
if (existingEncrypted) {
|
||||
targetHash = existingEncrypted.hash;
|
||||
}
|
||||
var selectResult = await selectChannel(targetHash, { userKey: keyHex, channelHashByte: hashByte, channelName: channelName });
|
||||
await selectChannel(targetHash, { userKey: keyHex, channelHashByte: hashByte, channelName: channelName });
|
||||
|
||||
// #1020: derive count from selectChannel's reported result, not from a
|
||||
// DOM scrape that can race with rendering.
|
||||
var msgCount = (selectResult && typeof selectResult.messageCount === 'number')
|
||||
? selectResult.messageCount
|
||||
: (Array.isArray(messages) ? messages.length : 0);
|
||||
var displayLabel = (typeof label === 'string' && label.trim()) ? label.trim() :
|
||||
(channelName.startsWith('psk:') ? 'Custom channel (' + channelName.substring(4) + ')' : channelName);
|
||||
if (selectResult && selectResult.wrongKey) {
|
||||
showAddStatus('Key does not match any packets for ' + displayLabel, 'error');
|
||||
} else if (msgCount > 0) {
|
||||
showAddStatus('Added ' + displayLabel + ' — ' + msgCount + ' messages decrypted', 'success');
|
||||
// Show success feedback (#759)
|
||||
var msgCount = document.querySelectorAll('#chMessages .ch-msg').length;
|
||||
var userDisplay = channelName.startsWith('psk:') ? 'Custom channel (' + channelName.substring(4) + ')' : channelName;
|
||||
if (msgCount > 0) {
|
||||
showAddStatus('Added ' + userDisplay + ' — ' + msgCount + ' messages decrypted', 'success');
|
||||
} else {
|
||||
showAddStatus('Added ' + displayLabel + ' — no messages found yet', 'warn');
|
||||
showAddStatus('No messages found for ' + userDisplay, 'warn');
|
||||
}
|
||||
} catch (err) {
|
||||
showAddStatus('Failed to decrypt', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// Merge user-stored keys into the channel list.
|
||||
// If a stored key matches a server-known channel, mark that channel as
|
||||
// userAdded so the ✕ button appears — otherwise the user has no way to
|
||||
// remove a key they added but that the server already knows about.
|
||||
// Merge user-stored keys into the channel list
|
||||
function mergeUserChannels() {
|
||||
var keys = ChannelDecrypt.getStoredKeys();
|
||||
var labels = (typeof ChannelDecrypt.getLabels === 'function') ? ChannelDecrypt.getLabels() : {};
|
||||
var names = Object.keys(keys);
|
||||
for (var i = 0; i < names.length; i++) {
|
||||
var name = names[i];
|
||||
var label = labels[name] || '';
|
||||
var matched = false;
|
||||
for (var j = 0; j < channels.length; j++) {
|
||||
var ch = channels[j];
|
||||
if (ch.name === name || ch.hash === name || ch.hash === ('user:' + name)) {
|
||||
ch.userAdded = true;
|
||||
if (label) ch.userLabel = label;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
// Check if channel already exists by name
|
||||
var exists = channels.some(function (ch) {
|
||||
return ch.name === name || ch.hash === name || ch.hash === ('user:' + name);
|
||||
});
|
||||
if (!exists) {
|
||||
channels.push({
|
||||
hash: 'user:' + name,
|
||||
name: name,
|
||||
userLabel: label,
|
||||
messageCount: 0,
|
||||
lastActivityMs: 0,
|
||||
lastSender: '',
|
||||
@@ -643,11 +622,6 @@
|
||||
aria-label="Channel name or hex key" spellcheck="false">
|
||||
<button type="submit" class="ch-add-btn" title="Add channel">+</button>
|
||||
</div>
|
||||
<div class="ch-add-row">
|
||||
<input type="text" id="chKeyLabelInput" class="ch-key-label-input"
|
||||
placeholder="optional name (e.g. My Crew)"
|
||||
aria-label="Optional display name for this channel" spellcheck="false">
|
||||
</div>
|
||||
<div class="ch-add-hint">e.g. #LongFast or 32-char hex key — decrypted in your browser.</div>
|
||||
<div id="chAddStatus" class="ch-add-status" style="display:none"></div>
|
||||
</form>
|
||||
@@ -696,13 +670,10 @@
|
||||
var submitHandler = async function (e) {
|
||||
e.preventDefault();
|
||||
var input = document.getElementById('chKeyInput');
|
||||
var labelInput = document.getElementById('chKeyLabelInput');
|
||||
var val = (input.value || '').trim();
|
||||
var label = labelInput ? (labelInput.value || '').trim() : '';
|
||||
if (!val) return;
|
||||
input.value = '';
|
||||
if (labelInput) labelInput.value = '';
|
||||
await addUserChannel(val, label);
|
||||
await addUserChannel(val);
|
||||
};
|
||||
chKeyForm.addEventListener('submit', submitHandler);
|
||||
var chKeyInput = document.getElementById('chKeyInput');
|
||||
@@ -778,50 +749,23 @@
|
||||
e.stopPropagation();
|
||||
var channelHash = removeBtn.getAttribute('data-remove-channel');
|
||||
if (!channelHash) return;
|
||||
// The localStorage key is the channel name. For user:-prefixed entries
|
||||
// strip the prefix; for server-known channels look up the channel
|
||||
// object so we use its display name (the hash itself isn't the key).
|
||||
var ch = channels.find(function (c) { return c.hash === channelHash; });
|
||||
var chName = channelHash.startsWith('user:')
|
||||
? channelHash.substring(5)
|
||||
: (ch && ch.name) || channelHash;
|
||||
var chName = channelHash.startsWith('user:') ? channelHash.substring(5) : channelHash;
|
||||
if (!confirm('Remove channel "' + chName + '"? This will clear saved keys and cached messages.')) return;
|
||||
ChannelDecrypt.removeKey(chName);
|
||||
if (channelHash.startsWith('user:')) {
|
||||
// Pure user-added channel — drop from the list entirely.
|
||||
channels = channels.filter(function (c) { return c.hash !== channelHash; });
|
||||
if (selectedHash === channelHash) {
|
||||
selectedHash = null;
|
||||
messages = [];
|
||||
history.replaceState(null, '', '#/channels');
|
||||
var msgEl2 = document.getElementById('chMessages');
|
||||
if (msgEl2) msgEl2.innerHTML = '<div class="ch-empty">Choose a channel from the sidebar to view messages</div>';
|
||||
var header2 = document.getElementById('chHeader');
|
||||
if (header2) header2.querySelector('.ch-header-text').textContent = 'Select a channel';
|
||||
}
|
||||
} else if (ch) {
|
||||
// Server-known channel: keep the row, just unmark as user-added so
|
||||
// the ✕ disappears until they re-add a key.
|
||||
ch.userAdded = false;
|
||||
// If this was the selected channel, clear decrypted messages since
|
||||
// the key is gone — they can't be re-decrypted without re-adding it.
|
||||
if (selectedHash === channelHash) {
|
||||
messages = [];
|
||||
var msgEl2 = document.getElementById('chMessages');
|
||||
if (msgEl2) msgEl2.innerHTML = '<div class="ch-empty">Key removed — add a key to decrypt messages</div>';
|
||||
}
|
||||
// Remove from channels array
|
||||
channels = channels.filter(function (c) { return c.hash !== channelHash; });
|
||||
if (selectedHash === channelHash) {
|
||||
selectedHash = null;
|
||||
messages = [];
|
||||
history.replaceState(null, '', '#/channels');
|
||||
var msgEl2 = document.getElementById('chMessages');
|
||||
if (msgEl2) msgEl2.innerHTML = '<div class="ch-empty">Choose a channel from the sidebar to view messages</div>';
|
||||
var header2 = document.getElementById('chHeader');
|
||||
if (header2) header2.querySelector('.ch-header-text').textContent = 'Select a channel';
|
||||
}
|
||||
renderChannelList();
|
||||
return;
|
||||
}
|
||||
// Color clear button — remove color without opening picker (#681)
|
||||
const clearBtn = e.target.closest('.ch-color-clear');
|
||||
if (clearBtn && window.ChannelColors) {
|
||||
e.stopPropagation();
|
||||
var clearCh = clearBtn.getAttribute('data-channel');
|
||||
if (clearCh) { window.ChannelColors.remove(clearCh); renderChannelList(); }
|
||||
return;
|
||||
}
|
||||
// Color dot click — open picker, don't select channel
|
||||
const dot = e.target.closest('.ch-color-dot');
|
||||
if (dot && window.ChannelColorPicker) {
|
||||
@@ -929,11 +873,6 @@
|
||||
if (!payload) continue;
|
||||
|
||||
var channelName = payload.channel || 'unknown';
|
||||
// For live-decrypted user-added (PSK) channels, decryptLivePSKBatch
|
||||
// also stamps payload.channelKey ("user:<name>") so we route the
|
||||
// message to the correct sidebar row and to the open chat view.
|
||||
// Falls back to channelName for server-known CHAN packets.
|
||||
var channelKey = payload.channelKey || channelName;
|
||||
var rawText = payload.text || '';
|
||||
var sender = payload.sender || null;
|
||||
var displayText = rawText;
|
||||
@@ -960,10 +899,10 @@
|
||||
var observer = m.data?.packet?.observer_name || m.data?.observer || null;
|
||||
|
||||
// Update channel list entry — only once per unique packet hash
|
||||
var isFirstObservation = pktHash && !seenHashes.has(pktHash + ':' + channelKey);
|
||||
if (pktHash) seenHashes.add(pktHash + ':' + channelKey);
|
||||
var isFirstObservation = pktHash && !seenHashes.has(pktHash + ':' + channelName);
|
||||
if (pktHash) seenHashes.add(pktHash + ':' + channelName);
|
||||
|
||||
var ch = channels.find(function (c) { return c.hash === channelKey; });
|
||||
var ch = channels.find(function (c) { return c.hash === channelName; });
|
||||
if (ch) {
|
||||
if (isFirstObservation) ch.messageCount = (ch.messageCount || 0) + 1;
|
||||
ch.lastActivityMs = Date.now();
|
||||
@@ -973,7 +912,7 @@
|
||||
} else if (isFirstObservation) {
|
||||
// New channel we haven't seen
|
||||
channels.push({
|
||||
hash: channelKey,
|
||||
hash: channelName,
|
||||
name: channelName,
|
||||
messageCount: 1,
|
||||
lastActivityMs: Date.now(),
|
||||
@@ -984,7 +923,7 @@
|
||||
}
|
||||
|
||||
// If this message is for the selected channel, append to messages
|
||||
if (selectedHash && channelKey === selectedHash) {
|
||||
if (selectedHash && channelName === selectedHash) {
|
||||
// Deduplicate by packet hash — same message seen by multiple observers
|
||||
var existing = pktHash ? messages.find(function (msg) { return msg.packetHash === pktHash; }) : null;
|
||||
if (existing) {
|
||||
@@ -1037,83 +976,8 @@
|
||||
processWSBatch(msgs, selectedRegions);
|
||||
}
|
||||
|
||||
// Pre-pass: rewrite encrypted GRP_TXT live packets into decrypted form
|
||||
// when a stored PSK key matches their channel hash byte (#1029 — live
|
||||
// PSK decrypt). Without this, users viewing a PSK-decrypted channel
|
||||
// had to refresh the page to see new messages.
|
||||
async function decryptLivePSKBatch(msgs) {
|
||||
if (typeof ChannelDecrypt === 'undefined' ||
|
||||
typeof ChannelDecrypt.tryDecryptLive !== 'function') {
|
||||
return;
|
||||
}
|
||||
// Quick scan: do any messages look like encrypted GRP_TXT?
|
||||
var anyEncrypted = false;
|
||||
for (var i = 0; i < msgs.length; i++) {
|
||||
var p = msgs[i] && msgs[i].data && msgs[i].data.decoded && msgs[i].data.decoded.payload;
|
||||
if (p && p.type === 'GRP_TXT' && p.encryptedData && p.mac) { anyEncrypted = true; break; }
|
||||
}
|
||||
if (!anyEncrypted) return;
|
||||
var keyMap;
|
||||
try { keyMap = await ChannelDecrypt.buildKeyMap(); } catch (e) { return; }
|
||||
if (!keyMap || keyMap.size === 0) return;
|
||||
for (var j = 0; j < msgs.length; j++) {
|
||||
var m = msgs[j];
|
||||
var payload = m && m.data && m.data.decoded && m.data.decoded.payload;
|
||||
if (!payload || payload.type !== 'GRP_TXT' || !payload.encryptedData || !payload.mac) continue;
|
||||
var dec;
|
||||
try { dec = await ChannelDecrypt.tryDecryptLive(payload, keyMap); } catch (e) { dec = null; }
|
||||
if (!dec) continue;
|
||||
// Rewrite payload into a CHAN-like shape so processWSBatch picks it
|
||||
// up as a real message instead of an encrypted blob. Keep the original
|
||||
// hash byte for any downstream consumer that wants it.
|
||||
payload.channel = dec.channelName;
|
||||
// For user-added PSK channels the sidebar entry & selectedHash use a
|
||||
// "user:<name>" key (see addUserChannel). Stamp the canonical key on
|
||||
// the payload so processWSBatch routes the live message to the
|
||||
// correct sidebar row and to the open chat view instead of dropping
|
||||
// it / creating a duplicate plain entry. Falls back to the raw name
|
||||
// for non-user channels (server-known CHAN paths still work).
|
||||
var userKey = 'user:' + dec.channelName;
|
||||
var hasUserCh = false;
|
||||
for (var ck = 0; ck < channels.length; ck++) {
|
||||
if (channels[ck].hash === userKey) { hasUserCh = true; break; }
|
||||
}
|
||||
payload.channelKey = hasUserCh ? userKey : dec.channelName;
|
||||
payload.sender = dec.sender;
|
||||
payload.text = dec.sender ? (dec.sender + ': ' + dec.text) : dec.text;
|
||||
payload.decryptedLocally = true;
|
||||
if (m.data.decoded.header) {
|
||||
// Leave payloadTypeName as GRP_TXT — processWSBatch already
|
||||
// accepts both 'message' and GRP_TXT-typed packet messages.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
wsHandler = debouncedOnWS(function (msgs) {
|
||||
var selectedRegions = getSelectedRegionsSnapshot();
|
||||
var prior = selectedHash;
|
||||
decryptLivePSKBatch(msgs).then(function () {
|
||||
// Bump unread for live-decrypted channels the user is NOT viewing.
|
||||
// Done here (not inside processWSBatch) so the count reflects ONLY
|
||||
// newly-decrypted live packets, not historical-fetch path.
|
||||
var bumped = false;
|
||||
for (var i = 0; i < msgs.length; i++) {
|
||||
var p = msgs[i] && msgs[i].data && msgs[i].data.decoded && msgs[i].data.decoded.payload;
|
||||
if (!p || !p.decryptedLocally) continue;
|
||||
// Use the canonical sidebar key stamped by decryptLivePSKBatch so
|
||||
// the comparison against `prior` (= selectedHash) actually matches
|
||||
// for user-added (user:*-prefixed) channels.
|
||||
var chKey = p.channelKey || p.channel;
|
||||
if (!chKey || chKey === prior) continue;
|
||||
var ch = channels.find(function (c) { return c.hash === chKey || c.name === chKey || c.hash === ('user:' + chKey); });
|
||||
if (ch) {
|
||||
ch.unread = (ch.unread || 0) + 1;
|
||||
bumped = true;
|
||||
}
|
||||
}
|
||||
processWSBatch(msgs, selectedRegions);
|
||||
if (bumped) renderChannelList();
|
||||
});
|
||||
handleWSBatch(msgs);
|
||||
});
|
||||
window._channelsHandleWSBatchForTest = handleWSBatch;
|
||||
window._channelsProcessWSBatchForTest = processWSBatch;
|
||||
@@ -1183,51 +1047,31 @@
|
||||
|
||||
el.innerHTML = sorted.map(ch => {
|
||||
const isEncrypted = ch.encrypted === true;
|
||||
const isUserAdded = ch.userAdded === true;
|
||||
// #1020: prefer user-supplied label over psk:<hex>
|
||||
const baseName = isEncrypted ? (ch.name || 'Unknown') : (ch.name || `Channel ${formatHashHex(ch.hash)}`);
|
||||
const name = (isUserAdded && ch.userLabel) ? ch.userLabel : baseName;
|
||||
const name = isEncrypted ? (ch.name || 'Unknown') : (ch.name || `Channel ${formatHashHex(ch.hash)}`);
|
||||
const color = isEncrypted ? 'var(--text-muted, #6b7280)' : getChannelColor(ch.hash);
|
||||
const time = ch.lastActivityMs ? formatSecondsAgo(Math.floor((Date.now() - ch.lastActivityMs) / 1000)) : '';
|
||||
const preview = isUserAdded
|
||||
? (ch.lastSender && ch.lastMessage
|
||||
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
|
||||
: `${ch.messageCount || 0} messages (your key)`)
|
||||
: isEncrypted
|
||||
? `${ch.messageCount} encrypted messages (no key configured)`
|
||||
: ch.lastSender && ch.lastMessage
|
||||
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
|
||||
: `${ch.messageCount} messages`;
|
||||
const preview = isEncrypted
|
||||
? `${ch.messageCount} encrypted messages (no key configured)`
|
||||
: ch.lastSender && ch.lastMessage
|
||||
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
|
||||
: `${ch.messageCount} messages`;
|
||||
const sel = selectedHash === ch.hash ? ' selected' : '';
|
||||
// #1020: distinct class so styling/tests can tell user-added apart
|
||||
// from server-known encrypted channels.
|
||||
const encClass = isUserAdded
|
||||
? ' ch-user-added'
|
||||
: (isEncrypted ? ' ch-encrypted' : '');
|
||||
// #1020: 🔓 marks "I have the key" vs 🔒 "encrypted, no key"
|
||||
const badgeIcon = isUserAdded ? '🔓' : (isEncrypted ? '🔒' : null);
|
||||
const abbr = badgeIcon || (name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase());
|
||||
const encClass = isEncrypted ? ' ch-encrypted' : '';
|
||||
const abbr = isEncrypted ? '🔒' : (name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase());
|
||||
// Channel color dot for color picker (#674)
|
||||
const chColor = window.ChannelColors ? window.ChannelColors.get(ch.hash) : null;
|
||||
const dotStyle = chColor ? ` style="background:${chColor}"` : '';
|
||||
// Left border for assigned color
|
||||
const borderStyle = chColor ? ` style="border-left:3px solid ${chColor}"` : '';
|
||||
// M4 / #1020: Remove button for user-added channels
|
||||
const removeBtn = isUserAdded ? ' <button class="ch-remove-btn" data-remove-channel="' + escapeHtml(ch.hash) + '" title="Remove channel and clear saved key" aria-label="Remove ' + escapeHtml(name) + '">✕</button>' : '';
|
||||
// #1020: explicit badge marker for "your key" so it's distinguishable
|
||||
// from server-known encrypted rows at a glance and for screen readers.
|
||||
const userBadge = isUserAdded ? ' <span class="ch-user-badge" title="You added this key" aria-label="Your key">🔑</span>' : '';
|
||||
// #1029 Unread badge — bumped by live PSK decrypt for channels not currently selected.
|
||||
const unreadBadge = (ch.unread && ch.unread > 0)
|
||||
? ' <span class="ch-unread-badge" data-unread-channel="' + escapeHtml(ch.hash) + '" title="' + ch.unread + ' new" aria-label="' + ch.unread + ' unread">' + (ch.unread > 99 ? '99+' : ch.unread) + '</span>'
|
||||
: '';
|
||||
// M4: Remove button for user-added channels
|
||||
const removeBtn = ch.userAdded ? ' <button class="ch-remove-btn" data-remove-channel="' + escapeHtml(ch.hash) + '" title="Remove channel" aria-label="Remove ' + escapeHtml(name) + '">✕</button>' : '';
|
||||
|
||||
return `<button class="ch-item${sel}${encClass}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}"${isEncrypted ? ' data-encrypted="true"' : ''}${isUserAdded ? ' data-user-added="true"' : ''}>
|
||||
<div class="ch-badge" style="background:${color}" aria-hidden="true">${badgeIcon ? badgeIcon : escapeHtml(abbr)}</div>
|
||||
return `<button class="ch-item${sel}${encClass}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}"${isEncrypted ? ' data-encrypted="true"' : ''}>
|
||||
<div class="ch-badge" style="background:${color}" aria-hidden="true">${isEncrypted ? '🔒' : escapeHtml(abbr)}</div>
|
||||
<div class="ch-item-body">
|
||||
<div class="ch-item-top">
|
||||
<span class="ch-item-name">${escapeHtml(name)}</span>${userBadge}${unreadBadge}
|
||||
<span class="ch-color-dot" data-channel="${escapeHtml(ch.hash)}"${dotStyle} title="Change channel color" aria-label="Change color for ${escapeHtml(name)}"></span>${chColor ? '<span class="ch-color-clear" data-channel="' + escapeHtml(ch.hash) + '" title="Clear color" aria-label="Clear color for ' + escapeHtml(name) + '">✕</span>' : ''}
|
||||
<span class="ch-item-name">${escapeHtml(name)}</span>
|
||||
<span class="ch-color-dot" data-channel="${escapeHtml(ch.hash)}"${dotStyle} title="Change channel color" aria-label="Change color for ${escapeHtml(name)}"></span>
|
||||
<span class="ch-item-time" data-channel-hash="${ch.hash}">${time}</span>${removeBtn}
|
||||
</div>
|
||||
<div class="ch-item-preview">${escapeHtml(preview)}</div>
|
||||
@@ -1240,9 +1084,6 @@
|
||||
const rp = RegionFilter.getRegionParam() || '';
|
||||
const request = beginMessageRequest(hash, rp);
|
||||
selectedHash = hash;
|
||||
// Clear unread badge on the channel we're about to view (#1029).
|
||||
var __selCh = channels.find(function (c) { return c.hash === hash; });
|
||||
if (__selCh && __selCh.unread) { __selCh.unread = 0; }
|
||||
history.replaceState(null, '', `#/channels/${encodeURIComponent(hash)}`);
|
||||
renderChannelList();
|
||||
const ch = channels.find(c => c.hash === hash);
|
||||
@@ -1269,14 +1110,14 @@
|
||||
}
|
||||
}
|
||||
});
|
||||
if (isStaleMessageRequest(request)) return { stale: true };
|
||||
if (isStaleMessageRequest(request)) return true;
|
||||
if (result.wrongKey) {
|
||||
msgEl.innerHTML = '<div class="ch-empty ch-wrong-key">🔒 Key does not match — no messages could be decrypted</div>';
|
||||
return { wrongKey: true, messageCount: 0 };
|
||||
return true;
|
||||
}
|
||||
if (result.error) {
|
||||
msgEl.innerHTML = '<div class="ch-empty">' + escapeHtml(result.error) + '</div>';
|
||||
return { error: result.error, messageCount: 0 };
|
||||
return true;
|
||||
}
|
||||
messages = result.messages || [];
|
||||
if (messages.length === 0) {
|
||||
@@ -1286,12 +1127,13 @@
|
||||
renderMessages();
|
||||
scrollToBottom();
|
||||
}
|
||||
return { messageCount: messages.length };
|
||||
return true;
|
||||
}
|
||||
|
||||
// Client-side decryption path (#725 M2)
|
||||
if (decryptOpts && decryptOpts.userKey) {
|
||||
return await decryptAndRender(decryptOpts.userKey, decryptOpts.channelHashByte, decryptOpts.channelName);
|
||||
await decryptAndRender(decryptOpts.userKey, decryptOpts.channelHashByte, decryptOpts.channelName);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if this is a user-added channel that needs decryption
|
||||
@@ -1323,40 +1165,6 @@
|
||||
return;
|
||||
}
|
||||
|
||||
// #811: Deep link to a `#`-named channel that's not in the loaded list.
|
||||
// If a stored key matches, decrypt. Otherwise we must distinguish an
|
||||
// encrypted-no-key channel (show lock) from an unencrypted channel that
|
||||
// simply isn't in the toggle-off list (#825 — must fall through to REST).
|
||||
if (hash.charAt(0) === '#') {
|
||||
if (storedKeys[hash]) {
|
||||
var keyHex2 = storedKeys[hash];
|
||||
var keyBytes2 = ChannelDecrypt.hexToBytes(keyHex2);
|
||||
var hashByte2 = await ChannelDecrypt.computeChannelHash(keyBytes2);
|
||||
await decryptAndRender(keyHex2, hashByte2, hash);
|
||||
return;
|
||||
}
|
||||
// #825: confirm encrypted-ness via an encrypted-included channel list
|
||||
// before assuming a lock state. Conservative on error — fall through.
|
||||
// Show a loading affordance so cold deep links don't display stale content
|
||||
// for the duration of the metadata RTT (cached 15s thereafter).
|
||||
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
|
||||
try {
|
||||
var rpInc = RegionFilter.getRegionParam();
|
||||
var paramsInc = ['includeEncrypted=true'];
|
||||
if (rpInc) paramsInc.push('region=' + encodeURIComponent(rpInc));
|
||||
var allCh = await api('/channels?' + paramsInc.join('&'), { ttl: CLIENT_TTL.channels });
|
||||
if (isStaleMessageRequest(request)) return;
|
||||
var foundCh = (allCh.channels || []).find(function (c) { return c.hash === hash; });
|
||||
if (foundCh && foundCh.encrypted === true) {
|
||||
msgEl.innerHTML = '<div class="ch-empty">🔒 This channel is encrypted and no decryption key is configured</div>';
|
||||
return;
|
||||
}
|
||||
// Unencrypted (or unknown) — fall through to the REST fetch below.
|
||||
} catch (e) {
|
||||
// ignore — fall through to REST fetch
|
||||
}
|
||||
}
|
||||
|
||||
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
|
||||
|
||||
try {
|
||||
|
||||
+5
-47
@@ -23,28 +23,8 @@ function comparePacketSets(hashesA, hashesB) {
|
||||
return { onlyA: onlyA, onlyB: onlyB, both: both };
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter packets by route type.
|
||||
* mode: 'all' | 'flood' | 'direct'
|
||||
* Flood = route_type 0 (TransportFlood) or 1 (Flood)
|
||||
* Direct = route_type 2 (Direct) or 3 (TransportDirect)
|
||||
*/
|
||||
function filterPacketsByRoute(packets, mode) {
|
||||
if (!packets || mode === 'all') return packets || [];
|
||||
if (mode === 'flood') {
|
||||
return packets.filter(function (p) { return p.route_type === 0 || p.route_type === 1; });
|
||||
}
|
||||
if (mode === 'direct') {
|
||||
return packets.filter(function (p) { return p.route_type === 2 || p.route_type === 3; });
|
||||
}
|
||||
return packets;
|
||||
}
|
||||
|
||||
// Expose for testing
|
||||
if (typeof window !== 'undefined') {
|
||||
window.comparePacketSets = comparePacketSets;
|
||||
window.filterPacketsByRoute = filterPacketsByRoute;
|
||||
}
|
||||
if (typeof window !== 'undefined') window.comparePacketSets = comparePacketSets;
|
||||
|
||||
(function () {
|
||||
var PAYLOAD_LABELS = { 0: 'Request', 1: 'Response', 2: 'Direct Msg', 3: 'ACK', 4: 'Advert', 5: 'Channel Msg', 7: 'Anon Req', 8: 'Path', 9: 'Trace', 11: 'Control' };
|
||||
@@ -56,7 +36,6 @@ if (typeof window !== 'undefined') {
|
||||
var packetsA = [];
|
||||
var packetsB = [];
|
||||
var currentView = 'summary';
|
||||
var routeFilter = 'all';
|
||||
|
||||
function init(app, routeParam) {
|
||||
// Parse preselected observers from URL: #/compare?a=ID1&b=ID2
|
||||
@@ -68,7 +47,6 @@ if (typeof window !== 'undefined') {
|
||||
packetsA = [];
|
||||
packetsB = [];
|
||||
currentView = 'summary';
|
||||
routeFilter = 'all';
|
||||
|
||||
app.innerHTML = '<div class="compare-page" style="padding:16px">' +
|
||||
'<div class="page-header" style="display:flex;align-items:center;gap:12px;margin-bottom:16px">' +
|
||||
@@ -98,7 +76,6 @@ if (typeof window !== 'undefined') {
|
||||
comparisonResult = null;
|
||||
packetsA = [];
|
||||
packetsB = [];
|
||||
routeFilter = 'all';
|
||||
}
|
||||
|
||||
async function loadObservers() {
|
||||
@@ -138,14 +115,6 @@ if (typeof window !== 'undefined') {
|
||||
'<select id="compareObsB" class="compare-select">' + optionsHtml + '</select>' +
|
||||
'</div>' +
|
||||
'<button id="compareBtn" class="compare-btn" disabled>Compare</button>' +
|
||||
'<div class="compare-select-group">' +
|
||||
'<label for="compareRouteFilter">Packet Type</label>' +
|
||||
'<select id="compareRouteFilter" class="compare-select">' +
|
||||
'<option value="all">All packets</option>' +
|
||||
'<option value="flood">Flood only</option>' +
|
||||
'<option value="direct">Direct only</option>' +
|
||||
'</select>' +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
|
||||
var ddA = document.getElementById('compareObsA');
|
||||
@@ -155,13 +124,6 @@ if (typeof window !== 'undefined') {
|
||||
if (selA) ddA.value = selA;
|
||||
if (selB) ddB.value = selB;
|
||||
|
||||
var ddRoute = document.getElementById('compareRouteFilter');
|
||||
ddRoute.value = routeFilter;
|
||||
ddRoute.addEventListener('change', function () {
|
||||
routeFilter = ddRoute.value;
|
||||
if (comparisonResult) runComparison();
|
||||
});
|
||||
|
||||
function updateBtn() {
|
||||
selA = ddA.value || null;
|
||||
selB = ddB.value || null;
|
||||
@@ -200,20 +162,16 @@ if (typeof window !== 'undefined') {
|
||||
packetsA = results[0].packets || [];
|
||||
packetsB = results[1].packets || [];
|
||||
|
||||
// Apply flood/direct filter (#928)
|
||||
var filteredA = filterPacketsByRoute(packetsA, routeFilter);
|
||||
var filteredB = filterPacketsByRoute(packetsB, routeFilter);
|
||||
|
||||
var hashesA = new Set(filteredA.map(function (p) { return p.hash; }));
|
||||
var hashesB = new Set(filteredB.map(function (p) { return p.hash; }));
|
||||
var hashesA = new Set(packetsA.map(function (p) { return p.hash; }));
|
||||
var hashesB = new Set(packetsB.map(function (p) { return p.hash; }));
|
||||
|
||||
comparisonResult = comparePacketSets(hashesA, hashesB);
|
||||
|
||||
// Build hash→packet lookups for detail rendering
|
||||
comparisonResult.packetMapA = new Map();
|
||||
comparisonResult.packetMapB = new Map();
|
||||
filteredA.forEach(function (p) { comparisonResult.packetMapA.set(p.hash, p); });
|
||||
filteredB.forEach(function (p) { comparisonResult.packetMapB.set(p.hash, p); });
|
||||
packetsA.forEach(function (p) { comparisonResult.packetMapA.set(p.hash, p); });
|
||||
packetsB.forEach(function (p) { comparisonResult.packetMapB.set(p.hash, p); });
|
||||
|
||||
currentView = 'summary';
|
||||
renderComparison();
|
||||
|
||||
+7
-44
@@ -33,7 +33,7 @@
|
||||
'meshcore-live-heatmap-opacity'
|
||||
];
|
||||
|
||||
var VALID_SECTIONS = ['branding', 'theme', 'themeDark', 'nodeColors', 'typeColors', 'home', 'timestamps', 'heatmapOpacity', 'liveHeatmapOpacity', 'distanceUnit', 'favorites', 'myNodes'];
|
||||
var VALID_SECTIONS = ['branding', 'theme', 'themeDark', 'nodeColors', 'typeColors', 'home', 'timestamps', 'heatmapOpacity', 'liveHeatmapOpacity', 'distanceUnit'];
|
||||
var OBJECT_SECTIONS = ['branding', 'theme', 'themeDark', 'nodeColors', 'typeColors', 'home', 'timestamps'];
|
||||
var SCALAR_SECTIONS = ['heatmapOpacity', 'liveHeatmapOpacity'];
|
||||
var DISTANCE_UNIT_VALUES = ['km', 'mi', 'auto'];
|
||||
@@ -313,17 +313,9 @@
|
||||
function readOverrides() {
|
||||
try {
|
||||
var raw = localStorage.getItem(STORAGE_KEY);
|
||||
var parsed = (raw != null) ? JSON.parse(raw) : {};
|
||||
if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) parsed = {};
|
||||
// Include favorites and claimed nodes from their own localStorage keys
|
||||
try {
|
||||
var favs = JSON.parse(localStorage.getItem('meshcore-favorites') || '[]');
|
||||
if (Array.isArray(favs) && favs.length) parsed.favorites = favs;
|
||||
} catch (e) { /* ignore */ }
|
||||
try {
|
||||
var myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]');
|
||||
if (Array.isArray(myNodes) && myNodes.length) parsed.myNodes = myNodes;
|
||||
} catch (e) { /* ignore */ }
|
||||
if (raw == null) return {};
|
||||
var parsed = JSON.parse(raw);
|
||||
if (parsed == null || typeof parsed !== 'object' || Array.isArray(parsed)) return {};
|
||||
return parsed;
|
||||
} catch (e) {
|
||||
return {};
|
||||
@@ -394,28 +386,14 @@
|
||||
|
||||
function writeOverrides(delta) {
|
||||
if (delta == null || typeof delta !== 'object') return;
|
||||
// Extract favorites/myNodes and store in their own localStorage keys
|
||||
if (Array.isArray(delta.favorites)) {
|
||||
try { localStorage.setItem('meshcore-favorites', JSON.stringify(delta.favorites)); } catch (e) { /* ignore */ }
|
||||
}
|
||||
if (Array.isArray(delta.myNodes)) {
|
||||
try { localStorage.setItem('meshcore-my-nodes', JSON.stringify(delta.myNodes)); } catch (e) { /* ignore */ }
|
||||
}
|
||||
// Build theme-only delta (without favorites/myNodes)
|
||||
var themeDelta = {};
|
||||
for (var k in delta) {
|
||||
if (delta.hasOwnProperty(k) && k !== 'favorites' && k !== 'myNodes') {
|
||||
themeDelta[k] = delta[k];
|
||||
}
|
||||
}
|
||||
// If empty, remove key entirely
|
||||
var keys = Object.keys(themeDelta);
|
||||
var keys = Object.keys(delta);
|
||||
if (keys.length === 0) {
|
||||
try { localStorage.removeItem(STORAGE_KEY); } catch (e) { /* ignore */ }
|
||||
_updateSaveStatus('saved');
|
||||
return;
|
||||
}
|
||||
var validated = _validateDelta(themeDelta);
|
||||
var validated = _validateDelta(delta);
|
||||
try {
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify(validated));
|
||||
_updateSaveStatus('saved');
|
||||
@@ -651,11 +629,7 @@
|
||||
}
|
||||
writeOverrides(delta);
|
||||
_runPipeline();
|
||||
// Skip re-render while the user is typing inside the panel — setting
|
||||
// innerHTML would destroy the focused input and collapse the mobile keyboard.
|
||||
if (!(_panelEl && _panelEl.contains(document.activeElement))) {
|
||||
_refreshPanel();
|
||||
}
|
||||
_refreshPanel();
|
||||
}, 300);
|
||||
}
|
||||
|
||||
@@ -780,17 +754,6 @@
|
||||
if (key === 'distanceUnit' && DISTANCE_UNIT_VALUES.indexOf(obj[key]) === -1) {
|
||||
errors.push('Invalid distanceUnit: "' + obj[key] + '" — must be km, mi, or auto');
|
||||
}
|
||||
// Validate favorites and myNodes arrays
|
||||
if (key === 'favorites') {
|
||||
if (!Array.isArray(obj[key])) {
|
||||
errors.push('"favorites" must be an array of public key strings');
|
||||
}
|
||||
}
|
||||
if (key === 'myNodes') {
|
||||
if (!Array.isArray(obj[key])) {
|
||||
errors.push('"myNodes" must be an array of node objects');
|
||||
}
|
||||
}
|
||||
}
|
||||
return { valid: errors.length === 0, errors: errors };
|
||||
}
|
||||
|
||||
@@ -26,12 +26,6 @@
|
||||
#btnCopy { padding: 6px 14px; background: #1a4a7a; color: #7ec8e3; border-radius: 6px; border: none; cursor: pointer; font-size: 0.85rem; white-space: nowrap; align-self: flex-end; }
|
||||
#btnCopy:hover { background: #2a6aaa; }
|
||||
#btnCopy.copied { background: #1a6a3a; color: #7effa0; }
|
||||
#btnSaveDraft { background: #1a5a3a; color: #7effa0; }
|
||||
#btnSaveDraft:hover { background: #2a7a4a; }
|
||||
#btnLoadDraft { background: #3a3a1a; color: #ffe07e; }
|
||||
#btnLoadDraft:hover { background: #5a5a2a; }
|
||||
#btnDownload { background: #1a4a7a; color: #7ec8e3; }
|
||||
#btnDownload:hover { background: #2a6aaa; }
|
||||
#counter { font-size: 0.8rem; color: #888; padding-top: 6px; white-space: nowrap; }
|
||||
.bufferRow { display: flex; align-items: center; gap: 8px; }
|
||||
.bufferRow label { font-size: 0.85rem; color: #aaa; }
|
||||
@@ -51,8 +45,6 @@
|
||||
<div class="controls">
|
||||
<button id="btnUndo">↩ Undo</button>
|
||||
<button id="btnClear">✕ Clear</button>
|
||||
<button id="btnSaveDraft">💾 Save Draft</button>
|
||||
<button id="btnLoadDraft">📂 Load Draft</button>
|
||||
</div>
|
||||
<div class="bufferRow">
|
||||
<label for="bufferKm">Buffer km:</label>
|
||||
@@ -71,18 +63,16 @@
|
||||
<div style="display:flex;flex-direction:column;gap:8px;align-items:flex-end">
|
||||
<span id="counter">0 points</span>
|
||||
<button id="btnCopy">Copy</button>
|
||||
<button id="btnDownload">⬇ Download</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Instructions: paste the output into config.json as a top-level "geo_filter" key, then restart the server -->
|
||||
<div id="help-bar">
|
||||
<strong>Save Draft</strong> preserves your polygon across sessions. <strong>Download</strong> exports a JSON snippet → paste as a top-level key in <code>config.json</code> → restart the server.
|
||||
Copy the JSON above → paste as a top-level key in <code>config.json</code> → restart the server.
|
||||
Nodes with no GPS fix always pass through. Remove the <code>geo_filter</code> block to disable filtering.
|
||||
· <a href="/geofilter-docs.html">Documentation</a>
|
||||
· <a href="https://github.com/Kpa-clawbot/CoreScope/blob/master/docs/user-guide/geofilter.md" target="_blank">Documentation ↗</a>
|
||||
</div>
|
||||
|
||||
<script src="geofilter-draft.js"></script>
|
||||
<script>
|
||||
const map = L.map('map').setView([50.5, 4.4], 8);
|
||||
|
||||
@@ -97,8 +87,7 @@ let polygon = null;
|
||||
let closingLine = null;
|
||||
|
||||
function latLonPair(latlng) {
|
||||
const w = latlng.wrap();
|
||||
return [parseFloat(w.lat.toFixed(6)), parseFloat(w.lng.toFixed(6))];
|
||||
return [parseFloat(latlng.lat.toFixed(6)), parseFloat(latlng.lng.toFixed(6))];
|
||||
}
|
||||
|
||||
function render() {
|
||||
@@ -176,40 +165,6 @@ document.getElementById('btnCopy').addEventListener('click', function() {
|
||||
setTimeout(() => { btn.textContent = 'Copy'; btn.classList.remove('copied'); }, 2000);
|
||||
});
|
||||
});
|
||||
|
||||
document.getElementById('btnSaveDraft').addEventListener('click', function() {
|
||||
if (points.length < 3) return;
|
||||
const bufferKm = parseFloat(document.getElementById('bufferKm').value) || 0;
|
||||
GeofilterDraft.saveDraft(points, bufferKm);
|
||||
const btn = document.getElementById('btnSaveDraft');
|
||||
btn.textContent = '✓ Saved';
|
||||
setTimeout(() => { btn.textContent = '💾 Save Draft'; }, 2000);
|
||||
});
|
||||
|
||||
document.getElementById('btnLoadDraft').addEventListener('click', function() {
|
||||
const draft = GeofilterDraft.loadDraft();
|
||||
if (!draft || !draft.polygon || draft.polygon.length < 3) return;
|
||||
// Clear current
|
||||
markers.forEach(m => map.removeLayer(m));
|
||||
markers = [];
|
||||
points = draft.polygon.slice();
|
||||
if (draft.bufferKm != null) document.getElementById('bufferKm').value = draft.bufferKm;
|
||||
// Recreate markers
|
||||
points.forEach(function(pt, i) {
|
||||
const marker = L.circleMarker([pt[0], pt[1]], {
|
||||
radius: 6, color: '#4a9eff', weight: 2, fillColor: '#4a9eff', fillOpacity: 0.9
|
||||
}).addTo(map).bindTooltip(String(i + 1), { permanent: true, direction: 'top', offset: [0, -8], className: 'pt-label' });
|
||||
markers.push(marker);
|
||||
});
|
||||
render();
|
||||
map.fitBounds(L.polygon(points).getBounds().pad(0.2));
|
||||
});
|
||||
|
||||
document.getElementById('btnDownload').addEventListener('click', function() {
|
||||
if (points.length < 3) return;
|
||||
const bufferKm = parseFloat(document.getElementById('bufferKm').value) || 0;
|
||||
GeofilterDraft.downloadConfig(points, bufferKm);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>GeoFilter Docs — CoreScope</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { font-family: system-ui, sans-serif; background: #1a1a2e; color: #e0e0e0; min-height: 100vh; display: flex; flex-direction: column; }
|
||||
header { padding: 12px 16px; background: #0f0f23; border-bottom: 1px solid #333; display: flex; align-items: center; gap: 16px; }
|
||||
header h1 { font-size: 1rem; font-weight: 600; color: #4a9eff; }
|
||||
#back-link { font-size: 0.8rem; color: #4a9eff; text-decoration: none; white-space: nowrap; }
|
||||
#back-link:hover { text-decoration: underline; }
|
||||
main { flex: 1; max-width: 800px; margin: 0 auto; padding: 32px 24px; width: 100%; }
|
||||
h2 { font-size: 1.1rem; font-weight: 600; color: #4a9eff; margin: 32px 0 12px; border-bottom: 1px solid #222; padding-bottom: 6px; }
|
||||
h2:first-of-type { margin-top: 0; }
|
||||
h3 { font-size: 0.95rem; font-weight: 600; color: #c0c0c0; margin: 20px 0 8px; }
|
||||
p { font-size: 0.9rem; line-height: 1.6; color: #ccc; margin-bottom: 10px; }
|
||||
ul { padding-left: 20px; margin-bottom: 10px; }
|
||||
li { font-size: 0.9rem; line-height: 1.7; color: #ccc; }
|
||||
code { font-family: monospace; font-size: 0.85rem; color: #7ec8e3; background: #111; border: 1px solid #333; border-radius: 3px; padding: 1px 5px; }
|
||||
pre { background: #111; border: 1px solid #333; border-radius: 6px; padding: 14px 16px; overflow-x: auto; margin: 10px 0 16px; }
|
||||
pre code { background: none; border: none; padding: 0; font-size: 0.82rem; color: #7ec8e3; }
|
||||
.note { background: #1a2a1a; border: 1px solid #2a4a2a; border-radius: 6px; padding: 10px 14px; margin: 12px 0; }
|
||||
.note p { color: #aaddaa; margin: 0; }
|
||||
.warn { background: #2a1a0a; border: 1px solid #5a3a0a; border-radius: 6px; padding: 10px 14px; margin: 12px 0; }
|
||||
.warn p { color: #ddbb88; margin: 0; }
|
||||
table { width: 100%; border-collapse: collapse; margin: 10px 0 16px; font-size: 0.88rem; }
|
||||
th { background: #0f0f23; color: #888; font-weight: 500; text-align: left; padding: 8px 12px; border: 1px solid #333; }
|
||||
td { padding: 8px 12px; border: 1px solid #222; color: #ccc; vertical-align: top; }
|
||||
td code { font-size: 0.82rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a href="/geofilter-builder.html" id="back-link">← GeoFilter Builder</a>
|
||||
<h1>GeoFilter Docs</h1>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
|
||||
<h2>How it works</h2>
|
||||
<p>Geographic filtering restricts which nodes are ingested and returned in API responses. It operates at two levels:</p>
|
||||
<ul>
|
||||
<li><strong>Ingest time</strong> — ADVERT packets carrying GPS coordinates are rejected by the ingestor if the node falls outside the configured area. The node never reaches the database.</li>
|
||||
<li><strong>API responses</strong> — Nodes already in the database are filtered from the <code>/api/nodes</code> response if they fall outside the area. This covers nodes ingested before the filter was configured.</li>
|
||||
</ul>
|
||||
<div class="note"><p>Nodes with no GPS fix (<code>lat=0, lon=0</code> or missing coordinates) always pass the filter regardless of configuration.</p></div>
|
||||
|
||||
<h2>Configuration</h2>
|
||||
<p>Add a <code>geo_filter</code> block to <code>config.json</code>:</p>
|
||||
<pre><code>"geo_filter": {
|
||||
"polygon": [
|
||||
[51.55, 3.80],
|
||||
[51.55, 5.90],
|
||||
[50.65, 5.90],
|
||||
[50.65, 3.80]
|
||||
],
|
||||
"bufferKm": 20
|
||||
}</code></pre>
|
||||
<table>
|
||||
<thead><tr><th>Field</th><th>Type</th><th>Description</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><code>polygon</code></td><td><code>[[lat, lon], ...]</code></td><td>Array of at least 3 coordinate pairs defining the boundary</td></tr>
|
||||
<tr><td><code>bufferKm</code></td><td>number</td><td>Extra distance (km) around the polygon edge that is also accepted. <code>0</code> = exact boundary</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>Both the server and the ingestor read <code>geo_filter</code> from <code>config.json</code>. Restart both after changing this section.</p>
|
||||
<p>To disable filtering entirely, remove the <code>geo_filter</code> block.</p>
|
||||
|
||||
<h2>Builder workflow: Save Draft, Load Draft, Download</h2>
|
||||
<p>The <a href="/geofilter-builder.html">GeoFilter Builder</a> lets you draw a polygon on a map and produce the <code>geo_filter</code> snippet without hand-editing JSON. Three buttons drive the workflow:</p>
|
||||
<ul>
|
||||
<li><strong>💾 Save Draft</strong> — writes the current polygon and <code>bufferKm</code> to your browser's <code>localStorage</code> under the key <code>geofilter-draft</code>. Drafts persist across page reloads and browser restarts so you can iterate on a shape over multiple sessions.</li>
|
||||
<li><strong>📂 Load Draft</strong> — restores the most recently saved draft into the builder. The current polygon is replaced. If no draft exists the button is a no-op.</li>
|
||||
<li><strong>⬇ Download</strong> — exports the current polygon and <code>bufferKm</code> as <code>geofilter-config-snippet.json</code> — a single JSON object containing a top-level <code>geo_filter</code> block. Open the file, copy the <code>geo_filter</code> entry, and paste it into your <code>config.json</code>.</li>
|
||||
</ul>
|
||||
<div class="note"><p>Drafts are stored locally in your browser only — they are not uploaded anywhere. Clearing site data or switching browsers will lose the draft. Use <strong>Download</strong> to keep a portable copy.</p></div>
|
||||
<p>After pasting the snippet into <code>config.json</code>, restart the server and ingestor for the new filter to take effect.</p>
|
||||
|
||||
<h2>Coordinate ordering</h2>
|
||||
<div class="warn"><p><strong>Important:</strong> Coordinates are <code>[lat, lon]</code> — latitude first, longitude second. This is the opposite of GeoJSON, which uses <code>[lon, lat]</code>. Swapping them will place your polygon in the wrong location.</p></div>
|
||||
|
||||
<h2>Multi-polygon</h2>
|
||||
<p>Only a single polygon is supported. If your deployment area consists of multiple disconnected regions, draw a single convex hull that covers all of them, or use the largest region with a generous <code>bufferKm</code> value.</p>
|
||||
|
||||
<h2>Examples</h2>
|
||||
<h3>Belgium (bounding rectangle)</h3>
|
||||
<pre><code>"geo_filter": {
|
||||
"polygon": [
|
||||
[51.55, 3.80],
|
||||
[51.55, 5.90],
|
||||
[50.65, 5.90],
|
||||
[50.65, 3.80]
|
||||
],
|
||||
"bufferKm": 20
|
||||
}</code></pre>
|
||||
<h3>Irregular shape</h3>
|
||||
<pre><code>"geo_filter": {
|
||||
"polygon": [
|
||||
[51.10, 3.70],
|
||||
[51.55, 4.20],
|
||||
[51.30, 5.10],
|
||||
[50.80, 5.50],
|
||||
[50.50, 4.80],
|
||||
[50.70, 3.90]
|
||||
],
|
||||
"bufferKm": 10
|
||||
}</code></pre>
|
||||
|
||||
<h2>Legacy bounding box</h2>
|
||||
<p>An older bounding box format is also supported as a fallback when no <code>polygon</code> is present:</p>
|
||||
<pre><code>"geo_filter": {
|
||||
"latMin": 50.65,
|
||||
"latMax": 51.55,
|
||||
"lonMin": 3.80,
|
||||
"lonMax": 5.90
|
||||
}</code></pre>
|
||||
<p>Prefer the polygon format — it supports irregular shapes and the <code>bufferKm</code> margin.</p>
|
||||
|
||||
<h2>Cleaning up historical nodes</h2>
|
||||
<p>The ingestor prevents new out-of-bounds nodes from being ingested, but does not retroactively remove nodes stored before the filter was configured. Use the prune script for that:</p>
|
||||
<pre><code># Dry run — shows what would be deleted without making any changes
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py --dry-run
|
||||
|
||||
# Default paths: /app/data/meshcore.db and /app/config.json
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py
|
||||
|
||||
# Custom paths
|
||||
python3 scripts/prune-nodes-outside-geo-filter.py /path/to/meshcore.db \
|
||||
--config /path/to/config.json
|
||||
|
||||
# In Docker — run inside the container
|
||||
docker exec -it meshcore-analyzer \
|
||||
python3 /app/scripts/prune-nodes-outside-geo-filter.py --dry-run</code></pre>
|
||||
<p>The script reads <code>geo_filter.polygon</code> and <code>geo_filter.bufferKm</code> from config, lists nodes that fall outside, then asks for <code>yes</code> confirmation before deleting. Nodes without coordinates are always kept.</p>
|
||||
<p>This is a one-time migration tool — run it once after first configuring <code>geo_filter</code> to clean up pre-filter data.</p>
|
||||
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,46 +0,0 @@
|
||||
// Geofilter draft save/load/download helpers.
|
||||
// Exposes GeofilterDraft global with: saveDraft, loadDraft, clearDraft, buildConfigSnippet, downloadConfig
|
||||
(function () {
|
||||
'use strict';
|
||||
var STORAGE_KEY = 'geofilter-draft';
|
||||
|
||||
function saveDraft(polygon, bufferKm) {
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify({ polygon: polygon, bufferKm: bufferKm }));
|
||||
}
|
||||
|
||||
function loadDraft() {
|
||||
var raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (!raw) return null;
|
||||
try { return JSON.parse(raw); } catch (e) { return null; }
|
||||
}
|
||||
|
||||
function clearDraft() {
|
||||
localStorage.removeItem(STORAGE_KEY);
|
||||
}
|
||||
|
||||
function buildConfigSnippet(polygon, bufferKm) {
|
||||
return JSON.stringify({ geo_filter: { bufferKm: bufferKm, polygon: polygon } }, null, 2);
|
||||
}
|
||||
|
||||
function downloadConfig(polygon, bufferKm) {
|
||||
var snippet = buildConfigSnippet(polygon, bufferKm);
|
||||
var blob = new Blob([snippet], { type: 'application/json' });
|
||||
var url = URL.createObjectURL(blob);
|
||||
var a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'geofilter-config-snippet.json';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
// Export
|
||||
(typeof window !== 'undefined' ? window : this).GeofilterDraft = {
|
||||
saveDraft: saveDraft,
|
||||
loadDraft: loadDraft,
|
||||
clearDraft: clearDraft,
|
||||
buildConfigSnippet: buildConfigSnippet,
|
||||
downloadConfig: downloadConfig
|
||||
};
|
||||
})();
|
||||
@@ -1,70 +0,0 @@
|
||||
/* hash-color.js — Deterministic HSL color from packet hash
|
||||
* IIFE attaching window.HashColor = { hashToHsl, hashToOutline }
|
||||
* Pure function: no DOM access, no state, works in Node vm.createContext sandbox.
|
||||
*/
|
||||
(function() {
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Derive a deterministic HSL color string from a hex hash.
|
||||
* Uses bytes 0-1 for hue, byte 2 for saturation, byte 3 for lightness.
|
||||
* Produces bright vivid fills; contrast is provided by a dark outline (hashToOutline).
|
||||
* @param {string|null|undefined} hashHex - Hex string (e.g. "a1b2c3d4...")
|
||||
* @param {string} theme - "light" or "dark"
|
||||
* @returns {string} CSS hsl() string
|
||||
*/
|
||||
function hashToHsl(hashHex, theme) {
|
||||
if (!hashHex || hashHex.length < 8) {
|
||||
return 'hsl(0, 0%, 50%)';
|
||||
}
|
||||
|
||||
var b0 = parseInt(hashHex.slice(0, 2), 16) || 0;
|
||||
var b1 = parseInt(hashHex.slice(2, 4), 16) || 0;
|
||||
var b2 = parseInt(hashHex.slice(4, 6), 16) || 0;
|
||||
var b3 = parseInt(hashHex.slice(6, 8), 16) || 0;
|
||||
|
||||
// Hue: 0-360 from bytes 0-1 (16-bit)
|
||||
var hue = Math.round(((b0 << 8) | b1) / 65535 * 360);
|
||||
// Saturation: 55-95% from byte 2
|
||||
var S = 55 + Math.round(b2 / 255 * 40);
|
||||
// Lightness: vivid range per theme from byte 3
|
||||
// Light: 50-65%, Dark: 55-72%
|
||||
var L;
|
||||
if (theme === 'dark') {
|
||||
L = 55 + Math.round(b3 / 255 * 17);
|
||||
} else {
|
||||
L = 50 + Math.round(b3 / 255 * 15);
|
||||
}
|
||||
|
||||
return 'hsl(' + hue + ', ' + S + '%, ' + L + '%)';
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a dark outline color (same hue) for contrast against backgrounds.
|
||||
* @param {string|null|undefined} hashHex - Hex string
|
||||
* @param {string} theme - "light" or "dark"
|
||||
* @returns {string} CSS hsl() string
|
||||
*/
|
||||
function hashToOutline(hashHex, theme) {
|
||||
if (!hashHex || hashHex.length < 8) {
|
||||
return 'hsl(0, 0%, 30%)';
|
||||
}
|
||||
|
||||
var b0 = parseInt(hashHex.slice(0, 2), 16) || 0;
|
||||
var b1 = parseInt(hashHex.slice(2, 4), 16) || 0;
|
||||
var hue = Math.round(((b0 << 8) | b1) / 65535 * 360);
|
||||
|
||||
// Dark outline: same hue, low lightness for contrast
|
||||
if (theme === 'dark') {
|
||||
return 'hsl(' + hue + ', 30%, 15%)';
|
||||
}
|
||||
return 'hsl(' + hue + ', 70%, 25%)';
|
||||
}
|
||||
|
||||
// Export
|
||||
if (typeof window !== 'undefined') {
|
||||
window.HashColor = { hashToHsl: hashToHsl, hashToOutline: hashToOutline };
|
||||
} else if (typeof module !== 'undefined') {
|
||||
module.exports = { hashToHsl: hashToHsl, hashToOutline: hashToOutline };
|
||||
}
|
||||
})();
|
||||
@@ -81,13 +81,9 @@ window.HopDisplay = (function() {
|
||||
const regionalConflicts = conflicts.filter(c => c.regional);
|
||||
const badgeCount = regionalConflicts.length > 0 ? regionalConflicts.length : (globalFallback ? conflicts.length : 0);
|
||||
const conflictData = escapeHtml(JSON.stringify({ h, conflicts, globalFallback }));
|
||||
const conflictBadge = badgeCount > 1
|
||||
const warnBadge = badgeCount > 1
|
||||
? ` <button class="hop-conflict-btn" data-conflict='${conflictData}' onclick="event.preventDefault();event.stopPropagation();HopDisplay._showFromBtn(this)" title="${badgeCount} candidates — click for details">⚠${badgeCount}</button>`
|
||||
: '';
|
||||
const unreliableBadge = unreliable
|
||||
? ' <button class="hop-unreliable-btn" aria-label="Unreliable name resolution" title="Unreliable name resolution — this hash\u2192name match is geographically inconsistent with the surrounding path hops. The repeater itself may be fine; this specific hop assignment is uncertain.">⚠️</button>'
|
||||
: '';
|
||||
const warnBadge = conflictBadge + unreliableBadge;
|
||||
|
||||
const cls = [
|
||||
'hop',
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user