Compare commits

..
Author SHA1 Message Date
you 68c669f4a4 Extract validateAdvertSignature to shared package, accept []byte, add server tests
- Extract signature validation to internal/sigvalidate/ shared package
- Change function signature to accept []byte instead of hex strings,
  eliminating unnecessary hex encode/decode round-trip
- Add signature validation tests to cmd/server/decoder_test.go
- Both cmd/server and cmd/ingestor now import from the shared package

Addresses all review feedback on PR #686.
2026-04-10 00:44:48 +00:00
you a7ada12f6d Add tests for ed25519 signature validation
Test validateAdvertSignature with valid signatures, tampered data,
wrong timestamps, malformed inputs, and wrong-length keys/signatures.
Also test decodeAdvert with validation enabled vs disabled.
2026-04-10 00:36:23 +00:00
you c182e799bf Clean up bool pointer allocation in signature validation
Replace &[]bool{false}[0] idiom with explicit variable for clarity.
2026-04-10 00:36:17 +00:00
you e900502c68 Add missing badge-success and badge-danger CSS classes
The signature validation UI uses badge-success and badge-danger classes
that were not defined in style.css, resulting in unstyled badges.
Add them using the same color patterns as existing badge-hash variants.
2026-04-10 00:36:12 +00:00
Jeff Copeland 8e8ff85f1c for BYOP mode, perform signature validation on advert packets and display status 2026-04-08 12:03:22 -04:00
103 changed files with 878 additions and 18232 deletions
+5 -60
View File
@@ -68,17 +68,6 @@ jobs:
echo "--- Go Ingestor Coverage ---"
go tool cover -func=ingestor-coverage.out | tail -1
- name: Build and test channel library + decrypt CLI
run: |
set -e -o pipefail
cd internal/channel
go test ./...
echo "--- Channel library tests passed ---"
cd ../../cmd/decrypt
CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt .
go test ./...
echo "--- Decrypt CLI tests passed ---"
- name: Verify proto syntax
run: |
set -e
@@ -290,10 +279,6 @@ jobs:
if: github.event_name == 'push'
uses: docker/setup-buildx-action@v3
- name: Set up QEMU (arm64 runtime stage)
if: github.event_name == 'push'
uses: docker/setup-qemu-action@v3
- name: Log in to GHCR
if: github.event_name == 'push'
uses: docker/login-action@v3
@@ -321,7 +306,7 @@ jobs:
with:
context: .
push: true
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
build-args: |
@@ -332,43 +317,7 @@ jobs:
cache-to: type=gha,mode=max
# ───────────────────────────────────────────────────────────────
# 4. Release Artifacts (tags only)
# ───────────────────────────────────────────────────────────────
release-artifacts:
name: "📦 Release Artifacts"
if: startsWith(github.ref, 'refs/tags/v')
needs: [go-test]
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Set up Go 1.22
uses: actions/setup-go@v6
with:
go-version: '1.22'
- name: Build corescope-decrypt (static, linux/amd64)
run: |
cd cmd/decrypt
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 .
- name: Build corescope-decrypt (static, linux/arm64)
run: |
cd cmd/decrypt
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 .
- name: Upload release assets
uses: softprops/action-gh-release@v2
with:
files: |
corescope-decrypt-linux-amd64
corescope-decrypt-linux-arm64
# ───────────────────────────────────────────────────────────────
# 4b. Deploy Staging (master only)
# 4. Deploy Staging (master only)
# ───────────────────────────────────────────────────────────────
deploy:
name: "🚀 Deploy Staging"
@@ -391,10 +340,7 @@ jobs:
- name: Deploy staging
run: |
# Force-remove the staging container regardless of how it was created
# (compose-managed OR manually created via docker run)
docker stop corescope-staging-go 2>/dev/null || true
docker rm -f corescope-staging-go 2>/dev/null || true
# Stop old container and release memory
docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --timeout 30 2>/dev/null || true
# Wait for container to be fully gone and OS to reclaim memory (3GB limit)
@@ -436,11 +382,10 @@ jobs:
- name: Smoke test staging API
run: |
PORT="${STAGING_GO_HTTP_PORT:-80}"
if curl -sf "http://localhost:${PORT}/api/stats" | grep -q engine; then
if curl -sf http://localhost:82/api/stats | grep -q engine; then
echo "Staging verified — engine field present ✅"
else
echo "Staging /api/stats did not return engine field (port ${PORT})"
echo "Staging /api/stats did not return engine field"
exit 1
fi
+7 -24
View File
@@ -1,42 +1,25 @@
# Build stage always runs natively on the builder's arch ($BUILDPLATFORM)
# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU.
FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
FROM golang:1.22-alpine AS builder
RUN apk add --no-cache build-base
ARG APP_VERSION=unknown
ARG GIT_COMMIT=unknown
ARG BUILD_TIME=unknown
# Provided by buildx for multi-arch builds
ARG TARGETOS
ARG TARGETARCH
# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly)
# Build server
WORKDIR /build/server
COPY cmd/server/go.mod cmd/server/go.sum ./
COPY internal/geofilter/ ../../internal/geofilter/
COPY internal/sigvalidate/ ../../internal/sigvalidate/
RUN go mod download
COPY cmd/server/ ./
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
# Build ingestor
WORKDIR /build/ingestor
COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./
COPY internal/geofilter/ ../../internal/geofilter/
COPY internal/sigvalidate/ ../../internal/sigvalidate/
RUN go mod download
COPY cmd/ingestor/ ./
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -o /corescope-ingestor .
# Build decrypt CLI
WORKDIR /build/decrypt
COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./
COPY internal/channel/ ../../internal/channel/
RUN go mod download
COPY cmd/decrypt/ ./
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -ldflags="-s -w" -o /corescope-decrypt .
RUN go build -o /corescope-ingestor .
# Runtime image
FROM alpine:3.20
@@ -46,7 +29,7 @@ RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget
WORKDIR /app
# Go binaries
COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/
COPY --from=builder /corescope-server /corescope-ingestor /app/
# Frontend assets + config
COPY public/ ./public/
+4 -15
View File
@@ -80,26 +80,15 @@ No build step required — just run:
```bash
docker run -d --name corescope \
--restart=unless-stopped \
-p 80:80 -p 1883:1883 \
-v /your/data:/app/data \
-p 80:80 \
-v corescope-data:/app/data \
ghcr.io/kpa-clawbot/corescope:latest
```
Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults.
For HTTPS with a custom domain, add `-p 443:443` and mount your Caddyfile:
```bash
docker run -d --name corescope \
--restart=unless-stopped \
-p 80:80 -p 443:443 -p 1883:1883 \
-v /your/data:/app/data \
-v /your/Caddyfile:/etc/caddy/Caddyfile:ro \
-v /your/caddy-data:/data/caddy \
ghcr.io/kpa-clawbot/corescope:latest
```
Disable built-in services with `-e DISABLE_MOSQUITTO=true` or `-e DISABLE_CADDY=true`, or drop a `.env` file in your data volume. See [docs/deployment.md](docs/deployment.md) for the full reference.
See [DEPLOY.md](DEPLOY.md) for image tags, Docker Compose, and migration from `manage.sh`.
See [docs/deployment.md](docs/deployment.md) for the full deployment guide — MQTT setup, HTTPS options, backups, monitoring, and troubleshooting.
### Build from Source
-142
View File
@@ -1,142 +0,0 @@
# corescope-decrypt
Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from a CoreScope SQLite database.
## Why
MeshCore hashtag channels use symmetric encryption where the key is derived deterministically from the channel name. The CoreScope ingestor stores **all** `GRP_TXT` packets in the database, including those it cannot decrypt at ingest time.
This tool enables:
- **Retroactive decryption** — decrypt historical messages for any channel whose name you learn after the fact
- **Forensics & analysis** — export channel traffic for offline review
- **Bulk export** — dump an entire channel's history as JSON, HTML, or plain text
## Installation
### From Docker image
The binary is included in the CoreScope Docker image at `/app/corescope-decrypt`:
```bash
docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
```
### From GitHub release
Download the static binary from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page:
```bash
# Linux amd64
curl -LO https://github.com/Kpa-clawbot/CoreScope/releases/latest/download/corescope-decrypt-linux-amd64
chmod +x corescope-decrypt-linux-amd64
./corescope-decrypt-linux-amd64 --help
```
### Build from source
```bash
cd cmd/decrypt
CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt .
```
The binary is statically linked — no dependencies, runs on any Linux.
## Usage
```
corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE]
```
Run `corescope-decrypt --help` for full flag documentation.
### JSON output (default)
Machine-readable, includes all metadata (observers, path hops, raw hex):
```bash
corescope-decrypt --channel "#wardriving" --db meshcore.db
```
```json
[
{
"hash": "a1b2c3...",
"timestamp": "2026-04-12T17:19:09Z",
"sender": "XMD Tag 1",
"message": "@[MapperBot] 37.76985, -122.40525 [0.3w]",
"channel": "#wardriving",
"raw_hex": "150206...",
"path": ["A3", "B0"],
"observers": [
{"name": "Observer1", "snr": 9.5, "rssi": -56, "timestamp": "2026-04-12T17:19:10Z"}
]
}
]
```
### HTML output
Self-contained interactive viewer — search, sortable columns, expandable detail rows:
```bash
corescope-decrypt --channel "#wardriving" --db meshcore.db --format html --output wardriving.html
open wardriving.html
```
No external dependencies. The JSON data is embedded directly in the HTML file.
### IRC / log output
Plain-text, one line per message — ideal for `grep`, `awk`, and piping:
```bash
corescope-decrypt --channel "#wardriving" --db meshcore.db --format irc
```
```
[2026-04-12 17:19:09] <XMD Tag 1> @[MapperBot] 37.76985, -122.40525 [0.3w]
[2026-04-12 17:20:25] <XMD Tag 1> @[MapperBot] 37.78075, -122.39774 [0.3w]
[2026-04-12 17:25:30] <mk 🤠> @[MapperBot] 35.32444, -120.62077
```
```bash
# Find all messages from a specific sender
corescope-decrypt --channel "#wardriving" --db meshcore.db --format irc | grep "KE6QR"
```
## How channel encryption works
MeshCore hashtag channels derive their encryption key from the channel name:
1. **Key derivation**: `AES-128 key = SHA-256("#channelname")[:16]` (first 16 bytes)
2. **Channel hash**: `SHA-256(key)[0]` — 1-byte identifier in the packet header, used for fast filtering
3. **Encryption**: AES-128-ECB
4. **MAC**: HMAC-SHA256 with a 32-byte secret (key + 16 zero bytes), truncated to 2 bytes
5. **Plaintext format**: `timestamp(4 LE) + flags(1) + "sender: message\0"`
See the firmware source at `firmware/src/helpers/BaseChatMesh.cpp` for the canonical implementation.
## Testing against the fixture DB
```bash
cd cmd/decrypt
go test ./...
# Manual test with the real fixture:
go run . --channel "#wardriving" --db ../../test-fixtures/e2e-fixture.db --format irc
```
The shared crypto library also has independent tests:
```bash
cd internal/channel
go test -v ./...
```
## Limitations
- **Hashtag channels only.** Only channels where the key is derived from `SHA-256("#name")` are supported. Custom PSK channels require the raw key (not implemented).
- **No DM decryption.** Direct messages (`TXT_MSG`) use per-peer asymmetric encryption and cannot be decrypted by this tool.
- **Read-only.** The tool opens the database in read-only mode and never modifies it.
- **Timestamps are UTC.** The sender's embedded timestamp is used when available, displayed in UTC.
-22
View File
@@ -1,22 +0,0 @@
module github.com/corescope/decrypt
go 1.22
require (
github.com/meshcore-analyzer/channel v0.0.0
modernc.org/sqlite v1.34.5
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/sys v0.22.0 // indirect
modernc.org/libc v1.55.3 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
)
replace github.com/meshcore-analyzer/channel => ../../internal/channel
-43
View File
@@ -1,43 +0,0 @@
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-467
View File
@@ -1,467 +0,0 @@
// corescope-decrypt decrypts and exports hashtag channel messages from a CoreScope SQLite database.
//
// Usage:
//
// corescope-decrypt --channel "#wardriving" --db meshcore.db [--format json|html] [--output file]
package main
import (
"database/sql"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"html"
"log"
"os"
"sort"
"strings"
"time"
"github.com/meshcore-analyzer/channel"
_ "modernc.org/sqlite"
)
// Version info (set via ldflags).
var version = "dev"
// ChannelMessage is a single decrypted channel message with metadata.
type ChannelMessage struct {
Hash string `json:"hash"`
Timestamp string `json:"timestamp"`
Sender string `json:"sender"`
Message string `json:"message"`
Channel string `json:"channel"`
RawHex string `json:"raw_hex"`
Path []string `json:"path"`
Observers []Observer `json:"observers"`
}
// Observer is a single observation of the transmission.
type Observer struct {
Name string `json:"name"`
SNR float64 `json:"snr"`
RSSI float64 `json:"rssi"`
Timestamp string `json:"timestamp"`
}
func main() {
channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")")
dbPath := flag.String("db", "", "Path to CoreScope SQLite database")
format := flag.String("format", "json", "Output format: json, html, irc (or log)")
output := flag.String("output", "", "Output file (default: stdout)")
showVersion := flag.Bool("version", false, "Print version and exit")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, `corescope-decrypt — Decrypt and export MeshCore hashtag channel messages
USAGE
corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE]
FLAGS
--channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving")
The "#" prefix is added automatically if missing.
--db PATH Path to a CoreScope SQLite database file (read-only access).
--format FORMAT Output format (default: json):
json — Machine-readable JSON array with full metadata
html — Self-contained HTML viewer with search and sorting
irc — Plain-text IRC-style log, one line per message
log — Alias for irc
--output FILE Write output to FILE instead of stdout.
--version Print version and exit.
EXAMPLES
# Export #wardriving messages as JSON
corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
# Generate an interactive HTML viewer
corescope-decrypt --channel wardriving --db meshcore.db --format html --output wardriving.html
# Greppable IRC log
corescope-decrypt --channel "#MeshCore" --db meshcore.db --format irc --output meshcore.log
grep "KE6QR" meshcore.log
# From the Docker container
docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
RETROACTIVE DECRYPTION
MeshCore hashtag channels use symmetric encryption — the key is derived from the
channel name. The CoreScope ingestor stores ALL GRP_TXT packets in the database,
even those it cannot decrypt at ingest time. This tool lets you retroactively
decrypt messages for any channel whose name you know, even if the ingestor was
never configured with that channel's key.
This means you can recover historical messages by simply knowing the channel name.
LIMITATIONS
- Only hashtag channels (shared-secret, name-derived key) are supported.
- Direct messages (TXT_MSG) use per-peer encryption and cannot be decrypted.
- Custom PSK channels (non-hashtag) require the raw key, not a channel name.
`)
}
flag.Parse()
if *showVersion {
fmt.Println("corescope-decrypt", version)
os.Exit(0)
}
if *channelName == "" || *dbPath == "" {
flag.Usage()
os.Exit(1)
}
// Normalize channel name
ch := *channelName
if !strings.HasPrefix(ch, "#") {
ch = "#" + ch
}
key := channel.DeriveKey(ch)
chHash := channel.ChannelHash(key)
db, err := sql.Open("sqlite", *dbPath+"?mode=ro")
if err != nil {
log.Fatalf("Failed to open database: %v", err)
}
defer db.Close()
// Query all GRP_TXT packets
rows, err := db.Query(`SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5`)
if err != nil {
log.Fatalf("Query failed: %v", err)
}
defer rows.Close()
var messages []ChannelMessage
decrypted, total := 0, 0
for rows.Next() {
var id int
var txHash, rawHex, firstSeen string
if err := rows.Scan(&id, &txHash, &rawHex, &firstSeen); err != nil {
log.Printf("Scan error: %v", err)
continue
}
total++
payload, err := extractGRPPayload(rawHex)
if err != nil {
continue
}
if len(payload) < 3 {
continue
}
// Check channel hash byte
if payload[0] != chHash {
continue
}
mac := payload[1:3]
ciphertext := payload[3:]
if len(ciphertext) < 5 || len(ciphertext)%16 != 0 {
// Pad ciphertext to block boundary for decryption attempt
if len(ciphertext) < 16 {
continue
}
// Truncate to block boundary
ciphertext = ciphertext[:len(ciphertext)/16*16]
}
plaintext, ok := channel.Decrypt(key, mac, ciphertext)
if !ok {
continue
}
ts, sender, msg, err := channel.ParsePlaintext(plaintext)
if err != nil {
continue
}
decrypted++
// Convert MeshCore timestamp
timestamp := time.Unix(int64(ts), 0).UTC().Format(time.RFC3339)
// Get path from decoded_json
path := getPathFromDB(db, id)
// Get observers
observers := getObservers(db, id)
messages = append(messages, ChannelMessage{
Hash: txHash,
Timestamp: timestamp,
Sender: sender,
Message: msg,
Channel: ch,
RawHex: rawHex,
Path: path,
Observers: observers,
})
}
// Sort by timestamp
sort.Slice(messages, func(i, j int) bool {
return messages[i].Timestamp < messages[j].Timestamp
})
log.Printf("Scanned %d GRP_TXT packets, decrypted %d for channel %s", total, decrypted, ch)
// Generate output
var out []byte
switch *format {
case "json":
out, err = json.MarshalIndent(messages, "", " ")
if err != nil {
log.Fatalf("JSON marshal: %v", err)
}
out = append(out, '\n')
case "html":
out = renderHTML(messages, ch)
case "irc", "log":
out = renderIRC(messages)
default:
log.Fatalf("Unknown format: %s (use json, html, irc, or log)", *format)
}
if *output != "" {
if err := os.WriteFile(*output, out, 0644); err != nil {
log.Fatalf("Write file: %v", err)
}
log.Printf("Written to %s", *output)
} else {
os.Stdout.Write(out)
}
}
// extractGRPPayload parses a raw hex packet and returns the GRP_TXT payload bytes.
func extractGRPPayload(rawHex string) ([]byte, error) {
buf, err := hex.DecodeString(strings.TrimSpace(rawHex))
if err != nil || len(buf) < 2 {
return nil, fmt.Errorf("invalid hex")
}
// Header byte
header := buf[0]
payloadType := int((header >> 2) & 0x0F)
if payloadType != 5 { // GRP_TXT
return nil, fmt.Errorf("not GRP_TXT")
}
routeType := int(header & 0x03)
offset := 1
// Transport codes (2 codes × 2 bytes) come BEFORE path for transport routes
if routeType == 0 || routeType == 3 {
offset += 4
}
// Path byte
if offset >= len(buf) {
return nil, fmt.Errorf("too short for path")
}
pathByte := buf[offset]
offset++
hashSize := int(pathByte>>6) + 1
hashCount := int(pathByte & 0x3F)
offset += hashSize * hashCount
if offset >= len(buf) {
return nil, fmt.Errorf("too short for payload")
}
return buf[offset:], nil
}
func getPathFromDB(db *sql.DB, txID int) []string {
var decodedJSON sql.NullString
err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = ?`, txID).Scan(&decodedJSON)
if err != nil || !decodedJSON.Valid {
return nil
}
var decoded struct {
Path struct {
Hops []string `json:"hops"`
} `json:"path"`
}
if json.Unmarshal([]byte(decodedJSON.String), &decoded) == nil {
return decoded.Path.Hops
}
return nil
}
func getObservers(db *sql.DB, txID int) []Observer {
rows, err := db.Query(`
SELECT o.name, obs.snr, obs.rssi, obs.timestamp
FROM observations obs
LEFT JOIN observers o ON o.id = CAST(obs.observer_idx AS TEXT)
WHERE obs.transmission_id = ?
ORDER BY obs.timestamp
`, txID)
if err != nil {
return nil
}
defer rows.Close()
var observers []Observer
for rows.Next() {
var name sql.NullString
var snr, rssi sql.NullFloat64
var ts int64
if err := rows.Scan(&name, &snr, &rssi, &ts); err != nil {
continue
}
obs := Observer{
Timestamp: time.Unix(ts, 0).UTC().Format(time.RFC3339),
}
if name.Valid {
obs.Name = name.String
}
if snr.Valid {
obs.SNR = snr.Float64
}
if rssi.Valid {
obs.RSSI = rssi.Float64
}
observers = append(observers, obs)
}
return observers
}
func renderIRC(messages []ChannelMessage) []byte {
var b strings.Builder
for _, m := range messages {
sender := m.Sender
if sender == "" {
sender = "???"
}
// Parse RFC3339 timestamp into a compact format
t, err := time.Parse(time.RFC3339, m.Timestamp)
if err != nil {
b.WriteString(fmt.Sprintf("[%s] <%s> %s\n", m.Timestamp, sender, m.Message))
continue
}
b.WriteString(fmt.Sprintf("[%s] <%s> %s\n", t.Format("2006-01-02 15:04:05"), sender, m.Message))
}
return []byte(b.String())
}
func renderHTML(messages []ChannelMessage, channelName string) []byte {
jsonData, _ := json.Marshal(messages)
var b strings.Builder
b.WriteString(`<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>CoreScope Channel Export — ` + html.EscapeString(channelName) + `</title>
<style>
*{box-sizing:border-box;margin:0;padding:0}
body{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif;background:#0d1117;color:#c9d1d9;padding:20px}
h1{color:#58a6ff;margin-bottom:16px;font-size:1.5em}
.stats{color:#8b949e;margin-bottom:16px;font-size:0.9em}
input[type=text]{width:100%;max-width:500px;padding:8px 12px;background:#161b22;border:1px solid #30363d;border-radius:6px;color:#c9d1d9;font-size:14px;margin-bottom:16px}
input[type=text]:focus{outline:none;border-color:#58a6ff}
table{width:100%;border-collapse:collapse;font-size:14px}
th{background:#161b22;color:#8b949e;text-align:left;padding:8px 12px;border-bottom:2px solid #30363d;cursor:pointer;user-select:none;white-space:nowrap}
th:hover{color:#58a6ff}
th.sorted-asc::after{content:" ▲"}
th.sorted-desc::after{content:" ▼"}
td{padding:8px 12px;border-bottom:1px solid #21262d;vertical-align:top}
tr:hover{background:#161b22}
tr.expanded{background:#161b22}
.detail-row td{padding:12px 24px;background:#0d1117;border-bottom:1px solid #21262d}
.detail-row pre{background:#161b22;padding:12px;border-radius:6px;overflow-x:auto;font-size:12px;color:#8b949e}
.detail-row .label{color:#58a6ff;font-weight:600;margin-top:8px;display:block}
.observer-tag{display:inline-block;background:#1f6feb22;color:#58a6ff;padding:2px 8px;border-radius:4px;margin:2px;font-size:12px}
.no-results{color:#8b949e;text-align:center;padding:40px;font-size:16px}
.sender{color:#d2a8ff;font-weight:600}
.timestamp{color:#8b949e;font-family:monospace;font-size:12px}
</style>
</head>
<body>
<h1>` + html.EscapeString(channelName) + ` — Channel Messages</h1>
<div class="stats" id="stats"></div>
<input type="text" id="search" placeholder="Search messages..." autocomplete="off">
<table>
<thead>
<tr>
<th data-col="timestamp">Timestamp</th>
<th data-col="sender">Sender</th>
<th data-col="message">Message</th>
<th data-col="observers">Observers</th>
</tr>
</thead>
<tbody id="tbody"></tbody>
</table>
<div class="no-results" id="no-results" style="display:none">No matching messages</div>
<script>
var DATA=` + string(jsonData) + `;
var sortCol="timestamp",sortAsc=true,expandedHash=null;
function init(){
document.getElementById("stats").textContent=DATA.length+" messages";
document.getElementById("search").addEventListener("input",render);
document.querySelectorAll("th[data-col]").forEach(function(th){
th.addEventListener("click",function(){
var col=th.dataset.col;
if(sortCol===col)sortAsc=!sortAsc;
else{sortCol=col;sortAsc=true}
render();
});
});
render();
}
function render(){
var q=document.getElementById("search").value.toLowerCase();
var filtered=DATA.filter(function(m){
if(!q)return true;
return(m.message||"").toLowerCase().indexOf(q)>=0||(m.sender||"").toLowerCase().indexOf(q)>=0;
});
filtered.sort(function(a,b){
var va=a[sortCol]||"",vb=b[sortCol]||"";
if(sortCol==="observers"){va=a.observers?a.observers.length:0;vb=b.observers?b.observers.length:0}
if(va<vb)return sortAsc?-1:1;
if(va>vb)return sortAsc?1:-1;
return 0;
});
document.querySelectorAll("th[data-col]").forEach(function(th){
th.className=th.dataset.col===sortCol?(sortAsc?"sorted-asc":"sorted-desc"):"";
});
var tb=document.getElementById("tbody");
tb.innerHTML="";
document.getElementById("no-results").style.display=filtered.length?"none":"block";
filtered.forEach(function(m){
var tr=document.createElement("tr");
tr.innerHTML='<td class="timestamp">'+esc(m.timestamp)+'</td><td class="sender">'+esc(m.sender||"—")+'</td><td>'+esc(m.message)+'</td><td>'+
(m.observers?m.observers.map(function(o){return'<span class="observer-tag">'+esc(o.name||"?")+" SNR:"+o.snr.toFixed(1)+'</span>'}).join(""):"—")+'</td>';
tr.style.cursor="pointer";
tr.addEventListener("click",function(){
expandedHash=expandedHash===m.hash?null:m.hash;
render();
});
tb.appendChild(tr);
if(expandedHash===m.hash){
tr.className="expanded";
var dr=document.createElement("tr");
dr.className="detail-row";
dr.innerHTML='<td colspan="4"><span class="label">Hash</span><pre>'+esc(m.hash)+'</pre>'+
'<span class="label">Raw Hex</span><pre>'+esc(m.raw_hex)+'</pre>'+
(m.path&&m.path.length?'<span class="label">Path</span><pre>'+esc(m.path.join(" → "))+'</pre>':'')+
'<span class="label">Observers</span><pre>'+esc(JSON.stringify(m.observers,null,2))+'</pre></td>';
tb.appendChild(dr);
}
});
}
function esc(s){var d=document.createElement("div");d.textContent=s;return d.innerHTML}
init();
</script>
</body>
</html>`)
return []byte(b.String())
}
-129
View File
@@ -1,129 +0,0 @@
package main
import (
"encoding/hex"
"encoding/json"
"os"
"strings"
"testing"
"github.com/meshcore-analyzer/channel"
)
func TestExtractGRPPayload(t *testing.T) {
// Build a minimal GRP_TXT packet: header(1) + path(1) + payload
// header: route=FLOOD(1), payload=GRP_TXT(5), version=0 → (5<<2)|1 = 0x15
// path: 0 hops, hash_size=1 → 0x00
payload := []byte{0x81, 0x12, 0x34} // channel_hash + mac + data
pkt := append([]byte{0x15, 0x00}, payload...)
rawHex := hex.EncodeToString(pkt)
result, err := extractGRPPayload(rawHex)
if err != nil {
t.Fatal(err)
}
if len(result) != 3 || result[0] != 0x81 {
t.Fatalf("payload mismatch: %x", result)
}
}
func TestExtractGRPPayloadTransport(t *testing.T) {
// Transport flood: route=0, 4 bytes transport codes BEFORE path byte
// header: (5<<2)|0 = 0x14
payload := []byte{0xAA, 0xBB, 0xCC}
// header + 4 transport bytes + path(0 hops) + payload
pkt := append([]byte{0x14, 0xFF, 0xFF, 0xFF, 0xFF, 0x00}, payload...)
rawHex := hex.EncodeToString(pkt)
result, err := extractGRPPayload(rawHex)
if err != nil {
t.Fatal(err)
}
if result[0] != 0xAA {
t.Fatalf("expected AA, got %02X", result[0])
}
}
func TestExtractGRPPayloadNotGRP(t *testing.T) {
// payload type = ADVERT (4): (4<<2)|1 = 0x11
rawHex := hex.EncodeToString([]byte{0x11, 0x00, 0x01, 0x02})
_, err := extractGRPPayload(rawHex)
if err == nil {
t.Fatal("expected error for non-GRP_TXT")
}
}
func TestKeyDerivationConsistency(t *testing.T) {
// Verify key derivation matches what the ingestor expects
key := channel.DeriveKey("#wardriving")
if len(key) != 16 {
t.Fatalf("key len %d", len(key))
}
ch := channel.ChannelHash(key)
if ch != 0x81 {
// We know from fixture data that #wardriving has channelHashHex "81"
t.Fatalf("channel hash %02X, expected 81", ch)
}
}
func TestRenderIRC(t *testing.T) {
msgs := []ChannelMessage{
{Timestamp: "2026-04-12T03:45:12Z", Sender: "NodeA", Message: "Hello"},
{Timestamp: "2026-04-12T03:46:01Z", Sender: "", Message: "No sender"},
}
out := string(renderIRC(msgs))
if !strings.Contains(out, "[2026-04-12 03:45:12] <NodeA> Hello") {
t.Fatalf("IRC output missing expected line: %s", out)
}
if !strings.Contains(out, "<???> No sender") {
t.Fatalf("IRC output should use ??? for empty sender: %s", out)
}
}
func TestRenderHTMLValid(t *testing.T) {
msgs := []ChannelMessage{
{Hash: "abc", Timestamp: "2026-04-12T00:00:00Z", Sender: "X", Message: "test", Channel: "#test"},
}
out := string(renderHTML(msgs, "#test"))
if !strings.Contains(out, "<!DOCTYPE html>") {
t.Fatal("not valid HTML")
}
if !strings.Contains(out, "#test") {
t.Fatal("channel name missing")
}
if !strings.Contains(out, "</html>") {
t.Fatal("HTML not closed")
}
}
func TestJSONOutputParseable(t *testing.T) {
msgs := []ChannelMessage{
{Hash: "abc", Timestamp: "2026-04-12T00:00:00Z", Sender: "X", Message: "hi", Channel: "#test"},
}
data, err := json.MarshalIndent(msgs, "", " ")
if err != nil {
t.Fatal(err)
}
var parsed []ChannelMessage
if err := json.Unmarshal(data, &parsed); err != nil {
t.Fatalf("JSON not parseable: %v", err)
}
if len(parsed) != 1 || parsed[0].Sender != "X" {
t.Fatalf("parsed mismatch: %+v", parsed)
}
}
// Integration test against fixture DB (skipped if DB not found)
func TestFixtureDecrypt(t *testing.T) {
dbPath := "../../test-fixtures/e2e-fixture.db"
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
t.Skip("fixture DB not found")
}
// We know the fixture has #wardriving messages with channelHash 0x81
key := channel.DeriveKey("#wardriving")
ch := channel.ChannelHash(key)
if ch != 0x81 {
t.Fatalf("unexpected channel hash: %02X", ch)
}
}
+3 -22
View File
@@ -39,8 +39,7 @@ type Config struct {
HashChannels []string `json:"hashChannels,omitempty"`
Retention *RetentionConfig `json:"retention,omitempty"`
Metrics *MetricsConfig `json:"metrics,omitempty"`
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
}
// GeoFilterConfig is an alias for the shared geofilter.Config type.
@@ -48,9 +47,8 @@ type GeoFilterConfig = geofilter.Config
// RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
type RetentionConfig struct {
NodeDays int `json:"nodeDays"`
ObserverDays int `json:"observerDays"`
MetricsDays int `json:"metricsDays"`
NodeDays int `json:"nodeDays"`
MetricsDays int `json:"metricsDays"`
}
// MetricsConfig controls observer metrics collection.
@@ -58,14 +56,6 @@ type MetricsConfig struct {
SampleIntervalSec int `json:"sampleIntervalSec"`
}
// ShouldValidateSignatures returns true (default) unless explicitly disabled.
func (c *Config) ShouldValidateSignatures() bool {
if c.ValidateSignatures != nil {
return *c.ValidateSignatures
}
return true
}
// MetricsSampleInterval returns the configured sample interval or 300s default.
func (c *Config) MetricsSampleInterval() int {
if c.Metrics != nil && c.Metrics.SampleIntervalSec > 0 {
@@ -90,15 +80,6 @@ func (c *Config) NodeDaysOrDefault() int {
return 7
}
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
// A value of -1 means observers are never removed.
func (c *Config) ObserverDaysOrDefault() int {
if c.Retention != nil && c.Retention.ObserverDays != 0 {
return c.Retention.ObserverDays
}
return 14
}
// LoadConfig reads configuration from a JSON file, with env var overrides.
// If the config file does not exist, sensible defaults are used (zero-config startup).
func LoadConfig(path string) (*Config, error) {
+31 -203
View File
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"encoding/json"
"testing"
"time"
)
// hmacSHA256 computes HMAC-SHA256 for test use.
@@ -158,7 +157,7 @@ func TestHandleMessageChannelMessage(t *testing.T) {
payload := []byte(`{"text":"Alice: Hello everyone","channel_idx":3,"SNR":5.0,"RSSI":-95,"score":10,"direction":"rx","sender_timestamp":1700000000}`)
msg := &mockMessage{topic: "meshcore/message/channel/2", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -204,13 +203,21 @@ func TestHandleMessageChannelMessage(t *testing.T) {
t.Errorf("direction=%v, want rx", direction)
}
// Sender node should NOT be created (see issue #665: synthetic "sender-" keys
// are unreachable from the claiming/health flow)
// Should create sender node
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 0 {
t.Errorf("nodes count=%d, want 0 (no phantom sender node)", count)
if count != 1 {
t.Errorf("nodes count=%d, want 1 (sender node)", count)
}
// Verify sender node name
var nodeName string
if err := store.db.QueryRow("SELECT name FROM nodes LIMIT 1").Scan(&nodeName); err != nil {
t.Fatal(err)
}
if nodeName != "Alice" {
t.Errorf("node name=%s, want Alice", nodeName)
}
}
@@ -218,7 +225,7 @@ func TestHandleMessageChannelMessageEmptyText(t *testing.T) {
store, source := newTestContext(t)
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":""}`)}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -233,7 +240,7 @@ func TestHandleMessageChannelNoSender(t *testing.T) {
store, source := newTestContext(t)
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":"no sender here"}`)}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
@@ -250,7 +257,7 @@ func TestHandleMessageDirectMessage(t *testing.T) {
payload := []byte(`{"text":"Bob: Hey there","sender_timestamp":1700000000,"SNR":3.0,"rssi":-100,"Score":8,"Direction":"tx"}`)
msg := &mockMessage{topic: "meshcore/message/direct/abc123", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -294,7 +301,7 @@ func TestHandleMessageDirectMessageEmptyText(t *testing.T) {
store, source := newTestContext(t)
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: []byte(`{"text":""}`)}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -309,7 +316,7 @@ func TestHandleMessageDirectNoSender(t *testing.T) {
store, source := newTestContext(t)
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: []byte(`{"text":"message with no colon"}`)}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -328,7 +335,7 @@ func TestHandleMessageUppercaseScoreDirection(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `","Score":9.0,"Direction":"tx"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var score *float64
var direction *string
@@ -349,7 +356,7 @@ func TestHandleMessageChannelLowercaseFields(t *testing.T) {
payload := []byte(`{"text":"Test: msg","snr":3.0,"rssi":-90,"Score":5,"Direction":"rx"}`)
msg := &mockMessage{topic: "meshcore/message/channel/0", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -365,7 +372,7 @@ func TestHandleMessageDirectLowercaseFields(t *testing.T) {
payload := []byte(`{"text":"Test: msg","snr":2.0,"rssi":-85,"score":7,"direction":"tx"}`)
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -388,7 +395,7 @@ func TestHandleMessageAdvertWithTelemetry(t *testing.T) {
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
// Should have created transmission, node, and observer
var txCount, nodeCount, obsCount int
@@ -428,7 +435,7 @@ func TestHandleMessageAdvertGeoFiltered(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{GeoFilter: gf})
handleMessage(store, "test", source, msg, nil, gf)
// Geo-filtered adverts should not create nodes
var nodeCount int
@@ -665,7 +672,7 @@ func TestHandleMessageCorruptedAdvertNoNode(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
@@ -687,7 +694,7 @@ func TestHandleMessageNonAdvertPacket(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -864,7 +871,7 @@ func TestHandleMessageChannelLongSender(t *testing.T) {
longText := "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: msg"
payload := []byte(`{"text":"` + longText + `"}`)
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
@@ -883,7 +890,7 @@ func TestHandleMessageDirectLongSender(t *testing.T) {
longText := "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB: msg"
payload := []byte(`{"text":"` + longText + `"}`)
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -900,7 +907,7 @@ func TestHandleMessageDirectUppercaseScoreDirection(t *testing.T) {
payload := []byte(`{"text":"X: hi","Score":6,"Direction":"rx"}`)
msg := &mockMessage{topic: "meshcore/message/direct/d1", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -930,7 +937,7 @@ func TestHandleMessageChannelUppercaseScoreDirection(t *testing.T) {
payload := []byte(`{"text":"Y: hi","Score":4,"Direction":"tx"}`)
msg := &mockMessage{topic: "meshcore/message/channel/5", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
@@ -961,7 +968,7 @@ func TestHandleMessageRawLowercaseScore(t *testing.T) {
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
payload := []byte(`{"raw":"` + rawHex + `","score":3.5}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var score *float64
if err := store.db.QueryRow("SELECT score FROM observations LIMIT 1").Scan(&score); err != nil {
@@ -980,7 +987,7 @@ func TestHandleMessageStatusNoOrigin(t *testing.T) {
topic: "meshcore/LAX/obs5/status",
payload: []byte(`{"model":"L1"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id = 'obs5'").Scan(&count); err != nil {
@@ -1139,182 +1146,3 @@ func TestDecodeTraceWithPath(t *testing.T) {
t.Errorf("flags=%v, want 3", p.TraceFlags)
}
}
// --- db.go: RemoveStaleObservers (soft-delete) ---
func TestRemoveStaleObservers(t *testing.T) {
store := newTestStore(t)
// Insert an observer with last_seen 30 days ago
err := store.UpsertObserver("obs-old", "OldObserver", "LAX", nil)
if err != nil {
t.Fatal(err)
}
// Override last_seen to 30 days ago
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old")
if err != nil {
t.Fatal(err)
}
// Insert a recent observer
err = store.UpsertObserver("obs-new", "NewObserver", "NYC", nil)
if err != nil {
t.Fatal(err)
}
removed, err := store.RemoveStaleObservers(14)
if err != nil {
t.Fatal(err)
}
if removed != 1 {
t.Errorf("removed=%d, want 1", removed)
}
// Observer should still be in the table (soft-delete), but marked inactive
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 2 {
t.Errorf("observers count=%d, want 2 (soft-delete preserves row)", count)
}
// Check that the old observer is marked inactive
var inactive int
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil {
t.Fatal(err)
}
if inactive != 1 {
t.Errorf("obs-old inactive=%d, want 1", inactive)
}
// Check that the recent observer is still active
var newInactive int
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil {
t.Fatal(err)
}
if newInactive != 0 {
t.Errorf("obs-new inactive=%d, want 0", newInactive)
}
}
func TestRemoveStaleObserversNone(t *testing.T) {
store := newTestStore(t)
removed, err := store.RemoveStaleObservers(14)
if err != nil {
t.Fatal(err)
}
if removed != 0 {
t.Errorf("removed=%d, want 0", removed)
}
}
func TestRemoveStaleObserversKeepForever(t *testing.T) {
store := newTestStore(t)
// Insert an old observer
err := store.UpsertObserver("obs-ancient", "AncientObserver", "LAX", nil)
if err != nil {
t.Fatal(err)
}
cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339)
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient")
if err != nil {
t.Fatal(err)
}
// observerDays = -1 means keep forever
removed, err := store.RemoveStaleObservers(-1)
if err != nil {
t.Fatal(err)
}
if removed != 0 {
t.Errorf("removed=%d, want 0 (keep forever)", removed)
}
var count int
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 1 {
t.Errorf("observers count=%d, want 1 (keep forever)", count)
}
// Observer should NOT be marked inactive
var inactive int
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil {
t.Fatal(err)
}
if inactive != 0 {
t.Errorf("obs-ancient inactive=%d, want 0 (keep forever)", inactive)
}
}
func TestRemoveStaleObserversReactivation(t *testing.T) {
store := newTestStore(t)
// Insert and stale-mark an observer
err := store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
if err != nil {
t.Fatal(err)
}
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test")
if err != nil {
t.Fatal(err)
}
removed, err := store.RemoveStaleObservers(14)
if err != nil {
t.Fatal(err)
}
if removed != 1 {
t.Errorf("removed=%d, want 1", removed)
}
// Verify it's inactive
var inactive int
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
t.Fatal(err)
}
if inactive != 1 {
t.Errorf("inactive=%d, want 1 after soft-delete", inactive)
}
// Now UpsertObserver should reactivate it
err = store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
if err != nil {
t.Fatal(err)
}
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
t.Fatal(err)
}
if inactive != 0 {
t.Errorf("inactive=%d, want 0 after reactivation", inactive)
}
}
func TestObserverDaysOrDefault(t *testing.T) {
tests := []struct {
name string
cfg *Config
want int
}{
{"nil retention", &Config{}, 14},
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.cfg.ObserverDaysOrDefault()
if got != tt.want {
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
}
})
}
}
+11 -167
View File
@@ -22,7 +22,6 @@ type DBStats struct {
NodeUpserts atomic.Int64
ObserverUpserts atomic.Int64
WriteErrors atomic.Int64
SignatureDrops atomic.Int64
}
// Store wraps the SQLite database for packet ingestion.
@@ -111,8 +110,7 @@ func applySchema(db *sql.DB) error {
radio TEXT,
battery_mv INTEGER,
uptime_secs INTEGER,
noise_floor REAL,
inactive INTEGER DEFAULT 0
noise_floor REAL
);
CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen);
@@ -197,7 +195,7 @@ func applySchema(db *sql.DB) error {
t.created_at
FROM observations o
JOIN transmissions t ON t.id = o.transmission_id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0)
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
`)
if vErr != nil {
return fmt.Errorf("packets_v view: %w", vErr)
@@ -337,19 +335,6 @@ func applySchema(db *sql.DB) error {
log.Println("[migration] observer_metrics timestamp index created")
}
// Migration: add inactive column to observers for soft-delete retention
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'")
if row.Scan(&migDone) != nil {
log.Println("[migration] Adding inactive column to observers...")
_, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`)
if err != nil {
// Column may already exist (e.g. fresh install with schema above)
log.Printf("[migration] observers.inactive: %v (may already exist)", err)
}
db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`)
log.Println("[migration] observers.inactive column added")
}
// Migration: add packets_sent and packets_recv columns to observer_metrics
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'")
if row.Scan(&migDone) != nil {
@@ -360,54 +345,6 @@ func applySchema(db *sql.DB) error {
log.Println("[migration] packets_sent/packets_recv columns added")
}
// Migration: add channel_hash column for fast channel queries (#762)
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'")
if row.Scan(&migDone) != nil {
log.Println("[migration] Adding channel_hash column to transmissions...")
db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`)
db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`)
// Backfill: extract channel name for decrypted (CHAN) packets
res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`)
if err == nil {
n, _ := res.RowsAffected()
log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n)
}
// Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_'
res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`)
if err == nil {
n, _ := res.RowsAffected()
log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n)
}
db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`)
log.Println("[migration] channel_hash column added and backfilled")
}
// Migration: dropped_packets table for signature validation failures (#793)
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'")
if row.Scan(&migDone) != nil {
log.Println("[migration] Creating dropped_packets table...")
_, err := db.Exec(`
CREATE TABLE IF NOT EXISTS dropped_packets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT,
raw_hex TEXT,
reason TEXT NOT NULL,
observer_id TEXT,
observer_name TEXT,
node_pubkey TEXT,
node_name TEXT,
dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id);
CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey);
`)
if err != nil {
return fmt.Errorf("dropped_packets schema: %w", err)
}
db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`)
log.Println("[migration] dropped_packets table created")
}
return nil
}
@@ -420,8 +357,8 @@ func (s *Store) prepareStatements() error {
}
s.stmtInsertTransmission, err = s.db.Prepare(`
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json)
VALUES (?, ?, ?, ?, ?, ?, ?)
`)
if err != nil {
return err
@@ -433,12 +370,8 @@ func (s *Store) prepareStatements() error {
}
s.stmtInsertObservation, err = s.db.Prepare(`
INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp)
INSERT OR IGNORE INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET
snr = COALESCE(excluded.snr, snr),
rssi = COALESCE(excluded.rssi, rssi),
score = COALESCE(excluded.score, score)
`)
if err != nil {
return err
@@ -548,7 +481,7 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
result, err := s.stmtInsertTransmission.Exec(
data.RawHex, hash, now,
data.RouteType, data.PayloadType, data.PayloadVersion,
data.DecodedJSON, nilIfEmpty(data.ChannelHash),
data.DecodedJSON,
)
if err != nil {
s.Stats.WriteErrors.Add(1)
@@ -689,13 +622,10 @@ func (s *Store) UpsertObserver(id, name, iata string, meta *ObserverMeta) error
)
if err != nil {
s.Stats.WriteErrors.Add(1)
return err
} else {
s.Stats.ObserverUpserts.Add(1)
}
s.Stats.ObserverUpserts.Add(1)
// Reactivate if this observer was previously marked inactive
s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = ? AND inactive = 1`, id)
return nil
return err
}
// Close checkpoints the WAL and closes the database.
@@ -789,14 +719,13 @@ func (s *Store) Checkpoint() {
// LogStats logs current operational metrics.
func (s *Store) LogStats() {
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d sig_drops=%d",
log.Printf("[stats] tx_inserted=%d tx_dupes=%d obs_inserted=%d node_upserts=%d observer_upserts=%d write_errors=%d",
s.Stats.TransmissionsInserted.Load(),
s.Stats.DuplicateTransmissions.Load(),
s.Stats.ObservationsInserted.Load(),
s.Stats.NodeUpserts.Load(),
s.Stats.ObserverUpserts.Load(),
s.Stats.WriteErrors.Load(),
s.Stats.SignatureDrops.Load(),
)
}
@@ -828,71 +757,6 @@ func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) {
return moved, nil
}
// RemoveStaleObservers marks observers that have not actively sent data in observerDays
// as inactive (soft-delete). This preserves JOIN integrity for observations.observer_idx
// and observer_metrics.observer_id — historical data still references the correct observer.
// An observer must actively send data to stay listed — being seen by another node does not count.
// observerDays <= -1 means never remove (keep forever).
func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) {
if observerDays <= -1 {
return 0, nil // keep forever
}
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
result, err := s.db.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
if err != nil {
return 0, fmt.Errorf("mark stale observers inactive: %w", err)
}
removed, _ := result.RowsAffected()
if removed > 0 {
// Clean up orphaned metrics for now-inactive observers
s.db.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
log.Printf("Marked %d observer(s) as inactive (not seen in %d days)", removed, observerDays)
}
return removed, nil
}
// DroppedPacket holds data for a packet rejected during ingest.
type DroppedPacket struct {
Hash string
RawHex string
Reason string
ObserverID string
ObserverName string
NodePubKey string
NodeName string
}
// InsertDroppedPacket records a rejected packet in the dropped_packets table.
func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error {
_, err := s.db.Exec(
`INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (?, ?, ?, ?, ?, ?, ?)`,
dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName,
)
if err != nil {
s.Stats.WriteErrors.Add(1)
return fmt.Errorf("insert dropped packet: %w", err)
}
s.Stats.SignatureDrops.Add(1)
return nil
}
// PruneDroppedPackets removes dropped_packets older than retentionDays.
func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) {
if retentionDays <= 0 {
return 0, nil
}
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339)
result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < ?`, cutoff)
if err != nil {
return 0, fmt.Errorf("prune dropped packets: %w", err)
}
n, _ := result.RowsAffected()
if n > 0 {
log.Printf("Pruned %d dropped packet(s) older than %d days", n, retentionDays)
}
return n, nil
}
// PacketData holds the data needed to insert a packet into the DB.
type PacketData struct {
RawHex string
@@ -909,15 +773,6 @@ type PacketData struct {
PayloadVersion int
PathJSON string
DecodedJSON string
ChannelHash string // grouping key for channel queries (#762)
}
// nilIfEmpty returns nil for empty strings (for nullable DB columns).
func nilIfEmpty(s string) interface{} {
if s == "" {
return nil
}
return s
}
// MQTTPacketMessage is the JSON payload from an MQTT raw packet message.
@@ -939,7 +794,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
pathJSON = string(b)
}
pd := &PacketData{
return &PacketData{
RawHex: msg.Raw,
Timestamp: now,
ObserverID: observerID,
@@ -955,15 +810,4 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
PathJSON: pathJSON,
DecodedJSON: PayloadJSON(&decoded.Payload),
}
// Populate channel_hash for fast channel queries (#762)
if decoded.Header.PayloadType == PayloadGRP_TXT {
if decoded.Payload.Type == "CHAN" && decoded.Payload.Channel != "" {
pd.ChannelHash = decoded.Payload.Channel
} else if decoded.Payload.Type == "GRP_TXT" && decoded.Payload.ChannelHashHex != "" {
pd.ChannelHash = "enc_" + decoded.Payload.ChannelHashHex
}
}
return pd
}
-86
View File
@@ -1882,89 +1882,3 @@ func TestExtractObserverMetaNewFields(t *testing.T) {
t.Errorf("RecvErrors = %v, want 3", meta.RecvErrors)
}
}
// TestInsertObservationSNRFillIn verifies that when the same observation is
// received twice — first without SNR, then with SNR — the SNR is filled in
// rather than silently discarded. The unique dedup index is
// (transmission_id, observer_idx, COALESCE(path_json, '')); observer_idx must
// be non-NULL for the conflict to fire (SQLite treats NULL != NULL).
func TestInsertObservationSNRFillIn(t *testing.T) {
s, err := OpenStore(tempDBPath(t))
if err != nil {
t.Fatal(err)
}
defer s.Close()
// Register the observer so observer_idx is non-NULL (required for dedup).
if err := s.UpsertObserver("pymc-obs1", "PyMC Observer", "SJC", nil); err != nil {
t.Fatal(err)
}
// First arrival: same observer, no SNR/RSSI (e.g. broker replay without RF fields).
data1 := &PacketData{
RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976",
Timestamp: "2026-04-20T00:00:00Z",
Hash: "snrfillin0001hash",
RouteType: 1,
ObserverID: "pymc-obs1",
SNR: nil,
RSSI: nil,
}
if _, err := s.InsertTransmission(data1); err != nil {
t.Fatal(err)
}
var snr1, rssi1 *float64
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr1, &rssi1)
if snr1 != nil || rssi1 != nil {
t.Fatalf("precondition: first insert should have nil SNR/RSSI, got snr=%v rssi=%v", snr1, rssi1)
}
// Second arrival: same packet, same observer, now WITH SNR/RSSI.
snr := 10.5
rssi := -88.0
data2 := &PacketData{
RawHex: data1.RawHex,
Timestamp: data1.Timestamp,
Hash: data1.Hash,
RouteType: data1.RouteType,
ObserverID: "pymc-obs1",
SNR: &snr,
RSSI: &rssi,
}
if _, err := s.InsertTransmission(data2); err != nil {
t.Fatal(err)
}
var snr2, rssi2 *float64
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr2, &rssi2)
if snr2 == nil || *snr2 != snr {
t.Errorf("SNR not filled in by second arrival: got %v, want %v", snr2, snr)
}
if rssi2 == nil || *rssi2 != rssi {
t.Errorf("RSSI not filled in by second arrival: got %v, want %v", rssi2, rssi)
}
// Third arrival: same packet again, SNR absent — must NOT overwrite existing SNR.
data3 := &PacketData{
RawHex: data1.RawHex,
Timestamp: data1.Timestamp,
Hash: data1.Hash,
RouteType: data1.RouteType,
ObserverID: "pymc-obs1",
SNR: nil,
RSSI: nil,
}
if _, err := s.InsertTransmission(data3); err != nil {
t.Fatal(err)
}
var snr3, rssi3 *float64
s.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr3, &rssi3)
if snr3 == nil || *snr3 != snr {
t.Errorf("SNR overwritten by null arrival: got %v, want %v", snr3, snr)
}
if rssi3 == nil || *rssi3 != rssi {
t.Errorf("RSSI overwritten by null arrival: got %v, want %v", rssi3, rssi)
}
}
+14 -52
View File
@@ -80,10 +80,9 @@ type TransportCodes struct {
// Path holds decoded path/hop information.
type Path struct {
HashSize int `json:"hashSize"`
HashCount int `json:"hashCount"`
Hops []string `json:"hops"`
HopsCompleted *int `json:"hopsCompleted,omitempty"`
HashSize int `json:"hashSize"`
HashCount int `json:"hashCount"`
Hops []string `json:"hops"`
}
// AdvertFlags holds decoded advert flag bits.
@@ -144,7 +143,6 @@ type DecodedPacket struct {
Path Path `json:"path"`
Payload Payload `json:"payload"`
Raw string `json:"raw"`
Anomaly string `json:"anomaly,omitempty"`
}
func decodeHeader(b byte) Header {
@@ -239,13 +237,8 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload {
}
if validateSignatures {
valid, err := sigvalidate.ValidateAdvert(buf[0:32], buf[36:100], timestamp, appdata)
if err != nil {
f := false
p.SignatureValid = &f
} else {
p.SignatureValid = &valid
}
valid := sigvalidate.ValidateAdvertSignature(buf[0:32], buf[36:100], timestamp, appdata)
p.SignatureValid = &valid
}
if len(appdata) > 0 {
@@ -588,35 +581,17 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
payload := decodePayload(header.PayloadType, payloadBuf, channelKeys, validateSignatures)
// TRACE packets store hop IDs in the payload (buf[9:]) rather than the header
// path field. Firmware always sends TRACE as DIRECT (route_type 2 or 3);
// FLOOD-routed TRACEs are anomalous but handled gracefully (parsed, but
// flagged). The TRACE flags byte (payload offset 8) encodes path_sz in
// bits 0-1 as a power-of-two exponent: hash_bytes = 1 << path_sz.
// NOT the header path byte's hash_size bits. The header path contains SNR
// bytes — one per hop that actually forwarded.
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
// how far the trace got vs the full intended route.
var anomaly string
// path field. The header path byte still encodes hashSize in bits 6-7, which
// we use to split the payload path data into individual hop prefixes.
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
// Flag anomalous routing — firmware only sends TRACE as DIRECT
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
anomaly = "TRACE packet with non-DIRECT routing (expected DIRECT or TRANSPORT_DIRECT)"
}
// The header path hops count represents SNR entries = completed hops
hopsCompleted := path.HashCount
pathBytes, err := hex.DecodeString(payload.PathData)
if err == nil && payload.TraceFlags != nil {
// path_sz from flags byte is a power-of-two exponent per firmware:
// hash_bytes = 1 << (flags & 0x03)
pathSz := 1 << (*payload.TraceFlags & 0x03)
hops := make([]string, 0, len(pathBytes)/pathSz)
for i := 0; i+pathSz <= len(pathBytes); i += pathSz {
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+pathSz])))
if err == nil && path.HashSize > 0 {
hops := make([]string, 0, len(pathBytes)/path.HashSize)
for i := 0; i+path.HashSize <= len(pathBytes); i += path.HashSize {
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+path.HashSize])))
}
path.Hops = hops
path.HashCount = len(hops)
path.HashSize = pathSz
path.HopsCompleted = &hopsCompleted
}
}
@@ -636,14 +611,12 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
Path: path,
Payload: payload,
Raw: strings.ToUpper(hexString),
Anomaly: anomaly,
}, nil
}
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
// route-independent identifier for the same logical packet. For TRACE packets,
// path_len is included in the hash to match firmware behavior.
// It hashes the header byte + payload (skipping path bytes) to produce a
// path-independent identifier for the same transmission.
func ComputeContentHash(rawHex string) string {
buf, err := hex.DecodeString(rawHex)
if err != nil || len(buf) < 2 {
@@ -679,18 +652,7 @@ func ComputeContentHash(rawHex string) string {
}
payload := buf[payloadStart:]
// Hash payload-type byte only (bits 2-5 of header), not the full header.
// Firmware: SHA256(payload_type + [path_len for TRACE] + payload)
// Using the full header caused different hashes for the same logical packet
// when route type or version bits differed. See issue #786.
payloadType := (headerByte >> 2) & 0x0F
toHash := []byte{payloadType}
if int(payloadType) == PayloadTRACE {
// Firmware uses uint16_t path_len (2 bytes, little-endian)
toHash = append(toHash, pathByte, 0x00)
}
toHash = append(toHash, payload...)
toHash := append([]byte{headerByte}, payload...)
h := sha256.Sum256(toHash)
return hex.EncodeToString(h[:])[:16]
+10 -137
View File
@@ -926,93 +926,6 @@ func TestComputeContentHashLongFallback(t *testing.T) {
}
}
// TestComputeContentHashRouteTypeIndependence verifies that the same logical
// packet produces the same content hash regardless of route type (issue #786).
func TestComputeContentHashRouteTypeIndependence(t *testing.T) {
// Same payload type (TXT_MSG=2, bits 2-5) with different route types.
// Header 0x08 = route_type 0 (TRANSPORT_FLOOD), payload_type 2
// Header 0x0A = route_type 2 (DIRECT), payload_type 2
// Header 0x09 = route_type 1 (FLOOD), payload_type 2
// pathByte=0x00, payload=D69FD7A5A7
payloadHex := "D69FD7A5A7"
// FLOOD: header=0x09 (route_type 1), pathByte=0x00
floodHex := "09" + "00" + payloadHex
// DIRECT: header=0x0A (route_type 2), pathByte=0x00
directHex := "0A" + "00" + payloadHex
hashFlood := ComputeContentHash(floodHex)
hashDirect := ComputeContentHash(directHex)
if hashFlood != hashDirect {
t.Errorf("same payload with different route types produced different hashes: flood=%s direct=%s", hashFlood, hashDirect)
}
}
// TestComputeContentHashTraceIncludesPathLen verifies TRACE packets include
// path_len in the hash (matching firmware behavior).
func TestComputeContentHashTraceIncludesPathLen(t *testing.T) {
// TRACE = payload_type 0x09, so header bits 2-5 = 0x09 → header = 0x09<<2 | route=2 = 0x26
// pathByte=0x01 (1 hop, 1-byte hash) → 1 path byte
traceHeader1 := "26" // route=2, payload_type=9
pathByte1 := "01"
pathData1 := "AA"
payload := "DEADBEEF"
hex1 := traceHeader1 + pathByte1 + pathData1 + payload
// Same but pathByte=0x02 (2 hops) → 2 path bytes
pathByte2 := "02"
pathData2 := "AABB"
hex2 := traceHeader1 + pathByte2 + pathData2 + payload
hash1 := ComputeContentHash(hex1)
hash2 := ComputeContentHash(hex2)
if hash1 == hash2 {
t.Error("TRACE packets with different path_len should produce different hashes (path_len is part of hash input)")
}
}
// TestComputeContentHashMatchesFirmware verifies hash output matches what the
// firmware would compute: SHA256(payload_type_byte + payload)[:16hex].
func TestComputeContentHashMatchesFirmware(t *testing.T) {
// header=0x0A → payload_type = (0x0A >> 2) & 0x0F = 2
// pathByte=0x00, payload = D69FD7A5A7475DB07337749AE61FA53A4788E976
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
hash := ComputeContentHash(rawHex)
// Manually compute expected: SHA256(0x02 + payload_bytes)
payloadBytes, _ := hex.DecodeString("D69FD7A5A7475DB07337749AE61FA53A4788E976")
toHash := append([]byte{0x02}, payloadBytes...)
expected := sha256.Sum256(toHash)
expectedHex := hex.EncodeToString(expected[:])[:16]
if hash != expectedHex {
t.Errorf("hash=%s, want %s (firmware-compatible)", hash, expectedHex)
}
}
// TestComputeContentHashTraceGoldenValue is a golden-value test that locks down
// the 2-byte path_len (uint16 LE) behavior for TRACE hashing. If anyone removes
// the 0x00 byte from the hash input, this test breaks.
//
// Packet: header=0x25 (FLOOD route=1, payload_type=TRACE=0x09), pathByte=0x02
// (2 hops, 1-byte hash), path=[AA,BB], payload=[DE,AD,BE,EF].
// Hash input: [0x09, 0x02, 0x00, 0xDE, 0xAD, 0xBE, 0xEF]
// → SHA256 = b1baaf3bf0d0726c2672b1ec9e2665dc...
// → first 16 hex chars = "b1baaf3bf0d0726c"
func TestComputeContentHashTraceGoldenValue(t *testing.T) {
// TRACE packet: header byte 0x25 = payload_type 9 (TRACE), route_type 1 (FLOOD)
// pathByte 0x02 = hash_size 1, hash_count 2
// 2 path bytes (AA, BB), then payload DEADBEEF
rawHex := "2502AABBDEADBEEF"
hash := ComputeContentHash(rawHex)
// Pre-computed: SHA256(0x09 0x02 0x00 0xDE 0xAD 0xBE 0xEF)[:16hex]
// The 0x00 is the high byte of uint16_t path_len (little-endian).
const golden = "b1baaf3bf0d0726c"
if hash != golden {
t.Errorf("TRACE golden hash = %s, want %s (2-byte path_len encoding)", hash, golden)
}
}
func TestDecodePacketWithWhitespace(t *testing.T) {
raw := "0A 00 D6 9F D7 A5 A7 47 5D B0 73 37 74 9A E6 1F A5 3A 47 88 E9 76"
pkt, err := DecodePacket(raw, nil, false)
@@ -1693,32 +1606,6 @@ func TestDirectNonZeroHopKeepsHashSize(t *testing.T) {
}
}
func TestZeroHopTransportDirectHashSize(t *testing.T) {
// TRANSPORT_DIRECT (RouteType=3) + REQ (PayloadType=0) → header byte = 0x03
// 4 bytes transport codes + pathByte=0x00 → hash_count=0 → should get HashSize=0
hex := "03" + "11223344" + "00" + repeatHex("AA", 20)
pkt, err := DecodePacket(hex, nil, false)
if err != nil {
t.Fatalf("DecodePacket failed: %v", err)
}
if pkt.Path.HashSize != 0 {
t.Errorf("TRANSPORT_DIRECT zero-hop: want HashSize=0, got %d", pkt.Path.HashSize)
}
}
func TestZeroHopTransportDirectHashSizeWithNonZeroUpperBits(t *testing.T) {
// TRANSPORT_DIRECT (RouteType=3) + REQ (PayloadType=0) → header byte = 0x03
// 4 bytes transport codes + pathByte=0xC0 → hash_count=0, hash_size bits=11 → should still get HashSize=0
hex := "03" + "11223344" + "C0" + repeatHex("AA", 20)
pkt, err := DecodePacket(hex, nil, false)
if err != nil {
t.Fatalf("DecodePacket failed: %v", err)
}
if pkt.Path.HashSize != 0 {
t.Errorf("TRANSPORT_DIRECT zero-hop with hash_size bits set: want HashSize=0, got %d", pkt.Path.HashSize)
}
}
func TestValidateAdvertSignature(t *testing.T) {
// Generate a real ed25519 key pair
pub, priv, err := ed25519.GenerateKey(nil)
@@ -1729,7 +1616,7 @@ func TestValidateAdvertSignature(t *testing.T) {
var timestamp uint32 = 1234567890
appdata := []byte{0x02, 0x11, 0x22} // flags + some data
// Build the signed message: pubKey + timestamp(LE) + appdata
// Build the message the same way ValidateAdvertSignature does
message := make([]byte, 32+4+len(appdata))
copy(message[0:32], pub)
binary.LittleEndian.PutUint32(message[32:36], timestamp)
@@ -1738,43 +1625,29 @@ func TestValidateAdvertSignature(t *testing.T) {
sig := ed25519.Sign(priv, message)
// Valid signature
valid, err := sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp, appdata)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !valid {
if !sigvalidate.ValidateAdvertSignature(pub, sig, timestamp, appdata) {
t.Error("expected valid signature")
}
// Tampered appdata → invalid
badAppdata := []byte{0x03, 0x11, 0x22}
valid, err = sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp, badAppdata)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if valid {
if sigvalidate.ValidateAdvertSignature(pub, sig, timestamp, badAppdata) {
t.Error("expected invalid signature with tampered appdata")
}
// Wrong timestamp → invalid
valid, err = sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp+1, appdata)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if valid {
if sigvalidate.ValidateAdvertSignature(pub, sig, timestamp+1, appdata) {
t.Error("expected invalid signature with wrong timestamp")
}
// Wrong length pubkey
_, err = sigvalidate.ValidateAdvert([]byte{0xAA, 0xBB}, sig, timestamp, appdata)
if err == nil {
t.Error("expected error for short pubkey")
// Wrong length pubkey → false
if sigvalidate.ValidateAdvertSignature([]byte{0xAA, 0xBB}, sig, timestamp, appdata) {
t.Error("expected false for short pubkey")
}
// Wrong length signature
_, err = sigvalidate.ValidateAdvert([]byte(pub), []byte{0xAA, 0xBB}, timestamp, appdata)
if err == nil {
t.Error("expected error for short signature")
// Wrong length signature → false
if sigvalidate.ValidateAdvertSignature(pub, []byte{0xAA, 0xBB}, timestamp, appdata) {
t.Error("expected false for short signature")
}
}
+1 -1
View File
@@ -5,7 +5,7 @@ go 1.22
require (
github.com/eclipse/paho.mqtt.golang v1.5.0
github.com/meshcore-analyzer/geofilter v0.0.0
github.com/meshcore-analyzer/sigvalidate v0.0.0
github.com/meshcore-analyzer/sigvalidate v0.0.0-00010101000000-000000000000
modernc.org/sqlite v1.34.5
)
+26 -65
View File
@@ -61,14 +61,9 @@ func main() {
nodeDays := cfg.NodeDaysOrDefault()
store.MoveStaleNodes(nodeDays)
// Observer retention: remove stale observers on startup
observerDays := cfg.ObserverDaysOrDefault()
store.RemoveStaleObservers(observerDays)
// Metrics retention: prune old metrics on startup
metricsDays := cfg.MetricsRetentionDays()
store.PruneOldMetrics(metricsDays)
store.PruneDroppedPackets(metricsDays)
// Daily ticker for node retention
retentionTicker := time.NewTicker(1 * time.Hour)
@@ -78,22 +73,11 @@ func main() {
}
}()
// Daily ticker for observer retention (every 24h, staggered 90s after startup)
observerRetentionTicker := time.NewTicker(24 * time.Hour)
go func() {
time.Sleep(90 * time.Second) // stagger after metrics prune
store.RemoveStaleObservers(observerDays)
for range observerRetentionTicker.C {
store.RemoveStaleObservers(observerDays)
}
}()
// Daily ticker for metrics retention (every 24h)
metricsRetentionTicker := time.NewTicker(24 * time.Hour)
go func() {
for range metricsRetentionTicker.C {
store.PruneOldMetrics(metricsDays)
store.PruneDroppedPackets(metricsDays)
}
}()
@@ -162,7 +146,7 @@ func main() {
// Capture source for closure
src := source
opts.SetDefaultPublishHandler(func(c mqtt.Client, m mqtt.Message) {
handleMessage(store, tag, src, m, channelKeys, cfg)
handleMessage(store, tag, src, m, channelKeys, cfg.GeoFilter)
})
client := mqtt.NewClient(opts)
@@ -197,7 +181,7 @@ func main() {
log.Println("Done.")
}
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, cfg *Config) {
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, geoFilter *GeoFilterConfig) {
defer func() {
if r := recover(); r != nil {
log.Printf("MQTT [%s] panic in handler: %v", tag, r)
@@ -207,6 +191,21 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
topic := m.Topic()
parts := strings.Split(topic, "/")
// IATA filter
if len(source.IATAFilter) > 0 && len(parts) > 1 {
region := parts[1]
matched := false
for _, f := range source.IATAFilter {
if f == region {
matched = true
break
}
}
if !matched {
return
}
}
var msg map[string]interface{}
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
return
@@ -218,9 +217,6 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
}
// Status topic: meshcore/<region>/<observer_id>/status
// IATA filter does NOT apply here — observer metadata (noise_floor, battery, etc.)
// is region-independent and should be accepted from all observers regardless of
// which IATA regions are configured for packet ingestion.
if len(parts) >= 4 && parts[3] == "status" {
observerID := parts[2]
name, _ := msg["origin"].(string)
@@ -249,26 +245,10 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
return
}
// IATA filter applies to packet messages only — not status messages above.
if len(source.IATAFilter) > 0 && len(parts) > 1 {
region := parts[1]
matched := false
for _, f := range source.IATAFilter {
if f == region {
matched = true
break
}
}
if !matched {
return
}
}
// Format 1: Raw packet (meshcoretomqtt / Cisien format)
rawHex, _ := msg["raw"].(string)
if rawHex != "" {
validateSigs := cfg.ShouldValidateSignatures()
decoded, err := DecodePacket(rawHex, channelKeys, validateSigs)
decoded, err := DecodePacket(rawHex, channelKeys, false)
if err != nil {
log.Printf("MQTT [%s] decode error: %v", tag, err)
return
@@ -328,27 +308,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
log.Printf("MQTT [%s] skipping corrupted ADVERT: %s", tag, reason)
return
}
// Signature validation: drop adverts with invalid ed25519 signatures
if validateSigs && decoded.Payload.SignatureValid != nil && !*decoded.Payload.SignatureValid {
hash := ComputeContentHash(rawHex)
truncPK := decoded.Payload.PubKey
if len(truncPK) > 16 {
truncPK = truncPK[:16]
}
log.Printf("MQTT [%s] DROPPED invalid signature: hash=%s name=%s observer=%s pubkey=%s",
tag, hash, decoded.Payload.Name, firstNonEmpty(mqttMsg.Origin, observerID), truncPK)
store.InsertDroppedPacket(&DroppedPacket{
Hash: hash,
RawHex: rawHex,
Reason: "invalid signature",
ObserverID: observerID,
ObserverName: mqttMsg.Origin,
NodePubKey: decoded.Payload.PubKey,
NodeName: decoded.Payload.Name,
})
return
}
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, cfg.GeoFilter) {
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, geoFilter) {
return
}
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
@@ -480,18 +440,19 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
PayloadType: 5, // GRP_TXT
PathJSON: "[]",
DecodedJSON: string(decodedJSON),
ChannelHash: channelName, // fast channel queries (#762)
}
if _, err := store.InsertTransmission(pktData); err != nil {
log.Printf("MQTT [%s] channel insert error: %v", tag, err)
}
// Note: we intentionally do NOT create a node entry for channel message senders.
// Channel messages don't carry the sender's real pubkey, so any entry we create
// would use a synthetic key ("sender-<name>") that doesn't match the real pubkey
// used for claiming/health lookups. The node will get a proper entry when it
// sends an advert. See issue #665.
// Upsert sender as a companion node
if sender != "" {
senderKey := "sender-" + strings.ToLower(sender)
if err := store.UpsertNode(senderKey, sender, "companion", nil, nil, now); err != nil {
log.Printf("MQTT [%s] sender node upsert error: %v", tag, err)
}
}
log.Printf("MQTT [%s] channel message: ch%s from %s", tag, channelIdx, firstNonEmpty(sender, "unknown"))
return
+22 -63
View File
@@ -130,7 +130,7 @@ func TestHandleMessageRawPacket(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"myobs"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -147,7 +147,7 @@ func TestHandleMessageRawPacketAdvert(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
// Should create a node from the ADVERT
var count int
@@ -169,7 +169,7 @@ func TestHandleMessageInvalidJSON(t *testing.T) {
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: []byte(`not json`)}
// Should not panic
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -186,7 +186,7 @@ func TestHandleMessageStatusTopic(t *testing.T) {
payload: []byte(`{"origin":"MyObserver"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var name, iata string
err := store.db.QueryRow("SELECT name, iata FROM observers WHERE id = 'obs1'").Scan(&name, &iata)
@@ -207,11 +207,11 @@ func TestHandleMessageSkipStatusTopics(t *testing.T) {
// meshcore/status should be skipped
msg1 := &mockMessage{topic: "meshcore/status", payload: []byte(`{"raw":"0A00"}`)}
handleMessage(store, "test", source, msg1, nil, &Config{})
handleMessage(store, "test", source, msg1, nil, nil)
// meshcore/events/connection should be skipped
msg2 := &mockMessage{topic: "meshcore/events/connection", payload: []byte(`{"raw":"0A00"}`)}
handleMessage(store, "test", source, msg2, nil, &Config{})
handleMessage(store, "test", source, msg2, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -230,7 +230,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -243,7 +243,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
topic: "meshcore/LAX/obs2/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg2, nil, &Config{})
handleMessage(store, "test", source, msg2, nil, nil)
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
if count != 1 {
@@ -261,7 +261,7 @@ func TestHandleMessageIATAFilterNoRegion(t *testing.T) {
topic: "meshcore",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
// No region part → filter doesn't apply, message goes through
// Actually the code checks len(parts) > 1 for IATA filter
@@ -277,7 +277,7 @@ func TestHandleMessageNoRawHex(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"type":"companion","data":"something"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -295,7 +295,7 @@ func TestHandleMessageBadRawHex(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"ZZZZ"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -312,7 +312,7 @@ func TestHandleMessageWithSNRRSSIAsNumbers(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"RSSI":-95}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var snr, rssi *float64
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
@@ -331,7 +331,7 @@ func TestHandleMessageMinimalTopic(t *testing.T) {
topic: "meshcore/SJC",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -352,7 +352,7 @@ func TestHandleMessageCorruptedAdvert(t *testing.T) {
topic: "meshcore/SJC/obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
// Transmission should be inserted (even if advert is invalid)
var count int
@@ -378,7 +378,7 @@ func TestHandleMessageNoObserverID(t *testing.T) {
topic: "packets",
payload: []byte(`{"raw":"` + rawHex + `","origin":"obs1"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -400,7 +400,7 @@ func TestHandleMessageSNRNotFloat(t *testing.T) {
// SNR as a string value — should not parse as float
payload := []byte(`{"raw":"` + rawHex + `","SNR":"bad","RSSI":"bad"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
@@ -416,7 +416,7 @@ func TestHandleMessageOriginExtraction(t *testing.T) {
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
payload := []byte(`{"raw":"` + rawHex + `","origin":"MyOrigin"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
// Verify origin was extracted to observer name
var name string
@@ -439,7 +439,7 @@ func TestHandleMessagePanicRecovery(t *testing.T) {
}
// Should not panic — the defer/recover should catch it
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
}
func TestHandleMessageStatusOriginFallback(t *testing.T) {
@@ -451,7 +451,7 @@ func TestHandleMessageStatusOriginFallback(t *testing.T) {
topic: "meshcore/SJC/obs1/status",
payload: []byte(`{"type":"status"}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var name string
err := store.db.QueryRow("SELECT name FROM observers WHERE id = 'obs1'").Scan(&name)
@@ -640,7 +640,7 @@ func TestHandleMessageWithLowercaseSNRRSSI(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `","snr":5.5,"rssi":-102}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var snr, rssi *float64
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
@@ -661,7 +661,7 @@ func TestHandleMessageSNRRSSIUppercaseWins(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"snr":1.0,"RSSI":-95,"rssi":-50}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var snr, rssi *float64
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
@@ -681,7 +681,7 @@ func TestHandleMessageNoSNRRSSI(t *testing.T) {
payload := []byte(`{"raw":"` + rawHex + `"}`)
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
handleMessage(store, "test", source, msg, nil, &Config{})
handleMessage(store, "test", source, msg, nil, nil)
var snr, rssi *float64
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
@@ -739,44 +739,3 @@ func TestToFloat64WithUnits(t *testing.T) {
}
}
}
// TestIATAFilterDoesNotDropStatusMessages verifies that status messages from
// out-of-region observers are still processed (noise_floor, battery, etc.)
// even when an IATA filter is configured for packet data.
func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
store := newTestStore(t)
source := MQTTSource{Name: "test", IATAFilter: []string{"SJC"}}
// BFL observer sends a status message with noise_floor — outside the IATA filter.
msg := &mockMessage{
topic: "meshcore/BFL/bfl-obs1/status",
payload: []byte(`{"origin":"BFLObserver","stats":{"noise_floor":-105.0}}`),
}
handleMessage(store, "test", source, msg, nil, &Config{})
var name string
var noiseFloor *float64
err := store.db.QueryRow("SELECT name, noise_floor FROM observers WHERE id = 'bfl-obs1'").Scan(&name, &noiseFloor)
if err != nil {
t.Fatalf("observer not found after status from out-of-region observer: %v", err)
}
if name != "BFLObserver" {
t.Errorf("name=%q, want BFLObserver", name)
}
if noiseFloor == nil || *noiseFloor != -105.0 {
t.Errorf("noise_floor=%v, want -105.0 — status message was dropped by IATA filter when it should not be", noiseFloor)
}
// Verify that a packet from BFL is still filtered.
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
pktMsg := &mockMessage{
topic: "meshcore/BFL/bfl-obs1/packets",
payload: []byte(`{"raw":"` + rawHex + `"}`),
}
handleMessage(store, "test", source, pktMsg, nil, &Config{})
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
if count != 0 {
t.Error("packet from out-of-region BFL should still be filtered by IATA")
}
}
-339
View File
@@ -1,339 +0,0 @@
package main
import (
"crypto/ed25519"
"encoding/binary"
"encoding/hex"
"strings"
"testing"
)
// buildAdvertHex constructs a full ADVERT packet hex string.
// header(1) + pathByte(1) + pubkey(32) + timestamp(4) + signature(64) + appdata
func buildAdvertHex(pubKey ed25519.PublicKey, privKey ed25519.PrivateKey, timestamp uint32, appdata []byte) string {
// Build signed message: pubkey(32) + timestamp(4 LE) + appdata
msg := make([]byte, 32+4+len(appdata))
copy(msg[0:32], pubKey)
binary.LittleEndian.PutUint32(msg[32:36], timestamp)
copy(msg[36:], appdata)
sig := ed25519.Sign(privKey, msg)
// Payload: pubkey(32) + timestamp(4) + signature(64) + appdata
payload := make([]byte, 0, 100+len(appdata))
payload = append(payload, pubKey...)
ts := make([]byte, 4)
binary.LittleEndian.PutUint32(ts, timestamp)
payload = append(payload, ts...)
payload = append(payload, sig...)
payload = append(payload, appdata...)
// Header: ADVERT (0x04 << 2) | FLOOD (1) = 0x11, pathByte=0 (no hops)
header := byte(0x11)
pathByte := byte(0x00)
pkt := append([]byte{header, pathByte}, payload...)
return hex.EncodeToString(pkt)
}
// makeAppdata builds minimal appdata: flags(1) + name
func makeAppdata(name string) []byte {
flags := byte(0x81) // hasName=true, type=companion(1)
data := []byte{flags}
data = append(data, []byte(name)...)
data = append(data, 0x00) // null terminator
return data
}
func TestSigValidation_ValidAdvertStored(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
appdata := makeAppdata("TestNode")
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
source := MQTTSource{Name: "test"}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"TestObs"}`)
cfg := &Config{}
handleMessage(store, "test", source, msg, nil, cfg)
// Verify packet was stored
var count int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
if count == 0 {
t.Fatal("valid advert should be stored, got 0 transmissions")
}
}
func TestSigValidation_TamperedSignatureDropped(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
appdata := makeAppdata("BadNode")
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
// Tamper with signature (flip a byte in the signature area)
// Signature starts at offset 2 (header+path) + 32 (pubkey) + 4 (timestamp) = 38
// That's byte 38 in the packet, hex chars 76-77
rawBytes := []byte(rawHex)
if rawBytes[76] == '0' {
rawBytes[76] = 'f'
} else {
rawBytes[76] = '0'
}
tamperedHex := string(rawBytes)
source := MQTTSource{Name: "test"}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
cfg := &Config{}
handleMessage(store, "test", source, msg, nil, cfg)
// Verify packet was NOT stored in transmissions
var txCount int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
if txCount != 0 {
t.Fatalf("tampered advert should be dropped, got %d transmissions", txCount)
}
// Verify it was recorded in dropped_packets
var dropCount int
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&dropCount)
if dropCount == 0 {
t.Fatal("tampered advert should be recorded in dropped_packets")
}
// Verify drop counter incremented
if store.Stats.SignatureDrops.Load() != 1 {
t.Fatalf("expected 1 signature drop, got %d", store.Stats.SignatureDrops.Load())
}
// Verify dropped_packets has correct fields
var reason, nodeKey, nodeName, obsID string
store.db.QueryRow("SELECT reason, node_pubkey, node_name, observer_id FROM dropped_packets LIMIT 1").Scan(&reason, &nodeKey, &nodeName, &obsID)
if reason != "invalid signature" {
t.Fatalf("expected reason 'invalid signature', got %q", reason)
}
if nodeKey == "" {
t.Fatal("dropped packet should have node_pubkey")
}
if !strings.Contains(nodeName, "BadNode") {
t.Fatalf("expected node_name to contain 'BadNode', got %q", nodeName)
}
if obsID != "obs1" {
t.Fatalf("expected observer_id 'obs1', got %q", obsID)
}
}
func TestSigValidation_TruncatedAppdataDropped(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
appdata := makeAppdata("TruncNode")
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
// Sign was computed with full appdata. Now truncate the raw hex to remove
// some appdata bytes, making the signature invalid.
// Truncate last 4 hex chars (2 bytes of appdata)
truncatedHex := rawHex[:len(rawHex)-4]
source := MQTTSource{Name: "test"}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+truncatedHex+`","origin":"TestObs"}`)
cfg := &Config{}
handleMessage(store, "test", source, msg, nil, cfg)
var txCount int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
if txCount != 0 {
t.Fatalf("truncated advert should be dropped, got %d transmissions", txCount)
}
}
func TestSigValidation_DisabledByConfig(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
appdata := makeAppdata("NoValNode")
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
// Tamper with signature
rawBytes := []byte(rawHex)
if rawBytes[76] == '0' {
rawBytes[76] = 'f'
} else {
rawBytes[76] = '0'
}
tamperedHex := string(rawBytes)
source := MQTTSource{Name: "test"}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
falseVal := false
cfg := &Config{ValidateSignatures: &falseVal}
handleMessage(store, "test", source, msg, nil, cfg)
// With validation disabled, tampered packet should be stored
var txCount int
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
if txCount == 0 {
t.Fatal("with validateSignatures=false, tampered advert should be stored")
}
}
func TestSigValidation_DropCounterIncrements(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
source := MQTTSource{Name: "test"}
cfg := &Config{}
for i := 0; i < 3; i++ {
appdata := makeAppdata("Node")
rawHex := buildAdvertHex(pub, priv, uint32(1700000000+i), appdata)
// Tamper
rawBytes := []byte(rawHex)
if rawBytes[76] == '0' {
rawBytes[76] = 'f'
} else {
rawBytes[76] = '0'
}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"Obs"}`)
handleMessage(store, "test", source, msg, nil, cfg)
}
if store.Stats.SignatureDrops.Load() != 3 {
t.Fatalf("expected 3 signature drops, got %d", store.Stats.SignatureDrops.Load())
}
}
func TestSigValidation_LogContainsFields(t *testing.T) {
// This test verifies the dropped_packets row has all required fields
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
pub, priv, _ := ed25519.GenerateKey(nil)
appdata := makeAppdata("LogTestNode")
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
// Tamper
rawBytes := []byte(rawHex)
if rawBytes[76] == '0' {
rawBytes[76] = 'f'
} else {
rawBytes[76] = '0'
}
source := MQTTSource{Name: "test"}
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"MyObserver"}`)
cfg := &Config{}
handleMessage(store, "test", source, msg, nil, cfg)
var hash, reason, obsID, obsName, pubkey, nodeName string
err = store.db.QueryRow("SELECT hash, reason, observer_id, observer_name, node_pubkey, node_name FROM dropped_packets LIMIT 1").
Scan(&hash, &reason, &obsID, &obsName, &pubkey, &nodeName)
if err != nil {
t.Fatal(err)
}
if hash == "" {
t.Error("dropped packet should have hash")
}
if reason != "invalid signature" {
t.Errorf("expected reason 'invalid signature', got %q", reason)
}
if obsID != "obs1" {
t.Errorf("expected observer_id 'obs1', got %q", obsID)
}
if obsName != "MyObserver" {
t.Errorf("expected observer_name 'MyObserver', got %q", obsName)
}
if pubkey == "" {
t.Error("dropped packet should have node_pubkey")
}
if !strings.Contains(nodeName, "LogTestNode") {
t.Errorf("expected node_name containing 'LogTestNode', got %q", nodeName)
}
}
func TestPruneDroppedPackets(t *testing.T) {
dbPath := t.TempDir() + "/test.db"
store, err := OpenStoreWithInterval(dbPath, 300)
if err != nil {
t.Fatal(err)
}
defer store.Close()
// Insert an old dropped packet
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`)
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`)
n, err := store.PruneDroppedPackets(30)
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("expected 1 pruned, got %d", n)
}
var count int
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
if count != 1 {
t.Fatalf("expected 1 remaining, got %d", count)
}
}
func TestShouldValidateSignatures_Default(t *testing.T) {
cfg := &Config{}
if !cfg.ShouldValidateSignatures() {
t.Fatal("default should be true")
}
falseVal := false
cfg2 := &Config{ValidateSignatures: &falseVal}
if cfg2.ShouldValidateSignatures() {
t.Fatal("explicit false should be false")
}
trueVal := true
cfg3 := &Config{ValidateSignatures: &trueVal}
if !cfg3.ShouldValidateSignatures() {
t.Fatal("explicit true should be true")
}
}
// newMockMsg creates a minimal mqtt.Message for testing.
func newMockMsg(topic, payload string) *mockMessage {
return &mockMessage{topic: topic, payload: []byte(payload)}
}
-321
View File
@@ -1,321 +0,0 @@
package main
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"time"
_ "modernc.org/sqlite"
)
// createTestDB creates a temporary SQLite database with N transmissions (1 obs each).
func createTestDB(t *testing.T, numTx int) string {
t.Helper()
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
createTestDBAt(t, dbPath, numTx)
return dbPath
}
// loadStore creates a PacketStore from a test DB with given maxMemoryMB.
func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore {
t.Helper()
db, err := OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
cfg := &PacketStoreConfig{MaxMemoryMB: maxMemMB}
store := NewPacketStore(db, cfg)
if err := store.Load(); err != nil {
t.Fatal(err)
}
return store
}
func TestBoundedLoad_LimitedMemory(t *testing.T) {
dbPath := createTestDB(t, 5000)
defer os.RemoveAll(filepath.Dir(dbPath))
// Use 1MB budget — should load far fewer than 5000 packets
store := loadStore(t, dbPath, 1)
defer store.db.conn.Close()
loaded := len(store.packets)
if loaded >= 5000 {
t.Errorf("expected bounded load to limit packets, got %d/5000", loaded)
}
if loaded < 1000 {
t.Errorf("expected at least 1000 packets (minimum), got %d", loaded)
}
t.Logf("Loaded %d/5000 packets with 1MB budget", loaded)
}
func TestBoundedLoad_NewestFirst(t *testing.T) {
dbPath := createTestDB(t, 5000)
defer os.RemoveAll(filepath.Dir(dbPath))
store := loadStore(t, dbPath, 1)
defer store.db.conn.Close()
loaded := len(store.packets)
if loaded >= 5000 {
t.Skip("all packets loaded, can't verify newest-first")
}
// The newest packet in DB has first_seen based on minute 5000.
// The loaded packets should be the newest ones.
// Last packet in store (sorted ASC) should be the newest in DB.
last := store.packets[loaded-1]
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
newestExpected := base.Add(5000 * time.Minute).Format(time.RFC3339)
if last.FirstSeen != newestExpected {
t.Errorf("expected last packet to be newest (%s), got %s", newestExpected, last.FirstSeen)
}
// First packet should NOT be the oldest in the DB (minute 1)
first := store.packets[0]
oldestAll := base.Add(1 * time.Minute).Format(time.RFC3339)
if first.FirstSeen == oldestAll {
t.Errorf("first loaded packet should not be the absolute oldest when bounded")
}
}
func TestBoundedLoad_OldestLoadedSet(t *testing.T) {
dbPath := createTestDB(t, 5000)
defer os.RemoveAll(filepath.Dir(dbPath))
store := loadStore(t, dbPath, 1)
defer store.db.conn.Close()
if store.oldestLoaded == "" {
t.Fatal("oldestLoaded should be set after bounded load")
}
if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen {
t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen)
}
t.Logf("oldestLoaded = %s", store.oldestLoaded)
}
func TestBoundedLoad_UnlimitedWithZero(t *testing.T) {
dbPath := createTestDB(t, 200)
defer os.RemoveAll(filepath.Dir(dbPath))
store := loadStore(t, dbPath, 0)
defer store.db.conn.Close()
if len(store.packets) != 200 {
t.Errorf("expected all 200 packets with maxMemoryMB=0, got %d", len(store.packets))
}
}
func TestBoundedLoad_AscendingOrder(t *testing.T) {
dbPath := createTestDB(t, 3000)
defer os.RemoveAll(filepath.Dir(dbPath))
store := loadStore(t, dbPath, 1)
defer store.db.conn.Close()
// Verify packets are in ascending first_seen order
for i := 1; i < len(store.packets); i++ {
if store.packets[i].FirstSeen < store.packets[i-1].FirstSeen {
t.Fatalf("packets not in ascending order at index %d: %s < %s",
i, store.packets[i].FirstSeen, store.packets[i-1].FirstSeen)
}
}
}
func TestEstimateStoreTxBytesTypical(t *testing.T) {
est := estimateStoreTxBytesTypical(10)
if est < 1000 {
t.Errorf("typical estimate too low: %d", est)
}
// Should be roughly proportional to observation count
est1 := estimateStoreTxBytesTypical(1)
est20 := estimateStoreTxBytesTypical(20)
if est20 <= est1 {
t.Errorf("estimate should grow with observations: 1obs=%d, 20obs=%d", est1, est20)
}
t.Logf("Typical estimate: 1obs=%d, 10obs=%d, 20obs=%d bytes", est1, est, est20)
}
func BenchmarkLoad_Bounded(b *testing.B) {
dir := b.TempDir()
dbPath := filepath.Join(dir, "bench.db")
createTestDBAt(b, dbPath, 5000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, _ := OpenDB(dbPath)
cfg := &PacketStoreConfig{MaxMemoryMB: 1}
store := NewPacketStore(db, cfg)
store.Load()
db.conn.Close()
}
}
func BenchmarkLoad_Unlimited(b *testing.B) {
dir := b.TempDir()
dbPath := filepath.Join(dir, "bench.db")
createTestDBAt(b, dbPath, 5000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, _ := OpenDB(dbPath)
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
store := NewPacketStore(db, cfg)
store.Load()
db.conn.Close()
}
}
// BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions
// and realistic observation counts (15 per transmission).
func BenchmarkLoad_30K_Bounded(b *testing.B) {
dir := b.TempDir()
dbPath := filepath.Join(dir, "bench30k.db")
createTestDBWithObs(b, dbPath, 30000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, _ := OpenDB(dbPath)
cfg := &PacketStoreConfig{MaxMemoryMB: 50}
store := NewPacketStore(db, cfg)
store.Load()
db.conn.Close()
}
}
// BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions
// and realistic observation counts (15 per transmission).
func BenchmarkLoad_30K_Unlimited(b *testing.B) {
dir := b.TempDir()
dbPath := filepath.Join(dir, "bench30k.db")
createTestDBWithObs(b, dbPath, 30000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, _ := OpenDB(dbPath)
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
store := NewPacketStore(db, cfg)
store.Load()
db.conn.Close()
}
}
// createTestDBAt is like createTestDB but writes to a specific path.
func createTestDBAt(tb testing.TB, dbPath string, numTx int) {
tb.Helper()
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
if err != nil {
tb.Fatal(err)
}
defer conn.Close()
execOrFail := func(sql string) {
if _, err := conn.Exec(sql); err != nil {
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql)
}
}
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
id INTEGER PRIMARY KEY,
raw_hex TEXT, hash TEXT, first_seen TEXT,
route_type INTEGER, payload_type INTEGER,
payload_version INTEGER, decoded_json TEXT
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
id INTEGER PRIMARY KEY,
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
direction TEXT, snr REAL, rssi REAL, score INTEGER,
path_json TEXT, timestamp TEXT
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
last_seen TEXT, first_seen TEXT, frequency REAL
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
if err != nil {
tb.Fatalf("test DB prepare transmissions insert: %v", err)
}
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
if err != nil {
tb.Fatalf("test DB prepare observations insert: %v", err)
}
defer txStmt.Close()
defer obsStmt.Close()
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
for i := 1; i <= numTx; i++ {
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
hash := fmt.Sprintf("h%04d", i)
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts)
}
}
// createTestDBWithObs creates a test DB with realistic observation counts (15 per tx).
func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) {
tb.Helper()
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
if err != nil {
tb.Fatal(err)
}
defer conn.Close()
execOrFail := func(sqlStr string) {
if _, err := conn.Exec(sqlStr); err != nil {
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr)
}
}
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT)`)
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
last_seen TEXT, first_seen TEXT, frequency REAL
)`)
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
if err != nil {
tb.Fatalf("test DB prepare transmissions: %v", err)
}
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
if err != nil {
tb.Fatalf("test DB prepare observations: %v", err)
}
defer txStmt.Close()
defer obsStmt.Close()
observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"}
obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"}
obsID := 1
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
for i := 1; i <= numTx; i++ {
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
hash := fmt.Sprintf("h%06d", i)
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i))
nObs := (i % 5) + 1 // 15 observations per transmission
for j := 0; j < nObs; j++ {
snr := -5.0 + float64(j)*2.5
rssi := -90.0 + float64(j)*5.0
obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts)
obsID++
}
}
}
+15 -196
View File
@@ -9,15 +9,14 @@ import (
func newTestStore(t *testing.T) *PacketStore {
t.Helper()
return &PacketStore{
rfCache: make(map[string]*cachedResult),
topoCache: make(map[string]*cachedResult),
hashCache: make(map[string]*cachedResult),
collisionCache: make(map[string]*cachedResult),
chanCache: make(map[string]*cachedResult),
distCache: make(map[string]*cachedResult),
subpathCache: make(map[string]*cachedResult),
rfCacheTTL: 15 * time.Second,
invCooldown: 10 * time.Second,
rfCache: make(map[string]*cachedResult),
topoCache: make(map[string]*cachedResult),
hashCache: make(map[string]*cachedResult),
chanCache: make(map[string]*cachedResult),
distCache: make(map[string]*cachedResult),
subpathCache: make(map[string]*cachedResult),
rfCacheTTL: 15 * time.Second,
invCooldown: 10 * time.Second,
}
}
@@ -30,7 +29,6 @@ func populateAllCaches(s *PacketStore) {
s.rfCache["global"] = dummy
s.topoCache["global"] = dummy
s.hashCache["global"] = dummy
s.collisionCache["global"] = dummy
s.chanCache["global"] = dummy
s.distCache["global"] = dummy
s.subpathCache["global"] = dummy
@@ -41,13 +39,12 @@ func cachePopulated(s *PacketStore) map[string]bool {
s.cacheMu.Lock()
defer s.cacheMu.Unlock()
return map[string]bool{
"rf": len(s.rfCache) > 0,
"topo": len(s.topoCache) > 0,
"hash": len(s.hashCache) > 0,
"collision": len(s.collisionCache) > 0,
"chan": len(s.chanCache) > 0,
"dist": len(s.distCache) > 0,
"subpath": len(s.subpathCache) > 0,
"rf": len(s.rfCache) > 0,
"topo": len(s.topoCache) > 0,
"hash": len(s.hashCache) > 0,
"chan": len(s.chanCache) > 0,
"dist": len(s.distCache) > 0,
"subpath": len(s.subpathCache) > 0,
}
}
@@ -93,8 +90,7 @@ func TestInvalidateCachesFor_NewTransmissionsOnly(t *testing.T) {
if pop["hash"] {
t.Error("hash cache should be cleared on new transmissions")
}
// collisionCache should NOT be cleared by transmissions alone (only by hasNewNodes)
for _, name := range []string{"rf", "topo", "collision", "chan", "dist", "subpath"} {
for _, name := range []string{"rf", "topo", "chan", "dist", "subpath"} {
if !pop[name] {
t.Errorf("%s cache should NOT be cleared on transmission-only ingest", name)
}
@@ -335,180 +331,3 @@ func BenchmarkCacheHitDuringIngestion(b *testing.B) {
}
b.ReportMetric(float64(hits)/float64(hits+misses)*100, "hit%")
}
// TestInvCooldownFromConfig verifies that invalidationDebounce from config
// is wired to invCooldown on PacketStore.
func TestInvCooldownFromConfig(t *testing.T) {
// Default without config
ps := NewPacketStore(nil, nil)
if ps.invCooldown != 300*time.Second {
t.Errorf("default invCooldown = %v, want 300s", ps.invCooldown)
}
// With config override
ct := map[string]interface{}{"invalidationDebounce": float64(60)}
ps2 := NewPacketStore(nil, nil, ct)
if ps2.invCooldown != 60*time.Second {
t.Errorf("configured invCooldown = %v, want 60s", ps2.invCooldown)
}
}
// TestCollisionCacheNotClearedByTransmissions verifies that collisionCache
// is only cleared by hasNewNodes, not hasNewTransmissions (fixes #720).
func TestCollisionCacheNotClearedByTransmissions(t *testing.T) {
s := newTestStore(t)
populateAllCaches(s)
s.invalidateCachesFor(cacheInvalidation{hasNewTransmissions: true})
pop := cachePopulated(s)
if !pop["collision"] {
t.Error("collisionCache should NOT be cleared by hasNewTransmissions alone")
}
if pop["hash"] {
t.Error("hashCache should be cleared by hasNewTransmissions")
}
}
// TestCollisionCacheClearedByNewNodes verifies that collisionCache IS cleared
// when genuinely new nodes are discovered.
func TestCollisionCacheClearedByNewNodes(t *testing.T) {
s := newTestStore(t)
populateAllCaches(s)
s.invalidateCachesFor(cacheInvalidation{hasNewNodes: true})
pop := cachePopulated(s)
if pop["collision"] {
t.Error("collisionCache should be cleared by hasNewNodes")
}
// Other caches should survive
for _, name := range []string{"rf", "topo", "hash", "chan", "dist", "subpath"} {
if !pop[name] {
t.Errorf("%s cache should NOT be cleared on new-nodes-only ingest", name)
}
}
}
// TestCacheSurvivesMultipleIngestCyclesWithinCooldown verifies that caches
// survive repeated ingest cycles during the cooldown period.
func TestCacheSurvivesMultipleIngestCyclesWithinCooldown(t *testing.T) {
s := newTestStore(t)
s.invCooldown = 200 * time.Millisecond
// First invalidation goes through (starts cooldown)
populateAllCaches(s)
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
pop := cachePopulated(s)
if pop["rf"] {
t.Error("rf should be cleared on first invalidation")
}
// Repopulate and simulate 5 rapid ingest cycles
populateAllCaches(s)
for i := 0; i < 5; i++ {
s.invalidateCachesFor(cacheInvalidation{
hasNewObservations: true,
hasNewTransmissions: true,
hasNewPaths: true,
})
}
// All caches should survive during cooldown
pop = cachePopulated(s)
for name, has := range pop {
if !has {
t.Errorf("%s cache should survive during cooldown period (ingest cycle %d)", name, 5)
}
}
}
// TestNewNodesAccumulatedDuringCooldown verifies that hasNewNodes flags
// accumulated during cooldown are applied when cooldown expires.
func TestNewNodesAccumulatedDuringCooldown(t *testing.T) {
s := newTestStore(t)
s.invCooldown = 100 * time.Millisecond
// First call starts cooldown
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
// During cooldown, accumulate hasNewNodes
s.invalidateCachesFor(cacheInvalidation{hasNewNodes: true})
// Verify accumulated
s.cacheMu.Lock()
if s.pendingInv == nil || !s.pendingInv.hasNewNodes {
t.Error("hasNewNodes should be accumulated in pendingInv")
}
s.cacheMu.Unlock()
// Wait for cooldown
time.Sleep(150 * time.Millisecond)
// Trigger flush
populateAllCaches(s)
s.invalidateCachesFor(cacheInvalidation{})
pop := cachePopulated(s)
if pop["collision"] {
t.Error("collisionCache should be cleared after pending hasNewNodes is flushed")
}
}
// BenchmarkAnalyticsLatencyCacheHitVsMiss benchmarks cache hit vs miss
// for analytics endpoints to demonstrate the performance impact.
func BenchmarkAnalyticsLatencyCacheHitVsMiss(b *testing.B) {
s := &PacketStore{
rfCache: make(map[string]*cachedResult),
topoCache: make(map[string]*cachedResult),
hashCache: make(map[string]*cachedResult),
collisionCache: make(map[string]*cachedResult),
chanCache: make(map[string]*cachedResult),
distCache: make(map[string]*cachedResult),
subpathCache: make(map[string]*cachedResult),
rfCacheTTL: 1800 * time.Second,
invCooldown: 300 * time.Second,
}
// Pre-populate cache
s.cacheMu.Lock()
s.rfCache["global"] = &cachedResult{
data: map[string]interface{}{"bins": make([]int, 100)},
expiresAt: time.Now().Add(time.Hour),
}
s.cacheMu.Unlock()
// Trigger initial invalidation to start cooldown
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
var hits, misses int64
for i := 0; i < b.N; i++ {
// Re-populate (simulates query filling cache)
s.cacheMu.Lock()
if len(s.rfCache) == 0 {
s.rfCache["global"] = &cachedResult{
data: map[string]interface{}{"bins": make([]int, 100)},
expiresAt: time.Now().Add(time.Hour),
}
}
s.cacheMu.Unlock()
// Simulate ingest (rate-limited)
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
// Check hit
s.cacheMu.Lock()
if len(s.rfCache) > 0 {
hits++
} else {
misses++
}
s.cacheMu.Unlock()
}
hitRate := float64(hits) / float64(hits+misses) * 100
b.ReportMetric(hitRate, "hit%")
if hitRate < 50 {
b.Errorf("hit rate %.1f%% is below 50%% target", hitRate)
}
}
-57
View File
@@ -1,57 +0,0 @@
package main
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
// TestPacketsChannelFilter verifies /api/packets?channel=... actually filters
// (regression test for #812).
func TestPacketsChannelFilter(t *testing.T) {
_, router := setupTestServer(t)
get := func(url string) map[string]interface{} {
req := httptest.NewRequest("GET", url, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("GET %s: expected 200, got %d", url, w.Code)
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("decode %s: %v", url, err)
}
return body
}
all := get("/api/packets?limit=50")
allTotal := int(all["total"].(float64))
if allTotal < 2 {
t.Fatalf("expected baseline >= 2 packets, got %d", allTotal)
}
test := get("/api/packets?limit=50&channel=%23test")
testTotal := int(test["total"].(float64))
if testTotal == 0 {
t.Fatalf("channel=#test: expected >= 1 match, got 0 (filter ignored?)")
}
if testTotal >= allTotal {
t.Fatalf("channel=#test: expected fewer packets than baseline (%d), got %d", allTotal, testTotal)
}
// Every returned packet must be a CHAN/GRP_TXT (payload_type=5) on #test.
pkts, _ := test["packets"].([]interface{})
for _, p := range pkts {
m := p.(map[string]interface{})
if pt, _ := m["payload_type"].(float64); int(pt) != 5 {
t.Errorf("channel=#test: returned non-GRP_TXT packet (payload_type=%v)", m["payload_type"])
}
}
none := get("/api/packets?limit=50&channel=nonexistentchannel")
if int(none["total"].(float64)) != 0 {
t.Fatalf("channel=nonexistentchannel: expected total=0, got %v", none["total"])
}
}
-748
View File
@@ -1,748 +0,0 @@
package main
import (
"math"
"sort"
"sync"
"time"
)
// ── Clock Skew Severity ────────────────────────────────────────────────────────
type SkewSeverity string
const (
SkewOK SkewSeverity = "ok" // < 5 min
SkewWarning SkewSeverity = "warning" // 5 min 1 hour
SkewCritical SkewSeverity = "critical" // 1 hour 30 days
SkewAbsurd SkewSeverity = "absurd" // > 30 days
SkewNoClock SkewSeverity = "no_clock" // > 365 days — uninitialized RTC
SkewBimodalClock SkewSeverity = "bimodal_clock" // mixed good+bad recent samples (flaky RTC)
)
// Default thresholds in seconds.
const (
skewThresholdWarnSec = 5 * 60 // 5 minutes
skewThresholdCriticalSec = 60 * 60 // 1 hour
skewThresholdAbsurdSec = 30 * 24 * 3600 // 30 days
skewThresholdNoClockSec = 365 * 24 * 3600 // 365 days — uninitialized RTC
// minDriftSamples is the minimum number of advert transmissions needed
// to compute a meaningful linear drift rate.
minDriftSamples = 5
// maxReasonableDriftPerDay caps drift display. Physically impossible
// drift rates (> 1 day/day) indicate insufficient or outlier samples.
maxReasonableDriftPerDay = 86400.0
// recentSkewWindowCount is the number of most-recent advert samples
// used to derive the "current" skew for severity classification (see
// issue #789). The all-time median is poisoned by historical bad
// samples (e.g. a node that was off and then GPS-corrected); severity
// must reflect current health, not lifetime statistics.
recentSkewWindowCount = 5
// recentSkewWindowSec bounds the recent-window in time as well: only
// samples from the last N seconds count as "recent" for severity.
// The effective window is min(recentSkewWindowCount, samples in 1h).
recentSkewWindowSec = 3600
// bimodalSkewThresholdSec is the absolute skew threshold (1 hour)
// above which a sample is considered "bad" — likely firmware emitting
// a nonsense timestamp from an uninitialized RTC, not real drift.
// Chosen to match the warning/critical severity boundary: real clock
// drift rarely exceeds 1 hour, while epoch-0 RTCs produce ~1.7B sec.
bimodalSkewThresholdSec = 3600.0
// maxPlausibleSkewJumpSec is the largest skew change between
// consecutive samples that we treat as physical drift. Anything larger
// (e.g. a GPS sync that jumps the clock by minutes/days) is rejected
// as an outlier when computing drift. Real microcontroller drift is
// fractions of a second per advert; 60s is a generous safety factor.
maxPlausibleSkewJumpSec = 60.0
// theilSenMaxPoints caps the number of points fed to Theil-Sen
// regression (O(n²) in pairs). For nodes with thousands of samples we
// keep the most-recent points, which are also the most relevant for
// current drift.
theilSenMaxPoints = 200
)
// classifySkew maps absolute skew (seconds) to a severity level.
// Float64 comparison is safe: inputs are rounded to 1 decimal via round(),
// and thresholds are integer multiples of 60 — no rounding artifacts.
func classifySkew(absSkewSec float64) SkewSeverity {
switch {
case absSkewSec >= skewThresholdNoClockSec:
return SkewNoClock
case absSkewSec >= skewThresholdAbsurdSec:
return SkewAbsurd
case absSkewSec >= skewThresholdCriticalSec:
return SkewCritical
case absSkewSec >= skewThresholdWarnSec:
return SkewWarning
default:
return SkewOK
}
}
// ── Data Types ─────────────────────────────────────────────────────────────────
// skewSample is a single raw skew measurement from one advert observation.
type skewSample struct {
advertTS int64 // node's advert Unix timestamp
observedTS int64 // observation Unix timestamp
observerID string // which observer saw this
hash string // transmission hash (for multi-observer grouping)
}
// ObserverCalibration holds the computed clock offset for an observer.
type ObserverCalibration struct {
ObserverID string `json:"observerID"`
OffsetSec float64 `json:"offsetSec"` // positive = observer clock ahead
Samples int `json:"samples"` // number of multi-observer packets used
}
// NodeClockSkew is the API response for a single node's clock skew data.
type NodeClockSkew struct {
Pubkey string `json:"pubkey"`
MeanSkewSec float64 `json:"meanSkewSec"` // corrected mean skew (positive = node ahead)
MedianSkewSec float64 `json:"medianSkewSec"` // corrected median skew
LastSkewSec float64 `json:"lastSkewSec"` // most recent corrected skew
RecentMedianSkewSec float64 `json:"recentMedianSkewSec"` // median across most-recent samples (drives severity, see #789)
DriftPerDaySec float64 `json:"driftPerDaySec"` // linear drift rate (sec/day)
Severity SkewSeverity `json:"severity"`
SampleCount int `json:"sampleCount"`
Calibrated bool `json:"calibrated"` // true if observer calibration was applied
LastAdvertTS int64 `json:"lastAdvertTS"` // most recent advert timestamp
LastObservedTS int64 `json:"lastObservedTS"` // most recent observation timestamp
Samples []SkewSample `json:"samples,omitempty"` // time-series for sparklines
GoodFraction float64 `json:"goodFraction"` // fraction of recent samples with |skew| <= 1h
RecentBadSampleCount int `json:"recentBadSampleCount"` // count of recent samples with |skew| > 1h
RecentSampleCount int `json:"recentSampleCount"` // total recent samples in window
NodeName string `json:"nodeName,omitempty"` // populated in fleet responses
NodeRole string `json:"nodeRole,omitempty"` // populated in fleet responses
}
// SkewSample is a single (timestamp, skew) point for sparkline rendering.
type SkewSample struct {
Timestamp int64 `json:"ts"` // Unix epoch of observation
SkewSec float64 `json:"skew"` // corrected skew in seconds
}
// txSkewResult maps tx hash → per-transmission skew stats. This is an
// intermediate result keyed by hash (not pubkey); the store maps hash → pubkey
// when building the final per-node view.
type txSkewResult = map[string]*NodeClockSkew
// ── Clock Skew Engine ──────────────────────────────────────────────────────────
// ClockSkewEngine computes and caches clock skew data for nodes and observers.
type ClockSkewEngine struct {
mu sync.RWMutex
observerOffsets map[string]float64 // observerID → calibrated offset (seconds)
observerSamples map[string]int // observerID → number of multi-observer packets used
nodeSkew txSkewResult
lastComputed time.Time
computeInterval time.Duration
}
func NewClockSkewEngine() *ClockSkewEngine {
return &ClockSkewEngine{
observerOffsets: make(map[string]float64),
observerSamples: make(map[string]int),
nodeSkew: make(txSkewResult),
computeInterval: 30 * time.Second,
}
}
// Recompute recalculates all clock skew data from the packet store.
// Called periodically or on demand. Holds store RLock externally.
// Uses read-copy-update: heavy computation runs outside the write lock,
// then results are swapped in under a brief lock.
func (e *ClockSkewEngine) Recompute(store *PacketStore) {
// Fast path: check under read lock if recompute is needed.
e.mu.RLock()
fresh := time.Since(e.lastComputed) < e.computeInterval
e.mu.RUnlock()
if fresh {
return
}
// Phase 1: Collect skew samples from ADVERT packets (store RLock held by caller).
samples := collectSamples(store)
// Phase 23: Compute outside the write lock.
var newOffsets map[string]float64
var newSamples map[string]int
var newNodeSkew txSkewResult
if len(samples) > 0 {
newOffsets, newSamples = calibrateObservers(samples)
newNodeSkew = computeNodeSkew(samples, newOffsets)
} else {
newOffsets = make(map[string]float64)
newSamples = make(map[string]int)
newNodeSkew = make(txSkewResult)
}
// Swap results under brief write lock.
e.mu.Lock()
// Re-check: another goroutine may have computed while we were working.
if time.Since(e.lastComputed) < e.computeInterval {
e.mu.Unlock()
return
}
e.observerOffsets = newOffsets
e.observerSamples = newSamples
e.nodeSkew = newNodeSkew
e.lastComputed = time.Now()
e.mu.Unlock()
}
// collectSamples extracts skew samples from ADVERT packets in the store.
// Must be called with store.mu held (at least RLock).
func collectSamples(store *PacketStore) []skewSample {
adverts := store.byPayloadType[PayloadADVERT]
if len(adverts) == 0 {
return nil
}
samples := make([]skewSample, 0, len(adverts)*2)
for _, tx := range adverts {
decoded := tx.ParsedDecoded()
if decoded == nil {
continue
}
// Extract advert timestamp from decoded JSON.
advertTS := extractTimestamp(decoded)
if advertTS <= 0 {
continue
}
// Sanity: skip timestamps before year 2020 or after year 2100.
if advertTS < 1577836800 || advertTS > 4102444800 {
continue
}
for _, obs := range tx.Observations {
obsTS := parseISO(obs.Timestamp)
if obsTS <= 0 {
continue
}
samples = append(samples, skewSample{
advertTS: advertTS,
observedTS: obsTS,
observerID: obs.ObserverID,
hash: tx.Hash,
})
}
}
return samples
}
// extractTimestamp gets the Unix timestamp from a decoded ADVERT payload.
func extractTimestamp(decoded map[string]interface{}) int64 {
// Try payload.timestamp first (nested in "payload" key).
if payload, ok := decoded["payload"]; ok {
if pm, ok := payload.(map[string]interface{}); ok {
if ts := jsonNumber(pm, "timestamp"); ts > 0 {
return ts
}
}
}
// Fallback: top-level timestamp.
if ts := jsonNumber(decoded, "timestamp"); ts > 0 {
return ts
}
return 0
}
// jsonNumber extracts an int64 from a JSON-parsed map (handles float64 and json.Number).
func jsonNumber(m map[string]interface{}, key string) int64 {
v, ok := m[key]
if !ok || v == nil {
return 0
}
switch n := v.(type) {
case float64:
return int64(n)
case int64:
return n
case int:
return int64(n)
}
return 0
}
// parseISO parses an ISO 8601 timestamp string to Unix seconds.
func parseISO(s string) int64 {
if s == "" {
return 0
}
t, err := time.Parse(time.RFC3339, s)
if err != nil {
// Try with fractional seconds.
t, err = time.Parse("2006-01-02T15:04:05.999999999Z07:00", s)
if err != nil {
return 0
}
}
return t.Unix()
}
// ── Phase 2: Observer Calibration ──────────────────────────────────────────────
// calibrateObservers computes each observer's clock offset using multi-observer
// packets. Returns offset map and sample count map.
func calibrateObservers(samples []skewSample) (map[string]float64, map[string]int) {
// Group observations by packet hash.
byHash := make(map[string][]skewSample)
for _, s := range samples {
byHash[s.hash] = append(byHash[s.hash], s)
}
// For each multi-observer packet, compute per-observer deviation from median.
deviations := make(map[string][]float64) // observerID → list of deviations
for _, group := range byHash {
if len(group) < 2 {
continue // single-observer packet, can't calibrate
}
// Compute median observation timestamp for this packet.
obsTimes := make([]float64, len(group))
for i, s := range group {
obsTimes[i] = float64(s.observedTS)
}
medianObs := median(obsTimes)
for _, s := range group {
dev := float64(s.observedTS) - medianObs
deviations[s.observerID] = append(deviations[s.observerID], dev)
}
}
// Each observer's offset = median of its deviations.
offsets := make(map[string]float64, len(deviations))
counts := make(map[string]int, len(deviations))
for obsID, devs := range deviations {
offsets[obsID] = median(devs)
counts[obsID] = len(devs)
}
return offsets, counts
}
// ── Phase 3: Per-Node Skew ─────────────────────────────────────────────────────
// computeNodeSkew calculates corrected skew statistics for each node.
func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) txSkewResult {
// Compute corrected skew per sample, grouped by hash (each hash = one
// node's advert transmission). The caller maps hash → pubkey via byNode.
type correctedSample struct {
skew float64
observedTS int64
calibrated bool
}
byHash := make(map[string][]correctedSample)
hashAdvertTS := make(map[string]int64)
for _, s := range samples {
obsOffset, hasCal := obsOffsets[s.observerID]
rawSkew := float64(s.advertTS - s.observedTS)
corrected := rawSkew
if hasCal {
// Observer offset = obs_ts - median(all_obs_ts). If observer is ahead,
// its obs_ts is inflated, making raw_skew too low. Add offset to correct.
corrected = rawSkew + obsOffset
}
byHash[s.hash] = append(byHash[s.hash], correctedSample{
skew: corrected,
observedTS: s.observedTS,
calibrated: hasCal,
})
hashAdvertTS[s.hash] = s.advertTS
}
// Each hash represents one advert from one node. Compute median corrected
// skew per hash (across multiple observers).
result := make(map[string]*NodeClockSkew) // keyed by hash for now
for hash, cs := range byHash {
skews := make([]float64, len(cs))
for i, c := range cs {
skews[i] = c.skew
}
medSkew := median(skews)
meanSkew := mean(skews)
// Find latest observation.
var latestObsTS int64
var anyCal bool
for _, c := range cs {
if c.observedTS > latestObsTS {
latestObsTS = c.observedTS
}
if c.calibrated {
anyCal = true
}
}
absMedian := math.Abs(medSkew)
result[hash] = &NodeClockSkew{
MeanSkewSec: round(meanSkew, 1),
MedianSkewSec: round(medSkew, 1),
LastSkewSec: round(cs[len(cs)-1].skew, 1),
Severity: classifySkew(absMedian),
SampleCount: len(cs),
Calibrated: anyCal,
LastAdvertTS: hashAdvertTS[hash],
LastObservedTS: latestObsTS,
}
}
return result
}
// ── Integration with PacketStore ───────────────────────────────────────────────
// GetNodeClockSkew returns the clock skew data for a specific node (acquires RLock).
func (s *PacketStore) GetNodeClockSkew(pubkey string) *NodeClockSkew {
s.mu.RLock()
defer s.mu.RUnlock()
return s.getNodeClockSkewLocked(pubkey)
}
// getNodeClockSkewLocked returns clock skew for a node.
// Must be called with s.mu held (at least RLock).
func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
s.clockSkew.Recompute(s)
txs := s.byNode[pubkey]
if len(txs) == 0 {
return nil
}
s.clockSkew.mu.RLock()
defer s.clockSkew.mu.RUnlock()
var allSkews []float64
var lastSkew float64
var lastObsTS, lastAdvTS int64
var totalSamples int
var anyCal bool
var tsSkews []tsSkewPair
for _, tx := range txs {
if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
continue
}
cs, ok := s.clockSkew.nodeSkew[tx.Hash]
if !ok {
continue
}
allSkews = append(allSkews, cs.MedianSkewSec)
totalSamples += cs.SampleCount
if cs.Calibrated {
anyCal = true
}
if cs.LastObservedTS > lastObsTS {
lastObsTS = cs.LastObservedTS
lastSkew = cs.LastSkewSec
lastAdvTS = cs.LastAdvertTS
}
tsSkews = append(tsSkews, tsSkewPair{ts: cs.LastObservedTS, skew: cs.MedianSkewSec})
}
if len(allSkews) == 0 {
return nil
}
medSkew := median(allSkews)
meanSkew := mean(allSkews)
// Severity is derived from RECENT samples only (issue #789). The
// all-time median is poisoned by historical bad data — a node that
// was off for hours and then GPS-corrected can have median = -59M sec
// while its current skew is -0.8s. Operators need severity to reflect
// current health, so they trust the dashboard.
//
// Sort tsSkews by time and take the last recentSkewWindowCount samples
// (or all samples within recentSkewWindowSec of the latest, whichever
// gives FEWER samples — we want the more-current view; a chatty node
// can fit dozens of samples in 1h, in which case the count cap wins).
sort.Slice(tsSkews, func(i, j int) bool { return tsSkews[i].ts < tsSkews[j].ts })
recentSkew := lastSkew
var recentVals []float64
if n := len(tsSkews); n > 0 {
latestTS := tsSkews[n-1].ts
// Index-based window: last K samples.
startByCount := n - recentSkewWindowCount
if startByCount < 0 {
startByCount = 0
}
// Time-based window: samples newer than latestTS - windowSec.
startByTime := n - 1
for i := n - 1; i >= 0; i-- {
if latestTS-tsSkews[i].ts <= recentSkewWindowSec {
startByTime = i
} else {
break
}
}
// Pick the narrower (larger-index) of the two windows — the most
// current view of the node's clock health.
start := startByCount
if startByTime > start {
start = startByTime
}
recentVals = make([]float64, 0, n-start)
for i := start; i < n; i++ {
recentVals = append(recentVals, tsSkews[i].skew)
}
if len(recentVals) > 0 {
recentSkew = median(recentVals)
}
}
// ── Bimodal detection (#845) ─────────────────────────────────────────
// Split recent samples into "good" (|skew| <= 1h, real clock) and
// "bad" (|skew| > 1h, firmware nonsense from uninitialized RTC).
// Classification order (first match wins):
// no_clock — goodFraction < 0.10 (essentially no real clock)
// bimodal_clock — 0.10 <= goodFraction < 0.80 AND badCount > 0
// ok/warn/etc. — goodFraction >= 0.80 (normal, outliers filtered)
var goodSamples []float64
for _, v := range recentVals {
if math.Abs(v) <= bimodalSkewThresholdSec {
goodSamples = append(goodSamples, v)
}
}
recentSampleCount := len(recentVals)
recentBadCount := recentSampleCount - len(goodSamples)
var goodFraction float64
if recentSampleCount > 0 {
goodFraction = float64(len(goodSamples)) / float64(recentSampleCount)
}
var severity SkewSeverity
if goodFraction < 0.10 {
// Essentially no real clock — classify as no_clock regardless
// of the raw skew magnitude.
severity = SkewNoClock
} else if goodFraction < 0.80 && recentBadCount > 0 {
// Bimodal: use median of GOOD samples as the "real" skew.
severity = SkewBimodalClock
if len(goodSamples) > 0 {
recentSkew = median(goodSamples)
}
} else {
// Normal path: if there are good samples, use their median
// (filters out rare outliers in ≥80% good case).
if len(goodSamples) > 0 && recentBadCount > 0 {
recentSkew = median(goodSamples)
}
severity = classifySkew(math.Abs(recentSkew))
}
// For no_clock / bimodal_clock nodes, skip drift when data is unreliable.
var drift float64
if severity != SkewNoClock && severity != SkewBimodalClock && len(tsSkews) >= minDriftSamples {
drift = computeDrift(tsSkews)
// Cap physically impossible drift rates.
if math.Abs(drift) > maxReasonableDriftPerDay {
drift = 0
}
}
// Build sparkline samples from tsSkews (already sorted by time above).
samples := make([]SkewSample, len(tsSkews))
for i, p := range tsSkews {
samples[i] = SkewSample{Timestamp: p.ts, SkewSec: round(p.skew, 1)}
}
return &NodeClockSkew{
Pubkey: pubkey,
MeanSkewSec: round(meanSkew, 1),
MedianSkewSec: round(medSkew, 1),
LastSkewSec: round(lastSkew, 1),
RecentMedianSkewSec: round(recentSkew, 1),
DriftPerDaySec: round(drift, 2),
Severity: severity,
SampleCount: totalSamples,
Calibrated: anyCal,
LastAdvertTS: lastAdvTS,
LastObservedTS: lastObsTS,
Samples: samples,
GoodFraction: round(goodFraction, 2),
RecentBadSampleCount: recentBadCount,
RecentSampleCount: recentSampleCount,
}
}
// GetFleetClockSkew returns clock skew data for all nodes that have skew data.
// Must NOT be called with s.mu held.
func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew {
s.mu.RLock()
defer s.mu.RUnlock()
// Build name/role lookup from DB cache (requires s.mu held).
allNodes, _ := s.getCachedNodesAndPM()
nameMap := make(map[string]nodeInfo, len(allNodes))
for _, ni := range allNodes {
nameMap[ni.PublicKey] = ni
}
var results []*NodeClockSkew
for pubkey := range s.byNode {
cs := s.getNodeClockSkewLocked(pubkey)
if cs == nil {
continue
}
// Enrich with node name/role.
if ni, ok := nameMap[pubkey]; ok {
cs.NodeName = ni.Name
cs.NodeRole = ni.Role
}
// Omit samples in fleet response (too much data).
cs.Samples = nil
results = append(results, cs)
}
return results
}
// GetObserverCalibrations returns the current observer clock offsets.
func (s *PacketStore) GetObserverCalibrations() []ObserverCalibration {
s.mu.RLock()
defer s.mu.RUnlock()
s.clockSkew.Recompute(s)
s.clockSkew.mu.RLock()
defer s.clockSkew.mu.RUnlock()
result := make([]ObserverCalibration, 0, len(s.clockSkew.observerOffsets))
for obsID, offset := range s.clockSkew.observerOffsets {
result = append(result, ObserverCalibration{
ObserverID: obsID,
OffsetSec: round(offset, 1),
Samples: s.clockSkew.observerSamples[obsID],
})
}
// Sort by absolute offset descending.
sort.Slice(result, func(i, j int) bool {
return math.Abs(result[i].OffsetSec) > math.Abs(result[j].OffsetSec)
})
return result
}
// ── Math Helpers ───────────────────────────────────────────────────────────────
func median(vals []float64) float64 {
if len(vals) == 0 {
return 0
}
sorted := make([]float64, len(vals))
copy(sorted, vals)
sort.Float64s(sorted)
n := len(sorted)
if n%2 == 0 {
return (sorted[n/2-1] + sorted[n/2]) / 2
}
return sorted[n/2]
}
func mean(vals []float64) float64 {
if len(vals) == 0 {
return 0
}
sum := 0.0
for _, v := range vals {
sum += v
}
return sum / float64(len(vals))
}
// tsSkewPair is a (timestamp, skew) pair for drift estimation.
type tsSkewPair struct {
ts int64
skew float64
}
// computeDrift estimates linear drift in seconds per day from time-ordered
// (timestamp, skew) pairs. Issue #789: a single GPS-correction event (huge
// skew jump in seconds) used to dominate ordinary least squares and produce
// absurd drift like 1.7M sec/day. We now:
//
// 1. Drop pairs whose consecutive skew jump exceeds maxPlausibleSkewJumpSec
// (clock corrections, not physical drift). This protects both OLS-style
// consumers and Theil-Sen.
// 2. Use Theil-Sen regression — the slope is the median of all pairwise
// slopes, naturally robust to remaining outliers (breakdown point ~29%).
//
// For very small samples after filtering we fall back to a simple slope
// between first and last calibrated samples.
func computeDrift(pairs []tsSkewPair) float64 {
if len(pairs) < 2 {
return 0
}
// Sort by timestamp.
sort.Slice(pairs, func(i, j int) bool {
return pairs[i].ts < pairs[j].ts
})
// Time span too short? Skip.
spanSec := float64(pairs[len(pairs)-1].ts - pairs[0].ts)
if spanSec < 3600 { // need at least 1 hour of data
return 0
}
// Outlier filter: drop samples where the skew jumps more than
// maxPlausibleSkewJumpSec from the running "stable" baseline.
// We anchor on the first sample, then accept each subsequent point
// that's within the threshold of the most recent accepted point —
// this preserves a slow drift while rejecting correction events.
filtered := make([]tsSkewPair, 0, len(pairs))
filtered = append(filtered, pairs[0])
for i := 1; i < len(pairs); i++ {
prev := filtered[len(filtered)-1]
if math.Abs(pairs[i].skew-prev.skew) <= maxPlausibleSkewJumpSec {
filtered = append(filtered, pairs[i])
}
}
// If the filter killed too much (e.g. unstable node), fall back to the
// raw series so we at least produce *something* — it'll be capped by
// maxReasonableDriftPerDay downstream.
if len(filtered) < 2 || float64(filtered[len(filtered)-1].ts-filtered[0].ts) < 3600 {
filtered = pairs
}
// Cap point count for Theil-Sen (O(n²) on pairs). Keep most-recent.
if len(filtered) > theilSenMaxPoints {
filtered = filtered[len(filtered)-theilSenMaxPoints:]
}
return theilSenSlope(filtered) * 86400 // sec/sec → sec/day
}
// theilSenSlope returns the Theil-Sen estimator: median of all pairwise
// slopes (yj - yi) / (tj - ti) for i < j. Naturally robust to outliers.
// Pairs must be sorted by timestamp ascending.
func theilSenSlope(pairs []tsSkewPair) float64 {
n := len(pairs)
if n < 2 {
return 0
}
// Pre-allocate: n*(n-1)/2 pairs.
slopes := make([]float64, 0, n*(n-1)/2)
for i := 0; i < n; i++ {
for j := i + 1; j < n; j++ {
dt := float64(pairs[j].ts - pairs[i].ts)
if dt <= 0 {
continue
}
slopes = append(slopes, (pairs[j].skew-pairs[i].skew)/dt)
}
}
if len(slopes) == 0 {
return 0
}
return median(slopes)
}
-956
View File
@@ -1,956 +0,0 @@
package main
import (
"fmt"
"math"
"testing"
"time"
)
// ── classifySkew ───────────────────────────────────────────────────────────────
func TestClassifySkew(t *testing.T) {
tests := []struct {
absSkew float64
expected SkewSeverity
}{
{0, SkewOK},
{60, SkewOK}, // 1 min
{299, SkewOK}, // just under 5 min
{300, SkewWarning}, // exactly 5 min
{1800, SkewWarning}, // 30 min
{3599, SkewWarning}, // just under 1 hour
{3600, SkewCritical}, // exactly 1 hour
{86400, SkewCritical}, // 1 day
{2592000 - 1, SkewCritical}, // just under 30 days
{2592000, SkewAbsurd}, // exactly 30 days
{86400 * 365 - 1, SkewAbsurd}, // just under 365 days
{86400 * 365, SkewNoClock}, // exactly 365 days
{86400 * 365 * 10, SkewNoClock}, // 10 years (epoch-0 style)
}
for _, tc := range tests {
got := classifySkew(tc.absSkew)
if got != tc.expected {
t.Errorf("classifySkew(%v) = %v, want %v", tc.absSkew, got, tc.expected)
}
}
}
// ── median ─────────────────────────────────────────────────────────────────────
func TestMedian(t *testing.T) {
tests := []struct {
vals []float64
expected float64
}{
{nil, 0},
{[]float64{}, 0},
{[]float64{5}, 5},
{[]float64{1, 3}, 2},
{[]float64{3, 1, 2}, 2},
{[]float64{4, 1, 3, 2}, 2.5},
{[]float64{-10, 0, 10}, 0},
}
for _, tc := range tests {
got := median(tc.vals)
if got != tc.expected {
t.Errorf("median(%v) = %v, want %v", tc.vals, got, tc.expected)
}
}
}
func TestMean(t *testing.T) {
tests := []struct {
vals []float64
expected float64
}{
{nil, 0},
{[]float64{10}, 10},
{[]float64{2, 4, 6}, 4},
}
for _, tc := range tests {
got := mean(tc.vals)
if got != tc.expected {
t.Errorf("mean(%v) = %v, want %v", tc.vals, got, tc.expected)
}
}
}
// ── parseISO ───────────────────────────────────────────────────────────────────
func TestParseISO(t *testing.T) {
tests := []struct {
input string
expected int64
}{
{"", 0},
{"garbage", 0},
{"2026-04-15T12:00:00Z", 1776254400},
{"2026-04-15T12:00:00+00:00", 1776254400},
}
for _, tc := range tests {
got := parseISO(tc.input)
if got != tc.expected {
t.Errorf("parseISO(%q) = %v, want %v", tc.input, got, tc.expected)
}
}
}
// ── extractTimestamp ────────────────────────────────────────────────────────────
func TestExtractTimestamp(t *testing.T) {
// Nested payload.timestamp
decoded := map[string]interface{}{
"payload": map[string]interface{}{
"timestamp": float64(1776340800),
},
}
got := extractTimestamp(decoded)
if got != 1776340800 {
t.Errorf("extractTimestamp (nested) = %v, want 1776340800", got)
}
// Top-level timestamp
decoded2 := map[string]interface{}{
"timestamp": float64(1776340900),
}
got2 := extractTimestamp(decoded2)
if got2 != 1776340900 {
t.Errorf("extractTimestamp (top-level) = %v, want 1776340900", got2)
}
// No timestamp
decoded3 := map[string]interface{}{"foo": "bar"}
got3 := extractTimestamp(decoded3)
if got3 != 0 {
t.Errorf("extractTimestamp (missing) = %v, want 0", got3)
}
}
// ── calibrateObservers ─────────────────────────────────────────────────────────
func TestCalibrateObservers_SingleObserver(t *testing.T) {
// Single-observer packets can't calibrate — should return empty.
samples := []skewSample{
{advertTS: 1000, observedTS: 1000, observerID: "obs1", hash: "h1"},
{advertTS: 2000, observedTS: 2000, observerID: "obs1", hash: "h2"},
}
offsets, _ := calibrateObservers(samples)
if len(offsets) != 0 {
t.Errorf("expected no offsets for single-observer, got %v", offsets)
}
}
func TestCalibrateObservers_MultiObserver(t *testing.T) {
// Packet h1 seen by 3 observers: obs1 at t=100, obs2 at t=110, obs3 at t=100.
// Median observation = 100. obs1=0, obs2=+10, obs3=0
// Packet h2 seen by 3 observers: obs1 at t=200, obs2 at t=210, obs3 at t=200.
// Median observation = 200. obs1=0, obs2=+10, obs3=0
samples := []skewSample{
{advertTS: 100, observedTS: 100, observerID: "obs1", hash: "h1"},
{advertTS: 100, observedTS: 110, observerID: "obs2", hash: "h1"},
{advertTS: 100, observedTS: 100, observerID: "obs3", hash: "h1"},
{advertTS: 200, observedTS: 200, observerID: "obs1", hash: "h2"},
{advertTS: 200, observedTS: 210, observerID: "obs2", hash: "h2"},
{advertTS: 200, observedTS: 200, observerID: "obs3", hash: "h2"},
}
offsets, _ := calibrateObservers(samples)
if offsets["obs1"] != 0 {
t.Errorf("obs1 offset = %v, want 0", offsets["obs1"])
}
if offsets["obs2"] != 10 {
t.Errorf("obs2 offset = %v, want 10", offsets["obs2"])
}
if offsets["obs3"] != 0 {
t.Errorf("obs3 offset = %v, want 0", offsets["obs3"])
}
}
// ── computeNodeSkew ────────────────────────────────────────────────────────────
func TestComputeNodeSkew_BasicCorrection(t *testing.T) {
// Validates observer offset correction direction.
//
// Setup: node is 60s ahead, obs1 accurate, obs2 is 10s ahead.
// With 2 observers, median obs_ts = 1005.
// obs1 offset = 1000 - 1005 = -5
// obs2 offset = 1010 - 1005 = +5
// Correction: corrected = raw_skew + obsOffset
// obs1: raw=60, corrected = 60 + (-5) = 55
// obs2: raw=50, corrected = 50 + 5 = 55
// Both converge to 55 (not exact 60 because with only 2 observers,
// the median can't fully distinguish which observer is drifted).
samples := []skewSample{
// Same packet seen by accurate obs1 and obs2 (+10s ahead)
{advertTS: 1060, observedTS: 1000, observerID: "obs1", hash: "h1"},
{advertTS: 1060, observedTS: 1010, observerID: "obs2", hash: "h1"},
}
offsets, _ := calibrateObservers(samples)
// median obs = 1005, obs1 offset = -5, obs2 offset = +5
// So the median approach finds obs2 is +5 ahead (relative to median)
// Now compute node skew with those offsets:
nodeSkew := computeNodeSkew(samples, offsets)
cs, ok := nodeSkew["h1"]
if !ok {
t.Fatal("expected skew data for hash h1")
}
// With only 2 observers, median obs_ts = 1005.
// obs1 offset = 1000-1005 = -5, obs2 offset = 1010-1005 = +5
// raw from obs1 = 60, corrected = 60 + (-5) = 55
// raw from obs2 = 50, corrected = 50 + 5 = 55
// median = 55
if cs.MedianSkewSec != 55 {
t.Errorf("median skew = %v, want 55", cs.MedianSkewSec)
}
}
func TestComputeNodeSkew_ThreeObservers(t *testing.T) {
// Node is exactly 60s ahead. obs1 accurate, obs2 accurate, obs3 +30s ahead.
// advertTS = 1060, real time = 1000
samples := []skewSample{
{advertTS: 1060, observedTS: 1000, observerID: "obs1", hash: "h1"},
{advertTS: 1060, observedTS: 1000, observerID: "obs2", hash: "h1"},
{advertTS: 1060, observedTS: 1030, observerID: "obs3", hash: "h1"},
}
offsets, _ := calibrateObservers(samples)
// median obs_ts = 1000. obs1=0, obs2=0, obs3=+30
if offsets["obs3"] != 30 {
t.Errorf("obs3 offset = %v, want 30", offsets["obs3"])
}
nodeSkew := computeNodeSkew(samples, offsets)
cs := nodeSkew["h1"]
if cs == nil {
t.Fatal("expected skew data for h1")
}
// raw from obs1 = 60, corrected = 60 + 0 = 60
// raw from obs2 = 60, corrected = 60 + 0 = 60
// raw from obs3 = 30, corrected = 30 + 30 = 60
// All three converge to 60.
if cs.MedianSkewSec != 60 {
t.Errorf("median skew = %v, want 60 (node is 60s ahead)", cs.MedianSkewSec)
}
}
// ── computeDrift ───────────────────────────────────────────────────────────────
func TestComputeDrift_Stable(t *testing.T) {
// Constant skew = no drift.
pairs := []tsSkewPair{
{ts: 0, skew: 60},
{ts: 7200, skew: 60},
{ts: 14400, skew: 60},
}
drift := computeDrift(pairs)
if drift != 0 {
t.Errorf("drift = %v, want 0 for stable skew", drift)
}
}
func TestComputeDrift_LinearDrift(t *testing.T) {
// 1 second drift per hour = 24 sec/day.
pairs := []tsSkewPair{
{ts: 0, skew: 0},
{ts: 3600, skew: 1},
{ts: 7200, skew: 2},
}
drift := computeDrift(pairs)
expected := 24.0
if math.Abs(drift-expected) > 0.1 {
t.Errorf("drift = %v, want ~%v", drift, expected)
}
}
func TestComputeDrift_TooFewSamples(t *testing.T) {
pairs := []tsSkewPair{{ts: 0, skew: 10}}
if computeDrift(pairs) != 0 {
t.Error("expected 0 drift for single sample")
}
}
func TestComputeDrift_TooShortSpan(t *testing.T) {
// Less than 1 hour apart.
pairs := []tsSkewPair{
{ts: 0, skew: 0},
{ts: 1800, skew: 10},
}
if computeDrift(pairs) != 0 {
t.Error("expected 0 drift for short time span")
}
}
// ── jsonNumber ─────────────────────────────────────────────────────────────────
func TestJsonNumber(t *testing.T) {
m := map[string]interface{}{
"a": float64(42),
"b": int64(99),
"c": "not a number",
"d": nil,
}
if jsonNumber(m, "a") != 42 {
t.Error("float64 case failed")
}
if jsonNumber(m, "b") != 99 {
t.Error("int64 case failed")
}
if jsonNumber(m, "c") != 0 {
t.Error("string case should return 0")
}
if jsonNumber(m, "d") != 0 {
t.Error("nil case should return 0")
}
if jsonNumber(m, "missing") != 0 {
t.Error("missing key should return 0")
}
}
// ── Integration: GetNodeClockSkew via PacketStore ──────────────────────────────
func TestGetNodeClockSkew_Integration(t *testing.T) {
ps := NewPacketStore(nil, nil)
// Simulate two ADVERT transmissions for the same node, seen by 2 observers each.
// Node "AABB" has clock 120s ahead.
pt := 4 // ADVERT
tx1 := &StoreTx{
Hash: "hash1",
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":1700002320}}`, // obs=1700002200, node ahead by 120s
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: "2023-11-14T22:50:00Z"}, // 1700002200
{ObserverID: "obs2", Timestamp: "2023-11-14T22:50:00Z"}, // 1700002200
},
}
tx2 := &StoreTx{
Hash: "hash2",
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":1700005920}}`, // obs=1700005800, node ahead by 120s
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: "2023-11-14T23:50:00Z"}, // 1700005800
{ObserverID: "obs2", Timestamp: "2023-11-14T23:50:00Z"}, // 1700005800
},
}
ps.mu.Lock()
ps.byNode["AABB"] = []*StoreTx{tx1, tx2}
ps.byPayloadType[4] = []*StoreTx{tx1, tx2}
// Force recompute by setting interval to 0.
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
result := ps.GetNodeClockSkew("AABB")
if result == nil {
t.Fatal("expected clock skew result for node AABB")
}
if result.Pubkey != "AABB" {
t.Errorf("pubkey = %q, want AABB", result.Pubkey)
}
// Both transmissions show 120s skew, so median should be 120.
if result.MedianSkewSec != 120 {
t.Errorf("median skew = %v, want 120", result.MedianSkewSec)
}
if result.SampleCount < 2 {
t.Errorf("sample count = %v, want >= 2", result.SampleCount)
}
if result.Severity != SkewOK {
t.Errorf("severity = %v, want ok (120s < 5min)", result.Severity)
}
// Drift should be ~0 since skew is constant.
if math.Abs(result.DriftPerDaySec) > 1 {
t.Errorf("drift = %v, want ~0 for constant skew", result.DriftPerDaySec)
}
}
func TestGetNodeClockSkew_NoData(t *testing.T) {
ps := NewPacketStore(nil, nil)
result := ps.GetNodeClockSkew("nonexistent")
if result != nil {
t.Error("expected nil for nonexistent node")
}
}
// ── Sanity check tests (#XXX — clock skew crazy stats) ────────────────────────
func TestGetNodeClockSkew_NoClock_EpochZero(t *testing.T) {
// Node with epoch-0 timestamp produces huge skew → no_clock severity, drift=0.
ps := NewPacketStore(nil, nil)
pt := 4 // ADVERT
// Epoch-ish advert: advertTS near start of 2020, observed in 2023 → |skew| > 365 days
var txs []*StoreTx
baseObs := int64(1700000000) // ~Nov 2023
for i := 0; i < 6; i++ {
obsTS := baseObs + int64(i)*7200
tx := &StoreTx{
Hash: "epoch-h" + string(rune('0'+i)),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":1577836800}}`, // Jan 1 2020 — valid but way off
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["EPOCH"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
result := ps.GetNodeClockSkew("EPOCH")
if result == nil {
t.Fatal("expected clock skew result for epoch-0 node")
}
if result.Severity != SkewNoClock {
t.Errorf("severity = %v, want no_clock", result.Severity)
}
if result.DriftPerDaySec != 0 {
t.Errorf("drift = %v, want 0 for no_clock node", result.DriftPerDaySec)
}
}
func TestGetNodeClockSkew_TooFewSamplesForDrift(t *testing.T) {
// Node with only 2 advert samples → drift should not be computed.
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 2; i++ {
obsTS := baseObs + int64(i)*7200
advTS := obsTS + 120 // 120s ahead
tx := &StoreTx{
Hash: "few-h" + string(rune('0'+i)),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["FEWSAMP"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
result := ps.GetNodeClockSkew("FEWSAMP")
if result == nil {
t.Fatal("expected clock skew result")
}
if result.DriftPerDaySec != 0 {
t.Errorf("drift = %v, want 0 for 2-sample node (minimum is %d)", result.DriftPerDaySec, minDriftSamples)
}
}
func TestGetNodeClockSkew_AbsurdDriftCapped(t *testing.T) {
// Node with wildly varying skew producing |drift| > 86400 s/day → drift capped to 0.
ps := NewPacketStore(nil, nil)
pt := 4
// Create 6 samples with extreme skew variation to produce absurd drift.
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 6; i++ {
obsTS := baseObs + int64(i)*3600
// Alternate between huge positive and negative skew offsets
skewOffset := int64(50000 * (1 - 2*(i%2))) // +50000 or -50000
advTS := obsTS + skewOffset
tx := &StoreTx{
Hash: "wild-h" + string(rune('0'+i)),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["WILD"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
result := ps.GetNodeClockSkew("WILD")
if result == nil {
t.Fatal("expected clock skew result")
}
if math.Abs(result.DriftPerDaySec) > maxReasonableDriftPerDay {
t.Errorf("drift = %v, should be capped (|drift| > %v)", result.DriftPerDaySec, maxReasonableDriftPerDay)
}
}
func TestGetNodeClockSkew_NormalNodeWithDrift(t *testing.T) {
// Normal node with 6 samples and consistent linear drift → drift computed correctly.
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 6; i++ {
obsTS := baseObs + int64(i)*7200 // every 2 hours
// Drift: 1 sec/hour = 24 sec/day
advTS := obsTS + 120 + int64(i) // skew grows by 1s per sample (2h apart)
tx := &StoreTx{
Hash: "norm-h" + string(rune('0'+i)),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["NORMAL"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
result := ps.GetNodeClockSkew("NORMAL")
if result == nil {
t.Fatal("expected clock skew result")
}
if result.Severity != SkewOK {
t.Errorf("severity = %v, want ok", result.Severity)
}
// 1s per 7200s = 12 s/day
if result.DriftPerDaySec == 0 {
t.Error("expected non-zero drift for linearly drifting node")
}
if math.Abs(result.DriftPerDaySec) > maxReasonableDriftPerDay {
t.Errorf("drift = %v, should be reasonable", result.DriftPerDaySec)
}
}
// formatInt64 is a test helper to format int64 as string for JSON embedding.
func formatInt64(n int64) string {
return fmt.Sprintf("%d", n)
}
// ── #789: Recent-window severity & robust drift ───────────────────────────────
// TestSeverityUsesRecentNotMedian: 100 historical bad samples (skew=-60s,
// each ~5min apart) followed by 5 fresh good samples (skew=-1s). All-time
// median is still huge-ish but recent-window severity must reflect the
// current healthy state.
func TestSeverityUsesRecentNotMedian(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 105; i++ {
obsTS := baseObs + int64(i)*300 // 5 min apart
var skew int64 = -60
if i >= 100 {
skew = -1 // good samples at the tail
}
advTS := obsTS + skew
tx := &StoreTx{
Hash: fmt.Sprintf("recent-h%03d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["RECENT"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("RECENT")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewOK {
t.Errorf("severity = %v, want ok (recent samples are healthy)", r.Severity)
}
if math.Abs(r.RecentMedianSkewSec) > 5 {
t.Errorf("recentMedianSkewSec = %v, want ~-1", r.RecentMedianSkewSec)
}
// Historical median should still be retained for context.
if math.Abs(r.MedianSkewSec) < 30 {
t.Errorf("medianSkewSec = %v, expected historical median to remain large", r.MedianSkewSec)
}
}
// TestDriftRejectsCorrectionJump: 30 minutes of clean linear drift, then a
// single 60-second skew jump. The pre-jump slope should win — drift must
// not be catastrophically inflated by the correction event.
func TestDriftRejectsCorrectionJump(t *testing.T) {
pairs := []tsSkewPair{}
// 30 min of stable, ~12 sec/day drift: 1s per 7200s.
for i := 0; i < 12; i++ {
ts := int64(i) * 300
skew := float64(i) * (1.0 / 24.0) // ~0.04s per 5min step → 12 s/day
pairs = append(pairs, tsSkewPair{ts: ts, skew: skew})
}
// Wait an hour, then a single 1000-sec correction jump (clearly outlier).
pairs = append(pairs, tsSkewPair{ts: 3600 + 12*300, skew: 1000})
drift := computeDrift(pairs)
// Without rejection this would be ~ (1000-0)/(end-0) * 86400 = enormous.
if math.Abs(drift) > 100 {
t.Errorf("drift = %v, expected small (~12 s/day), correction jump should be filtered", drift)
}
}
// TestTheilSenMatchesOLSWhenClean: on clean linear data Theil-Sen should
// produce essentially the OLS answer.
func TestTheilSenMatchesOLSWhenClean(t *testing.T) {
// 1 sec drift per hour = 24 sec/day, 20 evenly-spaced samples.
pairs := []tsSkewPair{}
for i := 0; i < 20; i++ {
pairs = append(pairs, tsSkewPair{
ts: int64(i) * 600,
skew: float64(i) * (600.0 / 3600.0),
})
}
drift := computeDrift(pairs)
if math.Abs(drift-24.0) > 0.25 { // ~1%
t.Errorf("drift = %v, want ~24", drift)
}
}
// TestReporterScenario_789: reproduce the exact scenario from issue #789.
// Reporter saw mean=-52565156, median=-59063561, last=-0.8, sample count
// 1662, drift +1793549.9 s/day, severity=absurd. After the fix, severity
// must be ok (recent samples are healthy) and drift must be sane.
func TestReporterScenario_789(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
// 1657 samples with the bad ~-683-day skew (the historical poison),
// then 5 freshly corrected samples at -0.8s — totals 1662.
for i := 0; i < 1662; i++ {
obsTS := baseObs + int64(i)*60 // 1 min apart
var skew int64
if i < 1657 {
skew = -59063561 // ~ -683 days
} else {
skew = -1 // corrected (rounded; reporter saw -0.8)
}
advTS := obsTS + skew
tx := &StoreTx{
Hash: fmt.Sprintf("rep-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["REPNODE"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("REPNODE")
if r == nil {
t.Fatal("nil result")
}
// Severity must reflect current health, not the all-time median.
if r.Severity != SkewOK && r.Severity != SkewWarning {
t.Errorf("severity = %v, want ok/warning (recent samples are healthy)", r.Severity)
}
if math.Abs(r.RecentMedianSkewSec) > 5 {
t.Errorf("recentMedianSkewSec = %v, want near 0", r.RecentMedianSkewSec)
}
// Drift must not be absurd. The historical jump is one event between
// the 1657th and 1658th sample; outlier rejection must contain it.
if math.Abs(r.DriftPerDaySec) > maxReasonableDriftPerDay {
t.Errorf("drift = %v, must be <= cap %v", r.DriftPerDaySec, maxReasonableDriftPerDay)
}
// And it should be close to zero (stable historical + stable corrected).
if math.Abs(r.DriftPerDaySec) > 1000 {
t.Errorf("drift = %v, expected near zero after outlier rejection", r.DriftPerDaySec)
}
// Historical median is preserved as context.
if math.Abs(r.MedianSkewSec) < 1e6 {
t.Errorf("medianSkewSec = %v, expected historical poison preserved as context", r.MedianSkewSec)
}
}
// TestBimodalClock_845: 60% good samples → bimodal_clock severity.
func TestBimodalClock_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
// 6 good samples (-5s each), 4 bad samples (-50000000s each) = 60% good
// Interleave so the recent window (last 5) captures both good and bad.
skews := []int64{-5, -5, -50000000, -5, -50000000, -5, -50000000, -5, -50000000, -5}
for i := 0; i < 10; i++ {
obsTS := baseObs + int64(i)*60
advTS := obsTS + skews[i]
tx := &StoreTx{
Hash: fmt.Sprintf("bimodal-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["BIMODAL"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("BIMODAL")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewBimodalClock {
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
}
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
t.Errorf("recentMedianSkewSec = %v, want ≈ -5 (median of good samples)", r.RecentMedianSkewSec)
}
if r.GoodFraction < 0.5 || r.GoodFraction > 0.7 {
t.Errorf("goodFraction = %v, want ~0.6", r.GoodFraction)
}
if r.RecentBadSampleCount < 1 {
t.Errorf("recentBadSampleCount = %v, want > 0", r.RecentBadSampleCount)
}
}
// TestAllBad_NoClock_845: all samples bad → no_clock.
func TestAllBad_NoClock_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 10; i++ {
obsTS := baseObs + int64(i)*60
advTS := obsTS - 50000000
tx := &StoreTx{
Hash: fmt.Sprintf("allbad-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["ALLBAD"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("ALLBAD")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewNoClock {
t.Errorf("severity = %v, want no_clock", r.Severity)
}
}
// TestMostlyGood_OK_845: 90% good 10% bad → ok (outlier filtered).
func TestMostlyGood_OK_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
// 9 good at -5s, 1 bad at -50000000s
for i := 0; i < 10; i++ {
obsTS := baseObs + int64(i)*60
var skew int64
if i < 9 {
skew = -5
} else {
skew = -50000000
}
advTS := obsTS + skew
tx := &StoreTx{
Hash: fmt.Sprintf("mostly-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["MOSTLY"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("MOSTLY")
if r == nil {
t.Fatal("nil result")
}
// 90% good → normal classification path, median of good samples = -5s → ok
if r.Severity != SkewOK {
t.Errorf("severity = %v, want ok", r.Severity)
}
if math.Abs(r.RecentMedianSkewSec-(-5)) > 1 {
t.Errorf("recentMedianSkewSec = %v, want ≈ -5", r.RecentMedianSkewSec)
}
}
// TestSingleSample_845: one good sample → ok.
func TestSingleSample_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
obsTS := int64(1700000000)
advTS := obsTS - 30 // 30s skew
tx := &StoreTx{
Hash: "single-0001",
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
ps.mu.Lock()
ps.byNode["SINGLE"] = []*StoreTx{tx}
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("SINGLE")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewOK {
t.Errorf("severity = %v, want ok", r.Severity)
}
if r.RecentSampleCount != 1 {
t.Errorf("recentSampleCount = %d, want 1", r.RecentSampleCount)
}
if r.GoodFraction != 1.0 {
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
}
}
// TestFiftyFifty_Bimodal_845: 50% good / 50% bad → bimodal_clock.
func TestFiftyFifty_Bimodal_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 10; i++ {
obsTS := baseObs + int64(i)*60
var skew int64
if i%2 == 0 {
skew = -10
} else {
skew = -50000000
}
tx := &StoreTx{
Hash: fmt.Sprintf("fifty-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS+skew) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["FIFTY"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("FIFTY")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewBimodalClock {
t.Errorf("severity = %v, want bimodal_clock", r.Severity)
}
if r.GoodFraction < 0.4 || r.GoodFraction > 0.6 {
t.Errorf("goodFraction = %v, want ~0.5", r.GoodFraction)
}
}
// TestAllGood_OK_845: all samples good → ok, no bimodal.
func TestAllGood_OK_845(t *testing.T) {
ps := NewPacketStore(nil, nil)
pt := 4
baseObs := int64(1700000000)
var txs []*StoreTx
for i := 0; i < 10; i++ {
obsTS := baseObs + int64(i)*60
tx := &StoreTx{
Hash: fmt.Sprintf("allgood-%04d", i),
PayloadType: &pt,
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(obsTS-3) + `}}`,
Observations: []*StoreObs{
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
},
}
txs = append(txs, tx)
}
ps.mu.Lock()
ps.byNode["ALLGOOD"] = txs
for _, tx := range txs {
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
}
ps.clockSkew.computeInterval = 0
ps.mu.Unlock()
r := ps.GetNodeClockSkew("ALLGOOD")
if r == nil {
t.Fatal("nil result")
}
if r.Severity != SkewOK {
t.Errorf("severity = %v, want ok", r.Severity)
}
if r.GoodFraction != 1.0 {
t.Errorf("goodFraction = %v, want 1.0", r.GoodFraction)
}
if r.RecentBadSampleCount != 0 {
t.Errorf("recentBadSampleCount = %v, want 0", r.RecentBadSampleCount)
}
}
-131
View File
@@ -1,131 +0,0 @@
package main
import (
"testing"
"time"
)
// TestCollisionDetailsIncludeNodePairs verifies that collision details contain
// the correct prefix and matching node pairs (#757).
func TestCollisionDetailsIncludeNodePairs(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
// Insert two repeater nodes with the same 3-byte prefix "AABB11"
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Node Alpha', 'repeater')`)
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11eeff334455', 'Node Beta', 'repeater')`)
// Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch)
store := NewPacketStore(db, nil)
store.Load()
result := store.GetAnalyticsHashCollisions("")
bySize, ok := result["by_size"].(map[string]interface{})
if !ok {
t.Fatal("expected by_size map")
}
size3, ok := bySize["3"].(map[string]interface{})
if !ok {
t.Fatal("expected by_size[3] map")
}
collisions, ok := size3["collisions"].([]collisionEntry)
if !ok {
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
}
// Find our collision
var found *collisionEntry
for i := range collisions {
if collisions[i].Prefix == "AABB11" {
found = &collisions[i]
break
}
}
if found == nil {
t.Fatal("expected collision with prefix AABB11")
}
if found.Appearances != 2 {
t.Errorf("expected 2 appearances, got %d", found.Appearances)
}
if len(found.Nodes) != 2 {
t.Fatalf("expected 2 nodes in collision, got %d", len(found.Nodes))
}
// Verify node pairs
pubkeys := map[string]bool{}
names := map[string]bool{}
for _, n := range found.Nodes {
pubkeys[n.PublicKey] = true
names[n.Name] = true
}
if !pubkeys["aabb11ccdd001122"] {
t.Error("expected node aabb11ccdd001122 in collision")
}
if !pubkeys["aabb11eeff334455"] {
t.Error("expected node aabb11eeff334455 in collision")
}
if !names["Node Alpha"] {
t.Error("expected Node Alpha in collision")
}
if !names["Node Beta"] {
t.Error("expected Node Beta in collision")
}
}
// TestCollisionDetailsEmptyWhenNoCollisions verifies that collision details are
// empty when there are no collisions (#757).
func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
// Insert one repeater node with 3-byte hash
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
store := NewPacketStore(db, nil)
store.Load()
result := store.GetAnalyticsHashCollisions("")
bySize, ok := result["by_size"].(map[string]interface{})
if !ok {
t.Fatal("expected by_size map")
}
size3, ok := bySize["3"].(map[string]interface{})
if !ok {
t.Fatal("expected by_size[3] map")
}
collisions, ok := size3["collisions"].([]collisionEntry)
if !ok {
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
}
if len(collisions) != 0 {
t.Errorf("expected 0 collisions, got %d", len(collisions))
}
}
+4 -54
View File
@@ -6,7 +6,6 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"github.com/meshcore-analyzer/geofilter"
)
@@ -17,17 +16,6 @@ type Config struct {
APIKey string `json:"apiKey"`
DBPath string `json:"dbPath"`
// NodeBlacklist is a list of public keys to exclude from all API responses.
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
// Use this to filter out trolls, nodes with offensive names, or nodes
// reporting deliberately false data (e.g. wrong GPS position) that the
// operator refuses to fix.
NodeBlacklist []string `json:"nodeBlacklist"`
// blacklistSetCached is the lazily-built set version of NodeBlacklist.
blacklistSetCached map[string]bool
blacklistOnce sync.Once
Branding map[string]interface{} `json:"branding"`
Theme map[string]interface{} `json:"theme"`
ThemeDark map[string]interface{} `json:"themeDark"`
@@ -115,18 +103,16 @@ type NeighborGraphConfig struct {
// PacketStoreConfig controls in-memory packet store limits.
type PacketStoreConfig struct {
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
MaxResolvedPubkeyIndexEntries int `json:"maxResolvedPubkeyIndexEntries"` // warning threshold for index size (0 = 5M default)
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
}
// GeoFilterConfig is an alias for the shared geofilter.Config type.
type GeoFilterConfig = geofilter.Config
type RetentionConfig struct {
NodeDays int `json:"nodeDays"`
ObserverDays int `json:"observerDays"`
PacketDays int `json:"packetDays"`
MetricsDays int `json:"metricsDays"`
NodeDays int `json:"nodeDays"`
PacketDays int `json:"packetDays"`
MetricsDays int `json:"metricsDays"`
}
// MetricsRetentionDays returns configured metrics retention or 30 days default.
@@ -179,15 +165,6 @@ func (c *Config) NodeDaysOrDefault() int {
return 7
}
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
// A value of -1 means observers are never removed.
func (c *Config) ObserverDaysOrDefault() int {
if c.Retention != nil && c.Retention.ObserverDays != 0 {
return c.Retention.ObserverDays
}
return 14
}
type HealthThresholds struct {
InfraDegradedHours float64 `json:"infraDegradedHours"`
InfraSilentHours float64 `json:"infraSilentHours"`
@@ -361,30 +338,3 @@ func (c *Config) PropagationBufferMs() int {
}
return 5000
}
// blacklistSet lazily builds and caches the nodeBlacklist as a set for O(1) lookups.
// Uses sync.Once to eliminate the data race on first concurrent access.
func (c *Config) blacklistSet() map[string]bool {
c.blacklistOnce.Do(func() {
if len(c.NodeBlacklist) == 0 {
return
}
m := make(map[string]bool, len(c.NodeBlacklist))
for _, pk := range c.NodeBlacklist {
trimmed := strings.ToLower(strings.TrimSpace(pk))
if trimmed != "" {
m[trimmed] = true
}
}
c.blacklistSetCached = m
})
return c.blacklistSetCached
}
// IsBlacklisted returns true if the given public key is in the nodeBlacklist.
func (c *Config) IsBlacklisted(pubkey string) bool {
if c == nil || len(c.NodeBlacklist) == 0 {
return false
}
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
}
-22
View File
@@ -365,25 +365,3 @@ func TestPropagationBufferMs(t *testing.T) {
}
})
}
func TestObserverDaysOrDefault(t *testing.T) {
tests := []struct {
name string
cfg *Config
want int
}{
{"nil retention", &Config{}, 14},
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.cfg.ObserverDaysOrDefault()
if got != tt.want {
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
}
})
}
}
+5 -285
View File
@@ -41,7 +41,7 @@ func setupTestDBv2(t *testing.T) *DB {
id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL,
hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL,
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
decoded_json TEXT, channel_hash TEXT DEFAULT NULL, created_at TEXT DEFAULT (datetime('now'))
decoded_json TEXT, created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -585,15 +585,12 @@ func TestHandlePacketsMultiNodeWithStore(t *testing.T) {
func TestHandlePacketDetailNoStore(t *testing.T) {
_, router := setupNoStoreServer(t)
// With no in-memory store, handlePacketDetail now falls back to the DB
// (#827). The seeded transmissions are present in the DB, so by-hash and
// by-ID lookups succeed; only truly absent IDs return 404.
t.Run("by hash", func(t *testing.T) {
req := httptest.NewRequest("GET", "/api/packets/abc123def4567890", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
if w.Code != 404 {
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
}
})
@@ -601,8 +598,8 @@ func TestHandlePacketDetailNoStore(t *testing.T) {
req := httptest.NewRequest("GET", "/api/packets/1", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200 (DB fallback), got %d: %s", w.Code, w.Body.String())
if w.Code != 404 {
t.Fatalf("expected 404 (no store), got %d: %s", w.Code, w.Body.String())
}
})
@@ -3220,189 +3217,6 @@ func TestGetNodeHashSizeInfoEdgeCases(t *testing.T) {
}
}
// TestHashSizeTransportRoutePathByteOffset verifies that transport routes (0, 3)
// read the path byte from offset 5 (after 4 transport code bytes), not offset 1.
// Regression test for #744 / #722.
func TestHashSizeTransportRoutePathByteOffset(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
// Route type 0 (TRANSPORT_FLOOD): header=0x04 (payload_type=1, route_type=0)
// 4 transport bytes + path byte at offset 5.
// Path byte 0x80 → hash_size bits = 10 → size 3
// If bug is present, code reads byte 1 (0xAA) → hash_size bits = 10 → size 3 (coincidence)
// Use path byte 0x40 (hash_size=2) and transport byte 0x01 at offset 1 (hash_size=1 if misread)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('100102030440aabb', 'tf_offset', ?, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch)
// Route type 3 (TRANSPORT_DIRECT): header=0x13 (payload_type=4, route_type=3)
// 4 transport bytes + path byte at offset 5.
// Path byte 0xC1 → hash_size bits = 11 → size 4, hop_count = 1 (not zero-hop)
// Byte 1 = 0x05 → hash_size bits = 00 → size 1 if misread
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1305060708C1bbcc', 'td_offset', ?, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
// Route type 1 (FLOOD): header=0x11 (payload_type=4, route_type=1)
// Path byte at offset 1. Path byte 0x80 → hash_size bits = 10 → size 3
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1180aabbccdd', 'flood_offset', ?, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch)
store := NewPacketStore(db, nil)
store.Load()
info := store.GetNodeHashSizeInfo()
// Transport flood node: path byte 0x40 → hash_size = 2
if ni, ok := info["aaaa000000000001"]; !ok {
t.Error("transport flood node missing from hash size info")
} else if ni.HashSize != 2 {
t.Errorf("transport flood node: want HashSize=2 (from path byte at offset 5), got %d", ni.HashSize)
}
// Transport direct node: path byte 0xC1 → hash_size = 4
if ni, ok := info["aaaa000000000002"]; !ok {
t.Error("transport direct node missing from hash size info")
} else if ni.HashSize != 4 {
t.Errorf("transport direct node: want HashSize=4 (from path byte at offset 5), got %d", ni.HashSize)
}
// Regular flood node: path byte 0x80 → hash_size = 3
if ni, ok := info["aaaa000000000003"]; !ok {
t.Error("regular flood node missing from hash size info")
} else if ni.HashSize != 3 {
t.Errorf("regular flood node: want HashSize=3 (from path byte at offset 1), got %d", ni.HashSize)
}
}
// TestHashSizeTransportDirectZeroHopSkipped verifies that RouteTransportDirect
// zero-hop adverts are skipped (same as RouteDirect). Regression test for #744.
func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
// RouteDirect (2) zero-hop: path byte 0x40 → hop_count=0, hash_size bits=01
// Should be skipped (existing behavior)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1240aabbccdd', 'direct_zh', ?, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch)
// RouteTransportDirect (3) zero-hop: 4 transport bytes + path byte 0x40 → hop_count=0
// Should ALSO be skipped (this was the missing case)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('130102030440aabb', 'tdirect_zh', ?, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
// RouteDirect (2) non-zero-hop: path byte 0x41 → hop_count=1
// Should NOT be skipped
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1241aabbccdd', 'direct_1h', ?, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch)
store := NewPacketStore(db, nil)
store.Load()
info := store.GetNodeHashSizeInfo()
// RouteDirect zero-hop should be absent
if _, ok := info["bbbb000000000001"]; ok {
t.Error("RouteDirect zero-hop advert should be skipped")
}
// RouteTransportDirect zero-hop should also be absent
if _, ok := info["bbbb000000000002"]; ok {
t.Error("RouteTransportDirect zero-hop advert should be skipped")
}
// RouteDirect non-zero-hop should be present with hash_size=2
if ni, ok := info["bbbb000000000003"]; !ok {
t.Error("RouteDirect non-zero-hop should be in hash size info")
} else if ni.HashSize != 2 {
t.Errorf("RouteDirect non-zero-hop: want HashSize=2, got %d", ni.HashSize)
}
}
// TestAnalyticsHashSizesZeroHopSkip verifies that computeAnalyticsHashSizes
// does not overwrite a node's hash_size with a zero-hop advert's unreliable value.
// Regression test for #744.
func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent)
pk := "cccc000000000001"
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'ZH-Analytics', 'repeater')`, pk)
decoded := `{"pubKey":"` + pk + `","name":"ZH-Analytics","type":"ADVERT"}`
// First: a flood advert with hashSize=2 (reliable, multi-hop)
// header 0x11 = route_type 1 (flood), payload_type 4
// pathByte 0x41 = hashSize bits 01 → size 2, hop_count 1
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1141aabbccdd', 'az_flood', ?, 1, 4, ?)`, recent, decoded)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -90, '["aabb"]', ?)`, recentEpoch)
// Second: a direct zero-hop advert with pathByte=0x00 → would give hashSize=1
// header 0x12 = route_type 2 (direct), payload_type 4
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('1200aabbccdd', 'az_direct', ?, 2, 4, ?)`, recent, decoded)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch)
store := NewPacketStore(db, nil)
store.Load()
result := store.GetAnalyticsHashSizes("")
// The node should appear in multiByteNodes (hashSize=2 from the flood advert)
// If the zero-hop bug is present, hashSize would be 1 and the node would NOT
// appear in multiByteNodes.
multiByteNodes, ok := result["multiByteNodes"].([]map[string]interface{})
if !ok {
t.Fatal("expected multiByteNodes slice in analytics hash sizes")
}
found := false
for _, n := range multiByteNodes {
if n["pubkey"] == pk {
found = true
if hs, ok := n["hashSize"].(int); ok && hs != 2 {
t.Errorf("expected hashSize=2 from flood advert, got %d", hs)
}
}
}
if !found {
t.Error("node should appear in multiByteNodes with hashSize=2; zero-hop advert should not overwrite to 1")
}
}
func TestHandleResolveHopsEdgeCases(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
@@ -4319,50 +4133,6 @@ func TestIndexByNodePreCheck(t *testing.T) {
})
}
// TestIndexByNodeResolvedPath tests that indexByNode only indexes decoded JSON pubkeys.
// After #800, resolved_path entries are handled via the decode-window, not indexByNode.
func TestIndexByNodeResolvedPath(t *testing.T) {
store := &PacketStore{
byNode: make(map[string][]*StoreTx),
nodeHashes: make(map[string]map[string]bool),
}
t.Run("decoded JSON pubkeys still indexed", func(t *testing.T) {
pk := "aabb1122334455ff"
tx := &StoreTx{
Hash: "rp1",
DecodedJSON: `{"pubKey":"` + pk + `"}`,
}
store.indexByNode(tx)
if len(store.byNode[pk]) != 1 {
t.Errorf("expected decoded pubkey indexed, got %d", len(store.byNode[pk]))
}
})
t.Run("resolved path pubkeys NOT indexed by indexByNode", func(t *testing.T) {
// After #800, indexByNode only handles decoded JSON fields.
// Resolved path pubkeys are handled by the decode-window.
tx := &StoreTx{
Hash: "rp2",
DecodedJSON: `{"type":"CHAN","text":"hello"}`, // no pubKey fields
}
store.indexByNode(tx)
// No new entries expected since there are no decoded pubkeys
})
t.Run("dedup within decoded JSON", func(t *testing.T) {
pk := "dedup0test0pk1234"
tx := &StoreTx{
Hash: "rp4",
DecodedJSON: `{"pubKey":"` + pk + `","destPubKey":"` + pk + `"}`,
}
store.indexByNode(tx)
if len(store.byNode[pk]) != 1 {
t.Errorf("expected dedup to keep 1 entry, got %d", len(store.byNode[pk]))
}
})
}
// BenchmarkIndexByNode measures indexByNode performance with and without pubkey
// fields to demonstrate the strings.Contains pre-check optimization.
func BenchmarkIndexByNode(b *testing.B) {
@@ -4616,53 +4386,3 @@ func TestHandleBatchObservations(t *testing.T) {
}
})
}
// TestIngestTraceBroadcastIncludesPath verifies that TRACE packet broadcasts
// include decoded.path with hopsCompleted (#683).
func TestIngestTraceBroadcastIncludesPath(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
store := NewPacketStore(db, nil)
store.Load()
initialMax := store.MaxTransmissionID()
// TRACE packet: header=0x25, path_byte=0x02 (2 SNR bytes), 2 SNR bytes,
// then payload: tag(4) + authCode(4) + flags(1) + 4 hop hashes (1-byte each)
traceHex := "2502AABB010000000200000000DEADBEEF"
now := time.Now().UTC().Format(time.RFC3339)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES (?, 'tracehash683test', ?, 1, 9, '')`, traceHex, now)
newTxID := 0
db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (?, 1, 5.0, -100, '["aa"]', ?)`, newTxID, time.Now().Unix())
broadcastMaps, _ := store.IngestNewFromDB(initialMax, 100)
if len(broadcastMaps) < 1 {
t.Fatal("expected >=1 broadcast maps")
}
bm := broadcastMaps[0]
decoded, ok := bm["decoded"].(map[string]interface{})
if !ok {
t.Fatal("broadcast map missing 'decoded'")
}
pathObj, ok := decoded["path"]
if !ok {
t.Fatal("decoded missing 'path' for TRACE packet — hopsCompleted not delivered to frontend (#683)")
}
// The path should be a Path struct with HopsCompleted = 2
pathStruct, ok := pathObj.(Path)
if !ok {
t.Fatalf("expected Path struct, got %T", pathObj)
}
if pathStruct.HopsCompleted == nil {
t.Fatal("path.HopsCompleted is nil for TRACE packet")
}
if *pathStruct.HopsCompleted != 2 {
t.Errorf("expected hopsCompleted=2, got %d", *pathStruct.HopsCompleted)
}
}
+80 -348
View File
@@ -8,7 +8,6 @@ import (
"math"
"os"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
@@ -20,12 +19,6 @@ type DB struct {
path string // filesystem path to the database file
isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2)
hasResolvedPath bool // observations table has resolved_path column
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
channelsCacheMu sync.Mutex
channelsCacheKey string
channelsCacheRes []map[string]interface{}
channelsCacheExp time.Time
}
// OpenDB opens a read-only SQLite connection with WAL mode.
@@ -384,7 +377,6 @@ type PacketQuery struct {
Until string
Region string
Node string
Channel string // channel_hash filter (#812). Plain names like "#test"/"public" or "enc_<HEX>" for encrypted
Order string // ASC or DESC
ExpandObservations bool // when true, include observation sub-maps in txToMap output
}
@@ -621,11 +613,6 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
where = append(where, "t.decoded_json LIKE ?")
args = append(args, "%"+pk+"%")
}
if q.Channel != "" {
// channel_hash column is indexed for payload_type = 5; filter is exact match.
where = append(where, "t.channel_hash = ?")
args = append(args, q.Channel)
}
if q.Observer != "" {
ids := strings.Split(q.Observer, ",")
placeholders := strings.Repeat("?,", len(ids))
@@ -692,20 +679,6 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) {
return nil, nil
}
// GetObservationsForHash returns all observations for the transmission with
// the given content hash. Used as a fallback by the packet-detail handler
// when the in-memory PacketStore has pruned the entry but the DB still has it.
func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} {
var txID int
err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?",
strings.ToLower(hash)).Scan(&txID)
if err != nil {
return nil
}
obsByTx := db.getObservationsForTransmissions([]int{txID})
return obsByTx[txID]
}
// GetNodes returns filtered, paginated node list.
func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortBy, region string) ([]map[string]interface{}, int, map[string]int, error) {
@@ -1180,219 +1153,69 @@ func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) {
// Queries transmissions directly (not a VIEW) to avoid observation-level
// duplicates that could cause stale lastMessage when an older message has
// a later re-observation timestamp.
func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) {
regionParam := ""
if len(region) > 0 {
regionParam = region[0]
}
// Check cache (60s TTL)
db.channelsCacheMu.Lock()
if db.channelsCacheRes != nil && db.channelsCacheKey == regionParam && time.Now().Before(db.channelsCacheExp) {
res := db.channelsCacheRes
db.channelsCacheMu.Unlock()
return res, nil
}
db.channelsCacheMu.Unlock()
regionCodes := normalizeRegionCodes(regionParam)
var querySQL string
args := make([]interface{}, 0, len(regionCodes))
if len(regionCodes) > 0 {
placeholders := make([]string, len(regionCodes))
for i, code := range regionCodes {
placeholders[i] = "?"
args = append(args, code)
}
regionPlaceholder := strings.Join(placeholders, ",")
if db.isV3 {
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
COUNT(*) AS msg_count,
MAX(t.first_seen) AS last_activity,
(SELECT t2.decoded_json FROM transmissions t2
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
FROM transmissions t
JOIN observations o ON o.transmission_id = t.id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
WHERE t.payload_type = 5
AND t.channel_hash IS NOT NULL
AND t.channel_hash NOT LIKE 'enc_%%'
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
GROUP BY t.channel_hash
ORDER BY last_activity DESC`, regionPlaceholder)
} else {
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
COUNT(*) AS msg_count,
MAX(t.first_seen) AS last_activity,
(SELECT t2.decoded_json FROM transmissions t2
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
FROM transmissions t
JOIN observations o ON o.transmission_id = t.id
WHERE t.payload_type = 5
AND t.channel_hash IS NOT NULL
AND t.channel_hash NOT LIKE 'enc_%%'
AND EXISTS (
SELECT 1 FROM observers obs
WHERE obs.id = o.observer_id
AND UPPER(TRIM(obs.iata)) IN (%s)
)
GROUP BY t.channel_hash
ORDER BY last_activity DESC`, regionPlaceholder)
}
} else {
querySQL = `SELECT channel_hash,
COUNT(*) AS msg_count,
MAX(first_seen) AS last_activity,
(SELECT t2.decoded_json FROM transmissions t2
WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5
ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json
FROM transmissions t
WHERE payload_type = 5
AND channel_hash IS NOT NULL
AND channel_hash NOT LIKE 'enc_%%'
GROUP BY channel_hash
ORDER BY last_activity DESC`
}
rows, err := db.conn.Query(querySQL, args...)
func (db *DB) GetChannels() ([]map[string]interface{}, error) {
rows, err := db.conn.Query(`SELECT decoded_json, first_seen FROM transmissions WHERE payload_type = 5 ORDER BY first_seen ASC`)
if err != nil {
return nil, err
}
defer rows.Close()
channels := make([]map[string]interface{}, 0)
channelMap := map[string]map[string]interface{}{}
for rows.Next() {
var chHash, lastActivity, sampleJSON sql.NullString
var msgCount int
if err := rows.Scan(&chHash, &msgCount, &lastActivity, &sampleJSON); err != nil {
var dj, fs sql.NullString
rows.Scan(&dj, &fs)
if !dj.Valid {
continue
}
channelName := nullStr(chHash)
var decoded map[string]interface{}
if json.Unmarshal([]byte(dj.String), &decoded) != nil {
continue
}
dtype, _ := decoded["type"].(string)
if dtype != "CHAN" {
continue
}
// Filter out garbage-decrypted channel names/messages (pre-#197 data still in DB)
chanStr, _ := decoded["channel"].(string)
textStr, _ := decoded["text"].(string)
if hasGarbageChars(chanStr) || hasGarbageChars(textStr) {
continue
}
channelName, _ := decoded["channel"].(string)
if channelName == "" {
continue
channelName = "unknown"
}
key := channelName
var lastMessage, lastSender interface{}
if sampleJSON.Valid {
var decoded map[string]interface{}
if json.Unmarshal([]byte(sampleJSON.String), &decoded) == nil {
if text, ok := decoded["text"].(string); ok && text != "" {
idx := strings.Index(text, ": ")
if idx > 0 {
lastMessage = text[idx+2:]
} else {
lastMessage = text
}
if sender, ok := decoded["sender"].(string); ok {
lastSender = sender
}
}
ch, exists := channelMap[key]
if !exists {
ch = map[string]interface{}{
"hash": key, "name": channelName,
"lastMessage": nil, "lastSender": nil,
"messageCount": 0, "lastActivity": nullStr(fs),
}
channelMap[key] = ch
}
ch["messageCount"] = ch["messageCount"].(int) + 1
if fs.Valid {
ch["lastActivity"] = fs.String
}
if text, ok := decoded["text"].(string); ok && text != "" {
idx := strings.Index(text, ": ")
if idx > 0 {
ch["lastMessage"] = text[idx+2:]
} else {
ch["lastMessage"] = text
}
if sender, ok := decoded["sender"].(string); ok {
ch["lastSender"] = sender
}
}
channels = append(channels, map[string]interface{}{
"hash": channelName, "name": channelName,
"lastMessage": lastMessage, "lastSender": lastSender,
"messageCount": msgCount, "lastActivity": nullStr(lastActivity),
})
}
// Store in cache (60s TTL)
db.channelsCacheMu.Lock()
db.channelsCacheRes = channels
db.channelsCacheKey = regionParam
db.channelsCacheExp = time.Now().Add(60 * time.Second)
db.channelsCacheMu.Unlock()
return channels, nil
}
// GetEncryptedChannels returns channels where all messages are undecryptable (no key).
// Uses channel_hash column (prefixed with 'enc_') for fast grouped queries.
func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, error) {
regionParam := ""
if len(region) > 0 {
regionParam = region[0]
}
regionCodes := normalizeRegionCodes(regionParam)
var querySQL string
args := make([]interface{}, 0, len(regionCodes))
if len(regionCodes) > 0 {
placeholders := make([]string, len(regionCodes))
for i, code := range regionCodes {
placeholders[i] = "?"
args = append(args, code)
}
regionPlaceholder := strings.Join(placeholders, ",")
if db.isV3 {
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
COUNT(*) AS msg_count,
MAX(t.first_seen) AS last_activity
FROM transmissions t
JOIN observations o ON o.transmission_id = t.id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
WHERE t.payload_type = 5
AND t.channel_hash LIKE 'enc_%%'
AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)
GROUP BY t.channel_hash
ORDER BY last_activity DESC`, regionPlaceholder)
} else {
querySQL = fmt.Sprintf(`SELECT t.channel_hash,
COUNT(*) AS msg_count,
MAX(t.first_seen) AS last_activity
FROM transmissions t
JOIN observations o ON o.transmission_id = t.id
WHERE t.payload_type = 5
AND t.channel_hash LIKE 'enc_%%'
AND EXISTS (
SELECT 1 FROM observers obs
WHERE obs.id = o.observer_id
AND UPPER(TRIM(obs.iata)) IN (%s)
)
GROUP BY t.channel_hash
ORDER BY last_activity DESC`, regionPlaceholder)
}
} else {
querySQL = `SELECT channel_hash,
COUNT(*) AS msg_count,
MAX(first_seen) AS last_activity
FROM transmissions
WHERE payload_type = 5
AND channel_hash LIKE 'enc_%%'
GROUP BY channel_hash
ORDER BY last_activity DESC`
}
rows, err := db.conn.Query(querySQL, args...)
if err != nil {
return nil, err
}
defer rows.Close()
channels := make([]map[string]interface{}, 0)
for rows.Next() {
var chHash, lastActivity sql.NullString
var msgCount int
if err := rows.Scan(&chHash, &msgCount, &lastActivity); err != nil {
continue
}
fullHash := nullStrVal(chHash) // e.g. "enc_3A"
hexPart := strings.TrimPrefix(fullHash, "enc_")
channels = append(channels, map[string]interface{}{
"hash": fullHash,
"name": "Encrypted (0x" + hexPart + ")",
"lastMessage": nil,
"lastSender": nil,
"messageCount": msgCount,
"lastActivity": nullStr(lastActivity),
"encrypted": true,
})
channels := make([]map[string]interface{}, 0, len(channelMap))
for _, ch := range channelMap {
channels = append(channels, ch)
}
return channels, nil
}
@@ -1421,16 +1244,15 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
regionPlaceholders = strings.Join(placeholders, ",")
}
// Fetch messages with channel_hash filter (pagination applied in Go after dedup)
var querySQL string
args := []interface{}{channelHash}
args := make([]interface{}, 0, len(regionArgs))
if db.isV3 {
querySQL = `SELECT o.id, t.hash, t.decoded_json, t.first_seen,
obs.id, obs.name, o.snr, o.path_json
FROM observations o
JOIN transmissions t ON t.id = o.transmission_id
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
WHERE t.channel_hash = ? AND t.payload_type = 5`
WHERE t.payload_type = 5`
if len(regionCodes) > 0 {
querySQL += fmt.Sprintf(" AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s)", regionPlaceholders)
args = append(args, regionArgs...)
@@ -1442,11 +1264,14 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
o.observer_id, o.observer_name, o.snr, o.path_json
FROM observations o
JOIN transmissions t ON t.id = o.transmission_id
WHERE t.channel_hash = ? AND t.payload_type = 5`
WHERE t.payload_type = 5`
if len(regionCodes) > 0 {
querySQL += fmt.Sprintf(` AND EXISTS (
SELECT 1 FROM observers obs WHERE obs.id = o.observer_id
AND UPPER(TRIM(obs.iata)) IN (%s))`, regionPlaceholders)
SELECT 1
FROM observers obs
WHERE obs.id = o.observer_id
AND UPPER(TRIM(obs.iata)) IN (%s)
)`, regionPlaceholders)
args = append(args, regionArgs...)
}
querySQL += `
@@ -1478,6 +1303,17 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
if json.Unmarshal([]byte(dj.String), &decoded) != nil {
continue
}
dtype, _ := decoded["type"].(string)
if dtype != "CHAN" {
continue
}
ch, _ := decoded["channel"].(string)
if ch == "" {
ch = "unknown"
}
if ch != channelHash {
continue
}
text, _ := decoded["text"].(string)
sender, _ := decoded["sender"].(string)
@@ -1537,18 +1373,18 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
}
}
// Return latest messages (tail) with pagination
msgTotal := len(msgOrder)
start := msgTotal - limit - offset
total := len(msgOrder)
// Return latest messages (tail)
start := total - limit - offset
if start < 0 {
start = 0
}
end := msgTotal - offset
end := total - offset
if end < 0 {
end = 0
}
if end > msgTotal {
end = msgTotal
if end > total {
end = total
}
messages := make([]map[string]interface{}, 0)
@@ -1559,7 +1395,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region .
messages = append(messages, m.Data)
}
return messages, msgTotal, nil
return messages, total, nil
}
@@ -1868,10 +1704,12 @@ func nullInt(ni sql.NullInt64) interface{} {
// Returns the number of transmissions deleted.
// Opens a separate read-write connection since the main connection is read-only.
func (db *DB) PruneOldPackets(days int) (int64, error) {
rw, err := openRW(db.path)
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=10000", db.path)
rw, err := sql.Open("sqlite", dsn)
if err != nil {
return 0, err
}
rw.SetMaxOpenConns(1)
defer rw.Close()
cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)
@@ -2215,10 +2053,12 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) {
// PruneOldMetrics deletes observer_metrics rows older than retentionDays.
func (db *DB) PruneOldMetrics(retentionDays int) (int64, error) {
rw, err := openRW(db.path)
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=10000", db.path)
rw, err := sql.Open("sqlite", dsn)
if err != nil {
return 0, err
}
rw.SetMaxOpenConns(1)
defer rw.Close()
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339)
@@ -2232,111 +2072,3 @@ func (db *DB) PruneOldMetrics(retentionDays int) (int64, error) {
}
return n, nil
}
// RemoveStaleObservers marks observers that have not actively sent data in observerDays
// as inactive (soft-delete). This preserves JOIN integrity for observations.observer_idx
// and observer_metrics.observer_id — historical data still references the correct observer.
// An observer must actively send data to stay listed — being seen by another node does not count.
// observerDays <= -1 means never remove (keep forever).
func (db *DB) RemoveStaleObservers(observerDays int) (int64, error) {
if observerDays <= -1 {
return 0, nil // keep forever
}
rw, err := openRW(db.path)
if err != nil {
return 0, err
}
defer rw.Close()
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
res, err := rw.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
if err != nil {
return 0, err
}
n, _ := res.RowsAffected()
if n > 0 {
// Clean up orphaned metrics for now-inactive observers
rw.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
log.Printf("[observers] Marked %d observer(s) as inactive (not seen in %d days)", n, observerDays)
}
return n, nil
}
// TouchNodeLastSeen updates last_seen for a node identified by full public key.
// Only updates if the new timestamp is newer than the existing value (or NULL).
// Returns nil even if no rows are affected (node doesn't exist).
func (db *DB) TouchNodeLastSeen(pubkey string, timestamp string) error {
_, err := db.conn.Exec(
"UPDATE nodes SET last_seen = ? WHERE public_key = ? AND (last_seen IS NULL OR last_seen < ?)",
timestamp, pubkey, timestamp,
)
return err
}
// GetDroppedPackets returns recently dropped packets, newest first.
func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map[string]interface{}, error) {
if limit <= 0 || limit > 500 {
limit = 100
}
query := `SELECT id, hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name, dropped_at FROM dropped_packets`
var conditions []string
var args []interface{}
if observerID != "" {
conditions = append(conditions, "observer_id = ?")
args = append(args, observerID)
}
if nodePubkey != "" {
conditions = append(conditions, "node_pubkey = ?")
args = append(args, nodePubkey)
}
if len(conditions) > 0 {
query += " WHERE " + strings.Join(conditions, " AND ")
}
query += " ORDER BY dropped_at DESC LIMIT ?"
args = append(args, limit)
rows, err := db.conn.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var results []map[string]interface{}
for rows.Next() {
var id int
var hash, rawHex, reason, obsID, obsName, pubkey, name, droppedAt sql.NullString
if err := rows.Scan(&id, &hash, &rawHex, &reason, &obsID, &obsName, &pubkey, &name, &droppedAt); err != nil {
continue
}
row := map[string]interface{}{
"id": id,
"hash": nullStr(hash),
"reason": nullStr(reason),
"observer_id": nullStr(obsID),
"observer_name": nullStr(obsName),
"node_pubkey": nullStr(pubkey),
"node_name": nullStr(name),
"dropped_at": nullStr(droppedAt),
}
// Only include raw_hex if explicitly requested (it's large)
if rawHex.Valid {
row["raw_hex"] = rawHex.String
}
results = append(results, row)
}
if results == nil {
results = []map[string]interface{}{}
}
return results, nil
}
// GetSignatureDropCount returns the total number of dropped packets.
func (db *DB) GetSignatureDropCount() int64 {
var count int64
// Table may not exist yet if ingestor hasn't run the migration
err := db.conn.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
if err != nil {
return 0
}
return count
}
+24 -81
View File
@@ -60,7 +60,6 @@ func setupTestDB(t *testing.T) *DB {
payload_type INTEGER,
payload_version INTEGER,
decoded_json TEXT,
channel_hash TEXT DEFAULT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
@@ -125,10 +124,10 @@ func seedTestData(t *testing.T, db *DB) {
VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, ?, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo)
// Seed transmissions
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test')`, recent)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}')`, recent)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, yesterday)
// Second ADVERT for same node with different hash_size (raw_hex byte 0x1F → hs=1 vs 0xBB → hs=3)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA1F', 'def456abc1230099', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}')`, yesterday)
@@ -736,12 +735,12 @@ func TestGetChannelMessagesRegionFiltering(t *testing.T) {
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`)
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', ' sfo ')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA', 'chanregion0001', ?, 1, 5,
'{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}', '#region')`, ts1)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
'{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}')`, ts1)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('BB', 'chanregion0002', ?, 1, 5,
'{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}', '#region')`, ts2)
'{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}')`, ts2)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 10.0, -90, '[]', ?)`, epoch1)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
@@ -1120,7 +1119,6 @@ func setupTestDBV2(t *testing.T) *DB {
payload_type INTEGER,
payload_version INTEGER,
decoded_json TEXT,
channel_hash TEXT DEFAULT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
@@ -1204,12 +1202,12 @@ func TestGetChannelMessagesDedup(t *testing.T) {
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', 'SFO')`)
// Insert two transmissions with same hash to test dedup
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA', 'chanmsg00000001', '2026-01-15T10:00:00Z', 1, 5,
'{"type":"CHAN","channel":"#general","text":"User1: Hello","sender":"User1"}', '#general')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
'{"type":"CHAN","channel":"#general","text":"User1: Hello","sender":"User1"}')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('BB', 'chanmsg00000002', '2026-01-15T10:01:00Z', 1, 5,
'{"type":"CHAN","channel":"#general","text":"User2: World","sender":"User2"}', '#general')`)
'{"type":"CHAN","channel":"#general","text":"User2: World","sender":"User2"}')`)
// Observations: first msg seen by two observers (dedup), second by one
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
@@ -1253,9 +1251,9 @@ func TestGetChannelMessagesNoSender(t *testing.T) {
defer db.Close()
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('CC', 'chanmsg00000003', '2026-01-15T10:02:00Z', 1, 5,
'{"type":"CHAN","channel":"#noname","text":"plain text no colon"}', '#noname')`)
'{"type":"CHAN","channel":"#noname","text":"plain text no colon"}')`)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, 1, 12.0, -90, null, 1736935300)`)
@@ -1358,9 +1356,9 @@ func TestGetChannelMessagesObserverFallback(t *testing.T) {
defer db.Close()
// Observer with ID but no name entry (observer_idx won't match)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA', 'chanmsg00000004', '2026-01-15T10:00:00Z', 1, 5,
'{"type":"CHAN","channel":"#obs","text":"Sender: Test","sender":"Sender"}', '#obs')`)
'{"type":"CHAN","channel":"#obs","text":"Sender: Test","sender":"Sender"}')`)
// Observation without observer (observer_idx = NULL)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (1, NULL, 12.0, -90, null, 1736935200)`)
@@ -1382,12 +1380,12 @@ func TestGetChannelsMultiple(t *testing.T) {
defer db.Close()
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer', 'SJC')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA', 'chan1hash', '2026-01-15T10:00:00Z', 1, 5,
'{"type":"CHAN","channel":"#alpha","text":"Alice: Hello","sender":"Alice"}', '#alpha')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
'{"type":"CHAN","channel":"#alpha","text":"Alice: Hello","sender":"Alice"}')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('BB', 'chan2hash', '2026-01-15T10:01:00Z', 1, 5,
'{"type":"CHAN","channel":"#beta","text":"Bob: World","sender":"Bob"}', '#beta')`)
'{"type":"CHAN","channel":"#beta","text":"Bob: World","sender":"Bob"}')`)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('CC', 'chan3hash', '2026-01-15T10:02:00Z', 1, 5,
'{"type":"CHAN","channel":"","text":"No channel"}')`)
@@ -1470,13 +1468,13 @@ func TestGetChannelsStaleMessage(t *testing.T) {
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer2', 'SFO')`)
// Older message (first_seen T1)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('AA', 'oldhash1', '2026-01-15T10:00:00Z', 1, 5,
'{"type":"CHAN","channel":"#test","text":"Alice: Old message","sender":"Alice"}', '#test')`)
'{"type":"CHAN","channel":"#test","text":"Alice: Old message","sender":"Alice"}')`)
// Newer message (first_seen T2 > T1)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('BB', 'newhash2', '2026-01-15T10:05:00Z', 1, 5,
'{"type":"CHAN","channel":"#test","text":"Bob: New message","sender":"Bob"}', '#test')`)
'{"type":"CHAN","channel":"#test","text":"Bob: New message","sender":"Bob"}')`)
// Observations: older message re-observed AFTER newer message (stale scenario)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
@@ -1506,61 +1504,6 @@ func TestGetChannelsStaleMessage(t *testing.T) {
}
}
func TestGetChannelsRegionFiltering(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer1', 'SJC')`)
db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer2', 'SFO')`)
// Channel message seen only in SJC
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('AA', 'hash1', '2026-01-15T10:00:00Z', 1, 5,
'{"type":"CHAN","channel":"#sjc-only","text":"Alice: Hello SJC","sender":"Alice"}', '#sjc-only')`)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
VALUES (1, 1, 12.0, -90, 1736935200)`)
// Channel message seen only in SFO
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('BB', 'hash2', '2026-01-15T10:05:00Z', 1, 5,
'{"type":"CHAN","channel":"#sfo-only","text":"Bob: Hello SFO","sender":"Bob"}', '#sfo-only')`)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, timestamp)
VALUES (2, 2, 14.0, -88, 1736935500)`)
// No region filter — both channels
all, err := db.GetChannels()
if err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("expected 2 channels without region filter, got %d", len(all))
}
// Filter SJC — only #sjc-only
sjc, err := db.GetChannels("SJC")
if err != nil {
t.Fatal(err)
}
if len(sjc) != 1 {
t.Fatalf("expected 1 channel for SJC, got %d", len(sjc))
}
if sjc[0]["name"] != "#sjc-only" {
t.Errorf("expected channel '#sjc-only', got %q", sjc[0]["name"])
}
// Filter SFO — only #sfo-only
sfo, err := db.GetChannels("SFO")
if err != nil {
t.Fatal(err)
}
if len(sfo) != 1 {
t.Fatalf("expected 1 channel for SFO, got %d", len(sfo))
}
if sfo[0]["name"] != "#sfo-only" {
t.Errorf("expected channel '#sfo-only', got %q", sfo[0]["name"])
}
}
func TestNodeTelemetryFields(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
+10 -43
View File
@@ -116,7 +116,6 @@ type DecodedPacket struct {
Path Path `json:"path"`
Payload Payload `json:"payload"`
Raw string `json:"raw"`
Anomaly string `json:"anomaly,omitempty"`
}
func decodeHeader(b byte) Header {
@@ -211,13 +210,8 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload {
}
if validateSignatures {
valid, err := sigvalidate.ValidateAdvert(buf[0:32], buf[36:100], timestamp, appdata)
if err != nil {
f := false
p.SignatureValid = &f
} else {
p.SignatureValid = &valid
}
valid := sigvalidate.ValidateAdvertSignature(buf[0:32], buf[36:100], timestamp, appdata)
p.SignatureValid = &valid
}
if len(appdata) > 0 {
@@ -389,34 +383,22 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
payload := decodePayload(header.PayloadType, payloadBuf, validateSignatures)
// TRACE packets store hop IDs in the payload (buf[9:]) rather than the header
// path field. Firmware always sends TRACE as DIRECT (route_type 2 or 3);
// FLOOD-routed TRACEs are anomalous but handled gracefully (parsed, but
// flagged). The TRACE flags byte (payload offset 8) encodes path_sz in
// bits 0-1 as a power-of-two exponent: hash_bytes = 1 << path_sz.
// NOT the header path byte's hash_size bits. The header path contains SNR
// bytes — one per hop that actually forwarded.
// path field. The header path byte still encodes hashSize in bits 6-7, which
// we use to split the payload path data into individual hop prefixes.
// The header path contains SNR bytes — one per hop that actually forwarded.
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
// how far the trace got vs the full intended route.
var anomaly string
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
// Flag anomalous routing — firmware only sends TRACE as DIRECT
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
anomaly = "TRACE packet with non-DIRECT routing (expected DIRECT or TRANSPORT_DIRECT)"
}
// The header path hops count represents SNR entries = completed hops
hopsCompleted := path.HashCount
pathBytes, err := hex.DecodeString(payload.PathData)
if err == nil && payload.TraceFlags != nil {
// path_sz from flags byte is a power-of-two exponent per firmware:
// hash_bytes = 1 << (flags & 0x03)
pathSz := 1 << (*payload.TraceFlags & 0x03)
hops := make([]string, 0, len(pathBytes)/pathSz)
for i := 0; i+pathSz <= len(pathBytes); i += pathSz {
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+pathSz])))
if err == nil && path.HashSize > 0 {
hops := make([]string, 0, len(pathBytes)/path.HashSize)
for i := 0; i+path.HashSize <= len(pathBytes); i += path.HashSize {
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+path.HashSize])))
}
path.Hops = hops
path.HashCount = len(hops)
path.HashSize = pathSz
path.HopsCompleted = &hopsCompleted
}
}
@@ -437,7 +419,6 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er
Path: path,
Payload: payload,
Raw: strings.ToUpper(hexString),
Anomaly: anomaly,
}, nil
}
@@ -542,9 +523,6 @@ func BuildBreakdown(hexString string) *Breakdown {
}
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
// route-independent identifier for the same logical packet. For TRACE packets,
// path_len is included in the hash to match firmware behavior.
func ComputeContentHash(rawHex string) string {
buf, err := hex.DecodeString(rawHex)
if err != nil || len(buf) < 2 {
@@ -580,18 +558,7 @@ func ComputeContentHash(rawHex string) string {
}
payload := buf[payloadStart:]
// Hash payload-type byte only (bits 2-5 of header), not the full header.
// Firmware: SHA256(payload_type + [path_len for TRACE] + payload)
// Using the full header caused different hashes for the same logical packet
// when route type or version bits differed. See issue #786.
payloadType := (headerByte >> 2) & 0x0F
toHash := []byte{payloadType}
if int(payloadType) == PayloadTRACE {
// Firmware uses uint16_t path_len (2 bytes, little-endian)
toHash = append(toHash, pathByte, 0x00)
}
toHash = append(toHash, payload...)
toHash := append([]byte{headerByte}, payload...)
h := sha256.Sum256(toHash)
return hex.EncodeToString(h[:])[:16]
+68 -156
View File
@@ -3,8 +3,9 @@ package main
import (
"crypto/ed25519"
"encoding/binary"
"encoding/hex"
"testing"
"github.com/meshcore-analyzer/sigvalidate"
)
func TestDecodeHeader_TransportFlood(t *testing.T) {
@@ -357,10 +358,6 @@ func TestDecodePacket_TraceHopsCompleted(t *testing.T) {
if *pkt.Path.HopsCompleted != 2 {
t.Errorf("expected HopsCompleted=2, got %d", *pkt.Path.HopsCompleted)
}
// FLOOD routing for TRACE is anomalous
if pkt.Anomaly == "" {
t.Error("expected anomaly flag for FLOOD-routed TRACE")
}
}
func TestDecodePacket_TraceNoSNR(t *testing.T) {
@@ -411,172 +408,87 @@ func TestDecodePacket_TraceFullyCompleted(t *testing.T) {
}
}
func TestDecodePacket_TraceFlags1_TwoBytePathSz(t *testing.T) {
// TRACE with flags=1 → path_sz = 1 << (1 & 0x03) = 2-byte hashes
// Firmware always sends TRACE as DIRECT (route_type=2), so header byte =
// (0<<6)|(9<<2)|2 = 0x26. path_length 0x00 = 0 SNR bytes.
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
"01000000" + // tag
"02000000" + // authCode
"01" + // flags = 1 → path_sz = 2
"AABBCCDD" // 4 bytes = 2 hops of 2-byte each
pkt, err := DecodePacket(hex, false)
if err != nil {
t.Fatalf("DecodePacket error: %v", err)
}
if len(pkt.Path.Hops) != 2 {
t.Errorf("expected 2 hops (2-byte path_sz), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
}
if pkt.Path.HashSize != 2 {
t.Errorf("expected HashSize=2, got %d", pkt.Path.HashSize)
}
if pkt.Anomaly != "" {
t.Errorf("expected no anomaly for DIRECT TRACE, got %q", pkt.Anomaly)
}
}
func TestDecodePacket_TraceFlags2_FourBytePathSz(t *testing.T) {
// TRACE with flags=2 → path_sz = 1 << (2 & 0x03) = 4-byte hashes
// DIRECT route_type (0x26)
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
"01000000" + // tag
"02000000" + // authCode
"02" + // flags = 2 → path_sz = 4
"AABBCCDD11223344" // 8 bytes = 2 hops of 4-byte each
pkt, err := DecodePacket(hex, false)
if err != nil {
t.Fatalf("DecodePacket error: %v", err)
}
if len(pkt.Path.Hops) != 2 {
t.Errorf("expected 2 hops (4-byte path_sz), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
}
if pkt.Path.HashSize != 4 {
t.Errorf("expected HashSize=4, got %d", pkt.Path.HashSize)
}
}
func TestDecodePacket_TracePathSzUnevenPayload(t *testing.T) {
// TRACE with flags=1 → path_sz=2, but 5 bytes of path data (not evenly divisible)
// Should produce 2 hops (4 bytes) and ignore the trailing byte
hex := "2600" + // header (DIRECT+TRACE) + path_length (0 SNR)
"01000000" + // tag
"02000000" + // authCode
"01" + // flags = 1 → path_sz = 2
"AABBCCDDEE" // 5 bytes → 2 hops, 1 byte remainder ignored
pkt, err := DecodePacket(hex, false)
if err != nil {
t.Fatalf("DecodePacket error: %v", err)
}
if len(pkt.Path.Hops) != 2 {
t.Errorf("expected 2 hops (trailing byte ignored), got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
}
}
func TestDecodePacket_TraceTransportDirect(t *testing.T) {
// TRACE via TRANSPORT_DIRECT (route_type=3) — includes 4 transport code bytes
// header: (0<<6)|(9<<2)|3 = 0x27
hex := "27" + // header (TRANSPORT_DIRECT+TRACE)
"AABB" + "CCDD" + // transport codes (2+2 bytes)
"02" + // path_length: hash_count=2 SNR bytes
"EEFF" + // 2 SNR bytes
"01000000" + // tag
"02000000" + // authCode
"00" + // flags = 0 → path_sz = 1
"112233" // 3 hops (1-byte each)
pkt, err := DecodePacket(hex, false)
if err != nil {
t.Fatalf("DecodePacket error: %v", err)
}
if pkt.TransportCodes == nil {
t.Fatal("expected transport codes for TRANSPORT_DIRECT")
}
if pkt.TransportCodes.Code1 != "AABB" {
t.Errorf("expected Code1=AABB, got %s", pkt.TransportCodes.Code1)
}
if len(pkt.Path.Hops) != 3 {
t.Errorf("expected 3 hops, got %d: %v", len(pkt.Path.Hops), pkt.Path.Hops)
}
if pkt.Path.HopsCompleted == nil || *pkt.Path.HopsCompleted != 2 {
t.Errorf("expected HopsCompleted=2, got %v", pkt.Path.HopsCompleted)
}
if pkt.Anomaly != "" {
t.Errorf("expected no anomaly for TRANSPORT_DIRECT TRACE, got %q", pkt.Anomaly)
}
}
func TestDecodePacket_TraceFloodRouteAnomaly(t *testing.T) {
// TRACE via FLOOD (route_type=1) — anomalous per firmware (firmware only
// sends TRACE as DIRECT). Should still parse but flag the anomaly.
hex := "2500" + // header (FLOOD+TRACE) + path_length (0 SNR)
"01000000" + // tag
"02000000" + // authCode
"01" + // flags = 1 → path_sz = 2
"AABBCCDD" // 4 bytes = 2 hops of 2-byte each
pkt, err := DecodePacket(hex, false)
if err != nil {
t.Fatalf("should not crash on anomalous FLOOD+TRACE: %v", err)
}
if len(pkt.Path.Hops) != 2 {
t.Errorf("expected 2 hops even for anomalous FLOOD route, got %d", len(pkt.Path.Hops))
}
if pkt.Anomaly == "" {
t.Error("expected anomaly flag for FLOOD-routed TRACE, got empty string")
}
}
func TestDecodeAdvertSignatureValidation(t *testing.T) {
func TestValidateAdvertSignature(t *testing.T) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
var timestamp uint32 = 1234567890
appdata := []byte{0x02} // flags: repeater, no extras
appdata := []byte{0x02, 0x11, 0x22}
// Build signed message: pubKey(32) + timestamp(4 LE) + appdata
msg := make([]byte, 32+4+len(appdata))
copy(msg[0:32], pub)
binary.LittleEndian.PutUint32(msg[32:36], timestamp)
copy(msg[36:], appdata)
sig := ed25519.Sign(priv, msg)
message := make([]byte, 32+4+len(appdata))
copy(message[0:32], pub)
binary.LittleEndian.PutUint32(message[32:36], timestamp)
copy(message[36:], appdata)
// Build a raw advert buffer: pubKey(32) + timestamp(4) + signature(64) + appdata
buf := make([]byte, 100+len(appdata))
copy(buf[0:32], pub)
binary.LittleEndian.PutUint32(buf[32:36], timestamp)
copy(buf[36:100], sig)
copy(buf[100:], appdata)
sig := ed25519.Sign(priv, message)
// With validation enabled
// Valid signature
if !sigvalidate.ValidateAdvertSignature(pub, sig, timestamp, appdata) {
t.Error("expected valid signature")
}
// Tampered appdata
if sigvalidate.ValidateAdvertSignature(pub, sig, timestamp, []byte{0x03, 0x11, 0x22}) {
t.Error("expected invalid with tampered appdata")
}
// Wrong timestamp
if sigvalidate.ValidateAdvertSignature(pub, sig, timestamp+1, appdata) {
t.Error("expected invalid with wrong timestamp")
}
// Short pubkey
if sigvalidate.ValidateAdvertSignature([]byte{0xAA}, sig, timestamp, appdata) {
t.Error("expected false for short pubkey")
}
// Short signature
if sigvalidate.ValidateAdvertSignature(pub, []byte{0xBB}, timestamp, appdata) {
t.Error("expected false for short signature")
}
}
func TestDecodeAdvertWithSignatureValidation(t *testing.T) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
var timestamp uint32 = 1000000
appdata := []byte{0x02} // repeater type
message := make([]byte, 32+4+len(appdata))
copy(message[0:32], pub)
binary.LittleEndian.PutUint32(message[32:36], timestamp)
copy(message[36:], appdata)
sig := ed25519.Sign(priv, message)
// Build advert buffer: pubkey(32) + timestamp(4) + signature(64) + appdata
buf := make([]byte, 0, 101)
buf = append(buf, pub...)
ts := make([]byte, 4)
binary.LittleEndian.PutUint32(ts, timestamp)
buf = append(buf, ts...)
buf = append(buf, sig...)
buf = append(buf, appdata...)
// With validation
p := decodeAdvert(buf, true)
if p.Error != "" {
t.Fatalf("decode error: %s", p.Error)
}
if p.SignatureValid == nil {
t.Fatal("expected SignatureValid to be set")
t.Fatal("SignatureValid should be set when validation enabled")
}
if !*p.SignatureValid {
t.Error("expected valid signature")
}
if p.PubKey != hex.EncodeToString(pub) {
t.Errorf("pubkey mismatch: got %s", p.PubKey)
}
// Tamper with signature → invalid
buf[40] ^= 0xFF
p = decodeAdvert(buf, true)
if p.SignatureValid == nil {
t.Fatal("expected SignatureValid to be set")
}
if *p.SignatureValid {
t.Error("expected invalid signature after tampering")
}
// Without validation → SignatureValid should be nil
p = decodeAdvert(buf, false)
if p.SignatureValid != nil {
t.Error("expected SignatureValid to be nil when validation disabled")
// Without validation
p2 := decodeAdvert(buf, false)
if p2.SignatureValid != nil {
t.Error("SignatureValid should be nil when validation disabled")
}
}
-145
View File
@@ -1,145 +0,0 @@
package main
import (
"encoding/json"
"net/http/httptest"
"testing"
"time"
)
// seedEncryptedChannelData adds undecryptable GRP_TXT packets to the test DB.
func seedEncryptedChannelData(t *testing.T, db *DB) {
t.Helper()
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
// Two encrypted GRP_TXT packets on channel hash "A1B2"
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('EE01', 'enc_hash_001', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
VALUES ('EE02', 'enc_hash_002', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent)
// Observations for both
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', ?)`, recentEpoch)
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', ?)`, recentEpoch)
}
func TestGetEncryptedChannels(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
seedTestData(t, db)
seedEncryptedChannelData(t, db)
channels, err := db.GetEncryptedChannels()
if err != nil {
t.Fatal(err)
}
if len(channels) != 1 {
t.Fatalf("expected 1 encrypted channel, got %d", len(channels))
}
ch := channels[0]
if ch["hash"] != "enc_A1B2" {
t.Errorf("expected hash enc_A1B2, got %v", ch["hash"])
}
if ch["encrypted"] != true {
t.Errorf("expected encrypted=true, got %v", ch["encrypted"])
}
if ch["messageCount"] != 2 {
t.Errorf("expected messageCount=2, got %v", ch["messageCount"])
}
}
func TestChannelsAPIExcludesEncrypted(t *testing.T) {
_, router := setupTestServer(t)
// Seed encrypted data into the server's DB
// setupTestServer uses seedTestData which has no encrypted packets,
// so default /api/channels should NOT include encrypted channels.
req := httptest.NewRequest("GET", "/api/channels", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
channels := body["channels"].([]interface{})
for _, ch := range channels {
m := ch.(map[string]interface{})
if enc, ok := m["encrypted"]; ok && enc == true {
t.Errorf("default /api/channels should not include encrypted channels, found: %v", m["hash"])
}
}
}
func TestChannelsAPIIncludesEncryptedWithParam(t *testing.T) {
srv, router := setupTestServer(t)
// Add encrypted data to the server's DB
seedEncryptedChannelData(t, srv.db)
// Reload store so in-memory also has the data
store := NewPacketStore(srv.db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load: %v", err)
}
srv.store = store
req := httptest.NewRequest("GET", "/api/channels?includeEncrypted=true", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
channels := body["channels"].([]interface{})
foundEncrypted := false
for _, ch := range channels {
m := ch.(map[string]interface{})
if enc, ok := m["encrypted"]; ok && enc == true {
foundEncrypted = true
break
}
}
if !foundEncrypted {
t.Error("expected encrypted channels with includeEncrypted=true, found none")
}
}
func TestChannelMessagesExcludesEncrypted(t *testing.T) {
srv, router := setupTestServer(t)
seedEncryptedChannelData(t, srv.db)
store := NewPacketStore(srv.db, nil)
if err := store.Load(); err != nil {
t.Fatalf("store.Load: %v", err)
}
srv.store = store
// Request messages for the encrypted channel — should return empty
req := httptest.NewRequest("GET", "/api/channels/enc_A1B2/messages", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
messages, ok := body["messages"].([]interface{})
if !ok {
// messages might be null/missing — that's fine, means no messages
return
}
// Encrypted messages should not be returned as readable messages
for _, msg := range messages {
m := msg.(map[string]interface{})
if text, ok := m["text"].(string); ok && text != "" {
t.Errorf("encrypted channel should not return readable messages, got text: %s", text)
}
}
}
+20 -320
View File
@@ -85,12 +85,6 @@ func makeTestStore(count int, startTime time.Time, intervalMin int) *PacketStore
// Subpath index
addTxToSubpathIndex(store.spIndex, tx)
// Track bytes for self-accounting
store.trackedBytes += estimateStoreTxBytes(tx)
for _, obs := range tx.Observations {
store.trackedBytes += estimateStoreObsBytes(obs)
}
}
return store
@@ -172,43 +166,43 @@ func TestEvictStale_MemoryBasedEviction(t *testing.T) {
// All packets are recent (1h old) so time-based won't trigger.
store.retentionHours = 24
store.maxMemoryMB = 3
// Set trackedBytes to simulate 6MB (over 3MB limit).
store.trackedBytes = 6 * 1048576
// Inject deterministic estimator: simulates 6MB (over 3MB limit).
// Uses packet count so it scales correctly after eviction.
store.memoryEstimator = func() float64 {
return float64(len(store.packets)*5120+store.totalObs*500) / 1048576.0
}
evicted := store.EvictStale()
if evicted == 0 {
t.Fatal("expected some evictions for memory cap")
}
// 25% safety cap should limit to 250 per pass
if evicted > 250 {
t.Fatalf("25%% safety cap violated: evicted %d", evicted)
}
// trackedBytes should have decreased
if store.trackedBytes >= 6*1048576 {
t.Fatal("trackedBytes should have decreased after eviction")
estMB := store.estimatedMemoryMB()
if estMB > 3.5 {
t.Fatalf("expected <=3.5MB after eviction, got %.1fMB", estMB)
}
}
// TestEvictStale_MemoryBasedEviction_UnderestimatedHeap verifies that the 25%
// safety cap prevents cascading eviction even when trackedBytes is very high.
// TestEvictStale_MemoryBasedEviction_UnderestimatedHeap verifies that eviction
// fires correctly when actual heap is much larger than a formula-based estimate
// would report — the scenario that caused OOM kills in production.
func TestEvictStale_MemoryBasedEviction_UnderestimatedHeap(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
store.retentionHours = 24
store.maxMemoryMB = 500
// Simulate trackedBytes 5x over budget.
store.trackedBytes = 2500 * 1048576
// Simulate actual heap 5x over budget (like production: ~5GB actual vs ~1GB limit).
store.memoryEstimator = func() float64 {
return 2500.0 // 2500MB actual vs 500MB limit
}
evicted := store.EvictStale()
if evicted == 0 {
t.Fatal("expected evictions when tracked is 5x over limit")
t.Fatal("expected evictions when heap is 5x over limit")
}
// Safety cap: max 25% per pass = 250
if evicted > 250 {
t.Fatalf("25%% safety cap violated: evicted %d of 1000", evicted)
}
if evicted != 250 {
t.Fatalf("expected exactly 250 evicted (25%% cap), got %d", evicted)
// Should keep roughly 500/2500 * 0.9 = 18% of packets → ~180 of 1000.
remaining := len(store.packets)
if remaining > 250 {
t.Fatalf("expected most packets evicted (heap 5x over), but %d of 1000 remain", remaining)
}
}
@@ -245,101 +239,6 @@ func TestEvictStale_CleansNodeIndexes(t *testing.T) {
}
}
func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) {
now := time.Now().UTC()
// Create a temp DB for on-demand SQL fetch during eviction
db := setupTestDB(t)
defer db.Close()
store := &PacketStore{
packets: make([]*StoreTx, 0),
byHash: make(map[string]*StoreTx),
byTxID: make(map[int]*StoreTx),
byObsID: make(map[int]*StoreObs),
byObserver: make(map[string][]*StoreObs),
byNode: make(map[string][]*StoreTx),
nodeHashes: make(map[string]map[string]bool),
byPayloadType: make(map[int][]*StoreTx),
spIndex: make(map[string]int),
distHops: make([]distHopRecord, 0),
distPaths: make([]distPathRecord, 0),
rfCache: make(map[string]*cachedResult),
topoCache: make(map[string]*cachedResult),
hashCache: make(map[string]*cachedResult),
chanCache: make(map[string]*cachedResult),
distCache: make(map[string]*cachedResult),
subpathCache: make(map[string]*cachedResult),
rfCacheTTL: 15 * time.Second,
retentionHours: 24,
db: db,
useResolvedPathIndex: true,
}
store.initResolvedPathIndex()
// Create a packet indexed via resolved_path pubkeys
relayPK := "relay0001abcdef"
txID := 1
obsID := 100
tx := &StoreTx{
ID: txID,
Hash: "hash_rp_001",
FirstSeen: now.Add(-48 * time.Hour).UTC().Format(time.RFC3339),
}
obs := &StoreObs{
ID: obsID,
TransmissionID: txID,
ObserverID: "obs0",
Timestamp: tx.FirstSeen,
}
tx.Observations = append(tx.Observations, obs)
// Insert into DB so on-demand SQL fetch works during eviction
db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)",
txID, tx.Hash, tx.FirstSeen)
db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)",
obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`)
store.packets = append(store.packets, tx)
store.byHash[tx.Hash] = tx
store.byTxID[tx.ID] = tx
store.byObsID[obs.ID] = obs
store.byObserver["obs0"] = append(store.byObserver["obs0"], obs)
// Index relay via decode-window simulation
store.addToByNode(tx, relayPK)
store.addToResolvedPubkeyIndex(txID, []string{relayPK})
// Verify indexed
if len(store.byNode[relayPK]) != 1 {
t.Fatalf("expected 1 entry in byNode[%s], got %d", relayPK, len(store.byNode[relayPK]))
}
if !store.nodeHashes[relayPK][tx.Hash] {
t.Fatalf("expected nodeHashes[%s] to contain %s", relayPK, tx.Hash)
}
evicted := store.RunEviction()
if evicted != 1 {
t.Fatalf("expected 1 evicted, got %d", evicted)
}
// Verify resolved_path entries are cleaned up
if len(store.byNode[relayPK]) != 0 {
t.Fatalf("expected byNode[%s] to be empty after eviction, got %d", relayPK, len(store.byNode[relayPK]))
}
if _, exists := store.nodeHashes[relayPK]; exists {
t.Fatalf("expected nodeHashes[%s] to be deleted after eviction", relayPK)
}
// Verify resolved pubkey index is cleaned up
h := resolvedPubkeyHash(relayPK)
if len(store.resolvedPubkeyIndex[h]) != 0 {
t.Fatalf("expected resolvedPubkeyIndex to be empty after eviction")
}
if _, exists := store.resolvedPubkeyReverse[txID]; exists {
t.Fatalf("expected resolvedPubkeyReverse to be empty after eviction")
}
}
func TestEvictStale_RunEvictionThreadSafe(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(20, now.Add(-48*time.Hour), 0)
@@ -403,202 +302,3 @@ func TestCacheTTLDefaults(t *testing.T) {
t.Fatalf("expected default rfCacheTTL=15s, got %v", store.rfCacheTTL)
}
}
// --- Self-accounting memory tracking tests ---
func TestTrackedBytes_IncreasesOnInsert(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(0, now, 0)
if store.trackedBytes != 0 {
t.Fatalf("expected 0 trackedBytes for empty store, got %d", store.trackedBytes)
}
store2 := makeTestStore(10, now, 1)
if store2.trackedBytes <= 0 {
t.Fatal("expected positive trackedBytes after inserting 10 packets")
}
// Each packet has 2 observations; should be roughly 10*(384+5*48) + 20*(192+2*48) = 10*624 + 20*288 = 12000
expectedMin := int64(10*600 + 20*250) // rough lower bound
if store2.trackedBytes < expectedMin {
t.Fatalf("trackedBytes %d seems too low (expected > %d)", store2.trackedBytes, expectedMin)
}
}
func TestTrackedBytes_DecreasesOnEvict(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(100, now.Add(-48*time.Hour), 0)
store.retentionHours = 24
beforeBytes := store.trackedBytes
if beforeBytes <= 0 {
t.Fatal("expected positive trackedBytes before eviction")
}
evicted := store.EvictStale()
if evicted != 100 {
t.Fatalf("expected 100 evicted, got %d", evicted)
}
if store.trackedBytes != 0 {
t.Fatalf("expected 0 trackedBytes after evicting all, got %d", store.trackedBytes)
}
}
func TestTrackedBytes_MatchesExpectedAfterMixedInsertEvict(t *testing.T) {
now := time.Now().UTC()
// Create 100 packets, 50 old + 50 recent
store := makeTestStore(100, now.Add(-48*time.Hour), 0)
for i := 50; i < 100; i++ {
store.packets[i].FirstSeen = now.Add(-1 * time.Hour).Format(time.RFC3339)
}
store.retentionHours = 24
totalBefore := store.trackedBytes
// Calculate expected bytes for first 50 packets (to be evicted)
var evictedBytes int64
for i := 0; i < 50; i++ {
tx := store.packets[i]
evictedBytes += estimateStoreTxBytes(tx)
for _, obs := range tx.Observations {
evictedBytes += estimateStoreObsBytes(obs)
}
}
store.EvictStale()
expectedAfter := totalBefore - evictedBytes
if store.trackedBytes != expectedAfter {
t.Fatalf("trackedBytes %d != expected %d (before=%d, evicted=%d)",
store.trackedBytes, expectedAfter, totalBefore, evictedBytes)
}
}
func TestWatermarkHysteresis(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
store.retentionHours = 0 // no time-based eviction
store.maxMemoryMB = 1 // 1MB budget
// Set trackedBytes to just above high watermark
highWatermark := int64(1 * 1048576)
lowWatermark := int64(float64(highWatermark) * 0.85)
store.trackedBytes = highWatermark + 1
evicted := store.EvictStale()
if evicted == 0 {
t.Fatal("expected eviction when above high watermark")
}
if store.trackedBytes > lowWatermark+1024 {
t.Fatalf("expected trackedBytes near low watermark after eviction, got %d (low=%d)",
store.trackedBytes, lowWatermark)
}
// Now set trackedBytes to just below high watermark — should NOT trigger
store.trackedBytes = highWatermark - 1
evicted2 := store.EvictStale()
if evicted2 != 0 {
t.Fatalf("expected no eviction below high watermark, got %d", evicted2)
}
}
func TestSafetyCap25Percent(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
store.retentionHours = 0
store.maxMemoryMB = 1
// Set trackedBytes way over limit to force maximum eviction
store.trackedBytes = 100 * 1048576 // 100MB vs 1MB limit
evicted := store.EvictStale()
// 25% of 1000 = 250
if evicted > 250 {
t.Fatalf("25%% safety cap violated: evicted %d of 1000 (max should be 250)", evicted)
}
if evicted != 250 {
t.Fatalf("expected exactly 250 evicted (25%% cap), got %d", evicted)
}
if len(store.packets) != 750 {
t.Fatalf("expected 750 remaining, got %d", len(store.packets))
}
}
func TestMultiplePassesConverge(t *testing.T) {
now := time.Now().UTC()
store := makeTestStore(1000, now.Add(-1*time.Hour), 0)
store.retentionHours = 0
// Set budget to half the actual tracked bytes — requires ~2 passes
actualBytes := store.trackedBytes
store.maxMemoryMB = int(float64(actualBytes) / 1048576.0 / 2)
if store.maxMemoryMB < 1 {
store.maxMemoryMB = 1
}
totalEvicted := 0
for pass := 0; pass < 20; pass++ {
evicted := store.EvictStale()
if evicted == 0 {
break
}
totalEvicted += evicted
}
// After convergence, trackedBytes should be at or below high watermark
// (may be between low and high due to hysteresis — that's fine)
highWatermark := int64(store.maxMemoryMB) * 1048576
if store.trackedBytes > highWatermark {
t.Fatalf("did not converge: trackedBytes=%d (%.1fMB) > highWatermark=%d after multiple passes",
store.trackedBytes, float64(store.trackedBytes)/1048576.0, highWatermark)
}
if totalEvicted == 0 {
t.Fatal("expected some evictions across multiple passes")
}
}
func TestEstimateStoreTxBytes(t *testing.T) {
tx := &StoreTx{
RawHex: "aabbcc",
Hash: "hash1234",
DecodedJSON: `{"pubKey":"pk1"}`,
PathJSON: `["aa","bb"]`,
}
est := estimateStoreTxBytes(tx)
// Manual calculation: base + string lengths + index entries + perTxMaps + path hops + subpaths
hops := int64(len(txGetParsedPath(tx)))
manualCalc := int64(storeTxBaseBytes) + int64(len(tx.RawHex)+len(tx.Hash)+len(tx.DecodedJSON)+len(tx.PathJSON)) + int64(numIndexesPerTx*indexEntryBytes)
manualCalc += perTxMapsBytes
manualCalc += hops * perPathHopBytes
if hops > 1 {
manualCalc += (hops * (hops - 1) / 2) * perSubpathEntryBytes
}
if est != manualCalc {
t.Fatalf("estimateStoreTxBytes = %d, want %d (manual calc)", est, manualCalc)
}
if est < 600 || est > 1200 {
t.Fatalf("estimateStoreTxBytes = %d, expected in range [600, 1200]", est)
}
}
func TestEstimateStoreObsBytes(t *testing.T) {
obs := &StoreObs{
ObserverID: "obs123",
PathJSON: `["aa"]`,
}
est := estimateStoreObsBytes(obs)
// storeObsBaseBytes(192) + len(ObserverID=6) + len(PathJSON=6) + 2*48(96) = 300
expected := int64(192 + 6 + 6 + 2*48)
if est != expected {
t.Fatalf("estimateStoreObsBytes = %d, want %d", est, expected)
}
}
func BenchmarkEviction100K(b *testing.B) {
now := time.Now().UTC()
for i := 0; i < b.N; i++ {
b.StopTimer()
store := makeTestStore(100000, now.Add(-48*time.Hour), 0)
store.retentionHours = 24
b.StartTimer()
store.EvictStale()
}
}
+1 -1
View File
@@ -6,7 +6,7 @@ require (
github.com/gorilla/mux v1.8.1
github.com/gorilla/websocket v1.5.3
github.com/meshcore-analyzer/geofilter v0.0.0
github.com/meshcore-analyzer/sigvalidate v0.0.0
github.com/meshcore-analyzer/sigvalidate v0.0.0-00010101000000-000000000000
modernc.org/sqlite v1.34.5
)
-119
View File
@@ -1,119 +0,0 @@
package main
import (
"log"
"time"
)
// migrateContentHashesAsync recomputes content hashes in batches after the
// server is already serving HTTP. Packets whose hash changes are updated in
// both the DB and the in-memory byHash index. The migration is idempotent:
// once all hashes match the current formula it completes instantly.
func migrateContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) {
defer func() {
if r := recover(); r != nil {
log.Printf("[hash-migrate] panic recovered: %v", r)
}
store.hashMigrationComplete.Store(true)
}()
// Snapshot the packet slice length under lock (packets only grow).
store.mu.RLock()
total := len(store.packets)
store.mu.RUnlock()
migrated := 0
for offset := 0; offset < total; offset += batchSize {
end := offset + batchSize
if end > total {
end = total
}
// Collect stale hashes in this batch under RLock.
type hashUpdate struct {
tx *StoreTx
oldHash string
newHash string
}
var updates []hashUpdate
store.mu.RLock()
for _, tx := range store.packets[offset:end] {
if tx.RawHex == "" {
continue
}
newHash := ComputeContentHash(tx.RawHex)
if newHash != tx.Hash {
updates = append(updates, hashUpdate{tx: tx, oldHash: tx.Hash, newHash: newHash})
}
}
store.mu.RUnlock()
if len(updates) == 0 {
continue
}
// Write batch to DB in a single transaction.
dbTx, err := store.db.conn.Begin()
if err != nil {
log.Printf("[hash-migrate] begin tx: %v", err)
continue
}
stmt, err := dbTx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?")
if err != nil {
log.Printf("[hash-migrate] prepare: %v", err)
dbTx.Rollback()
continue
}
for _, u := range updates {
if _, err := stmt.Exec(u.newHash, u.tx.ID); err != nil {
// UNIQUE constraint = two old hashes map to the same new hash (duplicate).
// Merge observations to the surviving tx, delete the duplicate.
log.Printf("[hash-migrate] tx %d collides — merging duplicate", u.tx.ID)
var survID int
if err2 := dbTx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil {
dbTx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.tx.ID)
dbTx.Exec("DELETE FROM transmissions WHERE id = ?", u.tx.ID)
u.newHash = "" // mark for in-memory removal only
}
}
}
stmt.Close()
if err := dbTx.Commit(); err != nil {
log.Printf("[hash-migrate] commit: %v", err)
continue
}
// Update in-memory index under write lock.
store.mu.Lock()
for _, u := range updates {
delete(store.byHash, u.oldHash)
if u.newHash == "" {
// Merged duplicate — remove from packets slice and indexes.
delete(store.byTxID, u.tx.ID)
// Move observations to survivor if present.
if surv := store.byHash[ComputeContentHash(u.tx.RawHex)]; surv != nil {
for _, obs := range u.tx.Observations {
surv.Observations = append(surv.Observations, obs)
surv.ObservationCount++
}
}
} else {
u.tx.Hash = u.newHash
store.byHash[u.newHash] = u.tx
}
}
store.mu.Unlock()
migrated += len(updates)
// Yield to let HTTP handlers run.
time.Sleep(yieldDuration)
}
if migrated > 0 {
log.Printf("[hash-migrate] Migrated %d content hashes to new formula", migrated)
}
}
-78
View File
@@ -1,78 +0,0 @@
package main
import (
"testing"
"time"
)
func TestMigrateContentHashesAsync(t *testing.T) {
db := setupTestDBv2(t)
store := NewPacketStore(db, nil)
// Insert a packet with a manually wrong hash (simulating old formula).
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
correctHash := ComputeContentHash(rawHex)
wrongHash := "deadbeef12345678"
_, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type)
VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash)
if err != nil {
t.Fatal(err)
}
if err := store.Load(); err != nil {
t.Fatal(err)
}
if store.byHash[wrongHash] == nil {
t.Fatal("expected packet under wrong hash before migration")
}
migrateContentHashesAsync(store, 100, time.Millisecond)
if !store.hashMigrationComplete.Load() {
t.Error("expected hashMigrationComplete to be true")
}
if store.byHash[wrongHash] != nil {
t.Error("old hash should be removed from index")
}
if store.byHash[correctHash] == nil {
t.Error("new hash should be in index")
}
var dbHash string
err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = ?", rawHex).Scan(&dbHash)
if err != nil {
t.Fatal(err)
}
if dbHash != correctHash {
t.Errorf("DB hash = %s, want %s", dbHash, correctHash)
}
}
func TestMigrateContentHashesAsync_NoOp(t *testing.T) {
db := setupTestDBv2(t)
store := NewPacketStore(db, nil)
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
correctHash := ComputeContentHash(rawHex)
_, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type)
VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, correctHash)
if err != nil {
t.Fatal(err)
}
if err := store.Load(); err != nil {
t.Fatal(err)
}
migrateContentHashesAsync(store, 100, time.Millisecond)
if !store.hashMigrationComplete.Load() {
t.Error("expected hashMigrationComplete to be true")
}
if store.byHash[correctHash] == nil {
t.Error("hash should remain in index")
}
}
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"encoding/json"
"testing"
"time"
_ "modernc.org/sqlite"
)
const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9"
// setupIssue673Store builds an in-memory store with one repeater node having:
// - one ADVERT packet (legitimately indexed in byNode)
// - one GRP_TXT packet whose decoded text contains the node's pubkey (false-positive candidate)
func setupIssue673Store(t *testing.T) (*PacketStore, *DB) {
t.Helper()
db := setupTestDB(t)
_, err := db.conn.Exec(
"INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)",
issue673NodePK, "Quail Hollow Park", "repeater",
)
if err != nil {
t.Fatal(err)
}
ps := NewPacketStore(db, nil)
now := time.Now().UTC().Format(time.RFC3339)
pt4 := 4 // ADVERT
pt5 := 5 // GRP_TXT
advertDecoded, _ := json.Marshal(map[string]interface{}{"pubKey": issue673NodePK})
advert := &StoreTx{
ID: 1,
Hash: "advert_hash_673",
PayloadType: &pt4,
DecodedJSON: string(advertDecoded),
FirstSeen: now,
}
otherPK := "aabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccddaabbccdd"
chatDecoded, _ := json.Marshal(map[string]interface{}{
"srcPubKey": otherPK,
"text": "Check out node " + issue673NodePK + " on the analyzer",
})
chat := &StoreTx{
ID: 2,
Hash: "chat_hash_673",
PayloadType: &pt5,
DecodedJSON: string(chatDecoded),
FirstSeen: now,
}
ps.mu.Lock()
ps.packets = append(ps.packets, advert, chat)
ps.byHash[advert.Hash] = advert
ps.byHash[chat.Hash] = chat
ps.byTxID[advert.ID] = advert
ps.byTxID[chat.ID] = chat
ps.byNode[issue673NodePK] = []*StoreTx{advert}
ps.mu.Unlock()
return ps, db
}
// TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText verifies that a GRP_TXT packet
// whose message text contains a node's pubkey is not counted in that node's analytics.
func TestGetNodeAnalytics_ExcludesGRPTXTWithPubkeyInText(t *testing.T) {
ps, db := setupIssue673Store(t)
defer db.Close()
analytics, err := ps.GetNodeAnalytics(issue673NodePK, 30)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if analytics == nil {
t.Fatal("expected analytics, got nil")
}
for _, ptc := range analytics.PacketTypeBreakdown {
if ptc.PayloadType == 5 {
t.Errorf("GRP_TXT (type 5) should not appear in analytics for repeater node, got count=%d", ptc.Count)
}
}
}
// TestFilterPackets_NodeQueryDoesNotMatchChatText verifies that the slow path of
// filterPackets (node filter combined with Since) does not return a GRP_TXT packet
// whose pubkey appears only in message text, not in a structured pubkey field.
func TestFilterPackets_NodeQueryDoesNotMatchChatText(t *testing.T) {
ps, db := setupIssue673Store(t)
defer db.Close()
yesterday := time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339)
result := ps.QueryPackets(PacketQuery{Node: issue673NodePK, Since: yesterday, Limit: 50})
if result.Total != 1 {
t.Errorf("expected 1 packet for node (ADVERT only), got %d", result.Total)
}
for _, pkt := range result.Packets {
if pkt["hash"] == "chat_hash_673" {
t.Errorf("GRP_TXT with pubkey in message text was incorrectly returned for node query")
}
}
}
-78
View File
@@ -1,78 +0,0 @@
package main
import (
"encoding/json"
"net/http/httptest"
"testing"
"time"
"github.com/gorilla/mux"
)
// TestRepro810 reproduces #810: when the longest-path observation has NULL
// resolved_path but a shorter-path observation has one, fetchResolvedPathForTxBest
// returns nil → /api/nodes/{pk}/health.recentPackets[].resolved_path is missing
// while /api/packets shows it.
func TestRepro810(t *testing.T) {
db := setupTestDB(t)
now := time.Now().UTC()
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
recentEpoch := now.Add(-1 * time.Hour).Unix()
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent)
db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent)
db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent)
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent)
// Longest-path obs WITHOUT resolved_path
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch)
// Shorter-path obs WITH resolved_path
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100)
cfg := &Config{Port: 3000}
hub := NewHub()
srv := NewServer(db, cfg, hub)
store := NewPacketStore(db, nil)
if err := store.Load(); err != nil {
t.Fatal(err)
}
srv.store = store
router := mux.NewRouter()
srv.RegisterRoutes(router)
// Sanity: /api/packets should show resolved_path for this tx.
reqP := httptest.NewRequest("GET", "/api/packets?limit=10", nil)
wP := httptest.NewRecorder()
router.ServeHTTP(wP, reqP)
var pktsBody map[string]interface{}
json.Unmarshal(wP.Body.Bytes(), &pktsBody)
pkts, _ := pktsBody["packets"].([]interface{})
hasOnPackets := false
for _, p := range pkts {
pm := p.(map[string]interface{})
if pm["hash"] == "testhash00000001" && pm["resolved_path"] != nil {
hasOnPackets = true
}
}
if !hasOnPackets {
t.Fatal("precondition: /api/packets must report resolved_path for tx")
}
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
rp, _ := body["recentPackets"].([]interface{})
if len(rp) == 0 {
t.Fatal("no recentPackets")
}
for _, p := range rp {
pm := p.(map[string]interface{})
if pm["hash"] == "testhash00000001" {
if pm["resolved_path"] == nil {
t.Fatal("BUG #810: /health.recentPackets resolved_path is nil despite /api/packets reporting it")
}
return
}
}
t.Fatal("tx not found in recentPackets")
}
-194
View File
@@ -1,194 +0,0 @@
package main
import (
"database/sql"
"testing"
"time"
"path/filepath"
_ "modernc.org/sqlite"
)
// setupTestDB871 creates a test DB with schema and returns a read-only *DB handle.
func setupTestDB871(t *testing.T) (*DB, *sql.DB) {
t.Helper()
dbPath := filepath.Join(t.TempDir(), "test871.db")
// Open writable connection for setup
rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
if err != nil {
t.Fatal(err)
}
_, err = rw.Exec(`
CREATE TABLE IF NOT EXISTS nodes (
public_key TEXT PRIMARY KEY,
name TEXT, role TEXT,
lat REAL, lon REAL,
last_seen TEXT, first_seen TEXT,
advert_count INTEGER DEFAULT 0,
battery_mv INTEGER, temperature_c REAL
);
CREATE TABLE IF NOT EXISTS transmissions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
raw_hex TEXT NOT NULL,
hash TEXT NOT NULL UNIQUE,
first_seen TEXT NOT NULL,
route_type INTEGER,
payload_type INTEGER,
payload_version INTEGER,
decoded_json TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS observers (
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
name TEXT
);
CREATE TABLE IF NOT EXISTS observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transmission_id INTEGER NOT NULL,
observer_id TEXT,
observer_name TEXT,
direction TEXT,
snr REAL, rssi REAL, score INTEGER,
path_json TEXT, timestamp TEXT
);
`)
if err != nil {
t.Fatal(err)
}
// Open read-only handle for the store
db, err := OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
db.Close()
rw.Close()
})
return db, rw
}
// TestEnrichObsFallbackToDB verifies that enrichObs falls back to the DB when
// the parent transmission has been evicted from memory (#871 root cause).
func TestEnrichObsFallbackToDB(t *testing.T) {
db, rw := setupTestDB871(t)
now := time.Now().UTC().Format(time.RFC3339)
_, err := rw.Exec(
`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type, decoded_json) VALUES (?, ?, ?, ?, ?)`,
"aabbcc", "abc123", now, 4, `{"pubKey":"pk1"}`,
)
if err != nil {
t.Fatal(err)
}
store := NewPacketStore(db, &PacketStoreConfig{})
// Observation references tx_id=1, but tx is NOT in byTxID (simulates eviction)
obs := &StoreObs{
ID: 1,
TransmissionID: 1,
ObserverID: "obs1",
ObserverName: "Observer1",
Timestamp: now,
}
result := store.enrichObs(obs)
// hash must be present from DB fallback
if result["hash"] == nil {
t.Errorf("enrichObs: hash is nil — DB fallback failed")
}
if h, ok := result["hash"].(string); !ok || h != "abc123" {
t.Errorf("enrichObs: expected hash 'abc123', got %v", result["hash"])
}
if result["payload_type"] == nil {
t.Errorf("enrichObs: payload_type is nil — DB fallback failed")
}
// When tx IS in memory, it should use the in-memory path
pt := 4
store.byTxID[1] = &StoreTx{
ID: 1, Hash: "abc123", FirstSeen: now,
PayloadType: &pt, RawHex: "aabbcc",
}
result2 := store.enrichObs(obs)
if result2["hash"] == nil {
t.Errorf("enrichObs with in-memory tx: hash is nil")
}
}
// TestGetNodeHealthRecentPacketsNoNilFields verifies that GetNodeHealth's
// recentPackets never contains entries with nil hash or timestamp.
func TestGetNodeHealthRecentPacketsNoNilFields(t *testing.T) {
db, rw := setupTestDB871(t)
now := time.Now().UTC().Format(time.RFC3339)
_, err := rw.Exec(
`INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)`,
"pk1", "TestNode", "repeater", now,
)
if err != nil {
t.Fatal(err)
}
store := NewPacketStore(db, &PacketStoreConfig{})
pt := 4
tx := &StoreTx{
ID: 1, Hash: "hash1", FirstSeen: now,
PayloadType: &pt, DecodedJSON: `{"pubKey":"pk1"}`,
obsKeys: make(map[string]bool), observerSet: make(map[string]bool),
}
store.byTxID[1] = tx
store.byHash["hash1"] = tx
store.byNode["pk1"] = []*StoreTx{tx}
store.nodeHashes["pk1"] = map[string]bool{"hash1": true}
result, err := store.GetNodeHealth("pk1")
if err != nil {
t.Fatal(err)
}
if result == nil {
t.Fatal("GetNodeHealth returned nil")
}
packets, ok := result["recentPackets"].([]map[string]interface{})
if !ok {
t.Fatal("recentPackets is not []map[string]interface{}")
}
for i, p := range packets {
if p["hash"] == nil {
t.Errorf("recentPackets[%d] has nil hash", i)
}
if p["timestamp"] == nil {
t.Errorf("recentPackets[%d] has nil timestamp", i)
}
}
}
// TestEnrichObsNilDB verifies enrichObs doesn't panic when db is nil.
func TestEnrichObsNilDB(t *testing.T) {
store := &PacketStore{
byTxID: make(map[int]*StoreTx),
byObsID: make(map[int]*StoreObs),
}
obs := &StoreObs{
ID: 1, TransmissionID: 999,
Timestamp: "2026-01-01T00:00:00Z",
}
// Should not panic
result := store.enrichObs(obs)
if result["hash"] != nil {
t.Errorf("expected nil hash when no DB and no in-memory tx, got %v", result["hash"])
}
}
-48
View File
@@ -111,14 +111,6 @@ func main() {
// Resolve DB path
resolvedDB := cfg.ResolveDBPath(configDir)
log.Printf("[config] port=%d db=%s public=%s", cfg.Port, resolvedDB, publicDir)
if len(cfg.NodeBlacklist) > 0 {
log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist))
for _, pk := range cfg.NodeBlacklist {
if trimmed := strings.ToLower(strings.TrimSpace(pk)); trimmed != "" {
log.Printf("[config] blacklisted: %s", trimmed)
}
}
}
// Open database
database, err := OpenDB(resolvedDB)
@@ -333,40 +325,6 @@ func main() {
log.Printf("[metrics-prune] auto-prune enabled: metrics older than %d days", metricsDays)
}
// Auto-prune stale observers
var stopObserverPrune func()
{
observerDays := cfg.ObserverDaysOrDefault()
if observerDays <= -1 {
// -1 means keep forever, skip
} else {
observerPruneTicker := time.NewTicker(24 * time.Hour)
observerPruneDone := make(chan struct{})
stopObserverPrune = func() {
observerPruneTicker.Stop()
close(observerPruneDone)
}
go func() {
defer func() {
if r := recover(); r != nil {
log.Printf("[observer-prune] panic recovered: %v", r)
}
}()
time.Sleep(3 * time.Minute) // stagger after metrics prune
database.RemoveStaleObservers(observerDays)
for {
select {
case <-observerPruneTicker.C:
database.RemoveStaleObservers(observerDays)
case <-observerPruneDone:
return
}
}
}()
log.Printf("[observer-prune] auto-prune enabled: observers not seen in %d days will be removed", observerDays)
}
}
// Auto-prune old neighbor edges
var stopEdgePrune func()
{
@@ -428,9 +386,6 @@ func main() {
if stopMetricsPrune != nil {
stopMetricsPrune()
}
if stopObserverPrune != nil {
stopObserverPrune()
}
if stopEdgePrune != nil {
stopEdgePrune()
}
@@ -457,9 +412,6 @@ func main() {
// Start async backfill in background — HTTP is now available.
go backfillResolvedPathsAsync(store, dbPath, 5000, 100*time.Millisecond, cfg.BackfillHours())
// Migrate old content hashes in background (one-time, idempotent).
go migrateContentHashesAsync(store, 5000, 100*time.Millisecond)
if err := httpServer.ListenAndServe(); err != http.ErrServerClosed {
log.Fatalf("[server] %v", err)
}
-132
View File
@@ -1,132 +0,0 @@
package main
import (
"os"
"strconv"
"strings"
"sync"
"time"
)
// MemorySnapshot is a point-in-time view of process memory across several
// vantage points. Values are in MB (1024*1024 bytes), rounded to one decimal.
//
// Field invariants (typical, not guaranteed under exotic conditions):
//
// processRSSMB >= goSysMB >= goHeapInuseMB >= storeDataMB
//
// - processRSSMB is what the kernel charges the process (resident set).
// Read from /proc/self/status `VmRSS:` on Linux; falls back to goSysMB
// on other platforms or when /proc is unavailable.
// - goSysMB is the total memory obtained from the OS by the Go runtime
// (heap, stacks, GC metadata, mspans, mcache, etc.). Includes
// fragmentation and unused-but-mapped span overhead.
// - goHeapInuseMB is the live, in-use Go heap (HeapInuse). Excludes
// idle spans and runtime overhead.
// - storeDataMB is the in-store packet byte estimate (transmissions +
// observations). Subset of HeapInuse. Does not include index maps,
// analytics caches, broadcast queues, or runtime overhead. Used as
// the input to the eviction watermark.
//
// processRSSMB and storeDataMB are monotonic only relative to ingest +
// eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB
// fluctuate with GC.
//
// cgoBytesMB intentionally absent: this build uses the pure-Go
// modernc.org/sqlite driver, so there is no cgo allocator to measure.
// Reintroduce only if we ever switch back to mattn/go-sqlite3.
type MemorySnapshot struct {
ProcessRSSMB float64 `json:"processRSSMB"`
GoHeapInuseMB float64 `json:"goHeapInuseMB"`
GoSysMB float64 `json:"goSysMB"`
StoreDataMB float64 `json:"storeDataMB"`
}
// rssCache rate-limits the /proc/self/status read. Go memory stats are
// already cached by Server.getMemStats (5s TTL). We use a tighter 1s TTL
// here so processRSSMB stays reasonably fresh during ops debugging
// without paying the syscall cost on every /api/stats hit.
var (
rssCacheMu sync.Mutex
rssCacheValueMB float64
rssCacheCachedAt time.Time
)
const rssCacheTTL = 1 * time.Second
// getMemorySnapshot composes a MemorySnapshot using the Server's existing
// runtime.MemStats cache (5s TTL, used by /api/health and /api/perf too)
// plus a rate-limited /proc RSS read. storeDataMB is supplied by the
// caller because the packet store is the source of truth.
func (s *Server) getMemorySnapshot(storeDataMB float64) MemorySnapshot {
ms := s.getMemStats()
rssCacheMu.Lock()
if time.Since(rssCacheCachedAt) > rssCacheTTL {
rssCacheValueMB = readProcRSSMB()
rssCacheCachedAt = time.Now()
}
rssMB := rssCacheValueMB
rssCacheMu.Unlock()
if rssMB <= 0 {
// Fallback when /proc is unavailable (non-Linux, sandboxes, etc.).
// runtime.Sys is an upper bound on Go-attributable memory and a
// reasonable proxy for pure-Go builds.
rssMB = float64(ms.Sys) / 1048576.0
}
return MemorySnapshot{
ProcessRSSMB: roundMB(rssMB),
GoHeapInuseMB: roundMB(float64(ms.HeapInuse) / 1048576.0),
GoSysMB: roundMB(float64(ms.Sys) / 1048576.0),
StoreDataMB: roundMB(storeDataMB),
}
}
// readProcRSSMB parses /proc/self/status for the VmRSS line. Returns 0 on
// any failure (file missing, malformed line, parse error) — the caller
// then uses a runtime fallback. Linux only; macOS/Windows return 0.
//
// Safety notes (djb): the file path is hard-coded, no untrusted input is
// concatenated. We bound the read at 8 KiB (the whole status file is
// well under 4 KiB on modern kernels) so a corrupt /proc can't OOM us.
// We only parse digits with strconv; no shell, no exec, no format strings.
func readProcRSSMB() float64 {
const maxStatusBytes = 8 * 1024
f, err := os.Open("/proc/self/status")
if err != nil {
return 0
}
defer f.Close()
buf := make([]byte, maxStatusBytes)
n, err := f.Read(buf)
if err != nil && n == 0 {
return 0
}
for _, line := range strings.Split(string(buf[:n]), "\n") {
if !strings.HasPrefix(line, "VmRSS:") {
continue
}
// Format: "VmRSS:\t 123456 kB"
fields := strings.Fields(line[len("VmRSS:"):])
if len(fields) < 2 {
return 0
}
kb, err := strconv.ParseFloat(fields[0], 64)
if err != nil || kb < 0 {
return 0
}
// Unit is kB per kernel convention; convert to MB.
return kb / 1024.0
}
return 0
}
func roundMB(v float64) float64 {
if v < 0 {
return 0
}
return float64(int64(v*10+0.5)) / 10.0
}
-435
View File
@@ -1,435 +0,0 @@
package main
import (
"database/sql"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
_ "modernc.org/sqlite"
)
// recentTS returns a timestamp string N hours ago, ensuring test data
// stays within the 7-day advert window used by computeNodeHashSizeInfo.
func recentTS(hoursAgo int) string {
return time.Now().UTC().Add(-time.Duration(hoursAgo) * time.Hour).Format("2006-01-02T15:04:05.000Z")
}
// setupCapabilityTestDB creates a minimal in-memory DB with nodes table.
func setupCapabilityTestDB(t *testing.T) *DB {
t.Helper()
conn, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
conn.SetMaxOpenConns(1)
conn.Exec(`CREATE TABLE nodes (
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT,
advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL
)`)
conn.Exec(`CREATE TABLE observers (
id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT,
first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT,
firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER,
uptime_secs INTEGER
)`)
return &DB{conn: conn}
}
// addTestPacket adds a StoreTx to the store's internal structures including
// the byPathHop index and byPayloadType index.
func addTestPacket(store *PacketStore, tx *StoreTx) {
store.mu.Lock()
defer store.mu.Unlock()
tx.ID = len(store.packets) + 1
if tx.Hash == "" {
tx.Hash = fmt.Sprintf("test-hash-%d", tx.ID)
}
store.packets = append(store.packets, tx)
store.byHash[tx.Hash] = tx
store.byTxID[tx.ID] = tx
if tx.PayloadType != nil {
store.byPayloadType[*tx.PayloadType] = append(store.byPayloadType[*tx.PayloadType], tx)
}
addTxToPathHopIndex(store.byPathHop, tx)
}
// buildPathByte returns a 2-char hex string for the path byte with given
// hashSize (1-3) and hopCount.
func buildPathByte(hashSize, hopCount int) string {
b := byte(((hashSize - 1) & 0x3) << 6) | byte(hopCount&0x3F)
return fmt.Sprintf("%02x", b)
}
// makeTestAdvert creates a StoreTx representing a flood advert packet.
func makeTestAdvert(pubkey string, hashSize int) *StoreTx {
decoded, _ := json.Marshal(map[string]interface{}{"pubKey": pubkey, "name": pubkey[:8]})
pt := 4
pathByte := buildPathByte(hashSize, 1)
prefix := strings.ToLower(pubkey[:hashSize*2])
rawHex := "01" + pathByte + prefix // flood header + path byte + hop prefix
return &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
DecodedJSON: string(decoded),
PathJSON: `["` + prefix + `"]`,
FirstSeen: recentTS(24),
}
}
// TestMultiByteCapability_Confirmed tests that a repeater advertising
// with hash_size >= 2 is classified as "confirmed".
func TestMultiByteCapability_Confirmed(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepA", "repeater", recentTS(24))
store := NewPacketStore(db, nil)
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 2))
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "confirmed" {
t.Errorf("expected confirmed, got %s", caps[0].Status)
}
if caps[0].Evidence != "advert" {
t.Errorf("expected advert evidence, got %s", caps[0].Evidence)
}
if caps[0].MaxHashSize != 2 {
t.Errorf("expected maxHashSize 2, got %d", caps[0].MaxHashSize)
}
}
// TestMultiByteCapability_Suspected tests that a repeater whose prefix
// appears in a multi-byte path is classified as "suspected".
func TestMultiByteCapability_Suspected(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepB", "repeater", recentTS(48))
store := NewPacketStore(db, nil)
// Non-advert packet with 2-byte hash in path, hop prefix matching node
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aabb"
pt := 1
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aabb"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "suspected" {
t.Errorf("expected suspected, got %s", caps[0].Status)
}
if caps[0].Evidence != "path" {
t.Errorf("expected path evidence, got %s", caps[0].Evidence)
}
if caps[0].MaxHashSize != 2 {
t.Errorf("expected maxHashSize 2, got %d", caps[0].MaxHashSize)
}
}
// TestMultiByteCapability_Unknown tests that a repeater with only 1-byte
// adverts and no multi-byte path appearances is classified as "unknown".
func TestMultiByteCapability_Unknown(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepC", "repeater", recentTS(72))
store := NewPacketStore(db, nil)
// Advert with 1-byte hash only
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 1))
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "unknown" {
t.Errorf("expected unknown, got %s", caps[0].Status)
}
if caps[0].MaxHashSize != 1 {
t.Errorf("expected maxHashSize 1, got %d", caps[0].MaxHashSize)
}
}
// TestMultiByteCapability_PrefixCollision tests that when two repeaters
// share the same prefix, one confirmed via advert, the other gets
// suspected (not confirmed) from path data alone.
func TestMultiByteCapability_PrefixCollision(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
// Two repeaters sharing 1-byte prefix "aa"
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabb000000000001", "RepConfirmed", "repeater", recentTS(24))
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aacc000000000002", "RepOther", "repeater", recentTS(24))
store := NewPacketStore(db, nil)
// RepConfirmed has a 2-byte advert
addTestPacket(store, makeTestAdvert("aabb000000000001", 2))
// A packet with 2-byte path containing 1-byte hop "aa" — both share this prefix
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aa"
pt := 1
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aa"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
caps := store.computeMultiByteCapability(nil)
if len(caps) != 2 {
t.Fatalf("expected 2 entries, got %d", len(caps))
}
capByName := map[string]MultiByteCapEntry{}
for _, c := range caps {
capByName[c.Name] = c
}
if capByName["RepConfirmed"].Status != "confirmed" {
t.Errorf("RepConfirmed expected confirmed, got %s", capByName["RepConfirmed"].Status)
}
if capByName["RepOther"].Status != "suspected" {
t.Errorf("RepOther expected suspected, got %s", capByName["RepOther"].Status)
}
}
// TestMultiByteCapability_TraceExcluded tests that TRACE packets (payload_type 8)
// do NOT contribute to "suspected" multi-byte capability. TRACE packets carry
// hash size in their own flags, so pre-1.14 repeaters can forward multi-byte
// TRACEs without actually supporting multi-byte hashes. See #714.
func TestMultiByteCapability_TraceExcluded(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepTrace", "repeater", recentTS(48))
store := NewPacketStore(db, nil)
// TRACE packet (payload_type 8) with 2-byte hash in path
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aabb"
pt := 8
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aabb"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "unknown" {
t.Errorf("expected unknown (TRACE excluded), got %s", caps[0].Status)
}
}
// TestMultiByteCapability_NonTraceStillSuspected verifies that non-TRACE packets
// with 2-byte paths still correctly mark a repeater as "suspected".
func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepNonTrace", "repeater", recentTS(48))
store := NewPacketStore(db, nil)
// GRP_TXT packet (payload_type 1) with 2-byte hash in path
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aabb"
pt := 1
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aabb"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "suspected" {
t.Errorf("expected suspected, got %s", caps[0].Status)
}
}
// TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion verifies that
// "confirmed" status from adverts is not affected by the TRACE exclusion.
func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepConfirmedTrace", "repeater", recentTS(24))
store := NewPacketStore(db, nil)
// Advert with 2-byte hash (confirms capability)
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 2))
// TRACE packet also present — should not downgrade confirmed status
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aabb"
pt := 8
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aabb"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "confirmed" {
t.Errorf("expected confirmed (unaffected by TRACE), got %s", caps[0].Status)
}
}
// TestMultiByteCapability_CompanionConfirmed tests that a companion with
// multi-byte advert is classified as "confirmed", not "unknown" (Bug 1, #754).
func TestMultiByteCapability_CompanionConfirmed(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "CompA", "companion", recentTS(24))
store := NewPacketStore(db, nil)
addTestPacket(store, makeTestAdvert("aabbccdd11223344", 2))
caps := store.computeMultiByteCapability(nil)
if len(caps) != 1 {
t.Fatalf("expected 1 entry, got %d", len(caps))
}
if caps[0].Status != "confirmed" {
t.Errorf("expected confirmed for companion, got %s", caps[0].Status)
}
if caps[0].Role != "companion" {
t.Errorf("expected role companion, got %s", caps[0].Role)
}
if caps[0].Evidence != "advert" {
t.Errorf("expected advert evidence, got %s", caps[0].Evidence)
}
}
// TestMultiByteCapability_RoleColumnPopulated tests that the Role field is
// populated for all node types (Bug 2, #754).
func TestMultiByteCapability_RoleColumnPopulated(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabb000000000001", "Rep1", "repeater", recentTS(24))
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"ccdd000000000002", "Comp1", "companion", recentTS(24))
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"eeff000000000003", "Room1", "room_server", recentTS(24))
store := NewPacketStore(db, nil)
addTestPacket(store, makeTestAdvert("aabb000000000001", 2))
addTestPacket(store, makeTestAdvert("ccdd000000000002", 2))
addTestPacket(store, makeTestAdvert("eeff000000000003", 1))
caps := store.computeMultiByteCapability(nil)
if len(caps) != 3 {
t.Fatalf("expected 3 entries, got %d", len(caps))
}
roleByName := map[string]string{}
for _, c := range caps {
roleByName[c.Name] = c.Role
}
if roleByName["Rep1"] != "repeater" {
t.Errorf("Rep1 role: expected repeater, got %s", roleByName["Rep1"])
}
if roleByName["Comp1"] != "companion" {
t.Errorf("Comp1 role: expected companion, got %s", roleByName["Comp1"])
}
if roleByName["Room1"] != "room_server" {
t.Errorf("Room1 role: expected room_server, got %s", roleByName["Room1"])
}
}
// TestMultiByteCapability_AdopterEvidenceTakesPrecedence tests that when
// adopter data shows hashSize >= 2 but path evidence says "suspected",
// the node is upgraded to "confirmed" (Bug 3, #754).
func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) {
db := setupCapabilityTestDB(t)
defer db.conn.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"aabbccdd11223344", "RepAdopter", "repeater", recentTS(24))
store := NewPacketStore(db, nil)
// Only a path-based packet (no advert) — would normally be "suspected"
pathByte := buildPathByte(2, 1)
rawHex := "01" + pathByte + "aabb"
pt := 1
pkt := &StoreTx{
RawHex: rawHex,
PayloadType: &pt,
PathJSON: `["aabb"]`,
FirstSeen: recentTS(48),
}
addTestPacket(store, pkt)
// Without adopter data: should be suspected
caps := store.computeMultiByteCapability(nil)
capByName := map[string]MultiByteCapEntry{}
for _, c := range caps {
capByName[c.Name] = c
}
if capByName["RepAdopter"].Status != "suspected" {
t.Errorf("without adopter data: expected suspected, got %s", capByName["RepAdopter"].Status)
}
// With adopter data showing hashSize 2: should be confirmed
adopterHS := map[string]int{"aabbccdd11223344": 2}
caps = store.computeMultiByteCapability(adopterHS)
capByName = map[string]MultiByteCapEntry{}
for _, c := range caps {
capByName[c.Name] = c
}
if capByName["RepAdopter"].Status != "confirmed" {
t.Errorf("with adopter data: expected confirmed, got %s", capByName["RepAdopter"].Status)
}
if capByName["RepAdopter"].Evidence != "advert" {
t.Errorf("with adopter data: expected advert evidence, got %s", capByName["RepAdopter"].Evidence)
}
}
-105
View File
@@ -94,10 +94,6 @@ func (s *Server) getNeighborGraph() *NeighborGraph {
func (s *Server) handleNodeNeighbors(w http.ResponseWriter, r *http.Request) {
pubkey := strings.ToLower(mux.Vars(r)["pubkey"])
if s.cfg.IsBlacklisted(pubkey) {
writeError(w, 404, "Not found")
return
}
minCount := 1
if v := r.URL.Query().Get("min_count"); v != "" {
@@ -191,10 +187,6 @@ func (s *Server) handleNodeNeighbors(w http.ResponseWriter, r *http.Request) {
entries = append(entries, entry)
}
// Defense-in-depth: deduplicate unresolved prefix entries that match
// resolved pubkey entries in the same neighbor set (fixes #698).
entries = dedupPrefixEntries(entries)
// Sort by score descending.
sort.Slice(entries, func(i, j int) bool {
return entries[i].Score > entries[j].Score
@@ -276,11 +268,6 @@ func (s *Server) handleNeighborGraph(w http.ResponseWriter, r *http.Request) {
}
}
// Filter blacklisted nodes from graph.
if s.cfg != nil && (s.cfg.IsBlacklisted(e.NodeA) || s.cfg.IsBlacklisted(e.NodeB)) {
continue
}
ge := GraphEdge{
Source: e.NodeA,
Target: e.NodeB,
@@ -382,97 +369,5 @@ func (s *Server) buildNodeInfoMap() map[string]nodeInfo {
for _, n := range nodes {
m[strings.ToLower(n.PublicKey)] = n
}
// Enrich observer-only nodes: if an observer pubkey isn't already in the
// map (i.e. it's not also a repeater/companion), add it with role "observer".
if s.db != nil {
rows, err := s.db.conn.Query("SELECT id, name FROM observers")
if err == nil {
defer rows.Close()
for rows.Next() {
var id, name string
if rows.Scan(&id, &name) != nil {
continue
}
key := strings.ToLower(id)
if _, exists := m[key]; !exists {
m[key] = nodeInfo{PublicKey: id, Name: name, Role: "observer"}
}
}
}
}
return m
}
// dedupPrefixEntries merges unresolved prefix entries with resolved pubkey entries
// where the prefix is a prefix of the resolved pubkey. Defense-in-depth for #698.
func dedupPrefixEntries(entries []NeighborEntry) []NeighborEntry {
if len(entries) < 2 {
return entries
}
// Mark indices of unresolved entries to remove after merging.
remove := make(map[int]bool)
for i := range entries {
if entries[i].Pubkey != nil {
continue // only check unresolved (no pubkey)
}
prefix := strings.ToLower(entries[i].Prefix)
if prefix == "" {
continue
}
// Find all resolved entries matching this prefix.
matchIdx := -1
matchCount := 0
for j := range entries {
if i == j || entries[j].Pubkey == nil {
continue
}
if strings.HasPrefix(strings.ToLower(*entries[j].Pubkey), prefix) {
matchIdx = j
matchCount++
}
}
// Only merge when exactly one resolved entry matches — ambiguous
// prefixes that match multiple resolved neighbors must not be
// arbitrarily assigned to one of them.
if matchCount != 1 {
continue
}
j := matchIdx
// Merge counts from unresolved into resolved.
entries[j].Count += entries[i].Count
// Preserve higher LastSeen.
if entries[i].LastSeen > entries[j].LastSeen {
entries[j].LastSeen = entries[i].LastSeen
}
// Merge observers.
obsSet := make(map[string]bool)
for _, o := range entries[j].Observers {
obsSet[o] = true
}
for _, o := range entries[i].Observers {
obsSet[o] = true
}
entries[j].Observers = observerList(obsSet)
remove[i] = true
}
if len(remove) == 0 {
return entries
}
result := make([]NeighborEntry, 0, len(entries)-len(remove))
for i, e := range entries {
if !remove[i] {
result = append(result, e)
}
}
return result
}
-68
View File
@@ -1,7 +1,6 @@
package main
import (
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -9,7 +8,6 @@ import (
"time"
"github.com/gorilla/mux"
_ "modernc.org/sqlite"
)
// ─── Helpers ───────────────────────────────────────────────────────────────────
@@ -459,69 +457,3 @@ func TestNeighborGraphAPI_ResponseShape(t *testing.T) {
}
}
}
// ─── Tests: buildNodeInfoMap observer enrichment (#753) ────────────────────────
func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) {
// Create a temp SQLite DB with nodes and observers tables.
tmpDir := t.TempDir()
dbPath := tmpDir + "/test.db"
conn, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Create tables
for _, stmt := range []string{
"CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)",
"CREATE TABLE observers (id TEXT, name TEXT)",
"INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)",
"INSERT INTO observers VALUES ('BBBB2222', 'Observer-Alpha')",
"INSERT INTO observers VALUES ('AAAA1111', 'Obs-also-repeater')",
} {
if _, err := conn.Exec(stmt); err != nil {
t.Fatalf("exec %q: %v", stmt, err)
}
}
conn.Close()
// Open via our DB wrapper
db, err := OpenDB(dbPath)
if err != nil {
t.Fatal(err)
}
defer db.conn.Close()
// Build a PacketStore with this DB (minimal — just need getCachedNodesAndPM)
store := NewPacketStore(db, nil)
store.Load()
srv := &Server{
db: db,
store: store,
perfStats: NewPerfStats(),
}
m := srv.buildNodeInfoMap()
// AAAA1111 should be from nodes table (repeater), NOT overwritten by observer
if info, ok := m["aaaa1111"]; !ok {
t.Error("expected aaaa1111 in map")
} else if info.Role != "repeater" {
t.Errorf("expected role=repeater for aaaa1111, got %q", info.Role)
}
// BBBB2222 should be enriched from observers table
if info, ok := m["bbbb2222"]; !ok {
t.Error("expected bbbb2222 in map (observer-only node)")
} else {
if info.Role != "observer" {
t.Errorf("expected role=observer for bbbb2222, got %q", info.Role)
}
if info.Name != "Observer-Alpha" {
t.Errorf("expected name=Observer-Alpha for bbbb2222, got %q", info.Name)
}
}
}
-527
View File
@@ -1,527 +0,0 @@
package main
import (
"strings"
"testing"
"time"
)
// ─── Phase 1.5: resolveAmbiguousEdges tests ───────────────────────────────────
// Test 1: Ambiguous edge resolved after Phase 1.5 when geo proximity succeeds.
func TestResolveAmbiguousEdges_GeoProximity(t *testing.T) {
// Node A at lat=45, lon=-122. Candidate B1 at lat=45.1, lon=-122.1 (close).
// Candidate B2 at lat=10, lon=10 (far away). Prefix "b0" matches both.
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "CloseNode", HasGPS: true, Lat: 45.1, Lon: -122.1}
nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "FarNode", HasGPS: true, Lat: 10.0, Lon: 10.0}
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2})
graph := NewNeighborGraph()
now := time.Now()
// Insert an ambiguous edge: NodeA ↔ prefix:b0
pseudoB := "prefix:b0"
key := makeEdgeKey("aaaa1111", pseudoB)
graph.edges[key] = &NeighborEdge{
NodeA: key.A,
NodeB: "",
Prefix: "b0",
Count: 50,
FirstSeen: now.Add(-1 * time.Hour),
LastSeen: now,
Observers: map[string]bool{"obs1": true},
Ambiguous: true,
Candidates: []string{"b0b1eeee", "b0c2ffff"},
}
graph.byNode["aaaa1111"] = append(graph.byNode["aaaa1111"], graph.edges[key])
resolveAmbiguousEdges(pm, graph)
// The ambiguous edge should be resolved to b0b1eeee (closest by geo).
graph.mu.RLock()
defer graph.mu.RUnlock()
if _, ok := graph.edges[key]; ok {
t.Error("ambiguous edge should have been removed")
}
resolvedKey := makeEdgeKey("aaaa1111", "b0b1eeee")
e, ok := graph.edges[resolvedKey]
if !ok {
t.Fatal("resolved edge not found")
}
if e.Ambiguous {
t.Error("resolved edge should not be ambiguous")
}
if e.Count != 50 {
t.Errorf("expected count 50, got %d", e.Count)
}
}
// Test 2: Ambiguous edge merged with existing resolved edge (count accumulation).
func TestResolveAmbiguousEdges_MergeWithExisting(t *testing.T) {
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB})
graph := NewNeighborGraph()
now := time.Now()
// Existing resolved edge: NodeA ↔ NodeB with count=10.
resolvedKey := makeEdgeKey("aaaa1111", "b0b1eeee")
resolvedEdge := &NeighborEdge{
NodeA: resolvedKey.A,
NodeB: resolvedKey.B,
Prefix: "b0b1",
Count: 10,
FirstSeen: now.Add(-2 * time.Hour),
LastSeen: now.Add(-30 * time.Minute),
Observers: map[string]bool{"obs1": true},
}
graph.edges[resolvedKey] = resolvedEdge
graph.byNode[resolvedKey.A] = append(graph.byNode[resolvedKey.A], resolvedEdge)
graph.byNode[resolvedKey.B] = append(graph.byNode[resolvedKey.B], resolvedEdge)
// Ambiguous edge: NodeA ↔ prefix:b0 with count=207.
pseudoB := "prefix:b0"
ambigKey := makeEdgeKey("aaaa1111", pseudoB)
ambigEdge := &NeighborEdge{
NodeA: ambigKey.A,
NodeB: "",
Prefix: "b0",
Count: 207,
FirstSeen: now.Add(-3 * time.Hour),
LastSeen: now, // more recent than resolved edge
Observers: map[string]bool{"obs2": true},
Ambiguous: true,
Candidates: []string{"b0b1eeee"},
}
graph.edges[ambigKey] = ambigEdge
graph.byNode["aaaa1111"] = append(graph.byNode["aaaa1111"], ambigEdge)
resolveAmbiguousEdges(pm, graph)
graph.mu.RLock()
defer graph.mu.RUnlock()
// Ambiguous edge should be gone.
if _, ok := graph.edges[ambigKey]; ok {
t.Error("ambiguous edge should have been removed")
}
// Resolved edge should have merged counts.
e := graph.edges[resolvedKey]
if e == nil {
t.Fatal("resolved edge not found")
}
if e.Count != 217 { // 10 + 207
t.Errorf("expected merged count 217, got %d", e.Count)
}
// LastSeen should be the max of both.
if !e.LastSeen.Equal(now) {
t.Errorf("expected LastSeen to be %v, got %v", now, e.LastSeen)
}
// Both observers should be present.
if !e.Observers["obs1"] || !e.Observers["obs2"] {
t.Error("expected both observers to be present after merge")
}
}
// Test 3: Ambiguous edge left as-is when resolution fails.
func TestResolveAmbiguousEdges_FailsNoChange(t *testing.T) {
// Two candidates, neither has GPS, no affinity data — resolution falls through.
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"}
nodeB1 := nodeInfo{PublicKey: "b0b1eeee", Name: "B1"}
nodeB2 := nodeInfo{PublicKey: "b0c2ffff", Name: "B2"}
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB1, nodeB2})
graph := NewNeighborGraph()
now := time.Now()
pseudoB := "prefix:b0"
key := makeEdgeKey("aaaa1111", pseudoB)
graph.edges[key] = &NeighborEdge{
NodeA: key.A,
NodeB: "",
Prefix: "b0",
Count: 5,
FirstSeen: now.Add(-1 * time.Hour),
LastSeen: now,
Observers: map[string]bool{"obs1": true},
Ambiguous: true,
Candidates: []string{"b0b1eeee", "b0c2ffff"},
}
graph.byNode["aaaa1111"] = append(graph.byNode["aaaa1111"], graph.edges[key])
resolveAmbiguousEdges(pm, graph)
graph.mu.RLock()
defer graph.mu.RUnlock()
// Edge should still be ambiguous — resolution falls to first_match which
// does resolve (it always picks something), but that's fine. Let's verify
// if it resolved or stayed. Actually, resolveWithContext returns first_match
// as fallback, so it WILL resolve. Let me adjust — the spec says "left as-is
// when resolution fails." For resolveWithContext to truly fail, we need
// no candidates at all in the prefix map.
// Actually the spec says resolution fails = "no_match" confidence. That
// only happens when pm.m has no entries for the prefix. With candidates
// in pm, it always returns something. Let me test the true no-match case.
}
// Test 3 (corrected): Resolution fails when prefix has no candidates in prefix map.
func TestResolveAmbiguousEdges_NoMatch(t *testing.T) {
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA"}
// pm has no entries matching prefix "zz"
pm := buildPrefixMap([]nodeInfo{nodeA})
graph := NewNeighborGraph()
now := time.Now()
pseudoB := "prefix:zz"
key := makeEdgeKey("aaaa1111", pseudoB)
graph.edges[key] = &NeighborEdge{
NodeA: key.A,
NodeB: "",
Prefix: "zz",
Count: 5,
FirstSeen: now.Add(-1 * time.Hour),
LastSeen: now,
Observers: map[string]bool{"obs1": true},
Ambiguous: true,
Candidates: []string{},
}
graph.byNode["aaaa1111"] = append(graph.byNode["aaaa1111"], graph.edges[key])
resolveAmbiguousEdges(pm, graph)
graph.mu.RLock()
defer graph.mu.RUnlock()
// Edge should still exist and be ambiguous.
e, ok := graph.edges[key]
if !ok {
t.Fatal("edge should still exist")
}
if !e.Ambiguous {
t.Error("edge should still be ambiguous")
}
}
// Test 6: Phase 1 edge collection unchanged (no regression).
func TestPhase1EdgeCollection_Unchanged(t *testing.T) {
// Build a simple graph and verify non-ambiguous edges are not touched.
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
nodeB := nodeInfo{PublicKey: "bbbb2222", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
ts := time.Now().UTC().Format(time.RFC3339)
payloadType := 4
obs := []*StoreObs{{
ObserverID: "cccc3333",
PathJSON: `["bbbb2222"]`,
Timestamp: ts,
}}
tx := &StoreTx{
ID: 1,
PayloadType: &payloadType,
DecodedJSON: `{"pubKey":"aaaa1111"}`,
Observations: obs,
}
store := ngTestStore([]nodeInfo{nodeA, nodeB, {PublicKey: "cccc3333", Name: "Observer"}}, []*StoreTx{tx})
graph := BuildFromStore(store)
edges := graph.Neighbors("aaaa1111")
found := false
for _, e := range edges {
if (e.NodeA == "aaaa1111" && e.NodeB == "bbbb2222") || (e.NodeA == "bbbb2222" && e.NodeB == "aaaa1111") {
found = true
if e.Ambiguous {
t.Error("resolved edge should not be ambiguous")
}
if e.Count != 1 {
t.Errorf("expected count 1, got %d", e.Count)
}
}
}
if !found {
t.Error("expected resolved edge between aaaa1111 and bbbb2222")
}
}
// Test 7: Merge preserves higher LastSeen timestamp.
func TestResolveAmbiguousEdges_PreservesHigherLastSeen(t *testing.T) {
nodeA := nodeInfo{PublicKey: "aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
nodeB := nodeInfo{PublicKey: "b0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
pm := buildPrefixMap([]nodeInfo{nodeA, nodeB})
graph := NewNeighborGraph()
later := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC)
earlier := time.Date(2026, 4, 9, 12, 0, 0, 0, time.UTC)
// Resolved edge has LATER LastSeen.
resolvedKey := makeEdgeKey("aaaa1111", "b0b1eeee")
re := &NeighborEdge{
NodeA: resolvedKey.A, NodeB: resolvedKey.B,
Count: 5, FirstSeen: earlier, LastSeen: later,
Observers: map[string]bool{"obs1": true},
}
graph.edges[resolvedKey] = re
graph.byNode[resolvedKey.A] = append(graph.byNode[resolvedKey.A], re)
graph.byNode[resolvedKey.B] = append(graph.byNode[resolvedKey.B], re)
// Ambiguous edge has EARLIER LastSeen.
pseudoB := "prefix:b0"
ambigKey := makeEdgeKey("aaaa1111", pseudoB)
ae := &NeighborEdge{
NodeA: ambigKey.A, NodeB: "",
Prefix: "b0", Count: 100,
FirstSeen: earlier.Add(-24 * time.Hour), LastSeen: earlier,
Observers: map[string]bool{"obs2": true},
Ambiguous: true,
Candidates: []string{"b0b1eeee"},
}
graph.edges[ambigKey] = ae
graph.byNode["aaaa1111"] = append(graph.byNode["aaaa1111"], ae)
resolveAmbiguousEdges(pm, graph)
graph.mu.RLock()
defer graph.mu.RUnlock()
e := graph.edges[resolvedKey]
if e == nil {
t.Fatal("resolved edge missing")
}
if !e.LastSeen.Equal(later) {
t.Errorf("expected LastSeen=%v (higher), got %v", later, e.LastSeen)
}
if !e.FirstSeen.Equal(earlier.Add(-24 * time.Hour)) {
t.Errorf("expected FirstSeen from ambiguous edge (earliest)")
}
}
// Test 5: Integration — node with both 1-byte and 2-byte prefix observations shows single entry.
func TestIntegration_DualPrefixSingleNeighbor(t *testing.T) {
nodeA := nodeInfo{PublicKey: "aaaa1111aaaa1111", Name: "NodeA", HasGPS: true, Lat: 45.0, Lon: -122.0}
nodeB := nodeInfo{PublicKey: "b0b1eeeeb0b1eeee", Name: "NodeB", HasGPS: true, Lat: 45.1, Lon: -122.1}
nodeB2 := nodeInfo{PublicKey: "b0c2ffffb0c2ffff", Name: "NodeB2", HasGPS: true, Lat: 10.0, Lon: 10.0}
observer := nodeInfo{PublicKey: "cccc3333cccc3333", Name: "Observer"}
ts := time.Now().UTC().Format(time.RFC3339)
pt := 4
// Observation 1: 1-byte prefix "b0" (ambiguous — matches both B and B2).
obs1 := []*StoreObs{{ObserverID: "cccc3333cccc3333", PathJSON: `["b0"]`, Timestamp: ts}}
tx1 := &StoreTx{ID: 1, PayloadType: &pt, DecodedJSON: `{"pubKey":"aaaa1111aaaa1111"}`, Observations: obs1}
// Observation 2: 4-byte prefix "b0b1" (unique — resolves to NodeB).
obs2 := []*StoreObs{{ObserverID: "cccc3333cccc3333", PathJSON: `["b0b1"]`, Timestamp: ts}}
tx2 := &StoreTx{ID: 2, PayloadType: &pt, DecodedJSON: `{"pubKey":"aaaa1111aaaa1111"}`, Observations: obs2}
store := ngTestStore([]nodeInfo{nodeA, nodeB, nodeB2, observer}, []*StoreTx{tx1, tx2})
graph := BuildFromStore(store)
edges := graph.Neighbors("aaaa1111aaaa1111")
// Count non-observer edges that point to NodeB or are ambiguous with b0 prefix.
resolvedToB := 0
ambiguousB0 := 0
for _, e := range edges {
other := e.NodeA
if strings.EqualFold(other, "aaaa1111aaaa1111") {
other = e.NodeB
}
if strings.EqualFold(other, "b0b1eeeeb0b1eeee") {
resolvedToB++
}
if e.Ambiguous && e.Prefix == "b0" {
ambiguousB0++
}
}
if ambiguousB0 > 0 {
t.Errorf("expected no ambiguous b0 edges after Phase 1.5, got %d", ambiguousB0)
}
if resolvedToB != 1 {
t.Errorf("expected exactly 1 resolved edge to NodeB, got %d", resolvedToB)
}
}
// ─── API dedup tests ───────────────────────────────────────────────────────────
// Test 4: API dedup merges unresolved prefix with resolved pubkey in response.
func TestDedupPrefixEntries_MergesUnresolved(t *testing.T) {
pk := "b0b1eeeeb0b1eeee"
name := "NodeB"
entries := []NeighborEntry{
{
Pubkey: nil, // unresolved
Prefix: "b0",
Count: 207,
LastSeen: "2026-04-10T12:00:00Z",
Observers: []string{"obs1"},
Ambiguous: true,
},
{
Pubkey: &pk,
Prefix: "b0b1",
Name: &name,
Count: 1,
LastSeen: "2026-04-09T12:00:00Z",
Observers: []string{"obs2"},
},
}
result := dedupPrefixEntries(entries)
if len(result) != 1 {
t.Fatalf("expected 1 entry after dedup, got %d", len(result))
}
if result[0].Pubkey == nil || *result[0].Pubkey != pk {
t.Error("expected resolved entry to remain")
}
if result[0].Count != 208 { // 1 + 207
t.Errorf("expected merged count 208, got %d", result[0].Count)
}
if result[0].LastSeen != "2026-04-10T12:00:00Z" {
t.Errorf("expected higher LastSeen, got %s", result[0].LastSeen)
}
// Both observers should be present.
obsMap := make(map[string]bool)
for _, o := range result[0].Observers {
obsMap[o] = true
}
if !obsMap["obs1"] || !obsMap["obs2"] {
t.Error("expected both observers after merge")
}
}
func TestDedupPrefixEntries_NoMatchNoChange(t *testing.T) {
pk := "dddd4444"
entries := []NeighborEntry{
{Pubkey: nil, Prefix: "b0", Count: 5, Ambiguous: true, Observers: []string{}},
{Pubkey: &pk, Prefix: "dd", Count: 10, Observers: []string{}},
}
result := dedupPrefixEntries(entries)
if len(result) != 2 {
t.Errorf("expected 2 entries (no match), got %d", len(result))
}
}
// ─── Benchmark ─────────────────────────────────────────────────────────────────
// Test 8: Benchmark Phase 1.5 with 500+ ambiguous edges to verify <100ms.
func BenchmarkResolveAmbiguousEdges_500(b *testing.B) {
// Create 600 nodes and 500 ambiguous edges.
var nodes []nodeInfo
for i := 0; i < 600; i++ {
pk := strings.ToLower(strings.Replace(
strings.Replace(
strings.Replace(
"xxxx0000xxxx0000", "xxxx", string(rune('a'+i/26))+string(rune('a'+i%26)), 1),
"0000", string(rune('0'+i/100))+string(rune('0'+(i/10)%10))+string(rune('0'+i%10))+"0", 1),
"xxxx0000", string(rune('a'+i/26))+string(rune('a'+i%26))+"ff"+string(rune('0'+i/100))+string(rune('0'+(i/10)%10))+string(rune('0'+i%10))+"0ff", 1))
// Use hex-safe pubkeys.
pk = hexPK(i)
nodes = append(nodes, nodeInfo{
PublicKey: pk,
Name: pk[:8],
HasGPS: true,
Lat: 45.0 + float64(i)*0.01,
Lon: -122.0 + float64(i)*0.01,
})
}
pm := buildPrefixMap(nodes)
b.ResetTimer()
for n := 0; n < b.N; n++ {
graph := NewNeighborGraph()
// Create 500 ambiguous edges.
for i := 0; i < 500; i++ {
knownPK := nodes[0].PublicKey
prefix := strings.ToLower(nodes[i+1].PublicKey[:2])
pseudoB := "prefix:" + prefix
key := makeEdgeKey(strings.ToLower(knownPK), pseudoB)
graph.edges[key] = &NeighborEdge{
NodeA: key.A,
NodeB: "",
Prefix: prefix,
Count: 10,
FirstSeen: time.Now(),
LastSeen: time.Now(),
Observers: map[string]bool{"obs": true},
Ambiguous: true,
Candidates: []string{strings.ToLower(nodes[i+1].PublicKey)},
}
graph.byNode[strings.ToLower(knownPK)] = append(
graph.byNode[strings.ToLower(knownPK)], graph.edges[key])
}
resolveAmbiguousEdges(pm, graph)
}
}
// hexPK generates a deterministic 16-char hex pubkey for index i.
func hexPK(i int) string {
const hexChars = "0123456789abcdef"
var b [16]byte
v := i
for j := 15; j >= 0; j-- {
b[j] = hexChars[v%16]
v /= 16
}
return string(b[:])
}
// Test: API dedup does NOT merge when prefix matches multiple resolved entries.
func TestDedupPrefixEntries_MultiMatchNoMerge(t *testing.T) {
pk1 := "b0b1eeeeb0b1eeee"
pk2 := "b0c2ffffb0c2ffff"
name1 := "NodeB1"
name2 := "NodeB2"
entries := []NeighborEntry{
{
Pubkey: nil, // unresolved
Prefix: "b0",
Count: 100,
LastSeen: "2026-04-10T12:00:00Z",
Observers: []string{"obs1"},
Ambiguous: true,
},
{
Pubkey: &pk1,
Prefix: "b0b1",
Name: &name1,
Count: 5,
LastSeen: "2026-04-09T12:00:00Z",
Observers: []string{"obs2"},
},
{
Pubkey: &pk2,
Prefix: "b0c2",
Name: &name2,
Count: 3,
LastSeen: "2026-04-08T12:00:00Z",
Observers: []string{"obs3"},
},
}
result := dedupPrefixEntries(entries)
if len(result) != 3 {
t.Fatalf("expected 3 entries (no merge for ambiguous prefix), got %d", len(result))
}
// Counts should be unchanged.
for _, e := range result {
if e.Pubkey != nil && *e.Pubkey == pk1 && e.Count != 5 {
t.Errorf("pk1 count should be unchanged at 5, got %d", e.Count)
}
if e.Pubkey != nil && *e.Pubkey == pk2 && e.Count != 3 {
t.Errorf("pk2 count should be unchanged at 3, got %d", e.Count)
}
}
}
+1 -69
View File
@@ -166,7 +166,7 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
// Phase 1: Extract edges from every transmission + observation.
for _, tx := range packets {
isAdvert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
fromNode := extractFromNode(tx)
// Pre-compute lowered originator once per tx (not per observation).
fromLower := ""
@@ -206,9 +206,6 @@ func BuildFromStoreWithLog(store *PacketStore, enableLog bool) *NeighborGraph {
}
}
// Phase 1.5: Resolve ambiguous edges using full graph context.
resolveAmbiguousEdges(pm, g)
// Phase 2: Disambiguation via Jaccard similarity.
g.disambiguate()
@@ -346,71 +343,6 @@ func (g *NeighborGraph) upsertEdgeWithCandidates(knownPK, prefix string, candida
}
}
// ─── Phase 1.5: Context-based resolution of ambiguous edges ────────────────────
// resolveAmbiguousEdges attempts to resolve ambiguous prefix edges using the
// fully-built graph context. Called after Phase 1 (edge collection) completes
// so that affinity and geo proximity tiers have full neighbor data.
func resolveAmbiguousEdges(pm *prefixMap, graph *NeighborGraph) {
// Step 1: Collect ambiguous edges under read lock.
graph.mu.RLock()
type ambiguousEntry struct {
key edgeKey
edge *NeighborEdge
knownNode string
prefix string
}
var ambiguous []ambiguousEntry
for key, e := range graph.edges {
if !e.Ambiguous {
continue
}
knownNode := e.NodeA
if strings.HasPrefix(e.NodeA, "prefix:") {
knownNode = e.NodeB
}
if knownNode == "" {
continue
}
ambiguous = append(ambiguous, ambiguousEntry{key, e, knownNode, e.Prefix})
}
graph.mu.RUnlock()
// Step 2: Resolve each (no lock needed — resolveWithContext takes its own RLock).
type resolution struct {
ambiguousEntry
resolvedPK string
}
var resolutions []resolution
for _, ae := range ambiguous {
resolved, confidence, _ := pm.resolveWithContext(ae.prefix, []string{ae.knownNode}, graph)
if resolved == nil || confidence == "no_match" || confidence == "first_match" || confidence == "gps_preference" {
continue
}
rpk := strings.ToLower(resolved.PublicKey)
if rpk == ae.knownNode {
continue // self-edge guard
}
resolutions = append(resolutions, resolution{ae, rpk})
}
// Step 3: Apply resolutions under write lock.
if len(resolutions) == 0 {
return
}
graph.mu.Lock()
for _, r := range resolutions {
// Verify edge still exists and is still ambiguous (could have been
// resolved by a prior iteration if two ambiguous edges resolve to same target).
e, ok := graph.edges[r.key]
if !ok || !e.Ambiguous {
continue
}
graph.resolveEdge(r.key, e, r.knownNode, r.resolvedPK)
}
graph.mu.Unlock()
}
// ─── Disambiguation ────────────────────────────────────────────────────────────
// disambiguate resolves ambiguous edges using Jaccard similarity of neighbor sets.
+11 -60
View File
@@ -381,13 +381,7 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
}
}
for _, obs := range tx.Observations {
// Check if this observation has been resolved: look up in the index.
// If the tx has no reverse-map entries AND path is non-empty, it needs backfill.
hasRP := false
if _, ok := store.resolvedPubkeyReverse[tx.ID]; ok {
hasRP = true
}
if !hasRP && obs.PathJSON != "" && obs.PathJSON != "[]" {
if obs.ResolvedPath == nil && obs.PathJSON != "" && obs.PathJSON != "[]" {
allPending = append(allPending, obsRef{
obsID: obs.ID,
pathJSON: obs.PathJSON,
@@ -488,61 +482,24 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int
}
}
// Update in-memory state: update resolved pubkey index, re-pick best observation,
// and invalidate LRU cache entries for backfilled observations (#800).
//
// Lock ordering: always take s.mu BEFORE lruMu. The read path
// (fetchResolvedPathForObs) takes lruMu independently of s.mu,
// so we must NOT hold s.mu while taking lruMu. Instead, collect
// obsIDs to invalidate under s.mu, release it, then take lruMu.
// Update in-memory state and re-pick best observation under a single
// write lock. The per-tx pickBestObservation is O(observations) which is
// typically <10 per tx — negligible cost vs. the race risk of splitting
// the lock (pollAndMerge can append to tx.Observations concurrently).
store.mu.Lock()
affectedSet := make(map[string]bool)
lruInvalidate := make([]int, 0, len(results))
for _, r := range results {
// Remove old index entries for this tx, then re-add with new pubkeys
if obs, ok := store.byObsID[r.obsID]; ok {
obs.ResolvedPath = r.rp
}
if !affectedSet[r.txHash] {
affectedSet[r.txHash] = true
if tx, ok := store.byHash[r.txHash]; ok {
store.removeFromResolvedPubkeyIndex(tx.ID)
pickBestObservation(tx)
}
}
// Add new resolved pubkeys to index
if tx, ok := store.byHash[r.txHash]; ok {
pks := extractResolvedPubkeys(r.rp)
store.addToResolvedPubkeyIndex(tx.ID, pks)
// Update byNode for relay nodes
for _, pk := range pks {
store.addToByNode(tx, pk)
}
// Update byPathHop resolved-key entries
hopsSeen := make(map[string]bool)
for _, hop := range txGetParsedPath(tx) {
hopsSeen[strings.ToLower(hop)] = true
}
for _, pk := range pks {
if !hopsSeen[pk] {
hopsSeen[pk] = true
store.byPathHop[pk] = append(store.byPathHop[pk], tx)
}
}
}
lruInvalidate = append(lruInvalidate, r.obsID)
}
// Re-pick best observation for affected transmissions
for txHash := range affectedSet {
if tx, ok := store.byHash[txHash]; ok {
pickBestObservation(tx)
}
}
store.mu.Unlock()
// Invalidate LRU entries AFTER releasing s.mu to maintain lock
// ordering (lruMu must never be taken while s.mu is held).
store.lruMu.Lock()
for _, obsID := range lruInvalidate {
store.lruDelete(obsID)
}
store.lruMu.Unlock()
}
totalProcessed += len(chunk)
@@ -568,7 +525,7 @@ type edgeCandidate struct {
// For ADVERTs: originator↔path[0] (if unambiguous). For ALL types: observer↔path[last] (if unambiguous).
// Also handles zero-hop ADVERTs (originator↔observer direct link).
func extractEdgesFromObs(obs *StoreObs, tx *StoreTx, pm *prefixMap) []edgeCandidate {
isAdvert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT
isAdvert := tx.PayloadType != nil && *tx.PayloadType == 4
fromNode := extractFromNode(tx)
path := parsePathJSON(obs.PathJSON)
observerPK := strings.ToLower(obs.ObserverID)
@@ -627,18 +584,12 @@ func extractEdgesFromObs(obs *StoreObs, tx *StoreTx, pm *prefixMap) []edgeCandid
// openRW opens a read-write SQLite connection (same pattern as PruneOldPackets).
func openRW(dbPath string) (*sql.DB, error) {
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL", dbPath)
dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=10000", dbPath)
rw, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, err
}
rw.SetMaxOpenConns(1)
// DSN _busy_timeout may not be honored by all drivers; set via PRAGMA
// to guarantee SQLite retries for up to 5s before returning SQLITE_BUSY.
if _, err := rw.Exec("PRAGMA busy_timeout = 5000"); err != nil {
rw.Close()
return nil, fmt.Errorf("set busy_timeout: %w", err)
}
return rw, nil
}
+51 -57
View File
@@ -27,7 +27,7 @@ func createTestDBWithSchema(t *testing.T) (*DB, string) {
id INTEGER PRIMARY KEY AUTOINCREMENT,
raw_hex TEXT, hash TEXT UNIQUE, first_seen TEXT,
route_type INTEGER, payload_type INTEGER, payload_version INTEGER,
decoded_json TEXT, channel_hash TEXT DEFAULT NULL
decoded_json TEXT
)`)
conn.Exec(`CREATE TABLE observers (
id TEXT PRIMARY KEY, name TEXT, iata TEXT
@@ -203,14 +203,14 @@ func TestLoadNeighborEdgesFromDB(t *testing.T) {
}
func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
// After #800 refactor, resolved_path is no longer stored on StoreTx/StoreObs structs.
// Broadcast maps carry resolved_path from the decode-window, not from struct fields.
// This test verifies pickBestObservation no longer sets ResolvedPath on tx.
// Verify resolved_path appears in broadcast maps
pk := "aabbccdd"
obs := &StoreObs{
ID: 1,
ObserverID: "obs1",
ObserverName: "Observer 1",
PathJSON: `["aa"]`,
ResolvedPath: []*string{&pk},
Timestamp: "2024-01-01T00:00:00Z",
}
@@ -221,26 +221,32 @@ func TestStoreObsResolvedPathInBroadcast(t *testing.T) {
}
pickBestObservation(tx)
// tx should NOT have a ResolvedPath field anymore (compile-time guard)
// Verify the best observation's fields are propagated correctly
if tx.ObserverID != "obs1" {
t.Errorf("expected ObserverID=obs1, got %s", tx.ObserverID)
if tx.ResolvedPath == nil {
t.Fatal("expected ResolvedPath to be set on tx after pickBestObservation")
}
if *tx.ResolvedPath[0] != "aabbccdd" {
t.Errorf("expected resolved path to be aabbccdd, got %s", *tx.ResolvedPath[0])
}
}
func TestResolvedPathInTxToMap(t *testing.T) {
// After #800, txToMap no longer includes resolved_path from the struct.
// resolved_path is only available via on-demand SQL fetch (txToMapWithRP).
pk := "aabbccdd"
tx := &StoreTx{
ID: 1,
Hash: "abc123",
PathJSON: `["aa"]`,
obsKeys: make(map[string]bool),
ID: 1,
Hash: "abc123",
PathJSON: `["aa"]`,
ResolvedPath: []*string{&pk},
obsKeys: make(map[string]bool),
}
m := txToMap(tx)
if _, ok := m["resolved_path"]; ok {
t.Error("resolved_path should not be in txToMap output (removed in #800)")
rp, ok := m["resolved_path"]
if !ok {
t.Fatal("resolved_path not in txToMap output")
}
rpSlice, ok := rp.([]*string)
if !ok || len(rpSlice) != 1 || *rpSlice[0] != "aabbccdd" {
t.Errorf("unexpected resolved_path: %v", rp)
}
}
@@ -359,21 +365,27 @@ func TestLoadWithResolvedPath(t *testing.T) {
t.Fatalf("expected 1 observation, got %d", len(tx.Observations))
}
// After #800, ResolvedPath is not stored on StoreObs struct.
// Instead, resolved pubkeys are in the membership index.
_ = tx.Observations[0] // obs exists
h := resolvedPubkeyHash("aabbccdd")
if len(store.resolvedPubkeyIndex[h]) != 1 {
t.Fatal("expected resolved pubkey to be indexed")
obs := tx.Observations[0]
if obs.ResolvedPath == nil {
t.Fatal("expected ResolvedPath to be loaded")
}
if len(obs.ResolvedPath) != 1 || *obs.ResolvedPath[0] != "aabbccdd" {
t.Errorf("unexpected ResolvedPath: %v", obs.ResolvedPath)
}
// Check that pickBestObservation propagated resolved_path to tx
if tx.ResolvedPath == nil || len(tx.ResolvedPath) != 1 {
t.Error("expected ResolvedPath to be propagated to tx")
}
}
func TestResolvedPathInAPIResponse(t *testing.T) {
// After #800, TransmissionResp no longer has ResolvedPath field.
// resolved_path is included dynamically in map-based API responses.
// Test that TransmissionResp properly marshals resolved_path
pk := "aabbccddee"
resp := TransmissionResp{
ID: 1,
Hash: "test",
ID: 1,
Hash: "test",
ResolvedPath: []*string{&pk, nil},
}
data, err := json.Marshal(resp)
@@ -384,9 +396,19 @@ func TestResolvedPathInAPIResponse(t *testing.T) {
var m map[string]interface{}
json.Unmarshal(data, &m)
// resolved_path should NOT be in the marshaled JSON
if _, ok := m["resolved_path"]; ok {
t.Error("resolved_path should not be in TransmissionResp JSON (#800)")
rp, ok := m["resolved_path"]
if !ok {
t.Fatal("resolved_path missing from JSON")
}
rpArr, ok := rp.([]interface{})
if !ok || len(rpArr) != 2 {
t.Fatalf("unexpected resolved_path shape: %v", rp)
}
if rpArr[0] != "aabbccddee" {
t.Errorf("first element wrong: %v", rpArr[0])
}
if rpArr[1] != nil {
t.Errorf("second element should be null: %v", rpArr[1])
}
}
@@ -510,31 +532,3 @@ func TestPersistSemaphoreTryAcquireSkipsBatch(t *testing.T) {
<-persistSem // release
}
func TestOpenRW_BusyTimeout(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
// Create the DB file first
db, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL")
if err != nil {
t.Fatal(err)
}
db.Exec("CREATE TABLE dummy (id INTEGER)")
db.Close()
// Open via openRW and verify busy_timeout is set
rw, err := openRW(dbPath)
if err != nil {
t.Fatalf("openRW failed: %v", err)
}
defer rw.Close()
var timeout int
if err := rw.QueryRow("PRAGMA busy_timeout").Scan(&timeout); err != nil {
t.Fatalf("query busy_timeout: %v", err)
}
if timeout != 5000 {
t.Errorf("expected busy_timeout=5000, got %d", timeout)
}
}
-311
View File
@@ -1,311 +0,0 @@
package main
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/gorilla/mux"
)
func TestConfigIsBlacklisted(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{"AA", "BB", "cc"},
}
tests := []struct {
pubkey string
want bool
}{
{"AA", true},
{"aa", true}, // case-insensitive
{"BB", true},
{"CC", true}, // lowercase "cc" matches uppercase
{"DD", false},
{"", false},
{"AAB", false},
}
for _, tt := range tests {
got := cfg.IsBlacklisted(tt.pubkey)
if got != tt.want {
t.Errorf("IsBlacklisted(%q) = %v, want %v", tt.pubkey, got, tt.want)
}
}
}
func TestConfigIsBlacklistedEmpty(t *testing.T) {
cfg := &Config{}
if cfg.IsBlacklisted("anything") {
t.Error("empty blacklist should not match anything")
}
if cfg.IsBlacklisted("") {
t.Error("empty blacklist should not match empty string")
}
}
func TestConfigBlacklistWhitespace(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{" AA ", "BB"},
}
if !cfg.IsBlacklisted("AA") {
t.Error("trimmed key should match")
}
if !cfg.IsBlacklisted(" AA ") {
t.Error("whitespace-padded key should match after trimming")
}
}
func TestConfigBlacklistEmptyEntries(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{"", " ", "AA"},
}
if !cfg.IsBlacklisted("AA") {
t.Error("non-empty entry should match")
}
if cfg.IsBlacklisted("") {
t.Error("empty blacklist entry should not match empty pubkey")
}
}
func TestBlacklistFiltersHandleNodes(t *testing.T) {
db := setupTestDB(t)
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))")
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
srv := NewServer(db, cfg, NewHub())
req := httptest.NewRequest("GET", "/api/nodes?limit=50", nil)
w := httptest.NewRecorder()
srv.RegisterRoutes(setupTestRouter(srv))
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
}
var resp NodeListResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
for _, node := range resp.Nodes {
if pk, _ := node["public_key"].(string); pk == "badnode" {
t.Error("blacklisted node should not appear in nodes list")
}
}
if resp.Total == 0 {
t.Error("expected at least one non-blacklisted node")
}
}
func TestBlacklistFiltersNodeDetail(t *testing.T) {
db := setupTestDB(t)
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
srv := NewServer(db, cfg, NewHub())
req := httptest.NewRequest("GET", "/api/nodes/badnode", nil)
w := httptest.NewRecorder()
srv.RegisterRoutes(setupTestRouter(srv))
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("expected 404 for blacklisted node, got %d", w.Code)
}
}
func TestBlacklistFiltersNodeSearch(t *testing.T) {
db := setupTestDB(t)
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', datetime('now'))")
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))")
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
srv := NewServer(db, cfg, NewHub())
req := httptest.NewRequest("GET", "/api/nodes/search?q=Troll", nil)
w := httptest.NewRecorder()
srv.RegisterRoutes(setupTestRouter(srv))
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", w.Code)
}
var resp NodeSearchResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
for _, node := range resp.Nodes {
if pk, _ := node["public_key"].(string); pk == "badnode" {
t.Error("blacklisted node should not appear in search results")
}
}
}
func TestNoBlacklistPassesAll(t *testing.T) {
db := setupTestDB(t)
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', datetime('now'))")
cfg := &Config{}
srv := NewServer(db, cfg, NewHub())
req := httptest.NewRequest("GET", "/api/nodes?limit=50", nil)
w := httptest.NewRecorder()
srv.RegisterRoutes(setupTestRouter(srv))
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", w.Code)
}
var resp NodeListResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
if resp.Total == 0 {
t.Error("without blacklist, node should appear")
}
}
// setupTestRouter creates a mux.Router and registers server routes.
func setupTestRouter(srv *Server) *mux.Router {
r := mux.NewRouter()
srv.RegisterRoutes(r)
srv.router = r
return r
}
func TestBlacklistFiltersNeighborGraph(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
db := setupTestDB(t)
srv := NewServer(db, cfg, NewHub())
srv.RegisterRoutes(setupTestRouter(srv))
req := httptest.NewRequest("GET", "/api/analytics/neighbor-graph", nil)
w := httptest.NewRecorder()
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
}
var resp map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
// Check edges don't contain blacklisted node
if edges, ok := resp["edges"].([]interface{}); ok {
for _, e := range edges {
if edge, ok := e.(map[string]interface{}); ok {
if src, _ := edge["source"].(string); src == "badnode" {
t.Error("blacklisted node should not appear as edge source in neighbor graph")
}
if tgt, _ := edge["target"].(string); tgt == "badnode" {
t.Error("blacklisted node should not appear as edge target in neighbor graph")
}
}
}
}
// Check nodes list doesn't contain blacklisted node
if nodes, ok := resp["nodes"].([]interface{}); ok {
for _, n := range nodes {
if node, ok := n.(map[string]interface{}); ok {
if pk, _ := node["pubkey"].(string); pk == "badnode" {
t.Error("blacklisted node should not appear in neighbor graph nodes")
}
}
}
}
}
func TestBlacklistFiltersResolveHops(t *testing.T) {
db := setupTestDB(t)
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))")
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
srv := NewServer(db, cfg, NewHub())
srv.RegisterRoutes(setupTestRouter(srv))
req := httptest.NewRequest("GET", "/api/resolve-hops?hops=badnode", nil)
w := httptest.NewRecorder()
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
}
var resp ResolveHopsResponse
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("failed to parse response: %v", err)
}
if hr, ok := resp.Resolved["badnode"]; ok {
for _, c := range hr.Candidates {
if c.Pubkey == "badnode" {
t.Error("blacklisted node should not appear as resolve-hops candidate")
}
}
}
}
func TestBlacklistFiltersSubpathDetail(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{"badnode"},
}
db := setupTestDB(t)
srv := NewServer(db, cfg, NewHub())
srv.RegisterRoutes(setupTestRouter(srv))
req := httptest.NewRequest("GET", "/api/analytics/subpath-detail?hops=badnode,othernode", nil)
w := httptest.NewRecorder()
srv.router.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
t.Errorf("expected 404 for subpath-detail with blacklisted hop, got %d", w.Code)
}
}
func TestBlacklistConcurrentIsBlacklisted(t *testing.T) {
cfg := &Config{
NodeBlacklist: []string{"AA", "BB", "CC"},
}
errc := make(chan error, 100)
for i := 0; i < 100; i++ {
go func() {
for j := 0; j < 100; j++ {
cfg.IsBlacklisted("AA")
cfg.IsBlacklisted("BB")
cfg.IsBlacklisted("DD")
}
}()
}
// If sync.Once is wrong, this would panic or race.
// We can't run the race detector on ARM, but at least verify no panics.
done := false
for !done {
select {
case <-errc:
t.Error("concurrent IsBlacklisted panicked")
default:
done = true
}
}
}
-475
View File
@@ -1,475 +0,0 @@
package main
// Lock ordering contract (MUST be followed everywhere):
//
// s.mu → s.lruMu (s.mu is the outer lock, lruMu is the inner lock)
//
// • Never acquire s.lruMu while holding s.mu.
// • fetchResolvedPathForObs takes lruMu independently — callers under s.mu
// must NOT call it directly; instead collect IDs under s.mu, release, then
// do LRU ops under lruMu separately.
// • The backfill path (backfillResolvedPathsAsync) follows this by collecting
// obsIDs to invalidate under s.mu, releasing it, then taking lruMu.
import (
"database/sql"
"hash/fnv"
"log"
"strings"
)
// resolvedPubkeyHash computes a fast 64-bit hash for membership index keying.
// Uses FNV-1a from stdlib — good distribution, no external dependency.
func resolvedPubkeyHash(pk string) uint64 {
h := fnv.New64a()
h.Write([]byte(strings.ToLower(pk)))
return h.Sum64()
}
// addToResolvedPubkeyIndex adds a txID under each resolved pubkey hash.
// Deduplicates both within a single call AND across calls — won't add the
// same (hash, txID) pair twice even when called multiple times for the same tx.
// Must be called under s.mu write lock.
func (s *PacketStore) addToResolvedPubkeyIndex(txID int, resolvedPubkeys []string) {
if !s.useResolvedPathIndex {
return
}
seen := make(map[uint64]bool, len(resolvedPubkeys))
for _, pk := range resolvedPubkeys {
if pk == "" {
continue
}
h := resolvedPubkeyHash(pk)
if seen[h] {
continue
}
seen[h] = true
// Cross-call dedup: check if (h, txID) already exists in forward index.
existing := s.resolvedPubkeyIndex[h]
alreadyPresent := false
for _, id := range existing {
if id == txID {
alreadyPresent = true
break
}
}
if alreadyPresent {
continue
}
s.resolvedPubkeyIndex[h] = append(existing, txID)
s.resolvedPubkeyReverse[txID] = append(s.resolvedPubkeyReverse[txID], h)
}
}
// removeFromResolvedPubkeyIndex removes all index entries for a txID using the reverse map.
// Must be called under s.mu write lock.
func (s *PacketStore) removeFromResolvedPubkeyIndex(txID int) {
if !s.useResolvedPathIndex {
return
}
hashes := s.resolvedPubkeyReverse[txID]
for _, h := range hashes {
list := s.resolvedPubkeyIndex[h]
// Remove ALL occurrences of txID (not just the first) to prevent orphans.
filtered := list[:0]
for _, id := range list {
if id != txID {
filtered = append(filtered, id)
}
}
if len(filtered) == 0 {
delete(s.resolvedPubkeyIndex, h)
} else {
s.resolvedPubkeyIndex[h] = filtered
}
}
delete(s.resolvedPubkeyReverse, txID)
}
// extractResolvedPubkeys extracts all non-nil, non-empty pubkeys from a resolved path.
func extractResolvedPubkeys(rp []*string) []string {
if len(rp) == 0 {
return nil
}
result := make([]string, 0, len(rp))
for _, p := range rp {
if p != nil && *p != "" {
result = append(result, *p)
}
}
return result
}
// mergeResolvedPubkeys collects unique non-empty pubkeys from multiple resolved paths.
func mergeResolvedPubkeys(paths ...[]*string) []string {
seen := make(map[string]bool)
var result []string
for _, rp := range paths {
for _, p := range rp {
if p != nil && *p != "" && !seen[*p] {
seen[*p] = true
result = append(result, *p)
}
}
}
return result
}
// nodeInResolvedPathViaIndex checks whether a transmission is associated with
// a target pubkey using the membership index + collision-safety SQL check.
// Must be called under s.mu RLock at minimum.
func (s *PacketStore) nodeInResolvedPathViaIndex(tx *StoreTx, targetPK string) bool {
if !s.useResolvedPathIndex {
// Flag off: can't disambiguate, keep candidate (conservative)
return true
}
// If this tx has no indexed pubkeys at all, we can't disambiguate —
// keep the candidate (same as old behavior for NULL resolved_path).
if _, hasReverse := s.resolvedPubkeyReverse[tx.ID]; !hasReverse {
return true
}
h := resolvedPubkeyHash(targetPK)
txIDs := s.resolvedPubkeyIndex[h]
// Check if this tx's ID is in the candidate list
for _, id := range txIDs {
if id == tx.ID {
// Found in index. Collision-safety: verify with SQL.
if s.db != nil && s.db.conn != nil {
return s.confirmResolvedPathContains(tx.ID, targetPK)
}
return true // no DB, trust the index
}
}
return false
}
// confirmResolvedPathContains verifies an exact pubkey match in resolved_path
// via SQL. This is the collision-safety fallback for the membership index.
func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool {
if s.db == nil || s.db.conn == nil {
return true
}
// Use INSTR with surrounding quotes for exact match — avoids LIKE escape issues.
// resolved_path format: ["pubkey1","pubkey2",...]
needle := `"` + strings.ToLower(pubkey) + `"`
var count int
err := s.db.conn.QueryRow(
`SELECT COUNT(*) FROM observations WHERE transmission_id = ? AND INSTR(LOWER(resolved_path), ?) > 0`,
txID, needle,
).Scan(&count)
if err != nil {
return true // on error, keep the candidate
}
return count > 0
}
// fetchResolvedPathsForTx fetches resolved_path from SQLite for all observations
// of a transmission. Used for on-demand API responses and eviction cleanup.
func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string {
if s.db == nil || s.db.conn == nil {
return nil
}
rows, err := s.db.conn.Query(
`SELECT id, resolved_path FROM observations WHERE transmission_id = ? AND resolved_path IS NOT NULL`,
txID,
)
if err != nil {
return nil
}
defer rows.Close()
result := make(map[int][]*string)
for rows.Next() {
var obsID int
var rpJSON sql.NullString
if err := rows.Scan(&obsID, &rpJSON); err != nil {
continue
}
if rpJSON.Valid && rpJSON.String != "" {
result[obsID] = unmarshalResolvedPath(rpJSON.String)
}
}
return result
}
// fetchResolvedPathForObs fetches resolved_path for a single observation,
// using the LRU cache.
func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string {
if s.db == nil || s.db.conn == nil {
return nil
}
// Check LRU cache first
s.lruMu.RLock()
if s.apiResolvedPathLRU != nil {
if entry, ok := s.apiResolvedPathLRU[obsID]; ok {
s.lruMu.RUnlock()
return entry
}
}
s.lruMu.RUnlock()
var rpJSON sql.NullString
err := s.db.conn.QueryRow(
`SELECT resolved_path FROM observations WHERE id = ?`, obsID,
).Scan(&rpJSON)
if err != nil || !rpJSON.Valid {
return nil
}
rp := unmarshalResolvedPath(rpJSON.String)
// Store in LRU
s.lruMu.Lock()
s.lruPut(obsID, rp)
s.lruMu.Unlock()
return rp
}
// fetchResolvedPathForTxBest returns the best observation's resolved_path for a tx.
//
// "Best" = the longest path_json among observations that actually have a stored
// resolved_path. Earlier versions picked the longest-path obs unconditionally
// and queried SQL for that single ID — if the longest-path obs had NULL
// resolved_path while a shorter sibling had one, the call returned nil and
// callers (e.g. /api/nodes/{pk}/health.recentPackets) lost the field. Fixes
// #810 by checking all observations and falling back to the longest sibling
// that has a stored path.
func (s *PacketStore) fetchResolvedPathForTxBest(tx *StoreTx) []*string {
if tx == nil || len(tx.Observations) == 0 {
return nil
}
// Fast path: try the longest-path obs first via the LRU/SQL helper.
longest := tx.Observations[0]
longestLen := pathLen(longest.PathJSON)
for _, obs := range tx.Observations[1:] {
if l := pathLen(obs.PathJSON); l > longestLen {
longest = obs
longestLen = l
}
}
if rp := s.fetchResolvedPathForObs(longest.ID); rp != nil {
return rp
}
// Fallback: longest-path obs has no stored resolved_path. Query all
// observations for this tx and pick the one with the longest path_json
// that actually has a stored resolved_path.
rpMap := s.fetchResolvedPathsForTx(tx.ID)
if len(rpMap) == 0 {
return nil
}
var bestRP []*string
bestObsID := 0
bestLen := -1
for _, obs := range tx.Observations {
rp, ok := rpMap[obs.ID]
if !ok || rp == nil {
continue
}
if l := pathLen(obs.PathJSON); l > bestLen {
bestLen = l
bestRP = rp
bestObsID = obs.ID
}
}
// Populate LRU so repeat lookups for this tx don't re-issue the multi-row
// SQL fallback (e.g. dashboard polling /api/nodes/{pk}/health).
if bestRP != nil && bestObsID != 0 {
s.lruMu.Lock()
s.lruPut(bestObsID, bestRP)
s.lruMu.Unlock()
}
return bestRP
}
// --- Simple LRU cache for resolved paths ---
const lruMaxSize = 10000
// lruPut adds an entry. Must be called under s.lruMu write lock.
func (s *PacketStore) lruPut(obsID int, rp []*string) {
if s.apiResolvedPathLRU == nil {
return
}
if _, exists := s.apiResolvedPathLRU[obsID]; exists {
return
}
// Compact lruOrder if stale entries exceed 50% of capacity.
// This prevents effective capacity degradation after bulk deletions.
if len(s.lruOrder) >= lruMaxSize && len(s.apiResolvedPathLRU) < lruMaxSize/2 {
compacted := make([]int, 0, len(s.apiResolvedPathLRU))
for _, id := range s.lruOrder {
if _, ok := s.apiResolvedPathLRU[id]; ok {
compacted = append(compacted, id)
}
}
s.lruOrder = compacted
}
if len(s.lruOrder) >= lruMaxSize {
// Evict oldest, skipping stale entries
for len(s.lruOrder) > 0 {
evictID := s.lruOrder[0]
s.lruOrder = s.lruOrder[1:]
if _, ok := s.apiResolvedPathLRU[evictID]; ok {
delete(s.apiResolvedPathLRU, evictID)
break
}
// stale entry — skip and continue
}
}
s.apiResolvedPathLRU[obsID] = rp
s.lruOrder = append(s.lruOrder, obsID)
}
// lruDelete removes an entry. Must be called under s.lruMu write lock.
func (s *PacketStore) lruDelete(obsID int) {
if s.apiResolvedPathLRU == nil {
return
}
delete(s.apiResolvedPathLRU, obsID)
// Don't scan lruOrder — eviction handles stale entries naturally.
}
// resolvedPubkeysForEvictionBatch fetches resolved pubkeys for multiple txIDs
// from SQL in a single batched query. Returns a map from txID to unique pubkeys.
// MUST be called WITHOUT holding s.mu — this is the whole point of the batch approach.
// Chunks queries to stay under SQLite's 500-parameter limit.
func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]string {
result := make(map[int][]string, len(txIDs))
if len(txIDs) == 0 || s.db == nil || s.db.conn == nil {
return result
}
const chunkSize = 499 // SQLite SQLITE_MAX_VARIABLE_NUMBER default is 999; stay well under
for start := 0; start < len(txIDs); start += chunkSize {
end := start + chunkSize
if end > len(txIDs) {
end = len(txIDs)
}
chunk := txIDs[start:end]
// Build query with placeholders
placeholders := make([]byte, 0, len(chunk)*2)
args := make([]interface{}, len(chunk))
for i, id := range chunk {
if i > 0 {
placeholders = append(placeholders, ',')
}
placeholders = append(placeholders, '?')
args[i] = id
}
query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" +
string(placeholders) + ") AND resolved_path IS NOT NULL"
rows, err := s.db.conn.Query(query, args...)
if err != nil {
continue
}
for rows.Next() {
var txID int
var rpJSON sql.NullString
if err := rows.Scan(&txID, &rpJSON); err != nil {
continue
}
if !rpJSON.Valid || rpJSON.String == "" {
continue
}
rp := unmarshalResolvedPath(rpJSON.String)
for _, p := range rp {
if p != nil && *p != "" {
result[txID] = append(result[txID], *p)
}
}
}
rows.Close()
}
// Deduplicate per-txID
for txID, pks := range result {
seen := make(map[string]bool, len(pks))
deduped := pks[:0]
for _, pk := range pks {
if !seen[pk] {
seen[pk] = true
deduped = append(deduped, pk)
}
}
result[txID] = deduped
}
return result
}
// initResolvedPathIndex initializes the resolved path index data structures.
func (s *PacketStore) initResolvedPathIndex() {
s.resolvedPubkeyIndex = make(map[uint64][]int, 4096)
s.resolvedPubkeyReverse = make(map[int][]uint64, 4096)
s.apiResolvedPathLRU = make(map[int][]*string, lruMaxSize)
s.lruOrder = make([]int, 0, lruMaxSize)
}
// CompactResolvedPubkeyIndex reclaims memory from the resolved pubkey index maps
// after eviction. It removes empty forward-index entries (shouldn't exist if
// removeFromResolvedPubkeyIndex is correct, but defense in depth) and clips
// oversized slice backing arrays where cap > 2*len.
// Must be called under s.mu write lock.
func (s *PacketStore) CompactResolvedPubkeyIndex() {
if !s.useResolvedPathIndex {
return
}
for h, ids := range s.resolvedPubkeyIndex {
if len(ids) == 0 {
delete(s.resolvedPubkeyIndex, h)
continue
}
// Clip oversized backing arrays: if cap > 2*len, reallocate.
if cap(ids) > 2*len(ids)+8 {
clipped := make([]int, len(ids))
copy(clipped, ids)
s.resolvedPubkeyIndex[h] = clipped
}
}
for txID, hashes := range s.resolvedPubkeyReverse {
if len(hashes) == 0 {
delete(s.resolvedPubkeyReverse, txID)
continue
}
if cap(hashes) > 2*len(hashes)+8 {
clipped := make([]uint64, len(hashes))
copy(clipped, hashes)
s.resolvedPubkeyReverse[txID] = clipped
}
}
}
// defaultMaxResolvedPubkeyIndexEntries is the default hard cap for the forward
// index. When exceeded, a warning is logged. No auto-eviction — that's the
// eviction ticker's job.
const defaultMaxResolvedPubkeyIndexEntries = 5_000_000
// CheckResolvedPubkeyIndexSize logs a warning if the resolved pubkey forward
// index exceeds the configured maximum entries. Must be called under s.mu
// read lock at minimum.
func (s *PacketStore) CheckResolvedPubkeyIndexSize() {
if !s.useResolvedPathIndex {
return
}
maxEntries := s.maxResolvedPubkeyIndexEntries
if maxEntries <= 0 {
maxEntries = defaultMaxResolvedPubkeyIndexEntries
}
fwdLen := len(s.resolvedPubkeyIndex)
revLen := len(s.resolvedPubkeyReverse)
if fwdLen > maxEntries || revLen > maxEntries {
log.Printf("[store] WARNING: resolvedPubkeyIndex size exceeds limit — forward=%d reverse=%d limit=%d",
fwdLen, revLen, maxEntries)
}
}
File diff suppressed because it is too large Load Diff
+32 -396
View File
@@ -124,7 +124,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
r.Handle("/api/perf/reset", s.requireAPIKey(http.HandlerFunc(s.handlePerfReset))).Methods("POST")
r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST")
r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET")
r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET")
// Packet endpoints
r.HandleFunc("/api/packets/observations", s.handleBatchObservations).Methods("POST")
@@ -143,9 +142,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
r.HandleFunc("/api/nodes/{pubkey}/health", s.handleNodeHealth).Methods("GET")
r.HandleFunc("/api/nodes/{pubkey}/paths", s.handleNodePaths).Methods("GET")
r.HandleFunc("/api/nodes/{pubkey}/analytics", s.handleNodeAnalytics).Methods("GET")
r.HandleFunc("/api/nodes/clock-skew", s.handleFleetClockSkew).Methods("GET")
r.HandleFunc("/api/nodes/{pubkey}/clock-skew", s.handleNodeClockSkew).Methods("GET")
r.HandleFunc("/api/observers/clock-skew", s.handleObserverClockSkew).Methods("GET")
r.HandleFunc("/api/nodes/{pubkey}/neighbors", s.handleNodeNeighbors).Methods("GET")
r.HandleFunc("/api/nodes/{pubkey}", s.handleNodeDetail).Methods("GET")
r.HandleFunc("/api/nodes", s.handleNodes).Methods("GET")
@@ -450,12 +446,10 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
// Real packet store stats
pktCount := 0
var pktEstMB float64
var pktTrackedMB float64
if s.store != nil {
ps := s.store.GetPerfStoreStatsTyped()
pktCount = ps.TotalLoaded
pktEstMB = ps.EstimatedMB
pktTrackedMB = ps.TrackedMB
}
// Real cache stats
@@ -521,7 +515,6 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
PacketStore: HealthPacketStoreStats{
Packets: pktCount,
EstimatedMB: pktEstMB,
TrackedMB: pktTrackedMB,
},
Perf: HealthPerfStats{
TotalRequests: int(perfRequests),
@@ -569,16 +562,6 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
backfillProgress = 1
}
// Memory accounting (#832). storeDataMB is the in-store packet byte
// estimate (the old "trackedMB"); processRSSMB / goHeapInuseMB / goSysMB
// give ops the breakdown needed to reason about real RSS. All values
// share a single 1s-cached snapshot to amortize ReadMemStats cost.
var storeDataMB float64
if s.store != nil {
storeDataMB = s.store.trackedMemoryMB()
}
mem := s.getMemorySnapshot(storeDataMB)
resp := &StatsResponse{
TotalPackets: stats.TotalPackets,
TotalTransmissions: &stats.TotalTransmissions,
@@ -598,16 +581,8 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
Companions: counts["companions"],
Sensors: counts["sensors"],
},
Backfilling: backfilling,
BackfillProgress: backfillProgress,
SignatureDrops: s.db.GetSignatureDropCount(),
HashMigrationComplete: s.store != nil && s.store.hashMigrationComplete.Load(),
TrackedMB: mem.StoreDataMB, // deprecated alias
StoreDataMB: mem.StoreDataMB,
ProcessRSSMB: mem.ProcessRSSMB,
GoHeapInuseMB: mem.GoHeapInuseMB,
GoSysMB: mem.GoSysMB,
Backfilling: backfilling,
BackfillProgress: backfillProgress,
}
s.statsMu.Lock()
@@ -790,7 +765,6 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
Until: r.URL.Query().Get("until"),
Region: r.URL.Query().Get("region"),
Node: r.URL.Query().Get("node"),
Channel: r.URL.Query().Get("channel"),
Order: "DESC",
ExpandObservations: r.URL.Query().Get("expand") == "observations",
}
@@ -893,11 +867,9 @@ func (s *Server) handleBatchObservations(w http.ResponseWriter, r *http.Request)
func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
param := mux.Vars(r)["id"]
var packet map[string]interface{}
fromDB := false
isHash := hashPattern.MatchString(strings.ToLower(param))
if s.store != nil {
if isHash {
if hashPattern.MatchString(strings.ToLower(param)) {
packet = s.store.GetPacketByHash(param)
}
if packet == nil {
@@ -910,25 +882,6 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
}
}
}
// DB fallback: in-memory PacketStore prunes old entries, but the SQLite
// DB retains them and is the source for /api/nodes recentAdverts. Without
// this fallback, links from node-detail pages 404 once the packet ages out.
if packet == nil && s.db != nil {
if isHash {
if dbPkt, err := s.db.GetPacketByHash(param); err == nil && dbPkt != nil {
packet = dbPkt
fromDB = true
}
}
if packet == nil {
if id, parseErr := strconv.Atoi(param); parseErr == nil {
if dbPkt, err := s.db.GetTransmissionByID(id); err == nil && dbPkt != nil {
packet = dbPkt
fromDB = true
}
}
}
}
if packet == nil {
writeError(w, 404, "Not found")
return
@@ -939,9 +892,6 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) {
if s.store != nil {
observations = s.store.GetObservationsForHash(hash)
}
if len(observations) == 0 && fromDB && s.db != nil && hash != "" {
observations = s.db.GetObservationsForHash(hash)
}
observationCount := len(observations)
if observationCount == 0 {
observationCount = 1
@@ -1095,17 +1045,6 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
total = len(filtered)
nodes = filtered
}
// Filter blacklisted nodes
if len(s.cfg.NodeBlacklist) > 0 {
filtered := nodes[:0]
for _, node := range nodes {
if pk, ok := node["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
filtered = append(filtered, node)
}
}
total = len(filtered)
nodes = filtered
}
writeJSON(w, NodeListResponse{Nodes: nodes, Total: total, Counts: counts})
}
@@ -1120,25 +1059,11 @@ func (s *Server) handleNodeSearch(w http.ResponseWriter, r *http.Request) {
writeError(w, 500, err.Error())
return
}
// Filter blacklisted nodes from search results
if len(s.cfg.NodeBlacklist) > 0 {
filtered := make([]map[string]interface{}, 0, len(nodes))
for _, node := range nodes {
if pk, ok := node["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
filtered = append(filtered, node)
}
}
nodes = filtered
}
writeJSON(w, NodeSearchResponse{Nodes: nodes})
}
func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) {
pubkey := mux.Vars(r)["pubkey"]
if s.cfg.IsBlacklisted(pubkey) {
writeError(w, 404, "Not found")
return
}
node, err := s.db.GetNodeByPubkey(pubkey)
if err != nil || node == nil {
writeError(w, 404, "Not found")
@@ -1164,10 +1089,6 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleNodeHealth(w http.ResponseWriter, r *http.Request) {
pubkey := mux.Vars(r)["pubkey"]
if s.cfg.IsBlacklisted(pubkey) {
writeError(w, 404, "Not found")
return
}
if s.store != nil {
result, err := s.store.GetNodeHealth(pubkey)
if err != nil || result == nil {
@@ -1188,19 +1109,7 @@ func (s *Server) handleBulkHealth(w http.ResponseWriter, r *http.Request) {
if s.store != nil {
region := r.URL.Query().Get("region")
results := s.store.GetBulkHealth(limit, region)
// Filter blacklisted nodes
if len(s.cfg.NodeBlacklist) > 0 {
filtered := make([]map[string]interface{}, 0, len(results))
for _, entry := range results {
if pk, ok := entry["public_key"].(string); !ok || !s.cfg.IsBlacklisted(pk) {
filtered = append(filtered, entry)
}
}
writeJSON(w, filtered)
return
}
writeJSON(w, results)
writeJSON(w, s.store.GetBulkHealth(limit, region))
return
}
@@ -1219,10 +1128,6 @@ func (s *Server) handleNetworkStatus(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
pubkey := mux.Vars(r)["pubkey"]
if s.cfg.IsBlacklisted(pubkey) {
writeError(w, 404, "Not found")
return
}
node, err := s.db.GetNodeByPubkey(pubkey)
if err != nil || node == nil {
writeError(w, 404, "Not found")
@@ -1274,52 +1179,14 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
// Post-filter: verify target node actually appears in each candidate's resolved_path.
// The byPathHop index uses short prefixes which can collide (e.g. "c0" matches multiple nodes).
// We lean on resolved_path (from neighbor affinity graph) to disambiguate.
//
// Collect candidate IDs and index membership under the read lock, then release
// the lock before running SQL queries (confirmResolvedPathContains does disk I/O).
type candidateCheck struct {
tx *StoreTx
hasReverse bool
inIndex bool
}
checks := make([]candidateCheck, len(candidates))
for i, tx := range candidates {
cc := candidateCheck{tx: tx}
if !s.store.useResolvedPathIndex {
cc.inIndex = true // flag off — keep all
} else if _, hasRev := s.store.resolvedPubkeyReverse[tx.ID]; !hasRev {
cc.inIndex = true // no indexed pubkeys — keep (conservative)
} else {
h := resolvedPubkeyHash(lowerPK)
for _, id := range s.store.resolvedPubkeyIndex[h] {
if id == tx.ID {
cc.hasReverse = true // needs SQL confirmation
break
}
}
// If not in index at all, it's a definite no
filtered := candidates[:0] // reuse backing array
for _, tx := range candidates {
if nodeInResolvedPath(tx, lowerPK) {
filtered = append(filtered, tx)
}
checks[i] = cc
}
s.store.mu.RUnlock()
// Now run SQL checks outside the lock for candidates that need confirmation.
filtered := candidates[:0]
for _, cc := range checks {
if cc.inIndex {
filtered = append(filtered, cc.tx)
} else if cc.hasReverse {
if s.store.confirmResolvedPathContains(cc.tx.ID, lowerPK) {
filtered = append(filtered, cc.tx)
}
}
// else: not in index → exclude
}
candidates = filtered
// Re-acquire read lock for the aggregation phase that reads store data.
s.store.mu.RLock()
type pathAgg struct {
Hops []PathHopResp
Count int
@@ -1424,10 +1291,6 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) {
pubkey := mux.Vars(r)["pubkey"]
if s.cfg.IsBlacklisted(pubkey) {
writeError(w, 404, "Not found")
return
}
days := queryInt(r, "days", 7)
if days < 1 {
days = 1
@@ -1449,36 +1312,6 @@ func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) {
writeError(w, 404, "Not found")
}
func (s *Server) handleNodeClockSkew(w http.ResponseWriter, r *http.Request) {
pubkey := mux.Vars(r)["pubkey"]
if s.store == nil {
writeError(w, 404, "Not found")
return
}
result := s.store.GetNodeClockSkew(pubkey)
if result == nil {
writeError(w, 404, "No clock skew data for this node")
return
}
writeJSON(w, result)
}
func (s *Server) handleObserverClockSkew(w http.ResponseWriter, r *http.Request) {
if s.store == nil {
writeJSON(w, []ObserverCalibration{})
return
}
writeJSON(w, s.store.GetObserverCalibrations())
}
func (s *Server) handleFleetClockSkew(w http.ResponseWriter, r *http.Request) {
if s.store == nil {
writeJSON(w, []*NodeClockSkew{})
return
}
writeJSON(w, s.store.GetFleetClockSkew())
}
// --- Analytics Handlers ---
func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
@@ -1504,11 +1337,7 @@ func (s *Server) handleAnalyticsRF(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleAnalyticsTopology(w http.ResponseWriter, r *http.Request) {
region := r.URL.Query().Get("region")
if s.store != nil {
data := s.store.GetAnalyticsTopology(region)
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
data = s.filterBlacklistedFromTopology(data)
}
writeJSON(w, data)
writeJSON(w, s.store.GetAnalyticsTopology(region))
return
}
writeJSON(w, TopologyResponse{
@@ -1596,11 +1425,7 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
}
maxLen := queryInt(r, "maxLen", 8)
limit := queryInt(r, "limit", 100)
data := s.store.GetAnalyticsSubpaths(region, minLen, maxLen, limit)
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
data = s.filterBlacklistedFromSubpaths(data)
}
writeJSON(w, data)
writeJSON(w, s.store.GetAnalyticsSubpaths(region, minLen, maxLen, limit))
return
}
writeJSON(w, SubpathsResponse{
@@ -1652,11 +1477,6 @@ func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Requ
}
results := s.store.GetAnalyticsSubpathsBulk(region, groups)
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
for i, r := range results {
results[i] = s.filterBlacklistedFromSubpaths(r)
}
}
writeJSON(w, map[string]interface{}{"results": results})
}
@@ -1676,15 +1496,6 @@ func (s *Server) handleAnalyticsSubpathDetail(w http.ResponseWriter, r *http.Req
writeJSON(w, ErrorResp{Error: "Need at least 2 hops"})
return
}
// Reject if any hop is a blacklisted node.
if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 {
for _, hop := range rawHops {
if s.cfg.IsBlacklisted(hop) {
writeError(w, 404, "Not found")
return
}
}
}
if s.store != nil {
writeJSON(w, s.store.GetSubpathDetail(rawHops))
return
@@ -1750,10 +1561,6 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
if pm != nil {
if matched, ok := pm.m[hopLower]; ok {
for _, ni := range matched {
// Skip blacklisted nodes from resolution results.
if s.cfg != nil && s.cfg.IsBlacklisted(ni.PublicKey) {
continue
}
c := HopCandidate{Pubkey: ni.PublicKey}
if ni.Name != "" {
c.Name = ni.Name
@@ -1822,8 +1629,7 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
}
// Use the resolved node as the default (best-effort pick).
// Skip if the best pick is a blacklisted node.
if best != nil && !(s.cfg != nil && s.cfg.IsBlacklisted(best.PublicKey)) {
if best != nil {
hr.Name = best.Name
hr.Pubkey = best.PublicKey
}
@@ -1846,35 +1652,18 @@ func (s *Server) handleResolveHops(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleChannels(w http.ResponseWriter, r *http.Request) {
region := r.URL.Query().Get("region")
includeEncrypted := r.URL.Query().Get("includeEncrypted") == "true"
// Prefer DB for full history (in-memory store has limited retention)
if s.db != nil {
channels, err := s.db.GetChannels(region)
if err != nil {
writeError(w, 500, err.Error())
return
}
if includeEncrypted {
encrypted, err := s.db.GetEncryptedChannels(region)
if err != nil {
log.Printf("WARN GetEncryptedChannels: %v", err)
} else {
channels = append(channels, encrypted...)
}
}
writeJSON(w, ChannelListResponse{Channels: channels})
return
}
if s.store != nil {
region := r.URL.Query().Get("region")
channels := s.store.GetChannels(region)
if includeEncrypted {
channels = append(channels, s.store.GetEncryptedChannels(region)...)
}
writeJSON(w, ChannelListResponse{Channels: channels})
return
}
writeJSON(w, ChannelListResponse{Channels: []map[string]interface{}{}})
channels, err := s.db.GetChannels()
if err != nil {
writeError(w, 500, err.Error())
return
}
writeJSON(w, ChannelListResponse{Channels: channels})
}
func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) {
@@ -1882,22 +1671,17 @@ func (s *Server) handleChannelMessages(w http.ResponseWriter, r *http.Request) {
limit := queryInt(r, "limit", 100)
offset := queryInt(r, "offset", 0)
region := r.URL.Query().Get("region")
// Prefer DB for full history (in-memory store has limited retention)
if s.db != nil {
messages, total, err := s.db.GetChannelMessages(hash, limit, offset, region)
if err != nil {
writeError(w, 500, err.Error())
return
}
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
return
}
if s.store != nil {
messages, total := s.store.GetChannelMessages(hash, limit, offset, region)
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
return
}
writeJSON(w, ChannelMessagesResponse{Messages: []map[string]interface{}{}, Total: 0})
messages, total, err := s.db.GetChannelMessages(hash, limit, offset, region)
if err != nil {
writeError(w, 500, err.Error())
return
}
writeJSON(w, ChannelMessagesResponse{Messages: messages, Total: total})
}
func (s *Server) handleObservers(w http.ResponseWriter, r *http.Request) {
@@ -2083,7 +1867,7 @@ func (s *Server) handleObserverAnalytics(w http.ResponseWriter, r *http.Request)
}
snrBuckets[bucket].Count++
}
if i < 20 && enriched["hash"] != nil {
if i < 20 {
recentPackets = append(recentPackets, enriched)
}
}
@@ -2366,6 +2150,9 @@ func mapSliceToTransmissions(maps []map[string]interface{}) []TransmissionResp {
tx.PathJSON = m["path_json"]
tx.Direction = m["direction"]
tx.Score = m["score"]
if rp, ok := m["resolved_path"].([]*string); ok {
tx.ResolvedPath = rp
}
result = append(result, tx)
}
return result
@@ -2386,10 +2173,10 @@ func mapSliceToObservations(maps []map[string]interface{}) []ObservationResp {
obs.SNR = m["snr"]
obs.RSSI = m["rssi"]
obs.PathJSON = m["path_json"]
obs.ResolvedPath = m["resolved_path"]
obs.Direction = m["direction"]
obs.RawHex = m["raw_hex"]
obs.Timestamp = m["timestamp"]
if rp, ok := m["resolved_path"].([]*string); ok {
obs.ResolvedPath = rp
}
result = append(result, obs)
}
return result
@@ -2540,167 +2327,16 @@ func (s *Server) handleAdminPrune(w http.ResponseWriter, r *http.Request) {
writeError(w, 400, "days parameter required (or set retention.packetDays in config)")
return
}
results := map[string]interface{}{}
// Prune old packets
n, err := s.db.PruneOldPackets(days)
if err != nil {
writeError(w, 500, err.Error())
return
}
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
results["packets_deleted"] = n
results["deleted"] = n // legacy alias
// Also mark stale observers as inactive if observerDays is configured
observerDays := s.cfg.ObserverDaysOrDefault()
if observerDays > 0 {
obsN, obsErr := s.db.RemoveStaleObservers(observerDays)
if obsErr != nil {
log.Printf("[prune] observer prune error: %v", obsErr)
} else {
results["observers_inactive"] = obsN
}
}
results["days"] = days
writeJSON(w, results)
writeJSON(w, map[string]interface{}{"deleted": n, "days": days})
}
// constantTimeEqual compares two strings in constant time to prevent timing attacks.
func constantTimeEqual(a, b string) bool {
return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
}
// filterBlacklistedFromTopology removes blacklisted node references from the
// topology analytics response (TopRepeaters, TopPairs, BestPathList, MultiObsNodes, PerObserverReach).
func (s *Server) filterBlacklistedFromTopology(data map[string]interface{}) map[string]interface{} {
// Filter TopRepeaters
if repeaters, ok := data["topRepeaters"]; ok {
if arr, ok := repeaters.([]TopRepeater); ok {
var filtered []TopRepeater
for _, r := range arr {
if pk, ok := r.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
continue
}
filtered = append(filtered, r)
}
data["topRepeaters"] = filtered
}
}
// Filter TopPairs
if pairs, ok := data["topPairs"]; ok {
if arr, ok := pairs.([]TopPair); ok {
var filtered []TopPair
for _, p := range arr {
if pkA, ok := p.PubkeyA.(string); ok && s.cfg.IsBlacklisted(pkA) {
continue
}
if pkB, ok := p.PubkeyB.(string); ok && s.cfg.IsBlacklisted(pkB) {
continue
}
filtered = append(filtered, p)
}
data["topPairs"] = filtered
}
}
// Filter BestPathList
if paths, ok := data["bestPathList"]; ok {
if arr, ok := paths.([]BestPathEntry); ok {
var filtered []BestPathEntry
for _, p := range arr {
if pk, ok := p.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
continue
}
filtered = append(filtered, p)
}
data["bestPathList"] = filtered
}
}
// Filter MultiObsNodes
if nodes, ok := data["multiObsNodes"]; ok {
if arr, ok := nodes.([]MultiObsNode); ok {
var filtered []MultiObsNode
for _, n := range arr {
if pk, ok := n.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
continue
}
filtered = append(filtered, n)
}
data["multiObsNodes"] = filtered
}
}
// Filter PerObserverReach
if reach, ok := data["perObserverReach"]; ok {
if m, ok := reach.(map[string]*ObserverReach); ok {
for k, v := range m {
for ri := range v.Rings {
var filteredNodes []ReachNode
for _, rn := range v.Rings[ri].Nodes {
if pk, ok := rn.Pubkey.(string); ok && s.cfg.IsBlacklisted(pk) {
continue
}
filteredNodes = append(filteredNodes, rn)
}
v.Rings[ri].Nodes = filteredNodes
}
m[k] = v
}
}
}
return data
}
// filterBlacklistedFromSubpaths removes blacklisted node references from
// the subpaths analytics response.
func (s *Server) filterBlacklistedFromSubpaths(data map[string]interface{}) map[string]interface{} {
if subpaths, ok := data["subpaths"]; ok {
if arr, ok := subpaths.([]interface{}); ok {
var filtered []interface{}
for _, item := range arr {
if m, ok := item.(map[string]interface{}); ok {
if hops, ok := m["hops"].([]interface{}); ok {
skip := false
for _, h := range hops {
if hp, ok := h.(string); ok && s.cfg.IsBlacklisted(hp) {
skip = true
break
}
}
if skip {
continue
}
}
}
filtered = append(filtered, item)
}
data["subpaths"] = filtered
}
}
return data
}
// handleDroppedPackets returns recently dropped packets for investigation.
func (s *Server) handleDroppedPackets(w http.ResponseWriter, r *http.Request) {
limit := 100
if v := r.URL.Query().Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
observerID := r.URL.Query().Get("observer")
nodePubkey := r.URL.Query().Get("pubkey")
results, err := s.db.GetDroppedPackets(limit, observerID, nodePubkey)
if err != nil {
writeError(w, 500, err.Error())
return
}
writeJSON(w, results)
}
+47 -275
View File
@@ -170,9 +170,6 @@ func TestHealthEndpoint(t *testing.T) {
if _, ok := pktStore["estimatedMB"]; !ok {
t.Error("expected estimatedMB in packetStore")
}
if _, ok := pktStore["trackedMB"]; !ok {
t.Error("expected trackedMB in packetStore")
}
// Verify eventLoop (GC pause metrics matching Node.js shape)
el, ok := body["eventLoop"].(map[string]interface{})
@@ -777,67 +774,6 @@ func TestNodeHealthNotFound(t *testing.T) {
}
}
// TestNodeHealthPartialFromPackets verifies that a node with packets in the
// in-memory store but no DB entry returns a partial 200 response instead of 404.
// This is the fix for issue #665 (companion nodes without adverts).
func TestNodeHealthPartialFromPackets(t *testing.T) {
srv, router := setupTestServer(t)
// Inject a packet into byNode for a pubkey that doesn't exist in the nodes table
ghostPubkey := "ghost_companion_no_advert"
now := time.Now().UTC().Format(time.RFC3339)
snr := 5.0
srv.store.mu.Lock()
if srv.store.byNode == nil {
srv.store.byNode = make(map[string][]*StoreTx)
}
if srv.store.nodeHashes == nil {
srv.store.nodeHashes = make(map[string]map[string]bool)
}
srv.store.byNode[ghostPubkey] = []*StoreTx{
{Hash: "abc123", FirstSeen: now, SNR: &snr, ObservationCount: 1},
}
srv.store.nodeHashes[ghostPubkey] = map[string]bool{"abc123": true}
srv.store.mu.Unlock()
req := httptest.NewRequest("GET", "/api/nodes/"+ghostPubkey+"/health", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200 for ghost companion, got %d (body: %s)", w.Code, w.Body.String())
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("json unmarshal: %v", err)
}
// Should have a synthetic node stub
node, ok := body["node"].(map[string]interface{})
if !ok || node == nil {
t.Fatal("expected node in response")
}
if node["role"] != "unknown" {
t.Errorf("expected role=unknown, got %v", node["role"])
}
if node["public_key"] != ghostPubkey {
t.Errorf("expected public_key=%s, got %v", ghostPubkey, node["public_key"])
}
// Should have stats from the packet
stats, ok := body["stats"].(map[string]interface{})
if !ok || stats == nil {
t.Fatal("expected stats in response")
}
if stats["totalPackets"] != 1.0 { // JSON numbers are float64
t.Errorf("expected totalPackets=1, got %v", stats["totalPackets"])
}
if stats["lastHeard"] == nil {
t.Error("expected lastHeard to be set")
}
}
func TestBulkHealthEndpoint(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/nodes/bulk-health?limit=10", nil)
@@ -2219,8 +2155,8 @@ pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk)
decoded := `{"name":"TestNode","pubKey":"` + pk + `"}`
raw1 := "11" + "01" + "aabb"
raw2 := "11" + "41" + "aabb"
raw1 := "04" + "00" + "aabb"
raw2 := "04" + "40" + "aabb"
payloadType := 4
for i := 0; i < 3; i++ {
@@ -2267,8 +2203,8 @@ pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk)
decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}`
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
raw1byte := "04" + "00" + "aabb" // pathByte=0x00 → hashSize=1 (direct send, no hops)
raw2byte := "04" + "40" + "aabb" // pathByte=0x40 → hashSize=2
payloadType := 4
// 1 packet with hashSize=1, 4 packets with hashSize=2 (latest is 2-byte)
@@ -2310,8 +2246,8 @@ func TestGetNodeHashSizeInfoLatestWins(t *testing.T) {
db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'LatestWins', 'repeater')", pk)
decoded := `{"name":"LatestWins","pubKey":"` + pk + `"}`
raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1
raw2byte := "11" + "41" + "aabb" // FLOOD, pathByte=0x41 → hashSize=2
raw1byte := "04" + "00" + "aabb" // pathByte=0x00 → hashSize=1
raw2byte := "04" + "40" + "aabb" // pathByte=0x40 → hashSize=2
payloadType := 4
// 4 historical 1-byte adverts, then 1 recent 2-byte advert (latest).
@@ -3681,55 +3617,67 @@ func TestNodePathsPrefixCollisionFilter(t *testing.T) {
func TestNodeInResolvedPath(t *testing.T) {
target := "aabbccdd11223344"
// After #800, nodeInResolvedPath is replaced by nodeInResolvedPathViaIndex
// which uses the membership index. Test the index-based approach.
store := &PacketStore{
byNode: make(map[string][]*StoreTx),
nodeHashes: make(map[string]map[string]bool),
useResolvedPathIndex: true,
}
store.initResolvedPathIndex()
// Case 1: tx indexed with target pubkey
tx1 := &StoreTx{ID: 1}
store.addToResolvedPubkeyIndex(1, []string{target})
if !store.nodeInResolvedPathViaIndex(tx1, target) {
t.Error("should match when index contains target")
// Case 1: tx.ResolvedPath contains target
pk := "aabbccdd11223344"
tx1 := &StoreTx{ResolvedPath: []*string{&pk}}
if !nodeInResolvedPath(tx1, target) {
t.Error("should match when ResolvedPath contains target")
}
// Case 2: tx indexed with different pubkey
tx2 := &StoreTx{ID: 2}
store.addToResolvedPubkeyIndex(2, []string{"aacafe0000000000"})
if store.nodeInResolvedPathViaIndex(tx2, target) {
t.Error("should not match when index contains different node")
// Case 2: tx.ResolvedPath contains different node
other := "aacafe0000000000"
tx2 := &StoreTx{ResolvedPath: []*string{&other}}
if nodeInResolvedPath(tx2, target) {
t.Error("should not match when ResolvedPath contains different node")
}
// Case 3: tx not in index at all — should match (no data to disambiguate)
tx3 := &StoreTx{ID: 3}
if !store.nodeInResolvedPathViaIndex(tx3, target) {
t.Error("should match when tx has no index entries (no data to disambiguate)")
// Case 3: nil ResolvedPath — should match (no data to disambiguate, keep it)
tx3 := &StoreTx{}
if !nodeInResolvedPath(tx3, target) {
t.Error("should match when ResolvedPath is nil (no data to disambiguate)")
}
// Case 4: ResolvedPath with nil elements only — has data but no match
tx4 := &StoreTx{ResolvedPath: []*string{nil, nil}}
if nodeInResolvedPath(tx4, target) {
t.Error("should not match when all ResolvedPath elements are nil")
}
// Case 5: target in observation but not in tx.ResolvedPath
tx5 := &StoreTx{
ResolvedPath: []*string{&other},
Observations: []*StoreObs{
{ResolvedPath: []*string{&pk}},
},
}
if !nodeInResolvedPath(tx5, target) {
t.Error("should match when observation's ResolvedPath contains target")
}
}
func TestPathHopIndexIncrementalUpdate(t *testing.T) {
// After #800, addTxToPathHopIndex only indexes raw hops (not resolved pubkeys).
// Resolved pubkeys are handled by the resolved pubkey membership index.
// Test that addTxToPathHopIndex and removeTxFromPathHopIndex work correctly
idx := make(map[string][]*StoreTx)
pk1 := "fullpubkey1"
tx1 := &StoreTx{
ID: 1,
PathJSON: `["ab","cd"]`,
ResolvedPath: []*string{&pk1, nil},
}
addTxToPathHopIndex(idx, tx1)
// Should be indexed under "ab" and "cd" only (no resolved pubkey)
// Should be indexed under "ab", "cd", and "fullpubkey1"
if len(idx["ab"]) != 1 {
t.Errorf("expected 1 entry for 'ab', got %d", len(idx["ab"]))
}
if len(idx["cd"]) != 1 {
t.Errorf("expected 1 entry for 'cd', got %d", len(idx["cd"]))
}
if len(idx["fullpubkey1"]) != 1 {
t.Errorf("expected 1 entry for resolved pubkey, got %d", len(idx["fullpubkey1"]))
}
// Add another tx with overlapping hop
tx2 := &StoreTx{
@@ -3754,6 +3702,9 @@ func TestPathHopIndexIncrementalUpdate(t *testing.T) {
if _, ok := idx["cd"]; ok {
t.Error("expected 'cd' key to be deleted after removal")
}
if _, ok := idx["fullpubkey1"]; ok {
t.Error("expected resolved pubkey key to be deleted after removal")
}
}
func TestMetricsAPIEndpoints(t *testing.T) {
@@ -3793,182 +3744,3 @@ func TestMetricsAPIEndpoints(t *testing.T) {
t.Errorf("expected 1 observer in summary, got %v", resp2["observers"])
}
}
// TestNodeHealth_RecentPackets_ResolvedPath verifies that recentPackets in the
// node health endpoint include resolved_path (regression for Codex review item #2).
func TestNodeHealth_RecentPackets_ResolvedPath(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344/health", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("json decode: %v", err)
}
rp, ok := body["recentPackets"].([]interface{})
if !ok || len(rp) == 0 {
t.Fatal("expected non-empty recentPackets")
}
// At least one packet should have resolved_path (tx 1 has observations with resolved_path)
found := false
for _, p := range rp {
pm, ok := p.(map[string]interface{})
if !ok {
continue
}
if pm["resolved_path"] != nil {
found = true
break
}
}
if !found {
t.Error("expected at least one recentPacket with resolved_path")
}
}
// TestPacketsExpand_ResolvedPath verifies that expandObservations=true includes
// resolved_path on expanded observations (regression for Codex review item #3).
func TestPacketsExpand_ResolvedPath(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/packets?expand=observations&limit=10", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("json decode: %v", err)
}
packets, ok := body["packets"].([]interface{})
if !ok || len(packets) == 0 {
t.Fatal("expected non-empty packets")
}
// Find a packet with observations that should have resolved_path
found := false
for _, p := range packets {
pm, ok := p.(map[string]interface{})
if !ok {
continue
}
obs, ok := pm["observations"].([]interface{})
if !ok {
continue
}
for _, o := range obs {
om, ok := o.(map[string]interface{})
if !ok {
continue
}
if om["resolved_path"] != nil {
found = true
break
}
}
if found {
break
}
}
if !found {
t.Error("expected at least one expanded observation with resolved_path")
}
}
// TestPacketDetailFallsBackToDBWhenStoreMisses verifies that handlePacketDetail
// serves transmissions present in the DB but absent from the in-memory store.
// This is the recentAdverts → "Not found" bug (#827).
func TestPacketDetailFallsBackToDBWhenStoreMisses(t *testing.T) {
srv, router := setupTestServer(t)
// Insert a transmission directly into the DB AFTER store.Load(), so the
// in-memory PacketStore won't see it. Mirrors the production case where
// the store has pruned an entry but the DB still has it.
const dbOnlyHash = "deadbeef00112233"
now := time.Now().UTC().Format(time.RFC3339)
if _, err := srv.db.conn.Exec(`INSERT INTO transmissions
(raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
VALUES ('FFEE', ?, ?, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil {
t.Fatalf("insert: %v", err)
}
var txID int
if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", dbOnlyHash).Scan(&txID); err != nil {
t.Fatalf("lookup tx id: %v", err)
}
if _, err := srv.db.conn.Exec(`INSERT INTO observations
(transmission_id, observer_idx, snr, rssi, path_json, timestamp)
VALUES (?, 1, 7.5, -99, '[]', ?)`, txID, time.Now().Unix()); err != nil {
t.Fatalf("insert obs: %v", err)
}
// Confirm the store really doesn't have it (precondition for the fix).
if got := srv.store.GetPacketByHash(dbOnlyHash); got != nil {
t.Fatalf("test precondition failed: store unexpectedly has %s", dbOnlyHash)
}
req := httptest.NewRequest("GET", "/api/packets/"+dbOnlyHash, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
pkt, ok := body["packet"].(map[string]interface{})
if !ok {
t.Fatal("expected packet object")
}
if pkt["hash"] != dbOnlyHash {
t.Errorf("expected hash %s, got %v", dbOnlyHash, pkt["hash"])
}
// Observations fallback should populate from DB too.
obs, _ := body["observations"].([]interface{})
if len(obs) == 0 {
t.Errorf("expected DB observations to be returned, got 0")
}
}
// TestPacketDetail404WhenAbsentFromBoth verifies that a hash present in
// neither store nor DB still returns 404 (no false positives from the fallback).
func TestPacketDetail404WhenAbsentFromBoth(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/packets/0011223344556677", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 404 {
t.Errorf("expected 404, got %d (body: %s)", w.Code, w.Body.String())
}
}
// TestPacketDetailPrefersStoreOverDB verifies the store result wins when the
// hash exists in both — the DB fallback must not double-fetch / overwrite.
func TestPacketDetailPrefersStoreOverDB(t *testing.T) {
srv, router := setupTestServer(t)
// abc123def4567890 is seeded in both DB and (after Load) the store.
const hash = "abc123def4567890"
if got := srv.store.GetPacketByHash(hash); got == nil {
t.Fatalf("test precondition failed: store should have %s", hash)
}
req := httptest.NewRequest("GET", "/api/packets/"+hash, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &body)
pkt, _ := body["packet"].(map[string]interface{})
if pkt == nil || pkt["hash"] != hash {
t.Fatalf("expected packet with hash %s, got %v", hash, pkt)
}
// observation_count comes from store observations (2 seeded for tx 1).
if cnt, _ := body["observation_count"].(float64); cnt != 2 {
t.Errorf("expected observation_count=2 (from store), got %v", body["observation_count"])
}
}
-95
View File
@@ -1,95 +0,0 @@
package main
import (
"encoding/json"
"net/http/httptest"
"strings"
"testing"
)
// TestStatsMemoryFields verifies that /api/stats exposes the new memory
// breakdown introduced for issue #832: storeDataMB, processRSSMB,
// goHeapInuseMB, goSysMB, plus the deprecated trackedMB alias.
//
// We assert presence, type, sign, and ordering invariants — but NOT
// "RSS within X% of true RSS" because that is flaky in CI under cgo,
// containerization, and shared-runner load.
func TestStatsMemoryFields(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/stats", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d", w.Code)
}
var body map[string]interface{}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("json decode: %v", err)
}
required := []string{"trackedMB", "storeDataMB", "processRSSMB", "goHeapInuseMB", "goSysMB"}
values := make(map[string]float64, len(required))
for _, k := range required {
v, ok := body[k]
if !ok {
t.Fatalf("missing field %q in /api/stats response", k)
}
f, ok := v.(float64)
if !ok {
t.Fatalf("field %q is %T, expected float64", k, v)
}
if f < 0 {
t.Errorf("field %q is negative: %v", k, f)
}
values[k] = f
}
// trackedMB is a deprecated alias for storeDataMB; they must match.
if values["trackedMB"] != values["storeDataMB"] {
t.Errorf("trackedMB (%v) != storeDataMB (%v); they must remain aliased",
values["trackedMB"], values["storeDataMB"])
}
// Ordering invariants. goSys is the runtime's view of total OS memory;
// HeapInuse is a subset of it. storeData is a subset of HeapInuse.
// processRSS may be 0 in environments without /proc — treat 0 as
// "unknown" rather than a failure.
if values["goHeapInuseMB"] > values["goSysMB"]+0.5 {
t.Errorf("invariant violated: goHeapInuseMB (%v) > goSysMB (%v)",
values["goHeapInuseMB"], values["goSysMB"])
}
if values["storeDataMB"] > values["goHeapInuseMB"]+0.5 && values["storeDataMB"] > 0 {
// In the test fixture storeDataMB is typically 0 (no packets in
// store); only enforce the bound when both are nonzero.
t.Errorf("invariant violated: storeDataMB (%v) > goHeapInuseMB (%v)",
values["storeDataMB"], values["goHeapInuseMB"])
}
if values["processRSSMB"] > 0 && values["goSysMB"] > 0 {
// goSys can briefly exceed RSS if pages are reserved-but-not-touched,
// so allow some slack.
if values["goSysMB"] > values["processRSSMB"]*4 {
t.Errorf("suspicious: goSysMB (%v) >> processRSSMB (%v)",
values["goSysMB"], values["processRSSMB"])
}
}
}
// TestStatsMemoryFieldsRawJSON spot-checks that the JSON wire format uses
// the documented camelCase names (no accidental rename through struct tags).
func TestStatsMemoryFieldsRawJSON(t *testing.T) {
_, router := setupTestServer(t)
req := httptest.NewRequest("GET", "/api/stats", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
body := w.Body.String()
for _, key := range []string{
`"trackedMB":`, `"storeDataMB":`,
`"processRSSMB":`, `"goHeapInuseMB":`, `"goSysMB":`,
} {
if !strings.Contains(body, key) {
t.Errorf("missing %s in raw response: %s", key, body)
}
}
}
+213 -1085
View File
File diff suppressed because it is too large Load Diff
-116
View File
@@ -1,116 +0,0 @@
package main
import (
"testing"
)
func f64(v float64) *float64 { return &v }
func TestDedupeTopHopsByPair(t *testing.T) {
hops := []distHopRecord{
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: f64(5.0), Hash: "h1", Timestamp: "t1"},
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: f64(8.0), Hash: "h2", Timestamp: "t2"},
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 80, Type: "R↔R", SNR: f64(3.0), Hash: "h3", Timestamp: "t3"},
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 70, Type: "R↔R", SNR: f64(6.0), Hash: "h4", Timestamp: "t4"},
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 60, Type: "R↔R", SNR: f64(4.0), Hash: "h5", Timestamp: "t5"},
{FromPk: "CCC", ToPk: "DDD", FromName: "C", ToName: "D", Dist: 50, Type: "C↔R", SNR: f64(7.0), Hash: "h6", Timestamp: "t6"},
}
result := dedupeHopsByPair(hops, 20)
if len(result) != 2 {
t.Fatalf("expected 2 entries, got %d", len(result))
}
// First entry: A↔B pair, max distance = 100, obsCount = 5
ab := result[0]
if ab["dist"].(float64) != 100 {
t.Errorf("expected dist 100, got %v", ab["dist"])
}
if ab["obsCount"].(int) != 5 {
t.Errorf("expected obsCount 5, got %v", ab["obsCount"])
}
if ab["hash"].(string) != "h1" {
t.Errorf("expected hash h1 (from max-dist record), got %v", ab["hash"])
}
if ab["bestSnr"].(float64) != 8.0 {
t.Errorf("expected bestSnr 8.0, got %v", ab["bestSnr"])
}
// medianSnr of [3,4,5,6,8] = 5.0
if ab["medianSnr"].(float64) != 5.0 {
t.Errorf("expected medianSnr 5.0, got %v", ab["medianSnr"])
}
// Second entry: C↔D pair
cd := result[1]
if cd["dist"].(float64) != 50 {
t.Errorf("expected dist 50, got %v", cd["dist"])
}
if cd["obsCount"].(int) != 1 {
t.Errorf("expected obsCount 1, got %v", cd["obsCount"])
}
}
func TestDedupeTopHopsReversePairMerges(t *testing.T) {
hops := []distHopRecord{
{FromPk: "BBB", ToPk: "AAA", FromName: "B", ToName: "A", Dist: 50, Type: "R↔R", Hash: "h1"},
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 80, Type: "R↔R", Hash: "h2"},
}
result := dedupeHopsByPair(hops, 20)
if len(result) != 1 {
t.Fatalf("expected 1 entry, got %d", len(result))
}
if result[0]["obsCount"].(int) != 2 {
t.Errorf("expected obsCount 2, got %v", result[0]["obsCount"])
}
if result[0]["dist"].(float64) != 80 {
t.Errorf("expected dist 80, got %v", result[0]["dist"])
}
}
func TestDedupeTopHopsNilSNR(t *testing.T) {
hops := []distHopRecord{
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 100, Type: "R↔R", SNR: nil, Hash: "h1"},
{FromPk: "AAA", ToPk: "BBB", FromName: "A", ToName: "B", Dist: 90, Type: "R↔R", SNR: nil, Hash: "h2"},
}
result := dedupeHopsByPair(hops, 20)
if len(result) != 1 {
t.Fatalf("expected 1 entry, got %d", len(result))
}
if result[0]["bestSnr"] != nil {
t.Errorf("expected bestSnr nil, got %v", result[0]["bestSnr"])
}
if result[0]["medianSnr"] != nil {
t.Errorf("expected medianSnr nil, got %v", result[0]["medianSnr"])
}
}
func TestDedupeTopHopsLimit(t *testing.T) {
// Generate 25 unique pairs, verify limit=20 caps output
hops := make([]distHopRecord, 25)
for i := range hops {
hops[i] = distHopRecord{
FromPk: "A", ToPk: string(rune('a' + i)),
Dist: float64(i), Type: "R↔R", Hash: "h",
}
}
result := dedupeHopsByPair(hops, 20)
if len(result) != 20 {
t.Errorf("expected 20 entries, got %d", len(result))
}
}
func TestDedupeTopHopsEvenMedian(t *testing.T) {
// Even count: median = avg of two middle values
hops := []distHopRecord{
{FromPk: "A", ToPk: "B", Dist: 10, Type: "R↔R", SNR: f64(2.0), Hash: "h1"},
{FromPk: "A", ToPk: "B", Dist: 20, Type: "R↔R", SNR: f64(4.0), Hash: "h2"},
{FromPk: "A", ToPk: "B", Dist: 30, Type: "R↔R", SNR: f64(6.0), Hash: "h3"},
{FromPk: "A", ToPk: "B", Dist: 40, Type: "R↔R", SNR: f64(8.0), Hash: "h4"},
}
result := dedupeHopsByPair(hops, 20)
// sorted SNR: [2,4,6,8], median = (4+6)/2 = 5.0
if result[0]["medianSnr"].(float64) != 5.0 {
t.Errorf("expected medianSnr 5.0, got %v", result[0]["medianSnr"])
}
}
+4 -13
View File
@@ -42,20 +42,14 @@
"type": {
"type": "string"
},
"snr": {
"type": "number"
},
"hash": {
"type": "string"
},
"timestamp": {
"type": "string"
},
"bestSnr": {
"type": "number"
},
"medianSnr": {
"type": "number"
},
"obsCount": {
"type": "number"
}
}
}
@@ -922,9 +916,6 @@
},
"estimatedMB": {
"type": "number"
},
"trackedMB": {
"type": "number"
}
}
},
@@ -1586,4 +1577,4 @@
}
}
}
}
}
-126
View File
@@ -1,126 +0,0 @@
package main
import (
"database/sql"
"testing"
"time"
_ "modernc.org/sqlite"
)
func TestTouchNodeLastSeen_UpdatesDB(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
// Insert a node with no last_seen
db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", "abc123", "relay1", "REPEATER")
err := db.TouchNodeLastSeen("abc123", "2026-04-12T04:00:00Z")
if err != nil {
t.Fatalf("TouchNodeLastSeen returned error: %v", err)
}
var lastSeen sql.NullString
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "abc123").Scan(&lastSeen)
if !lastSeen.Valid || lastSeen.String != "2026-04-12T04:00:00Z" {
t.Fatalf("expected last_seen=2026-04-12T04:00:00Z, got %v", lastSeen)
}
}
func TestTouchNodeLastSeen_DoesNotGoBackwards(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)",
"abc123", "relay1", "REPEATER", "2026-04-12T05:00:00Z")
// Try to set an older timestamp
err := db.TouchNodeLastSeen("abc123", "2026-04-12T04:00:00Z")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var lastSeen string
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "abc123").Scan(&lastSeen)
if lastSeen != "2026-04-12T05:00:00Z" {
t.Fatalf("last_seen went backwards: got %s", lastSeen)
}
}
func TestTouchNodeLastSeen_NonExistentNode(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
// Should not error for non-existent node
err := db.TouchNodeLastSeen("nonexistent", "2026-04-12T04:00:00Z")
if err != nil {
t.Fatalf("unexpected error for non-existent node: %v", err)
}
}
func TestTouchRelayLastSeen_Debouncing(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", "relay1", "R1", "REPEATER")
s := &PacketStore{
db: db,
lastSeenTouched: make(map[string]time.Time),
}
// After #800, touchRelayLastSeen takes a []string of pubkeys (from decode-window)
pks := []string{"relay1"}
now := time.Now()
s.touchRelayLastSeen(pks, now)
// Verify it was written
var lastSeen sql.NullString
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
if !lastSeen.Valid {
t.Fatal("expected last_seen to be set after first touch")
}
// Reset last_seen to check debounce prevents second write
db.conn.Exec("UPDATE nodes SET last_seen = NULL WHERE public_key = ?", "relay1")
// Call again within 5 minutes — should be debounced (no write)
s.touchRelayLastSeen(pks, now.Add(2*time.Minute))
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
if lastSeen.Valid {
t.Fatal("expected debounce to prevent second write within 5 minutes")
}
// Call after 5 minutes — should write again
s.touchRelayLastSeen(pks, now.Add(6*time.Minute))
db.conn.QueryRow("SELECT last_seen FROM nodes WHERE public_key = ?", "relay1").Scan(&lastSeen)
if !lastSeen.Valid {
t.Fatal("expected write after debounce interval expired")
}
}
func TestTouchRelayLastSeen_SkipsEmptyPubkeys(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
s := &PacketStore{
db: db,
lastSeenTouched: make(map[string]time.Time),
}
// Empty pubkeys — should not panic or error
s.touchRelayLastSeen([]string{}, time.Now())
s.touchRelayLastSeen(nil, time.Now())
}
func TestTouchRelayLastSeen_NilDB(t *testing.T) {
s := &PacketStore{
db: nil,
lastSeenTouched: make(map[string]time.Time),
}
// Should not panic with nil db
s.touchRelayLastSeen([]string{"abc"}, time.Now())
}
-166
View File
@@ -1,166 +0,0 @@
package main
import (
"testing"
"time"
)
// TestEstimateStoreTxBytes_ReasonableValues verifies the estimate function
// returns reasonable values for different packet sizes.
func TestEstimateStoreTxBytes_ReasonableValues(t *testing.T) {
tx := &StoreTx{
Hash: "abcdef1234567890",
RawHex: "deadbeef",
DecodedJSON: `{"type":"GRP_TXT"}`,
PathJSON: `["hop1","hop2","hop3"]`,
parsedPath: []string{"hop1", "hop2", "hop3"},
pathParsed: true,
}
got := estimateStoreTxBytes(tx)
// Should be at least base (384) + maps (200) + indexes + path/subpath costs
if got < 700 {
t.Errorf("estimate too low for 3-hop tx: %d", got)
}
if got > 5000 {
t.Errorf("estimate unreasonably high for 3-hop tx: %d", got)
}
}
// TestEstimateStoreTxBytes_ManyHopsSubpaths verifies that packets with many
// hops estimate significantly more due to O(path²) subpath index entries.
func TestEstimateStoreTxBytes_ManyHopsSubpaths(t *testing.T) {
tx2 := &StoreTx{
Hash: "aabb",
parsedPath: []string{"a", "b"},
pathParsed: true,
}
tx10 := &StoreTx{
Hash: "aabb",
parsedPath: []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"},
pathParsed: true,
}
est2 := estimateStoreTxBytes(tx2)
est10 := estimateStoreTxBytes(tx10)
// 10 hops → 45 subpath combos × 40 = 1800 bytes just for subpaths
if est10 <= est2 {
t.Errorf("10-hop (%d) should estimate more than 2-hop (%d)", est10, est2)
}
if est10 < est2+1500 {
t.Errorf("10-hop (%d) should estimate at least 1500 more than 2-hop (%d)", est10, est2)
}
}
// TestEstimateStoreObsBytes_AfterRefactor verifies that after #800 refactor,
// observations no longer have ResolvedPath overhead in their estimate.
func TestEstimateStoreObsBytes_AfterRefactor(t *testing.T) {
obs := &StoreObs{
ObserverID: "obs1",
PathJSON: `["a","b"]`,
}
est := estimateStoreObsBytes(obs)
if est <= 0 {
t.Errorf("estimate should be positive, got %d", est)
}
// After #800, all obs estimates should be the same (no RP field variation)
obs2 := &StoreObs{
ObserverID: "obs1",
PathJSON: `["a","b"]`,
}
est2 := estimateStoreObsBytes(obs2)
if est != est2 {
t.Errorf("estimates should be equal after #800 (no RP field), got %d vs %d", est, est2)
}
}
// TestEstimateStoreObsBytes_ManyObservations verifies that 15 observations
// estimate significantly more than 1.
func TestEstimateStoreObsBytes_ManyObservations(t *testing.T) {
est1 := estimateStoreObsBytes(&StoreObs{ObserverID: "a", PathJSON: `["x"]`})
est15 := int64(0)
for i := 0; i < 15; i++ {
est15 += estimateStoreObsBytes(&StoreObs{ObserverID: "a", PathJSON: `["x"]`})
}
if est15 <= est1*10 {
t.Errorf("15 obs total (%d) should be >10x single obs (%d)", est15, est1)
}
}
// TestTrackedBytesMatchesSumAfterInsert verifies that trackedBytes equals the
// sum of individual estimates after inserting packets via makeTestStore.
func TestTrackedBytesMatchesSumAfterInsert(t *testing.T) {
store := makeTestStore(20, time.Now().Add(-2*time.Hour), 5)
// Manually compute trackedBytes as sum of estimates
var expectedSum int64
for _, tx := range store.packets {
expectedSum += estimateStoreTxBytes(tx)
for _, obs := range tx.Observations {
expectedSum += estimateStoreObsBytes(obs)
}
}
if store.trackedBytes != expectedSum {
t.Errorf("trackedBytes=%d, expected sum=%d", store.trackedBytes, expectedSum)
}
}
// TestEvictionTriggersWithImprovedEstimates verifies that eviction triggers
// at the right point with the improved (higher) estimates.
func TestEvictionTriggersWithImprovedEstimates(t *testing.T) {
store := makeTestStore(100, time.Now().Add(-10*time.Hour), 5)
// trackedBytes for 100 packets is small — artificially set maxMemoryMB
// so highWatermark is just below trackedBytes to trigger eviction.
highWatermarkBytes := store.trackedBytes - 1000
if highWatermarkBytes < 1 {
highWatermarkBytes = 1
}
// maxMemoryMB * 1048576 = highWatermark, so maxMemoryMB = ceil(highWatermarkBytes / 1048576)
// But that'll be 0 for small values. Instead, directly set trackedBytes high.
store.trackedBytes = 6 * 1048576 // 6MB
store.maxMemoryMB = 3 // 3MB limit
beforeCount := len(store.packets)
store.RunEviction()
afterCount := len(store.packets)
if afterCount >= beforeCount {
t.Errorf("expected eviction to remove packets: before=%d, after=%d, trackedBytes=%d, maxMB=%d",
beforeCount, afterCount, store.trackedBytes, store.maxMemoryMB)
}
// trackedBytes should have decreased
if store.trackedBytes >= 6*1048576 {
t.Errorf("trackedBytes should have decreased after eviction")
}
}
// BenchmarkEstimateStoreTxBytes verifies the estimate function is fast.
func BenchmarkEstimateStoreTxBytes(b *testing.B) {
tx := &StoreTx{
Hash: "abcdef1234567890",
RawHex: "deadbeefdeadbeef",
DecodedJSON: `{"type":"GRP_TXT","payload":"hello"}`,
PathJSON: `["hop1","hop2","hop3","hop4","hop5"]`,
parsedPath: []string{"hop1", "hop2", "hop3", "hop4", "hop5"},
pathParsed: true,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
estimateStoreTxBytes(tx)
}
}
// BenchmarkEstimateStoreObsBytes verifies the obs estimate function is fast.
func BenchmarkEstimateStoreObsBytes(b *testing.B) {
obs := &StoreObs{
ObserverID: "observer1234",
PathJSON: `["a","b","c"]`,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
estimateStoreObsBytes(obs)
}
}
+5 -30
View File
@@ -68,26 +68,8 @@ type StatsResponse struct {
Commit string `json:"commit"`
BuildTime string `json:"buildTime"`
Counts RoleCounts `json:"counts"`
Backfilling bool `json:"backfilling"`
BackfillProgress float64 `json:"backfillProgress"`
SignatureDrops int64 `json:"signatureDrops,omitempty"`
HashMigrationComplete bool `json:"hashMigrationComplete"`
// Memory accounting (issue #832). All values in MB.
//
// StoreDataMB ("trackedMB" historically) is the in-store packet byte
// estimate — useful packet bytes only. Subset of HeapInuse. Used as
// the eviction watermark input. NOT a proxy for RSS; ops dashboards
// should prefer ProcessRSSMB for capacity decisions.
//
// Old field name TrackedMB is retained for backward compatibility
// with pre-v3.6 consumers; it carries the same value as StoreDataMB
// and is deprecated.
TrackedMB float64 `json:"trackedMB"` // deprecated alias for storeDataMB
StoreDataMB float64 `json:"storeDataMB"` // in-store packet bytes (subset of heap)
ProcessRSSMB float64 `json:"processRSSMB"` // process RSS from /proc (Linux) or runtime.Sys fallback
GoHeapInuseMB float64 `json:"goHeapInuseMB"` // runtime.MemStats.HeapInuse
GoSysMB float64 `json:"goSysMB"` // runtime.MemStats.Sys (total Go-managed)
Backfilling bool `json:"backfilling"`
BackfillProgress float64 `json:"backfillProgress"`
}
// ─── Health ────────────────────────────────────────────────────────────────────
@@ -133,7 +115,6 @@ type WebSocketStatsResp struct {
type HealthPacketStoreStats struct {
Packets int `json:"packets"`
EstimatedMB float64 `json:"estimatedMB"`
TrackedMB float64 `json:"trackedMB"`
}
type SlowQuery struct {
@@ -193,8 +174,6 @@ type PerfPacketStoreStats struct {
SqliteOnly bool `json:"sqliteOnly"`
MaxPackets int `json:"maxPackets"`
EstimatedMB float64 `json:"estimatedMB"`
TrackedMB float64 `json:"trackedMB"`
AvgBytesPerPacket int64 `json:"avgBytesPerPacket"`
MaxMB int `json:"maxMB"`
Indexes PacketStoreIndexes `json:"indexes"`
}
@@ -263,6 +242,7 @@ type TransmissionResp struct {
SNR interface{} `json:"snr"`
RSSI interface{} `json:"rssi"`
PathJSON interface{} `json:"path_json"`
ResolvedPath []*string `json:"resolved_path,omitempty"`
Direction interface{} `json:"direction"`
Score interface{} `json:"score,omitempty"`
Observations []ObservationResp `json:"observations,omitempty"`
@@ -277,9 +257,7 @@ type ObservationResp struct {
SNR interface{} `json:"snr"`
RSSI interface{} `json:"rssi"`
PathJSON interface{} `json:"path_json"`
ResolvedPath interface{} `json:"resolved_path,omitempty"`
Direction interface{} `json:"direction,omitempty"`
RawHex interface{} `json:"raw_hex,omitempty"`
ResolvedPath []*string `json:"resolved_path,omitempty"`
Timestamp interface{} `json:"timestamp"`
}
@@ -488,7 +466,6 @@ type NodeAnalyticsResponse struct {
PeerInteractions []PeerInteraction `json:"peerInteractions"`
UptimeHeatmap []HeatmapCell `json:"uptimeHeatmap"`
ComputedStats ComputedNodeStats `json:"computedStats"`
ClockSkew *NodeClockSkew `json:"clockSkew,omitempty"`
}
// ─── Analytics — RF ────────────────────────────────────────────────────────────
@@ -681,9 +658,7 @@ type DistanceHop struct {
ToPk string `json:"toPk"`
Dist float64 `json:"dist"`
Type string `json:"type"`
BestSnr interface{} `json:"bestSnr"`
MedianSnr interface{} `json:"medianSnr"`
ObsCount int `json:"obsCount"`
SNR interface{} `json:"snr"`
Hash string `json:"hash"`
Timestamp string `json:"timestamp"`
}
+2 -15
View File
@@ -1,13 +1,10 @@
{
"port": 3000,
"apiKey": "your-secret-api-key-here",
"nodeBlacklist": [],
"_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.",
"retention": {
"nodeDays": 7,
"observerDays": 14,
"packetDays": 30,
"_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)."
"_comment": "nodeDays: nodes not seen in N days are moved to inactive_nodes (default 7). packetDays: transmissions+observations older than N days are deleted daily (0 = disabled)."
},
"https": {
"cert": "/path/to/cert.pem",
@@ -128,7 +125,7 @@
}
],
"channelKeys": {
"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72"
"public": "8b3387e9c5cdea6ac9e5edbaa115cd72"
},
"hashChannels": [
"#LongFast",
@@ -156,16 +153,6 @@
],
"zoom": 9
},
"geo_filter": {
"polygon": [
[37.80, -122.52],
[37.80, -121.80],
[37.20, -121.80],
[37.20, -122.52]
],
"bufferKm": 20,
"_comment": "Optional. Restricts ingestion and API responses to nodes within the polygon + bufferKm. Polygon is an array of [lat, lon] pairs (minimum 3). Use tools/geofilter-builder.html to draw a polygon visually. Remove this section to disable filtering. Nodes with no GPS fix are always allowed through."
},
"regions": {
"SJC": "San Jose, US",
"SFO": "San Francisco, US",
-1
View File
@@ -29,7 +29,6 @@ services:
- NODE_ENV=staging
- ENABLE_PPROF=true
- DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false}
- DISABLE_CADDY=${DISABLE_CADDY:-false}
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"]
interval: 30s
-1
View File
@@ -29,7 +29,6 @@ services:
environment:
- NODE_ENV=production
- DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false}
- DISABLE_CADDY=${DISABLE_CADDY:-false}
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"]
interval: 30s
-7
View File
@@ -14,13 +14,6 @@ if [ -f /app/data/theme.json ]; then
ln -sf /app/data/theme.json /app/theme.json
fi
# Source .env from data volume if present (works with any launch method)
if [ -f /app/data/.env ]; then
set -a
. /app/data/.env
set +a
fi
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
if [ "${DISABLE_MOSQUITTO:-false}" = "true" ] && [ "${DISABLE_CADDY:-false}" = "true" ]; then
echo "[config] internal MQTT broker disabled (DISABLE_MOSQUITTO=true)"
+2 -51
View File
@@ -48,52 +48,9 @@ No `config.json` is required. The server starts with sensible defaults:
- Ingestor connects to `mqtt://localhost:1883` automatically
- SQLite database at `/app/data/meshcore.db`
### Full `docker run` Reference (recommended)
### Docker Compose (recommended for production)
The bare `docker run` command is the primary deployment method. One image, documented parameters — run it however you want.
```bash
docker run -d --name corescope \
--restart=unless-stopped \
-p 80:80 -p 443:443 -p 1883:1883 \
-e DISABLE_MOSQUITTO=false \
-e DISABLE_CADDY=false \
-v /your/data:/app/data \
-v /your/Caddyfile:/etc/caddy/Caddyfile:ro \
-v /your/caddy-data:/data/caddy \
ghcr.io/kpa-clawbot/corescope:latest
```
#### Parameters
| Parameter | Required | Description |
|-----------|----------|-------------|
| `-p 80:80` | Yes | HTTP web UI |
| `-p 443:443` | No | HTTPS (only if using built-in Caddy with a domain) |
| `-p 1883:1883` | No | MQTT broker (expose if external gateways connect directly) |
| `-v /your/data:/app/data` | Yes | Persistent data: SQLite DB, config.json, theme.json |
| `-v /your/Caddyfile:/etc/caddy/Caddyfile:ro` | No | Custom Caddyfile for HTTPS |
| `-v /your/caddy-data:/data/caddy` | No | Caddy TLS certificate storage |
| `-e DISABLE_MOSQUITTO=true` | No | Skip the internal Mosquitto broker (use your own) |
| `-e DISABLE_CADDY=true` | No | Skip the built-in Caddy reverse proxy |
| `-e MQTT_BROKER=mqtt://host:1883` | No | Override MQTT broker URL |
#### `/app/data/.env` convenience file
Instead of passing `-e` flags, you can drop a `.env` file in your data volume:
```bash
# /your/data/.env
DISABLE_MOSQUITTO=true
DISABLE_CADDY=true
MQTT_BROKER=mqtt://my-broker:1883
```
The entrypoint sources this file before starting services. This works with any launch method (`docker run`, compose, or manage.sh).
### Docker Compose (legacy alternative)
Docker Compose files are maintained for backward compatibility but are no longer the recommended approach.
Download the example compose file:
```bash
curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \
@@ -108,11 +65,6 @@ docker compose up -d
| `HTTP_PORT` | `80` | Host port for the web UI |
| `DATA_DIR` | `./data` | Host path for persistent data |
| `DISABLE_MOSQUITTO` | `false` | Set `true` to use an external MQTT broker |
| `DISABLE_CADDY` | `false` | Set `true` to skip the built-in Caddy proxy |
### manage.sh (legacy alternative)
The `manage.sh` wrapper script provides a setup wizard and convenience commands. It uses Docker Compose internally. See [DEPLOY.md](../DEPLOY.md) for usage. New deployments should prefer bare `docker run`.
### Image tags
@@ -159,7 +111,6 @@ CoreScope uses a layered configuration system (highest priority wins):
| `MQTT_TOPIC` | `meshcore/#` | MQTT topic subscription pattern |
| `DB_PATH` | `data/meshcore.db` | SQLite database path |
| `DISABLE_MOSQUITTO` | `false` | Skip the internal Mosquitto broker |
| `DISABLE_CADDY` | `false` | Skip the built-in Caddy reverse proxy |
### config.json
-13
View File
@@ -176,19 +176,6 @@ Lower values = fresher data but more server load.
Provide cert and key paths to enable HTTPS.
## Geographic filtering
```json
"geo_filter": {
"polygon": [[51.55, 3.80], [51.55, 5.90], [50.65, 5.90], [50.65, 3.80]],
"bufferKm": 20
}
```
Restricts ingestion and API responses to nodes within the polygon plus a buffer margin. Remove the block to disable filtering. Nodes with no GPS fix always pass through.
See [Geographic Filtering](geofilter.md) for the full guide including the visual polygon builder and the prune script for cleaning up historical data.
## Home page
The `home` section customizes the onboarding experience. See `config.example.json` for the full structure including `steps`, `checklist`, and `footerLinks`.
-6
View File
@@ -66,12 +66,6 @@ Click **Import JSON** and paste a previously exported theme. The customizer load
Click **Reset to Defaults** to restore all settings to the built-in defaults.
## GeoFilter Builder
The Export tab includes a **GeoFilter Builder →** link. Click it to open a Leaflet map where you can draw a polygon boundary for your deployment area. The tool generates a `geo_filter` block you can paste directly into `config.json`.
See [Geographic Filtering](geofilter.md) for full details on what geo filtering does and how to configure it.
## How it works
The customizer writes CSS custom properties (variables) to override the defaults. Exported JSON maps directly to the `theme`, `nodeColors`, `branding`, and `home` sections of [config.json](configuration.md).
-114
View File
@@ -1,114 +0,0 @@
# Geographic Filtering
CoreScope supports geographic filtering to restrict which nodes are ingested and returned in API responses. This is useful for public-facing deployments that should only show activity in a specific region.
## How it works
Geographic filtering operates at two levels:
- **Ingest time** — ADVERT packets carrying GPS coordinates are rejected by the ingestor if the node falls outside the configured area. The node never reaches the database.
- **API responses** — Nodes already in the database are filtered from the `/api/nodes` response if they fall outside the area. This covers nodes ingested before the filter was configured.
Nodes with no GPS fix (`lat=0, lon=0` or missing coordinates) always pass the filter regardless of configuration.
## Configuration
Add a `geo_filter` block to `config.json`:
```json
"geo_filter": {
"polygon": [
[51.55, 3.80],
[51.55, 5.90],
[50.65, 5.90],
[50.65, 3.80]
],
"bufferKm": 20
}
```
| Field | Type | Description |
|-------|------|-------------|
| `polygon` | `[[lat, lon], ...]` | Array of at least 3 coordinate pairs defining the boundary |
| `bufferKm` | number | Extra distance (km) around the polygon edge that is also accepted. `0` = exact boundary |
Both the server and the ingestor read `geo_filter` from `config.json`. Restart both after changing this section.
To disable filtering entirely, remove the `geo_filter` block.
### Legacy bounding box
An older bounding box format is also supported as a fallback when no `polygon` is present:
```json
"geo_filter": {
"latMin": 50.65,
"latMax": 51.55,
"lonMin": 3.80,
"lonMax": 5.90
}
```
Prefer the polygon format — it supports irregular shapes and the `bufferKm` margin.
## API endpoint
The current geo filter configuration is exposed at:
```
GET /api/config/geo-filter
```
The frontend reads this endpoint to display the active filter. No authentication is required (the endpoint returns config, not private data).
## GeoFilter Builder
The simplest way to create a polygon is the included visual builder:
**File:** `tools/geofilter-builder.html`
Open it directly in a browser — it runs entirely client-side, no server required:
```bash
# From the project root
open tools/geofilter-builder.html # macOS
xdg-open tools/geofilter-builder.html # Linux
start tools/geofilter-builder.html # Windows
```
**Workflow:**
1. The map opens centered on Belgium by default. Navigate to your region.
2. Click on the map to add polygon vertices. Each click adds a numbered point.
3. Add at least 3 points to form a closed polygon.
4. Adjust **Buffer km** (default 20) to add a margin around the polygon edge.
5. The generated JSON block appears at the bottom of the page — copy it directly into `config.json`.
6. Use **↩ Undo** to remove the last point, **✕ Clear** to start over.
The output is a complete `{ "geo_filter": { ... } }` block ready to paste into `config.json`.
## Cleaning up historical nodes
The ingestor prevents new out-of-bounds nodes from being ingested, but it does not retroactively remove nodes that were stored before the filter was configured. For that, use the prune script.
**File:** `scripts/prune-nodes-outside-geo-filter.py`
```bash
# Dry run — shows what would be deleted without making any changes
python3 scripts/prune-nodes-outside-geo-filter.py --dry-run
# Default paths: /app/data/meshcore.db and /app/config.json
python3 scripts/prune-nodes-outside-geo-filter.py
# Custom paths
python3 scripts/prune-nodes-outside-geo-filter.py /path/to/meshcore.db \
--config /path/to/config.json
# In Docker — run inside the container
docker exec -it meshcore-analyzer \
python3 /app/scripts/prune-nodes-outside-geo-filter.py --dry-run
```
The script reads `geo_filter.polygon` and `geo_filter.bufferKm` from config, lists the nodes that fall outside, then asks for `yes` confirmation before deleting. Nodes without coordinates are always kept.
This is a **one-time migration tool** — run it once after first configuring `geo_filter` to clean up pre-filter data. The ingestor handles all subsequent filtering automatically at ingest time.
-98
View File
@@ -1,98 +0,0 @@
// Package channel provides MeshCore hashtag channel key derivation,
// decryption (HMAC-SHA256 MAC + AES-128-ECB), and plaintext parsing.
package channel
import (
"crypto/aes"
"crypto/hmac"
"crypto/sha256"
"encoding/binary"
"fmt"
"strings"
"unicode/utf8"
)
// DeriveKey derives an AES-128 key from a channel name (e.g. "#wardriving").
// Returns 16 bytes: SHA-256(channelName)[:16].
func DeriveKey(channelName string) []byte {
h := sha256.Sum256([]byte(channelName))
return h[:16]
}
// ChannelHash returns the 1-byte channel hash used as the first byte of GRP_TXT payloads.
// It is the first byte of SHA-256 of the 16-byte key.
func ChannelHash(key []byte) byte {
h := sha256.Sum256(key)
return h[0]
}
// Decrypt verifies the 2-byte HMAC-SHA256 MAC and performs AES-128-ECB decryption.
// mac must be exactly 2 bytes. ciphertext must be a multiple of 16 bytes.
// Returns the plaintext and true if MAC verification succeeded, or nil and false otherwise.
func Decrypt(key []byte, mac []byte, ciphertext []byte) ([]byte, bool) {
if len(key) != 16 || len(mac) != 2 || len(ciphertext) == 0 || len(ciphertext)%aes.BlockSize != 0 {
return nil, false
}
// 32-byte channel secret: 16-byte key + 16 zero bytes
channelSecret := make([]byte, 32)
copy(channelSecret, key)
// Verify HMAC-SHA256 (first 2 bytes must match)
h := hmac.New(sha256.New, channelSecret)
h.Write(ciphertext)
calculatedMac := h.Sum(nil)
if calculatedMac[0] != mac[0] || calculatedMac[1] != mac[1] {
return nil, false
}
// AES-128-ECB decrypt
block, err := aes.NewCipher(key)
if err != nil {
return nil, false
}
plaintext := make([]byte, len(ciphertext))
for i := 0; i < len(ciphertext); i += aes.BlockSize {
block.Decrypt(plaintext[i:i+aes.BlockSize], ciphertext[i:i+aes.BlockSize])
}
return plaintext, true
}
// ParsePlaintext parses decrypted plaintext into timestamp, sender, and message.
// Format: timestamp(4 LE) + flags(1) + "sender: message\0..."
func ParsePlaintext(plaintext []byte) (timestamp uint32, sender string, message string, err error) {
if len(plaintext) < 5 {
return 0, "", "", fmt.Errorf("plaintext too short (%d bytes)", len(plaintext))
}
timestamp = binary.LittleEndian.Uint32(plaintext[0:4])
text := string(plaintext[5:])
if idx := strings.IndexByte(text, 0); idx >= 0 {
text = text[:idx]
}
if !utf8.ValidString(text) || countNonPrintable(text) > 2 {
return 0, "", "", fmt.Errorf("decrypted text contains non-printable characters")
}
// Parse "sender: message" format
if colonIdx := strings.Index(text, ": "); colonIdx > 0 && colonIdx < 50 {
potentialSender := text[:colonIdx]
if !strings.ContainsAny(potentialSender, ":[]") {
return timestamp, potentialSender, text[colonIdx+2:], nil
}
}
return timestamp, "", text, nil
}
func countNonPrintable(s string) int {
count := 0
for _, r := range s {
if r < 32 && r != '\n' && r != '\r' && r != '\t' {
count++
}
}
return count
}
-161
View File
@@ -1,161 +0,0 @@
package channel
import (
"crypto/aes"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"testing"
)
func TestDeriveKey(t *testing.T) {
key := DeriveKey("#wardriving")
h := sha256.Sum256([]byte("#wardriving"))
expected := h[:16]
if len(key) != 16 {
t.Fatalf("key length %d, want 16", len(key))
}
for i := range key {
if key[i] != expected[i] {
t.Fatalf("DeriveKey mismatch at byte %d", i)
}
}
}
func TestChannelHash(t *testing.T) {
key := DeriveKey("#wardriving")
ch := ChannelHash(key)
h := sha256.Sum256(key)
if ch != h[0] {
t.Fatalf("ChannelHash %02x, want %02x", ch, h[0])
}
}
func testECBEncrypt(t *testing.T, key, plaintext []byte) []byte {
t.Helper()
block, err := aes.NewCipher(key)
if err != nil {
t.Fatal(err)
}
ct := make([]byte, len(plaintext))
for i := 0; i < len(plaintext); i += aes.BlockSize {
block.Encrypt(ct[i:i+aes.BlockSize], plaintext[i:i+aes.BlockSize])
}
return ct
}
func testComputeMAC(key, ciphertext []byte) []byte {
secret := make([]byte, 32)
copy(secret, key)
h := hmac.New(sha256.New, secret)
h.Write(ciphertext)
sum := h.Sum(nil)
return sum[:2]
}
func TestDecryptValidMAC(t *testing.T) {
key := DeriveKey("#test")
padded := make([]byte, 16)
copy(padded, []byte{0x01, 0x00, 0x00, 0x00, 0x00})
ciphertext := testECBEncrypt(t, key, padded)
mac := testComputeMAC(key, ciphertext)
result, ok := Decrypt(key, mac, ciphertext)
if !ok {
t.Fatal("Decrypt returned false for valid MAC")
}
if len(result) != 16 {
t.Fatalf("result length %d, want 16", len(result))
}
}
func TestDecryptInvalidMAC(t *testing.T) {
key := DeriveKey("#test")
ciphertext := make([]byte, 16)
mac := []byte{0xFF, 0xFF}
_, ok := Decrypt(key, mac, ciphertext)
if ok {
t.Fatal("Decrypt should reject wrong MAC")
}
}
func TestDecryptWrongChannel(t *testing.T) {
key1 := DeriveKey("#channel1")
key2 := DeriveKey("#channel2")
padded := make([]byte, 16)
copy(padded, []byte{0x01, 0x00, 0x00, 0x00, 0x00, 'h', 'i'})
ciphertext := testECBEncrypt(t, key1, padded)
mac := testComputeMAC(key1, ciphertext)
_, ok := Decrypt(key2, mac, ciphertext)
if ok {
t.Fatal("Decrypt should reject wrong channel key")
}
}
func TestParsePlaintext(t *testing.T) {
plain := []byte{100, 0, 0, 0, 0}
plain = append(plain, []byte("Alice: Hello\x00")...)
ts, sender, msg, err := ParsePlaintext(plain)
if err != nil {
t.Fatal(err)
}
if ts != 100 {
t.Fatalf("timestamp %d, want 100", ts)
}
if sender != "Alice" {
t.Fatalf("sender %q, want Alice", sender)
}
if msg != "Hello" {
t.Fatalf("message %q, want Hello", msg)
}
}
func TestParsePlaintextNoSender(t *testing.T) {
plain := []byte{1, 0, 0, 0, 0}
plain = append(plain, []byte("just a message\x00")...)
_, sender, msg, err := ParsePlaintext(plain)
if err != nil {
t.Fatal(err)
}
if sender != "" {
t.Fatalf("sender %q, want empty", sender)
}
if msg != "just a message" {
t.Fatalf("message %q", msg)
}
}
func TestDeriveKeyMatchesIngestor(t *testing.T) {
channelName := "#MeshCore"
key := DeriveKey(channelName)
hexKey := hex.EncodeToString(key)
h := sha256.Sum256([]byte(channelName))
expected := hex.EncodeToString(h[:16])
if hexKey != expected {
t.Fatalf("key hex %s != expected %s", hexKey, expected)
}
}
func TestRoundTrip(t *testing.T) {
key := DeriveKey("#test")
original := make([]byte, 32)
copy(original, []byte{0x64, 0x00, 0x00, 0x00, 0x00})
copy(original[5:], []byte("Bob: world\x00"))
ciphertext := testECBEncrypt(t, key, original)
mac := testComputeMAC(key, ciphertext)
plaintext, ok := Decrypt(key, mac, ciphertext)
if !ok {
t.Fatal("round-trip MAC failed")
}
ts, sender, msg, err := ParsePlaintext(plaintext)
if err != nil {
t.Fatal(err)
}
if ts != 100 || sender != "Bob" || msg != "world" {
t.Fatalf("got ts=%d sender=%q msg=%q", ts, sender, msg)
}
}
-3
View File
@@ -1,3 +0,0 @@
module github.com/meshcore-analyzer/channel
go 1.22
+8 -12
View File
@@ -1,21 +1,17 @@
// Package sigvalidate provides ed25519 signature validation for MeshCore advert packets.
// Package sigvalidate provides Ed25519 signature validation for MeshCore adverts.
package sigvalidate
import (
"crypto/ed25519"
"encoding/binary"
"fmt"
)
// ValidateAdvert verifies the ed25519 signature on a MeshCore advert.
// pubKey must be 32 bytes, signature must be 64 bytes.
// The signed message is: pubKey (32) + timestamp (4 LE) + appdata.
func ValidateAdvert(pubKey, signature []byte, timestamp uint32, appdata []byte) (bool, error) {
if len(pubKey) != 32 {
return false, fmt.Errorf("invalid pubkey length: %d", len(pubKey))
}
if len(signature) != 64 {
return false, fmt.Errorf("invalid signature length: %d", len(signature))
// ValidateAdvertSignature verifies an Ed25519 signature over a MeshCore advert.
// The signed message is: pubKey (32 bytes) || timestamp (4 bytes LE) || appdata.
// Returns false if pubKey is not 32 bytes or signature is not 64 bytes.
func ValidateAdvertSignature(pubKey, signature []byte, timestamp uint32, appdata []byte) bool {
if len(pubKey) != 32 || len(signature) != 64 {
return false
}
message := make([]byte, 32+4+len(appdata))
@@ -23,5 +19,5 @@ func ValidateAdvert(pubKey, signature []byte, timestamp uint32, appdata []byte)
binary.LittleEndian.PutUint32(message[32:36], timestamp)
copy(message[36:], appdata)
return ed25519.Verify(ed25519.PublicKey(pubKey), message, signature), nil
return ed25519.Verify(ed25519.PublicKey(pubKey), message, signature)
}
-63
View File
@@ -1,63 +0,0 @@
package sigvalidate
import (
"crypto/ed25519"
"encoding/binary"
"testing"
)
func TestValidateAdvert_ValidSignature(t *testing.T) {
pub, priv, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
var timestamp uint32 = 1234567890
appdata := []byte{0x02, 0x10, 0x20}
// Build the signed message: pubKey + timestamp(LE) + appdata
msg := make([]byte, 32+4+len(appdata))
copy(msg[0:32], pub)
binary.LittleEndian.PutUint32(msg[32:36], timestamp)
copy(msg[36:], appdata)
sig := ed25519.Sign(priv, msg)
valid, err := ValidateAdvert([]byte(pub), sig, timestamp, appdata)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !valid {
t.Fatal("expected valid signature")
}
}
func TestValidateAdvert_InvalidSignature(t *testing.T) {
pub, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
badSig := make([]byte, 64)
valid, err := ValidateAdvert([]byte(pub), badSig, 100, []byte{0x01})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if valid {
t.Fatal("expected invalid signature")
}
}
func TestValidateAdvert_BadPubkeyLength(t *testing.T) {
_, err := ValidateAdvert([]byte{1, 2, 3}, make([]byte, 64), 0, nil)
if err == nil {
t.Fatal("expected error for short pubkey")
}
}
func TestValidateAdvert_BadSignatureLength(t *testing.T) {
_, err := ValidateAdvert(make([]byte, 32), []byte{1, 2, 3}, 0, nil)
if err == nil {
t.Fatal("expected error for short signature")
}
}
+42 -320
View File
@@ -28,7 +28,7 @@
function barChart(data, labels, colors, w = 800, h = 220, pad = 40) {
const max = Math.max(...data, 1);
const barW = Math.max(1, Math.min((w - pad * 2) / data.length - 2, 30));
const barW = Math.min((w - pad * 2) / data.length - 2, 30);
let svg = `<svg viewBox="0 0 ${w} ${h}" style="width:100%;max-height:${h}px" role="img" aria-label="Bar chart showing data distribution"><title>Bar chart showing data distribution</title>`;
// Grid
for (let i = 0; i <= 4; i++) {
@@ -87,7 +87,6 @@
<button class="tab-btn" data-tab="distance">Distance</button>
<button class="tab-btn" data-tab="neighbor-graph">Neighbor Graph</button>
<button class="tab-btn" data-tab="rf-health">RF Health</button>
<button class="tab-btn" data-tab="clock-health">Clock Health</button>
<button class="tab-btn" data-tab="prefix-tool">Prefix Tool</button>
</div>
</div>
@@ -182,7 +181,6 @@
case 'distance': await renderDistanceTab(el); break;
case 'neighbor-graph': await renderNeighborGraphTab(el); break;
case 'rf-health': await renderRFHealthTab(el); break;
case 'clock-health': await renderClockHealthTab(el); break;
case 'prefix-tool': await renderPrefixTool(el); break;
}
// Auto-apply column resizing to all analytics tables
@@ -263,25 +261,7 @@
<div class="analytics-row">
<div class="analytics-card flex-1">
<h3>📈 Packets / Hour</h3>
${(() => {
const pph = rf.packetsPerHour;
const counts = pph.map(h => h.count);
// Decimate x-axis labels to avoid overlap
const totalHours = pph.length;
// Pick label interval: <=24h show every 6h, <=72h every 12h, else every 24h
const labelInterval = totalHours <= 24 ? 6 : totalHours <= 72 ? 12 : 24;
const labels = pph.map((h, i) => {
const hh = h.hour.slice(11, 13); // "HH"
const hourNum = parseInt(hh, 10);
if (hourNum % labelInterval === 0) {
// For multi-day ranges, show date on 00h boundaries
if (totalHours > 48 && hourNum === 0) return h.hour.slice(5, 10);
return hh + 'h';
}
return ''; // skip label
});
return barChart(counts, labels, 'var(--accent)');
})()}
${barChart(rf.packetsPerHour.map(h=>h.count), rf.packetsPerHour.map(h=>h.hour.slice(11)+'h'), 'var(--accent)')}
</div>
</div>
@@ -642,13 +622,14 @@
if (!data || !data.rings.length) return '<div class="text-muted">No path data for this observer</div>';
let html = `<div class="reach-rings">`;
data.rings.forEach(ring => {
const opacity = Math.max(0.3, 1 - ring.hops * 0.06);
const nodeLinks = ring.nodes.slice(0, 8).map(n => {
const label = n.name ? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>` : `<span class="mono">${n.hop}</span>`;
const detail = n.distRange ? ` <span class="text-muted">(${n.distRange})</span>` : '';
return label + detail;
}).join(', ');
const extra = ring.nodes.length > 8 ? ` <span class="text-muted">+${ring.nodes.length - 8} more</span>` : '';
html += `<div class="reach-ring">
html += `<div class="reach-ring" style="opacity:${opacity}">
<div class="reach-hop">${ring.hops} hop${ring.hops > 1 ? 's' : ''}</div>
<div class="reach-nodes">${nodeLinks}${extra}</div>
<div class="reach-count">${ring.nodes.length} node${ring.nodes.length > 1 ? 's' : ''}</div>
@@ -692,6 +673,7 @@
});
let html = '<div class="reach-rings">';
Object.entries(byDist).sort((a, b) => +a[0] - +b[0]).forEach(([dist, nodes]) => {
const opacity = Math.max(0.3, 1 - (+dist) * 0.06);
const nodeLinks = nodes.slice(0, 10).map(n => {
const label = n.name
? `<a href="#/nodes/${encodeURIComponent(n.pubkey)}" class="analytics-link">${esc(n.name)}</a>`
@@ -699,7 +681,7 @@
return label + ` <span class="text-muted">via ${esc(n.observer_name)}</span>`;
}).join(', ');
const extra = nodes.length > 10 ? ` <span class="text-muted">+${nodes.length - 10} more</span>` : '';
html += `<div class="reach-ring">
html += `<div class="reach-ring" style="opacity:${opacity}">
<div class="reach-hop">${dist} hop${+dist > 1 ? 's' : ''}</div>
<div class="reach-nodes">${nodeLinks}${extra}</div>
<div class="reach-count">${nodes.length} node${nodes.length > 1 ? 's' : ''}</div>
@@ -856,44 +838,29 @@
}
}
var CHANNEL_TIMELINE_MAX_SERIES = 8;
function renderChannelTimeline(data) {
if (!data.length) return '<div class="text-muted">No data</div>';
var hours = []; var hourSet = {};
var channelList = []; var channelSet = {};
var lookup = {};
var channelVolume = {};
var maxCount = 1;
for (var i = 0; i < data.length; i++) {
var d = data[i];
if (!hourSet[d.hour]) { hourSet[d.hour] = 1; hours.push(d.hour); }
if (!channelSet[d.channel]) { channelSet[d.channel] = 1; channelList.push(d.channel); }
lookup[d.hour + '|' + d.channel] = d.count;
channelVolume[d.channel] = (channelVolume[d.channel] || 0) + d.count;
if (d.count > maxCount) maxCount = d.count;
}
hours.sort();
// Sort channels by total volume descending, cap to top N
channelList.sort(function(a, b) { return channelVolume[b] - channelVolume[a]; });
var hiddenCount = Math.max(0, channelList.length - CHANNEL_TIMELINE_MAX_SERIES);
var visibleChannels = channelList.slice(0, CHANNEL_TIMELINE_MAX_SERIES);
var maxCount = 1;
for (var vi = 0; vi < visibleChannels.length; vi++) {
for (var hi2 = 0; hi2 < hours.length; hi2++) {
var c = lookup[hours[hi2] + '|' + visibleChannels[vi]] || 0;
if (c > maxCount) maxCount = c;
}
}
var colors = ['#ef4444','#22c55e','#3b82f6','#f59e0b','#8b5cf6','#ec4899','#14b8a6','#64748b'];
var w = 600, h = 180, pad = 35;
var xScale = (w - pad * 2) / Math.max(hours.length - 1, 1);
var yScale = (h - pad * 2) / maxCount;
var svg = '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:100%;max-height:180px" role="img" aria-label="Channel message activity over time"><title>Channel message activity over time</title>';
for (var ci = 0; ci < visibleChannels.length; ci++) {
for (var ci = 0; ci < channelList.length; ci++) {
var pts = [];
for (var hi = 0; hi < hours.length; hi++) {
var count = lookup[hours[hi] + '|' + visibleChannels[ci]] || 0;
var count = lookup[hours[hi] + '|' + channelList[ci]] || 0;
var x = pad + hi * xScale;
var y = h - pad - count * yScale;
pts.push(x + ',' + y);
@@ -907,11 +874,8 @@
}
svg += '</svg>';
var legendParts = [];
for (var lci = 0; lci < visibleChannels.length; lci++) {
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(visibleChannels[lci]) + '</span>');
}
if (hiddenCount > 0) {
legendParts.push('<span class="text-muted">+' + hiddenCount + ' more</span>');
for (var lci = 0; lci < channelList.length; lci++) {
legendParts.push('<span><span class="legend-dot" style="background:' + colors[lci % colors.length] + '"></span>' + esc(channelList[lci]) + '</span>');
}
svg += '<div class="timeline-legend">' + legendParts.join('') + '</div>';
return svg;
@@ -982,9 +946,25 @@
</div>
</div>
${renderMultiByteAdopters(data.multiByteNodes, data.multiByteCapability || [])}
<div class="analytics-row">
<div class="analytics-card flex-1">
<h3>Multi-Byte Hash Adopters</h3>
<p class="text-muted">Nodes advertising with 2+ byte hash paths</p>
${data.multiByteNodes.length ? `
<table class="analytics-table">
<thead><tr><th scope="col">Node</th><th scope="col">Hash Size</th><th scope="col">Adverts</th><th scope="col">Last Seen</th></tr></thead>
<tbody>
${data.multiByteNodes.map(n => `<tr class="clickable-row" data-action="navigate" data-value="#/nodes/${n.pubkey ? encodeURIComponent(n.pubkey) : ''}" tabindex="0" role="row">
<td><strong>${esc(n.name)}</strong></td>
<td><span class="badge badge-hash-${n.hashSize}">${n.hashSize}-byte</span></td>
<td>${n.packets}</td>
<td>${timeAgo(n.lastSeen)}</td>
</tr>`).join('')}
</tbody>
</table>
` : '<div class="text-muted" style="padding:16px">No multi-byte adopters found</div>'}
</div>
<div class="analytics-card flex-1">
<h3>Top Path Hops</h3>
<table class="analytics-table">
@@ -1006,136 +986,6 @@
`;
}
function renderMultiByteAdopters(nodes, caps) {
// Merge capability status into adopter nodes
var capByPubkey = {};
(caps || []).forEach(function(c) { capByPubkey[c.pubkey] = c; });
var statusIcon = { confirmed: '✅', suspected: '⚠️', unknown: '❓' };
var statusLabel = { confirmed: 'Confirmed', suspected: 'Suspected', unknown: 'Unknown' };
var statusColor = { confirmed: 'var(--success, #22c55e)', suspected: 'var(--warning, #eab308)', unknown: 'var(--text-muted, #888)' };
// Build merged rows: each adopter node gets a capability status
var rows = (nodes || []).map(function(n) {
var cap = capByPubkey[n.pubkey] || {};
return {
name: n.name, pubkey: n.pubkey || '', role: n.role || '',
hashSize: n.hashSize, packets: n.packets, lastSeen: n.lastSeen,
status: cap.status || 'unknown', evidence: cap.evidence || ''
};
});
// Count statuses
var counts = { confirmed: 0, suspected: 0, unknown: 0 };
rows.forEach(function(r) { counts[r.status] = (counts[r.status] || 0) + 1; });
function buildTableContent(rows, filter) {
var filtered = filter === 'all' ? rows : rows.filter(function(r) { return r.status === filter; });
return (filtered.length ? '<table class="analytics-table" id="mbAdoptersTable" style="margin-top:12px">' +
'<thead><tr>' +
'<th scope="col" data-sort="name">Node</th>' +
'<th scope="col" data-sort="role">Role</th>' +
'<th scope="col" data-sort="status">Status</th>' +
'<th scope="col" data-sort="hashSize">Hash Size</th>' +
'<th scope="col" data-sort="packets">Adverts</th>' +
'<th scope="col" data-sort="lastSeen">Last Seen</th>' +
'</tr></thead>' +
'<tbody>' +
filtered.map(function(r) {
var roleColor = (window.ROLE_COLORS || {})[r.role] || '#6b7280';
return '<tr class="clickable-row" data-action="navigate" data-value="#/nodes/' + encodeURIComponent(r.pubkey) + '" tabindex="0" role="row">' +
'<td><strong>' + esc(r.name) + '</strong></td>' +
'<td><span class="badge" style="background:' + roleColor + '20;color:' + roleColor + '">' + esc(r.role || 'unknown') + '</span></td>' +
'<td><span style="color:' + (statusColor[r.status] || statusColor.unknown) + '">' +
(statusIcon[r.status] || '❓') + ' ' + (statusLabel[r.status] || 'Unknown') + '</span></td>' +
'<td><span class="badge badge-hash-' + r.hashSize + '">' + r.hashSize + '-byte</span></td>' +
'<td>' + r.packets + '</td>' +
'<td>' + (r.lastSeen ? timeAgo(r.lastSeen) : '—') + '</td>' +
'</tr>';
}).join('') +
'</tbody>' +
'</table>' : '<div class="text-muted" style="padding:16px">No adopters match this filter.</div>');
}
if (!rows.length) return '<div class="analytics-row"><div class="analytics-card flex-1">' +
'<h3>Multi-Byte Hash Adopters</h3>' +
'<div class="text-muted" style="padding:16px">No multi-byte adopters found</div></div></div>';
var html = '<div class="analytics-row"><div class="analytics-card flex-1" id="mbAdoptersSection">' +
'<div style="display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:8px">' +
'<div>' +
'<h3 style="margin:0">Multi-Byte Hash Adopters</h3>' +
'<p class="text-muted" style="margin:4px 0 0;font-size:0.8em">Nodes advertising with 2+ byte hash paths. ' +
'<strong>Confirmed</strong> = seen advertising with multi-byte hash. ' +
'<strong>Suspected</strong> = prefix appeared in a multi-byte path. ' +
'<strong>Unknown</strong> = no multi-byte evidence yet.</p>' +
'</div>' +
'<div style="display:flex;gap:4px;flex-wrap:wrap" id="mbCapFilters">' +
'<button class="tab-btn active" data-mb-filter="all">All (' + rows.length + ')</button>' +
'<button class="tab-btn" data-mb-filter="confirmed" style="--filter-color:var(--success, #22c55e)">✅ Confirmed (' + counts.confirmed + ')</button>' +
'<button class="tab-btn" data-mb-filter="suspected" style="--filter-color:var(--warning, #eab308)">⚠️ Suspected (' + counts.suspected + ')</button>' +
'<button class="tab-btn" data-mb-filter="unknown" style="--filter-color:var(--text-muted, #888)">❓ Unknown (' + counts.unknown + ')</button>' +
'</div>' +
'</div>' +
'<div id="mbAdoptersTableWrap">' + buildTableContent(rows, 'all') + '</div>' +
'</div></div>';
// Use setTimeout for event delegation on the stable section container
setTimeout(function() {
var section = document.getElementById('mbAdoptersSection');
if (!section) return;
var currentFilter = 'all';
section.addEventListener('click', function handler(e) {
var btn = e.target.closest('[data-mb-filter]');
if (btn) {
currentFilter = btn.dataset.mbFilter;
// Update active state on buttons (no DOM replacement needed)
var buttons = section.querySelectorAll('[data-mb-filter]');
buttons.forEach(function(b) { b.classList.toggle('active', b.dataset.mbFilter === currentFilter); });
// Replace only the table content, not the whole section
var wrap = section.querySelector('#mbAdoptersTableWrap');
if (wrap) wrap.innerHTML = buildTableContent(rows, currentFilter);
return;
}
var th = e.target.closest('[data-sort]');
if (th) {
var tbody = section.querySelector('tbody');
if (!tbody) return;
var sortRows = Array.from(tbody.querySelectorAll('tr'));
var col = th.dataset.sort;
var colIdx = { name: 0, status: 1, hashSize: 2, packets: 3, lastSeen: 4 };
var statusWeight = { 'confirmed': 0, 'suspected': 1, 'unknown': 2 };
sortRows.sort(function(a, b) {
var va = a.children[colIdx[col]] ? a.children[colIdx[col]].textContent.trim() : '';
var vb = b.children[colIdx[col]] ? b.children[colIdx[col]].textContent.trim() : '';
if (col === 'status') {
va = statusWeight[va.toLowerCase().split(' ').pop()] !== undefined ? statusWeight[va.toLowerCase().split(' ').pop()] : 2;
vb = statusWeight[vb.toLowerCase().split(' ').pop()] !== undefined ? statusWeight[vb.toLowerCase().split(' ').pop()] : 2;
}
if (col === 'hashSize' || col === 'packets') { va = parseInt(va) || 0; vb = parseInt(vb) || 0; }
if (va < vb) return -1;
if (va > vb) return 1;
return 0;
});
sortRows.forEach(function(r) { tbody.appendChild(r); });
}
});
}, 100);
return html;
}
// Legacy alias for tests — delegates to renderMultiByteAdopters with empty nodes
function renderMultiByteCapability(caps) {
if (!caps.length) return '';
// Convert caps to adopter-style rows for backward compat
var fakeNodes = caps.map(function(c) {
return { name: c.name, pubkey: c.pubkey, role: c.role, hashSize: c.maxHashSize, packets: 0, lastSeen: c.lastSeen };
});
return renderMultiByteAdopters(fakeNodes, caps);
}
async function renderCollisionTab(el, data, collisionData) {
el.innerHTML = `
<nav id="hashIssuesToc" style="display:flex;gap:12px;margin-bottom:12px;font-size:13px;flex-wrap:wrap">
@@ -1229,10 +1079,10 @@
else matrixDesc.textContent = '3-byte prefix space is too large to visualize as a matrix — collision table is shown below.';
}
renderHashMatrixFromServer(cData.by_size[String(bytes)], bytes);
// Show collision risk section for all byte sizes
// Hide collision risk card for 3-byte — stats are shown in the matrix panel
const riskCard = document.getElementById('collisionRiskSection');
if (riskCard) riskCard.style.display = '';
renderCollisionsFromServer(cData.by_size[String(bytes)], bytes);
if (riskCard) riskCard.style.display = bytes === 3 ? 'none' : '';
if (bytes !== 3) renderCollisionsFromServer(cData.by_size[String(bytes)], bytes);
}
// Wire up selector
@@ -1324,9 +1174,9 @@
<div class="analytics-stat-value" style="font-size:16px">${pctStr}%</div>
<div style="font-size:10px;color:var(--text-muted);margin-top:2px">${usedCount > 256 ? usedCount + ' of ' : 'of '}${spaceLabel} possible</div>
</div>
<div class="analytics-stat-card" style="flex:1;min-width:110px;border-color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--border)'}${collisionCount > 0 ? ';cursor:pointer' : ''}" ${collisionCount > 0 ? 'onclick="document.getElementById(\'collisionRiskSection\')?.scrollIntoView({behavior:\'smooth\',block:\'start\'})"' : ''} ${collisionCount > 0 ? 'title="Click to see collision details"' : ''}>
<div class="analytics-stat-card" style="flex:1;min-width:110px;border-color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--border)'}">
<div class="analytics-stat-label">Prefix collisions</div>
<div class="analytics-stat-value" style="color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--status-green)'}">${collisionCount}${collisionCount > 0 ? ' <span style="font-size:11px;opacity:0.7">▼</span>' : ''}</div>
<div class="analytics-stat-value" style="color:${collisionCount > 0 ? 'var(--status-red)' : 'var(--status-green)'}">${collisionCount}</div>
</div>
</div>`;
}
@@ -1401,7 +1251,7 @@
// 3-byte: show a summary panel instead of a matrix
if (bytes === 3) {
el.innerHTML = hashStatCardsHtml(totalNodes, stats.using_this_size || 0, '3-byte', 16777216, stats.unique_prefixes || 0, stats.collision_count || 0) +
`<p class="text-muted" style="margin:0;font-size:0.8em">The 3-byte prefix space (16.7M values) is too large to visualize as a grid.${(stats.collision_count || 0) > 0 ? ' See collision details below.' : ''}</p>` +
`<p class="text-muted" style="margin:0;font-size:0.8em">The 3-byte prefix space (16.7M values) is too large to visualize as a grid.</p>` +
`<p class="text-muted" style="margin:8px 0 0;font-size:0.8em">️ This tab only counts collisions among repeaters configured for this hash size. The <a href="#/analytics?tab=prefix-tool" style="color:var(--accent)">Prefix Tool</a> checks all repeaters regardless of configured hash size.</p>`;
return;
}
@@ -1971,18 +1821,15 @@
}
// Top hops leaderboard
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">Obs</th><th scope="col">Best SNR</th><th scope="col">Median SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
html += `<div class="analytics-section"><h3>🏆 Top 20 Longest Hops</h3><table class="data-table"><thead><tr><th scope="col">#</th><th scope="col">From</th><th scope="col">To</th><th scope="col">Distance (${distUnitLabel})</th><th scope="col">Type</th><th scope="col">SNR</th><th scope="col">Packet</th><th scope="col"></th></tr></thead><tbody>`;
const top20 = data.topHops.slice(0, 20);
top20.forEach((h, i) => {
const fromLink = h.fromPk ? `<a href="#/nodes/${encodeURIComponent(h.fromPk)}" class="analytics-link">${esc(h.fromName)}</a>` : esc(h.fromName || '?');
const toLink = h.toPk ? `<a href="#/nodes/${encodeURIComponent(h.toPk)}" class="analytics-link">${esc(h.toName)}</a>` : esc(h.toName || '?');
const bestSnr = h.bestSnr != null ? Number(h.bestSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
const medianSnr = h.medianSnr != null ? Number(h.medianSnr).toFixed(1) + ' dB' : '<span class="text-muted">—</span>';
const obs = h.obsCount != null ? h.obsCount : 1;
const snr = h.snr != null ? h.snr + ' dB' : '<span class="text-muted">—</span>';
const pktLink = h.hash ? `<a href="#/packet/${encodeURIComponent(h.hash)}" class="analytics-link mono" style="font-size:0.85em">${esc(h.hash.slice(0, 12))}…</a>` : '—';
const mapBtn = h.fromPk && h.toPk ? `<button class="btn-icon dist-map-hop" data-from="${esc(h.fromPk)}" data-to="${esc(h.toPk)}" title="View on map">🗺️</button>` : '';
const tsTitle = h.timestamp ? `Best observation: ${h.timestamp}` : '';
html += `<tr title="${esc(tsTitle)}"><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${obs}</td><td>${bestSnr}</td><td>${medianSnr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
html += `<tr><td>${i+1}</td><td>${fromLink}</td><td>${toLink}</td><td><strong>${formatDistance(h.dist)}</strong></td><td>${esc(h.type)}</td><td>${snr}</td><td>${pktLink}</td><td>${mapBtn}</td></tr>`;
});
html += `</tbody></table></div>`;
@@ -2035,10 +1882,6 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
window._analyticsChannelTbodyHtml = channelTbodyHtml;
window._analyticsChannelTheadHtml = channelTheadHtml;
window._analyticsRfNFColumnChart = rfNFColumnChart;
window._analyticsRenderMultiByteCapability = renderMultiByteCapability;
window._analyticsRenderMultiByteAdopters = renderMultiByteAdopters;
window._analyticsHashStatCardsHtml = hashStatCardsHtml;
window._analyticsRenderCollisionsFromServer = renderCollisionsFromServer;
}
// ─── Neighbor Graph Tab ─────────────────────────────────────────────────────
@@ -2053,8 +1896,8 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
<label style="font-size:13px">Roles:
<span id="ngRoleChecks" style="margin-left:4px"></span>
</label>
<label style="font-size:13px">Min Score: <input type="range" id="ngMinScore" min="0" max="100" value="70" style="width:100px;vertical-align:middle">
<span id="ngMinScoreVal">0.70</span>
<label style="font-size:13px">Min Score: <input type="range" id="ngMinScore" min="0" max="100" value="10" style="width:100px;vertical-align:middle">
<span id="ngMinScoreVal">0.10</span>
</label>
<label style="font-size:13px">Confidence:
<select id="ngConfidence" style="font-size:12px;padding:2px 4px">
@@ -2082,11 +1925,6 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
const color = (window.ROLE_COLORS || {})[r] || '#888';
rcEl.innerHTML += `<label style="font-size:12px;margin-right:8px"><input type="checkbox" data-role="${r}" checked> <span style="color:${esc(color)}">${esc(r)}</span></label>`;
});
// Observer checkbox — unchecked by default (observers create hub-and-spoke noise)
{
const color = (window.ROLE_COLORS || {}).observer || '#8b5cf6';
rcEl.innerHTML += `<label style="font-size:12px;margin-right:8px"><input type="checkbox" data-role="observer"> <span style="color:${esc(color)}">observer</span></label>`;
}
// Load data
const rqs = RegionFilter.regionQueryString();
@@ -2104,17 +1942,8 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
startGraphRenderer();
// Filter listeners
// Restore saved min score from localStorage
var savedScore = localStorage.getItem('ng-min-score');
if (savedScore !== null) {
document.getElementById('ngMinScore').value = savedScore;
document.getElementById('ngMinScoreVal').textContent = (savedScore / 100).toFixed(2);
applyNGFilters();
}
document.getElementById('ngMinScore').addEventListener('input', function() {
document.getElementById('ngMinScoreVal').textContent = (this.value / 100).toFixed(2);
localStorage.setItem('ng-min-score', this.value);
applyNGFilters();
});
document.getElementById('ngConfidence').addEventListener('change', applyNGFilters);
@@ -2153,7 +1982,7 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
// Filter nodes by role
const visibleNodes = _ngState.allNodes.filter(n => {
const role = (n.role || 'unknown').toLowerCase();
return checkedRoles.has(role) || role === 'unknown';
return checkedRoles.has(role) || role === 'unknown' || role === 'observer';
});
const visiblePKs = new Set(visibleNodes.map(n => n.pubkey));
@@ -3460,112 +3289,5 @@ function destroy() { _analyticsData = {}; _channelData = null; if (_ngState && _
return svg;
}
// #690 — Clock Health fleet view (M3)
async function renderClockHealthTab(el) {
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">Loading clock health data…</div>';
try {
var data = await (await fetch('/api/nodes/clock-skew')).json();
if (!Array.isArray(data) || !data.length) {
el.innerHTML = '<div class="text-center text-muted" style="padding:40px">No clock skew data available. Nodes need recent adverts for clock analysis.</div>';
return;
}
// State
var activeFilter = 'all';
var sortKey = 'severity';
var sortDir = 'asc'; // severity worst-first
function render() {
// Filter
var filtered = activeFilter === 'all' ? data : data.filter(function(n) { return n.severity === activeFilter; });
// Sort
filtered = filtered.slice().sort(function(a, b) {
var v;
if (sortKey === 'severity') {
v = (SKEW_SEVERITY_ORDER[a.severity] || 9) - (SKEW_SEVERITY_ORDER[b.severity] || 9);
} else if (sortKey === 'skew') {
v = Math.abs(window.currentSkewValue(b) || 0) - Math.abs(window.currentSkewValue(a) || 0);
} else if (sortKey === 'name') {
v = (a.nodeName || '').localeCompare(b.nodeName || '');
} else if (sortKey === 'drift') {
v = Math.abs(b.driftPerDaySec || 0) - Math.abs(a.driftPerDaySec || 0);
}
return sortDir === 'desc' ? -v : v;
});
// Summary
var counts = { ok: 0, warning: 0, critical: 0, absurd: 0 };
data.forEach(function(n) { if (counts[n.severity] !== undefined) counts[n.severity]++; });
// Filter buttons (also serve as summary — no separate stats pills needed)
var filterColors = { ok: 'var(--status-green)', warning: 'var(--status-yellow)', critical: 'var(--status-orange)', absurd: 'var(--status-purple)', no_clock: 'var(--text-muted)' };
var filters = ['all', 'ok', 'warning', 'critical', 'absurd', 'no_clock'];
var filterHtml = '<div style="margin-bottom:10px">' + filters.map(function(f) {
var dot = f !== 'all' ? '<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:' + filterColors[f] + ';margin-right:4px;vertical-align:middle"></span>' : '';
return '<button class="clock-filter-btn' + (activeFilter === f ? ' active' : '') + '" data-filter="' + f + '">' +
dot + (f === 'all' ? 'All (' + data.length + ')' : (SKEW_SEVERITY_LABELS[f] || f) + ' (' + (counts[f] || 0) + ')') +
'</button>';
}).join('') + '</div>';
// Table
var rowsHtml = filtered.map(function(n) {
var rowClass = 'clock-fleet-row--' + (n.severity || 'ok');
var lastAdv = n.lastObservedTS ? new Date(n.lastObservedTS * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC') : '—';
var skewVal = window.currentSkewValue(n);
var skewText = n.severity === 'no_clock' ? 'No Clock' : formatSkew(skewVal);
var driftText = n.severity === 'no_clock' || !n.driftPerDaySec ? '' : formatDrift(n.driftPerDaySec);
return '<tr class="' + rowClass + '" data-pubkey="' + esc(n.pubkey) + '" style="cursor:pointer">' +
'<td><strong>' + esc(n.nodeName || n.pubkey.slice(0, 12)) + '</strong></td>' +
'<td style="font-family:var(--mono,monospace)">' + skewText + '</td>' +
'<td>' + renderSkewBadge(n.severity, skewVal, n) + '</td>' +
'<td style="font-family:var(--mono,monospace)">' + driftText + '</td>' +
'<td style="font-size:11px">' + lastAdv + '</td>' +
'</tr>';
}).join('');
el.innerHTML = '<h3 style="margin:0 0 10px">⏰ Clock Health</h3>' +
filterHtml +
'<table class="data-table analytics-table" id="clock-health-table">' +
'<thead><tr>' +
'<th data-sort-col="name" style="cursor:pointer">Name</th>' +
'<th data-sort-col="skew" style="cursor:pointer">Skew</th>' +
'<th data-sort-col="severity" style="cursor:pointer">Severity</th>' +
'<th data-sort-col="drift" style="cursor:pointer">Drift Rate</th>' +
'<th>Last Advert</th>' +
'</tr></thead><tbody>' + rowsHtml + '</tbody></table>';
// Bind filter clicks
el.querySelectorAll('.clock-filter-btn').forEach(function(btn) {
btn.addEventListener('click', function() {
activeFilter = btn.dataset.filter;
render();
});
});
// Bind header sort clicks
el.querySelectorAll('[data-sort-col]').forEach(function(th) {
th.addEventListener('click', function() {
var col = th.dataset.sortCol;
if (sortKey === col) { sortDir = sortDir === 'asc' ? 'desc' : 'asc'; }
else { sortKey = col; sortDir = 'asc'; }
render();
});
});
// Bind row clicks → navigate to node
el.querySelectorAll('tr[data-pubkey]').forEach(function(tr) {
tr.addEventListener('click', function() {
location.hash = '#/nodes/' + encodeURIComponent(tr.dataset.pubkey);
});
});
}
render();
} catch (err) {
el.innerHTML = '<div class="text-center" style="color:var(--status-red);padding:40px">Failed to load clock health data: ' + esc(String(err)) + '</div>';
}
}
registerPage('analytics', { init, destroy });
})();
+1 -2
View File
@@ -10,8 +10,6 @@ function routeTypeName(n) { return ROUTE_TYPES[n] || 'UNKNOWN'; }
function payloadTypeName(n) { return PAYLOAD_TYPES[n] || 'UNKNOWN'; }
function payloadTypeColor(n) { return PAYLOAD_COLORS[n] || 'unknown'; }
function isTransportRoute(rt) { return rt === 0 || rt === 3; }
/** Byte offset of path_len in raw_hex: 5 for transport routes (4 bytes of next/last hop codes precede it), 1 otherwise. */
function getPathLenOffset(routeType) { return isTransportRoute(routeType) ? 5 : 1; }
function transportBadge(rt) { return isTransportRoute(rt) ? ' <span class="badge badge-transport" title="' + routeTypeName(rt) + '">T</span>' : ''; }
// --- Utilities ---
@@ -1029,6 +1027,7 @@ function makeColumnsResizable(tableSelector, storageKey) {
// Add resize handles
ths.forEach((th, i) => {
if (i === ths.length - 1) return;
th.style.position = 'relative';
const handle = document.createElement('div');
handle.className = 'col-resize-handle';
handle.addEventListener('mousedown', (e) => {
-295
View File
@@ -1,295 +0,0 @@
/**
* Client-side MeshCore channel decryption module.
*
* Implements the same crypto as internal/channel/channel.go:
* - Key derivation: SHA-256("#channelname")[:16]
* - Channel hash: SHA-256(key)[0]
* - MAC: HMAC-SHA256 with 32-byte secret (key + 16 zero bytes), truncated to 2 bytes
* - Encryption: AES-128-ECB (block-by-block)
* - Plaintext: timestamp(4 LE) + flags(1) + "sender: message\0"
*
* Keys NEVER leave the browser. No fetch/XHR/network calls in this module.
*/
/* eslint-disable no-var */
window.ChannelDecrypt = (function () {
'use strict';
var STORAGE_KEY = 'corescope_channel_keys';
var CACHE_KEY = 'corescope_channel_cache';
// ---- Hex utilities ----
function bytesToHex(bytes) {
var hex = '';
for (var i = 0; i < bytes.length; i++) {
hex += (bytes[i] < 16 ? '0' : '') + bytes[i].toString(16);
}
return hex;
}
function hexToBytes(hex) {
var bytes = new Uint8Array(hex.length / 2);
for (var i = 0; i < hex.length; i += 2) {
bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16);
}
return bytes;
}
// ---- Key derivation ----
/**
* Derive AES-128 key from channel name: SHA-256("#channelname")[:16].
* @param {string} channelName - e.g. "#LongFast"
* @returns {Promise<Uint8Array>} 16-byte key
*/
async function deriveKey(channelName) {
var enc = new TextEncoder();
var hash = await crypto.subtle.digest('SHA-256', enc.encode(channelName));
return new Uint8Array(hash).slice(0, 16);
}
/**
* Compute the 1-byte channel hash: SHA-256(key)[0].
* @param {Uint8Array} key - 16-byte key
* @returns {Promise<number>} single byte (0-255)
*/
async function computeChannelHash(key) {
var hash = await crypto.subtle.digest('SHA-256', key);
return new Uint8Array(hash)[0];
}
// ---- AES-128-ECB via Web Crypto (CBC with zero IV, block-by-block) ----
/**
* Decrypt AES-128-ECB by decrypting each 16-byte block independently
* using AES-CBC with a zero IV (equivalent to ECB for single blocks).
* @param {Uint8Array} key - 16-byte AES key
* @param {Uint8Array} ciphertext - must be multiple of 16 bytes
* @returns {Promise<Uint8Array>} plaintext
*/
async function decryptECB(key, ciphertext) {
if (ciphertext.length === 0 || ciphertext.length % 16 !== 0) {
return null;
}
var cryptoKey = await crypto.subtle.importKey(
'raw', key, { name: 'AES-CBC' }, false, ['decrypt']
);
var zeroIV = new Uint8Array(16);
var plaintext = new Uint8Array(ciphertext.length);
for (var i = 0; i < ciphertext.length; i += 16) {
var block = ciphertext.slice(i, i + 16);
// Append a dummy block (16 bytes of 0x10 = PKCS7 padding for empty next block)
// so Web Crypto doesn't complain about padding
var padded = new Uint8Array(32);
padded.set(block, 0);
// Second block is PKCS7 padding: 16 bytes of 0x10
for (var j = 16; j < 32; j++) padded[j] = 16;
var decrypted = await crypto.subtle.decrypt(
{ name: 'AES-CBC', iv: zeroIV }, cryptoKey, padded
);
var decBytes = new Uint8Array(decrypted);
plaintext.set(decBytes.slice(0, 16), i);
}
return plaintext;
}
// ---- MAC verification ----
/**
* Verify HMAC-SHA256 MAC (first 2 bytes) using 32-byte secret (key + 16 zero bytes).
* @param {Uint8Array} key - 16-byte AES key
* @param {Uint8Array} ciphertext - encrypted data
* @param {string} macHex - 4-char hex string (2 bytes)
* @returns {Promise<boolean>}
*/
async function verifyMAC(key, ciphertext, macHex) {
// Build 32-byte channel secret: key + 16 zero bytes
var secret = new Uint8Array(32);
secret.set(key, 0);
// remaining 16 bytes are already 0
var cryptoKey = await crypto.subtle.importKey(
'raw', secret, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']
);
var sig = await crypto.subtle.sign('HMAC', cryptoKey, ciphertext);
var sigBytes = new Uint8Array(sig);
var macBytes = hexToBytes(macHex);
return sigBytes[0] === macBytes[0] && sigBytes[1] === macBytes[1];
}
// ---- Plaintext parsing ----
/**
* Parse decrypted plaintext: timestamp(4 LE) + flags(1) + "sender: message\0..."
* @param {Uint8Array} plaintext
* @returns {{ timestamp: number, flags: number, sender: string, message: string } | null}
*/
function parsePlaintext(plaintext) {
if (!plaintext || plaintext.length < 5) return null;
var timestamp = plaintext[0] | (plaintext[1] << 8) | (plaintext[2] << 16) | ((plaintext[3] << 24) >>> 0);
var flags = plaintext[4];
// Extract text up to first null byte
var textBytes = plaintext.slice(5);
var nullIdx = -1;
for (var i = 0; i < textBytes.length; i++) {
if (textBytes[i] === 0) { nullIdx = i; break; }
}
var text = new TextDecoder().decode(nullIdx >= 0 ? textBytes.slice(0, nullIdx) : textBytes);
// Count non-printable characters
var nonPrintable = 0;
for (var c = 0; c < text.length; c++) {
var code = text.charCodeAt(c);
if (code < 32 && code !== 10 && code !== 13 && code !== 9) nonPrintable++;
}
if (nonPrintable > 2) return null;
// Parse "sender: message" format
var colonIdx = text.indexOf(': ');
if (colonIdx > 0 && colonIdx < 50) {
var potentialSender = text.substring(0, colonIdx);
if (potentialSender.indexOf(':') < 0 && potentialSender.indexOf('[') < 0 && potentialSender.indexOf(']') < 0) {
return { timestamp: timestamp, flags: flags, sender: potentialSender, message: text.substring(colonIdx + 2) };
}
}
return { timestamp: timestamp, flags: flags, sender: '', message: text };
}
// ---- Full decrypt pipeline ----
/**
* Verify MAC, decrypt, and parse a single packet.
* @param {Uint8Array} keyBytes - 16-byte key
* @param {string} macHex - 4-char hex MAC
* @param {string} encryptedHex - hex-encoded ciphertext
* @returns {Promise<{ sender: string, message: string, timestamp: number } | null>}
*/
async function decrypt(keyBytes, macHex, encryptedHex) {
var ciphertext = hexToBytes(encryptedHex);
if (ciphertext.length === 0 || ciphertext.length % 16 !== 0) return null;
var macOk = await verifyMAC(keyBytes, ciphertext, macHex);
if (!macOk) return null;
var plaintext = await decryptECB(keyBytes, ciphertext);
if (!plaintext) return null;
return parsePlaintext(plaintext);
}
// Alias used by channels.js
var decryptPacket = decrypt;
// ---- Key storage (localStorage) ----
function saveKey(channelName, keyHex) {
var keys = getKeys();
keys[channelName] = keyHex;
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
}
// Alias used by channels.js
var storeKey = saveKey;
function getKeys() {
try {
var raw = localStorage.getItem(STORAGE_KEY);
return raw ? JSON.parse(raw) : {};
} catch (e) { return {}; }
}
// Alias used by channels.js
var getStoredKeys = getKeys;
function removeKey(channelName) {
var keys = getKeys();
delete keys[channelName];
try { localStorage.setItem(STORAGE_KEY, JSON.stringify(keys)); } catch (e) { /* quota */ }
// Also clear cached messages for this channel
clearChannelCache(channelName);
}
/** Remove cached messages for a specific channel (by name or hash). */
function clearChannelCache(channelKey) {
try {
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
delete cache[channelKey];
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
} catch (e) { /* quota */ }
}
// ---- Message cache (localStorage) ----
function cacheMessages(channelHash, messages) {
try {
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
cache[channelHash] = { messages: messages, ts: Date.now() };
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
} catch (e) { /* quota */ }
}
function getCachedMessages(channelHash) {
try {
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
var entry = cache[channelHash];
return entry ? entry.messages : null;
} catch (e) { return null; }
}
// Cache with lastTimestamp and count (used by channels.js via getCache/setCache)
var MAX_CACHED_MESSAGES = 1000;
function setCache(key, messages, lastTimestamp, totalCount) {
try {
// Enforce cache size limit: only keep most recent MAX_CACHED_MESSAGES
var toStore = messages;
if (messages.length > MAX_CACHED_MESSAGES) {
toStore = messages.slice(messages.length - MAX_CACHED_MESSAGES);
}
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
cache[key] = {
messages: toStore,
lastTimestamp: lastTimestamp,
count: totalCount || toStore.length,
ts: Date.now()
};
localStorage.setItem(CACHE_KEY, JSON.stringify(cache));
} catch (e) { /* quota */ }
}
function getCache(key) {
try {
var cache = JSON.parse(localStorage.getItem(CACHE_KEY) || '{}');
return cache[key] || null;
} catch (e) { return null; }
}
return {
deriveKey: deriveKey,
decrypt: decrypt,
decryptPacket: decryptPacket,
decryptECB: decryptECB,
verifyMAC: verifyMAC,
parsePlaintext: parsePlaintext,
computeChannelHash: computeChannelHash,
bytesToHex: bytesToHex,
hexToBytes: hexToBytes,
saveKey: saveKey,
storeKey: storeKey,
getKeys: getKeys,
getStoredKeys: getStoredKeys,
removeKey: removeKey,
clearChannelCache: clearChannelCache,
cacheMessages: cacheMessages,
getCachedMessages: getCachedMessages,
setCache: setCache,
getCache: getCache
};
})();
+11 -491
View File
@@ -318,290 +318,6 @@
let regionChangeHandler = null;
// --- Client-side channel decryption (#725 M2) ---
// Check if input is a valid hex string (32 hex chars = 16 bytes)
function isHexKey(val) {
return /^[0-9a-fA-F]{32}$/.test(val);
}
// Show status message in the add-channel form (#759)
var statusTimer = null;
function showAddStatus(msg, type) {
var el = document.getElementById('chAddStatus');
if (!el) return;
el.textContent = msg;
el.className = 'ch-add-status ch-add-status--' + (type || 'info');
el.style.display = '';
clearTimeout(statusTimer);
if (type !== 'loading') {
statusTimer = setTimeout(function () { el.style.display = 'none'; }, 5000);
}
}
// Add a user channel by name (#channelname) or hex key
async function addUserChannel(val) {
var displayName = val.startsWith('#') ? val : (isHexKey(val) ? val.substring(0, 8) + '…' : '#' + val);
showAddStatus('Decrypting ' + displayName + ' messages…', 'loading');
var channelName, keyHex;
try {
if (val.startsWith('#')) {
channelName = val;
var keyBytes = await ChannelDecrypt.deriveKey(channelName);
keyHex = ChannelDecrypt.bytesToHex(keyBytes);
} else if (isHexKey(val)) {
keyHex = val.toLowerCase();
channelName = 'psk:' + keyHex.substring(0, 8);
} else {
// Try with # prefix if user forgot
channelName = '#' + val;
var keyBytes2 = await ChannelDecrypt.deriveKey(channelName);
keyHex = ChannelDecrypt.bytesToHex(keyBytes2);
}
ChannelDecrypt.storeKey(channelName, keyHex);
// Compute channel hash byte to find matching encrypted channels
var keyBytes3 = ChannelDecrypt.hexToBytes(keyHex);
var hashByte = await ChannelDecrypt.computeChannelHash(keyBytes3);
// Add to sidebar or merge with existing encrypted channel
mergeUserChannels();
renderChannelList();
// Auto-select and start decrypting
var targetHash = 'user:' + channelName;
// Check if there's an existing encrypted channel with this hash byte
var existingEncrypted = channels.find(function (ch) {
return ch.encrypted && String(ch.hash) === String(hashByte);
});
if (existingEncrypted) {
targetHash = existingEncrypted.hash;
}
await selectChannel(targetHash, { userKey: keyHex, channelHashByte: hashByte, channelName: channelName });
// Show success feedback (#759)
var msgCount = document.querySelectorAll('#chMessages .ch-msg').length;
var userDisplay = channelName.startsWith('psk:') ? 'Custom channel (' + channelName.substring(4) + ')' : channelName;
if (msgCount > 0) {
showAddStatus('Added ' + userDisplay + ' — ' + msgCount + ' messages decrypted', 'success');
} else {
showAddStatus('No messages found for ' + userDisplay, 'warn');
}
} catch (err) {
showAddStatus('Failed to decrypt', 'error');
}
}
// Merge user-stored keys into the channel list
function mergeUserChannels() {
var keys = ChannelDecrypt.getStoredKeys();
var names = Object.keys(keys);
for (var i = 0; i < names.length; i++) {
var name = names[i];
// Check if channel already exists by name
var exists = channels.some(function (ch) {
return ch.name === name || ch.hash === name || ch.hash === ('user:' + name);
});
if (!exists) {
channels.push({
hash: 'user:' + name,
name: name,
messageCount: 0,
lastActivityMs: 0,
lastSender: '',
lastMessage: 'Encrypted — click to decrypt',
encrypted: true,
userAdded: true
});
}
}
}
// Fetch and decrypt GRP_TXT packets client-side (M5: delta fetch + cache)
async function fetchAndDecryptChannel(keyHex, channelHashByte, channelName, opts) {
opts = opts || {};
var keyBytes = ChannelDecrypt.hexToBytes(keyHex);
// M5: Check cache first — serve cached messages immediately
var cacheKey = channelName || String(channelHashByte);
var cached = ChannelDecrypt.getCache(cacheKey);
var cachedMsgs = cached ? cached.messages : [];
var lastTs = cached ? cached.lastTimestamp : '';
var cachedCount = cached ? (cached.count || 0) : 0;
// If we have cached messages and caller wants instant render, return them first
if (cachedMsgs.length > 0 && !opts.forceFullDecrypt) {
// Signal caller to render cache immediately, then do delta fetch
if (opts.onCacheHit) opts.onCacheHit(cachedMsgs);
}
// Fetch packets from API — get all payload_type=5 (GRP_TXT/CHAN)
var rp = RegionFilter.getRegionParam();
var qs = rp ? '&region=' + encodeURIComponent(rp) : '';
var data;
try {
data = await api('/packets?limit=1000&payloadType=5' + qs, { ttl: 10000 });
} catch (e) {
return { messages: cachedMsgs, error: 'Failed to fetch packets: ' + e.message, fromCache: cachedMsgs.length > 0 };
}
var packets = data.packets || [];
// Filter for GRP_TXT (encrypted) packets matching our channel hash byte
var candidates = [];
for (var i = 0; i < packets.length; i++) {
var p = packets[i];
var dj;
try { dj = typeof p.decoded_json === 'string' ? JSON.parse(p.decoded_json) : p.decoded_json; }
catch (e) { continue; }
if (!dj) continue;
if (dj.type === 'CHAN' && dj.channel === channelName) {
candidates.push({ type: 'already_decrypted', decoded: dj, packet: p });
} else if (dj.type === 'GRP_TXT' && dj.encryptedData && dj.mac) {
if (dj.channelHash === channelHashByte) {
candidates.push({ type: 'encrypted', decoded: dj, packet: p });
}
}
}
// M5: Cache invalidation — if total candidate count changed, re-decrypt everything
var totalCandidates = candidates.length;
var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt;
// M5: Delta fetch — only decrypt packets newer than lastTs
if (!needFullDecrypt && cachedMsgs.length > 0 && lastTs) {
// Filter candidates to only those newer than cached lastTimestamp
var newCandidates = candidates.filter(function (c) {
var ts = c.packet.first_seen || c.packet.timestamp || '';
return ts > lastTs;
});
if (newCandidates.length === 0) {
// Nothing new — return cache as-is
return { messages: cachedMsgs, fromCache: true };
}
// Decrypt only new candidates
var newDecrypted = await decryptCandidates(keyBytes, newCandidates);
if (newDecrypted.wrongKey) {
return { messages: cachedMsgs, wrongKey: true };
}
// Merge: cached + new, deduplicate by packetHash, sort chronologically
var merged = deduplicateAndMerge(cachedMsgs, newDecrypted.messages);
var newLastTs = merged.length ? merged[merged.length - 1].timestamp : lastTs;
ChannelDecrypt.setCache(cacheKey, merged, newLastTs, totalCandidates);
return { messages: merged, deltaCount: newDecrypted.messages.length };
}
if (candidates.length === 0) {
return { messages: cachedMsgs, empty: true };
}
// Full decrypt
var result = await decryptCandidates(keyBytes, candidates);
if (result.wrongKey) {
return { messages: result.messages, wrongKey: true };
}
var decrypted = result.messages;
// Sort chronologically (oldest first)
decrypted.sort(function (a, b) {
var ta = a.timestamp || '';
var tb = b.timestamp || '';
return ta.localeCompare(tb);
});
// M5: Cache results
var newLastTimestamp = decrypted.length ? decrypted[decrypted.length - 1].timestamp : '';
ChannelDecrypt.setCache(cacheKey, decrypted, newLastTimestamp, totalCandidates);
return { messages: decrypted };
}
/** Decrypt an array of candidate packets. Returns { messages, wrongKey }. */
async function decryptCandidates(keyBytes, candidates) {
// Sort newest first for progressive rendering
candidates.sort(function (a, b) {
var ta = a.packet.first_seen || a.packet.timestamp || '';
var tb = b.packet.first_seen || b.packet.timestamp || '';
return tb.localeCompare(ta);
});
var decrypted = [];
var macFailCount = 0;
var macCheckCount = 0;
for (var j = 0; j < candidates.length; j++) {
var c = candidates[j];
if (c.type === 'already_decrypted') {
var d = c.decoded;
var sender = d.sender || 'Unknown';
var text = d.text || '';
var ci = text.indexOf(': ');
if (ci > 0 && ci < 50 && text.substring(0, ci) === sender) {
text = text.substring(ci + 2);
}
decrypted.push({
sender: sender, text: text,
timestamp: c.packet.first_seen || c.packet.timestamp,
sender_timestamp: d.sender_timestamp || null,
packetHash: c.packet.hash, packetId: c.packet.id,
hops: d.path_len || 0, snr: c.packet.snr || null,
observers: c.packet.observer_name ? [c.packet.observer_name] : [],
repeats: 1
});
continue;
}
macCheckCount++;
var result = await ChannelDecrypt.decryptPacket(keyBytes, c.decoded.mac, c.decoded.encryptedData);
if (result) {
macFailCount = 0;
decrypted.push({
sender: result.sender, text: result.message,
timestamp: c.packet.first_seen || c.packet.timestamp,
sender_timestamp: result.timestamp || null,
packetHash: c.packet.hash, packetId: c.packet.id,
hops: 0, snr: c.packet.snr || null,
observers: c.packet.observer_name ? [c.packet.observer_name] : [],
repeats: 1
});
} else {
macFailCount++;
if (macCheckCount >= 10 && macFailCount >= macCheckCount) {
return { messages: decrypted, wrongKey: true };
}
}
}
return { messages: decrypted, wrongKey: false };
}
/** Merge cached and new messages, deduplicate by packetHash, sort chronologically. */
function deduplicateAndMerge(cached, newMsgs) {
var seen = {};
var merged = [];
// Add cached first
for (var i = 0; i < cached.length; i++) {
var key = cached[i].packetHash || ('idx:' + i);
if (!seen[key]) { seen[key] = true; merged.push(cached[i]); }
}
// Add new
for (var j = 0; j < newMsgs.length; j++) {
var key2 = newMsgs[j].packetHash || ('new:' + j);
if (!seen[key2]) { seen[key2] = true; merged.push(newMsgs[j]); }
}
merged.sort(function (a, b) {
var ta = a.timestamp || '';
var tb = b.timestamp || '';
return ta.localeCompare(tb);
});
return merged;
}
function init(app, routeParam) {
var _initUrlParams = getHashParams();
var _pendingNode = _initUrlParams.get('node');
@@ -610,21 +326,6 @@
<div class="ch-sidebar" aria-label="Channel list">
<div class="ch-sidebar-header">
<div class="ch-sidebar-title"><span class="ch-icon">💬</span> Channels</div>
<label class="ch-encrypted-toggle" title="Show encrypted channels (no key configured)">
<input type="checkbox" id="chShowEncrypted"> <span class="ch-toggle-label">🔒 No key</span>
</label>
</div>
<div class="ch-key-input-wrap" style="padding:4px 8px">
<form id="chKeyForm" autocomplete="off" class="ch-add-form">
<div class="ch-add-row">
<input type="text" id="chKeyInput" class="ch-key-input"
placeholder="#channelname"
aria-label="Channel name or hex key" spellcheck="false">
<button type="submit" class="ch-add-btn" title="Add channel">+</button>
</div>
<div class="ch-add-hint">e.g. #LongFast or 32-char hex key decrypted in your browser.</div>
<div id="chAddStatus" class="ch-add-status" style="display:none"></div>
</form>
</div>
<div id="chRegionFilter" class="region-filter-container" style="padding:0 8px"></div>
<div class="ch-channel-list" id="chList" role="listbox" aria-label="Channels">
@@ -646,17 +347,6 @@
</div>`;
RegionFilter.init(document.getElementById('chRegionFilter'));
// Encrypted channels toggle (#727)
var showEncryptedCb = document.getElementById('chShowEncrypted');
var showEncrypted = localStorage.getItem('channels-show-encrypted') === 'true';
showEncryptedCb.checked = showEncrypted;
showEncryptedCb.addEventListener('change', function () {
showEncrypted = showEncryptedCb.checked;
localStorage.setItem('channels-show-encrypted', showEncrypted ? 'true' : 'false');
loadChannels(true);
});
regionChangeHandler = RegionFilter.onChange(function () {
loadChannels(true).then(async function () {
if (!selectedHash) return;
@@ -664,38 +354,8 @@
});
});
// Channel key input handler (#725 M2, improved UX #759)
var chKeyForm = document.getElementById('chKeyForm');
if (chKeyForm) {
var submitHandler = async function (e) {
e.preventDefault();
var input = document.getElementById('chKeyInput');
var val = (input.value || '').trim();
if (!val) return;
input.value = '';
await addUserChannel(val);
};
chKeyForm.addEventListener('submit', submitHandler);
var chKeyInput = document.getElementById('chKeyInput');
if (chKeyInput) {
chKeyInput.addEventListener('focus', function () {
var st = document.getElementById('chAddStatus');
if (st) { st.style.display = 'none'; clearTimeout(statusTimer); statusTimer = null; }
});
}
}
// Auto-enable encrypted toggle if deep-linking to an encrypted channel
if (routeParam && routeParam.startsWith('enc_') && !showEncrypted) {
showEncrypted = true;
showEncryptedCb.checked = true;
localStorage.setItem('channels-show-encrypted', 'true');
}
loadObserverRegions();
loadChannels().then(async function () {
// Also load user-added encrypted channels into the sidebar
mergeUserChannels();
if (routeParam) await selectChannel(routeParam);
if (_pendingNode && _pendingNode.length < 200) await showNodeDetail(_pendingNode);
});
@@ -743,29 +403,6 @@
// Event delegation for channel selection (touch-friendly)
document.getElementById('chList').addEventListener('click', (e) => {
// M4: Remove channel button
const removeBtn = e.target.closest('[data-remove-channel]');
if (removeBtn) {
e.stopPropagation();
var channelHash = removeBtn.getAttribute('data-remove-channel');
if (!channelHash) return;
var chName = channelHash.startsWith('user:') ? channelHash.substring(5) : channelHash;
if (!confirm('Remove channel "' + chName + '"? This will clear saved keys and cached messages.')) return;
ChannelDecrypt.removeKey(chName);
// Remove from channels array
channels = channels.filter(function (c) { return c.hash !== channelHash; });
if (selectedHash === channelHash) {
selectedHash = null;
messages = [];
history.replaceState(null, '', '#/channels');
var msgEl2 = document.getElementById('chMessages');
if (msgEl2) msgEl2.innerHTML = '<div class="ch-empty">Choose a channel from the sidebar to view messages</div>';
var header2 = document.getElementById('chHeader');
if (header2) header2.querySelector('.ch-header-text').textContent = 'Select a channel';
}
renderChannelList();
return;
}
// Color dot click — open picker, don't select channel
const dot = e.target.closest('.ch-color-dot');
if (dot && window.ChannelColorPicker) {
@@ -1015,11 +652,7 @@
async function loadChannels(silent) {
try {
const rp = RegionFilter.getRegionParam();
var showEnc = localStorage.getItem('channels-show-encrypted') === 'true';
var params = [];
if (rp) params.push('region=' + encodeURIComponent(rp));
if (showEnc) params.push('includeEncrypted=true');
const qs = params.length ? '?' + params.join('&') : '';
const qs = rp ? '?region=' + encodeURIComponent(rp) : '';
const data = await api('/channels' + qs, { ttl: CLIENT_TTL.channels });
channels = (data.channels || []).map(ch => {
ch.lastActivityMs = ch.lastActivity ? new Date(ch.lastActivity).getTime() : 0;
@@ -1046,33 +679,27 @@
});
el.innerHTML = sorted.map(ch => {
const isEncrypted = ch.encrypted === true;
const name = isEncrypted ? (ch.name || 'Unknown') : (ch.name || `Channel ${formatHashHex(ch.hash)}`);
const color = isEncrypted ? 'var(--text-muted, #6b7280)' : getChannelColor(ch.hash);
const name = ch.name || `Channel ${formatHashHex(ch.hash)}`;
const color = getChannelColor(ch.hash);
const time = ch.lastActivityMs ? formatSecondsAgo(Math.floor((Date.now() - ch.lastActivityMs) / 1000)) : '';
const preview = isEncrypted
? `${ch.messageCount} encrypted messages (no key configured)`
: ch.lastSender && ch.lastMessage
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
: `${ch.messageCount} messages`;
const preview = ch.lastSender && ch.lastMessage
? `${ch.lastSender}: ${truncate(ch.lastMessage, 28)}`
: `${ch.messageCount} messages`;
const sel = selectedHash === ch.hash ? ' selected' : '';
const encClass = isEncrypted ? ' ch-encrypted' : '';
const abbr = isEncrypted ? '🔒' : (name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase());
const abbr = name.startsWith('#') ? name.slice(0, 3) : name.slice(0, 2).toUpperCase();
// Channel color dot for color picker (#674)
const chColor = window.ChannelColors ? window.ChannelColors.get(ch.hash) : null;
const dotStyle = chColor ? ` style="background:${chColor}"` : '';
// Left border for assigned color
const borderStyle = chColor ? ` style="border-left:3px solid ${chColor}"` : '';
// M4: Remove button for user-added channels
const removeBtn = ch.userAdded ? ' <button class="ch-remove-btn" data-remove-channel="' + escapeHtml(ch.hash) + '" title="Remove channel" aria-label="Remove ' + escapeHtml(name) + '">✕</button>' : '';
return `<button class="ch-item${sel}${encClass}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}"${isEncrypted ? ' data-encrypted="true"' : ''}>
<div class="ch-badge" style="background:${color}" aria-hidden="true">${isEncrypted ? '🔒' : escapeHtml(abbr)}</div>
return `<button class="ch-item${sel}" data-hash="${ch.hash}"${borderStyle} type="button" role="option" aria-selected="${selectedHash === ch.hash ? 'true' : 'false'}" aria-label="${escapeHtml(name)}">
<div class="ch-badge" style="background:${color}" aria-hidden="true">${escapeHtml(abbr)}</div>
<div class="ch-item-body">
<div class="ch-item-top">
<span class="ch-item-name">${escapeHtml(name)}</span>
<span class="ch-color-dot" data-channel="${escapeHtml(ch.hash)}"${dotStyle} title="Change channel color" aria-label="Change color for ${escapeHtml(name)}"></span>
<span class="ch-item-time" data-channel-hash="${ch.hash}">${time}</span>${removeBtn}
<span class="ch-item-time" data-channel-hash="${ch.hash}">${time}</span>
</div>
<div class="ch-item-preview">${escapeHtml(preview)}</div>
</div>
@@ -1080,7 +707,7 @@
}).join('');
}
async function selectChannel(hash, decryptOpts) {
async function selectChannel(hash) {
const rp = RegionFilter.getRegionParam() || '';
const request = beginMessageRequest(hash, rp);
selectedHash = hash;
@@ -1095,110 +722,6 @@
document.querySelector('.ch-layout')?.classList.add('ch-show-main');
const msgEl = document.getElementById('chMessages');
// Shared helper: fetch, decrypt, and render messages for a channel key (M5: cache-first)
async function decryptAndRender(keyHex, channelHashByte, channelName) {
msgEl.innerHTML = '<div class="ch-loading">Decrypting messages…</div>';
var result = await fetchAndDecryptChannel(keyHex, channelHashByte, channelName, {
onCacheHit: function (cachedMsgs) {
// M5: Render cached messages immediately while delta fetch runs
messages = cachedMsgs;
if (messages.length > 0) {
header.querySelector('.ch-header-text').textContent = name + ' — ' + messages.length + ' messages (cached)';
renderMessages();
scrollToBottom();
}
}
});
if (isStaleMessageRequest(request)) return true;
if (result.wrongKey) {
msgEl.innerHTML = '<div class="ch-empty ch-wrong-key">🔒 Key does not match — no messages could be decrypted</div>';
return true;
}
if (result.error) {
msgEl.innerHTML = '<div class="ch-empty">' + escapeHtml(result.error) + '</div>';
return true;
}
messages = result.messages || [];
if (messages.length === 0) {
msgEl.innerHTML = '<div class="ch-empty">No encrypted messages found for this channel</div>';
} else {
header.querySelector('.ch-header-text').textContent = `${name}${messages.length} messages (decrypted)`;
renderMessages();
scrollToBottom();
}
return true;
}
// Client-side decryption path (#725 M2)
if (decryptOpts && decryptOpts.userKey) {
await decryptAndRender(decryptOpts.userKey, decryptOpts.channelHashByte, decryptOpts.channelName);
return;
}
// Check if this is a user-added channel that needs decryption
var storedKeys = typeof ChannelDecrypt !== 'undefined' ? ChannelDecrypt.getStoredKeys() : {};
if (hash.startsWith('user:')) {
var chName = hash.substring(5);
if (storedKeys[chName]) {
var keyHex = storedKeys[chName];
var keyBytes = ChannelDecrypt.hexToBytes(keyHex);
var hashByte = await ChannelDecrypt.computeChannelHash(keyBytes);
await decryptAndRender(keyHex, hashByte, chName);
return;
}
}
// Also check if an encrypted channel hash matches a stored key
if (ch && ch.encrypted) {
for (var kn in storedKeys) {
var kh = storedKeys[kn];
var kb = ChannelDecrypt.hexToBytes(kh);
var hb = await ChannelDecrypt.computeChannelHash(kb);
if (String(hb) === String(hash) || String(ch.hash) === String(hb)) {
await decryptAndRender(kh, hb, kn);
return;
}
}
// #781: No matching key found — show lock message instead of fetching gibberish
msgEl.innerHTML = '<div class="ch-empty">🔒 This channel is encrypted and no decryption key is configured</div>';
return;
}
// #811: Deep link to a `#`-named channel that's not in the loaded list.
// If a stored key matches, decrypt. Otherwise we must distinguish an
// encrypted-no-key channel (show lock) from an unencrypted channel that
// simply isn't in the toggle-off list (#825 — must fall through to REST).
if (hash.charAt(0) === '#') {
if (storedKeys[hash]) {
var keyHex2 = storedKeys[hash];
var keyBytes2 = ChannelDecrypt.hexToBytes(keyHex2);
var hashByte2 = await ChannelDecrypt.computeChannelHash(keyBytes2);
await decryptAndRender(keyHex2, hashByte2, hash);
return;
}
// #825: confirm encrypted-ness via an encrypted-included channel list
// before assuming a lock state. Conservative on error — fall through.
// Show a loading affordance so cold deep links don't display stale content
// for the duration of the metadata RTT (cached 15s thereafter).
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
try {
var rpInc = RegionFilter.getRegionParam();
var paramsInc = ['includeEncrypted=true'];
if (rpInc) paramsInc.push('region=' + encodeURIComponent(rpInc));
var allCh = await api('/channels?' + paramsInc.join('&'), { ttl: CLIENT_TTL.channels });
if (isStaleMessageRequest(request)) return;
var foundCh = (allCh.channels || []).find(function (c) { return c.hash === hash; });
if (foundCh && foundCh.encrypted === true) {
msgEl.innerHTML = '<div class="ch-empty">🔒 This channel is encrypted and no decryption key is configured</div>';
return;
}
// Unencrypted (or unknown) — fall through to the REST fetch below.
} catch (e) {
// ignore — fall through to REST fetch
}
}
msgEl.innerHTML = '<div class="ch-loading">Loading messages…</div>';
try {
@@ -1220,9 +743,6 @@
async function refreshMessages(opts) {
if (!selectedHash) return;
// Skip refresh for encrypted channels — no messages to fetch
var selCh = channels.find(function (c) { return c.hash === selectedHash; });
if (selCh && selCh.encrypted) return;
opts = opts || {};
const msgEl = document.getElementById('chMessages');
if (!msgEl) return;
-4
View File
@@ -1173,10 +1173,6 @@
'<details style="margin-top:12px"><summary style="font-size:12px;font-weight:600;cursor:pointer;color:var(--text-muted)">Raw JSON</summary>' +
'<textarea id="cv2ExportJson" style="width:100%;min-height:200px;font-family:var(--mono);font-size:12px;background:var(--surface-1);border:1px solid var(--border);border-radius:6px;padding:12px;color:var(--text);resize:vertical;box-sizing:border-box;margin-top:8px">' + esc(json) + '</textarea>' +
'</details>' +
'<p class="cust-section-title" style="margin-top:20px">Tools</p>' +
'<p style="font-size:12px;color:var(--text-muted);margin-bottom:10px">Server-side configuration helpers.</p>' +
'<a href="/geofilter-builder.html" target="_blank" style="display:inline-block;padding:7px 14px;background:var(--surface-1);border:1px solid var(--border);border-radius:6px;color:var(--accent);font-size:13px;text-decoration:none;font-weight:500">🗺️ GeoFilter Builder →</a>' +
'<p style="font-size:11px;color:var(--text-muted);margin-top:6px">Draw a polygon on the map to generate a <code style="font-family:var(--mono)">geo_filter</code> block for <code style="font-family:var(--mono)">config.json</code>.</p>' +
'</div>';
}
-216
View File
@@ -1,216 +0,0 @@
/* drag-manager.js Free-form panel dragging (#608 M1)
* State machine: IDLE PENDING DRAGGING IDLE
* Pointer events on .panel-header, transform: translate() during drag,
* snap-to-edge on release, z-index on focus, viewport % persistence.
*/
(function () {
'use strict';
var DEAD_ZONE = 5; // px — disambiguate click vs drag
var SNAP_THRESHOLD = 20; // px — snap to edge on release
var SNAP_MARGIN = 12; // px — margin when snapped
function DragManager() {
this.state = 'IDLE';
this.activePanel = null;
this.startX = 0;
this.startY = 0;
this.panelStartX = 0;
this.panelStartY = 0;
this.preTransform = '';
this.enabled = true;
this.zCounter = 1000;
this._panels = [];
this._onKeyDown = this._handleKeyDown.bind(this);
}
DragManager.prototype.register = function (panel) {
if (!panel) return;
var header = panel.querySelector('.panel-header');
if (!header) return;
this._panels.push(panel);
var self = this;
header.addEventListener('pointerdown', function (e) {
if (!self.enabled) return;
if (e.button !== 0) return;
if (e.target.closest('button')) return;
e.preventDefault();
header.setPointerCapture(e.pointerId);
self.state = 'PENDING';
self.activePanel = panel;
self.startX = e.clientX;
self.startY = e.clientY;
var rect = panel.getBoundingClientRect();
self.panelStartX = rect.left;
self.panelStartY = rect.top;
self.preTransform = panel.style.transform || '';
document.addEventListener('keydown', self._onKeyDown);
});
header.addEventListener('pointermove', function (e) {
if (self.state === 'IDLE') return;
if (self.activePanel !== panel) return;
var dx = e.clientX - self.startX;
var dy = e.clientY - self.startY;
if (self.state === 'PENDING') {
if (Math.hypot(dx, dy) < DEAD_ZONE) return;
self.state = 'DRAGGING';
panel.classList.add('is-dragging');
panel.style.zIndex = ++self.zCounter;
self._detachFromCorner(panel);
}
panel.style.transform = 'translate(' + dx + 'px, ' + dy + 'px)';
});
header.addEventListener('pointerup', function (e) {
if (self.activePanel !== panel) return;
header.releasePointerCapture(e.pointerId);
if (self.state === 'DRAGGING') {
panel.classList.remove('is-dragging');
self._finalizePosition(panel);
}
self._reset();
});
header.addEventListener('pointercancel', function () {
if (self.activePanel !== panel) return;
panel.classList.remove('is-dragging');
if (self.state === 'DRAGGING') {
self._finalizePosition(panel);
}
self._reset();
});
};
DragManager.prototype._handleKeyDown = function (e) {
if (e.key === 'Escape' && this.state === 'DRAGGING' && this.activePanel) {
this.activePanel.classList.remove('is-dragging');
this.activePanel.style.transform = this.preTransform;
// Revert: re-attach to corner if it was cornered before
var saved = localStorage.getItem('panel-drag-' + this.activePanel.id);
if (!saved) {
// Was in corner mode — restore corner CSS
delete this.activePanel.dataset.dragged;
this.activePanel.style.top = '';
this.activePanel.style.left = '';
this.activePanel.style.right = '';
this.activePanel.style.bottom = '';
this.activePanel.style.transform = '';
// Re-apply corner position from M0
var corner = localStorage.getItem('panel-corner-' + this.activePanel.id);
if (corner) this.activePanel.setAttribute('data-position', corner);
} else {
// Was already dragged — revert to pre-drag position
this.activePanel.style.transform = 'none';
}
this._reset();
}
};
DragManager.prototype._reset = function () {
document.removeEventListener('keydown', this._onKeyDown);
this.state = 'IDLE';
this.activePanel = null;
};
DragManager.prototype._detachFromCorner = function (panel) {
var rect = panel.getBoundingClientRect();
panel.removeAttribute('data-position');
panel.dataset.dragged = 'true';
panel.style.position = 'fixed';
panel.style.top = rect.top + 'px';
panel.style.left = rect.left + 'px';
panel.style.right = 'auto';
panel.style.bottom = 'auto';
panel.style.transform = 'none';
};
DragManager.prototype._finalizePosition = function (panel) {
var rect = panel.getBoundingClientRect();
var vw = window.innerWidth;
var vh = window.innerHeight;
var x = Math.max(0, Math.min(rect.left, vw - 40));
var y = Math.max(0, Math.min(rect.top, vh - 40));
// Snap to edge
if (x < SNAP_THRESHOLD) x = SNAP_MARGIN;
if (y < SNAP_THRESHOLD) y = SNAP_MARGIN;
if (x + rect.width > vw - SNAP_THRESHOLD) x = vw - rect.width - SNAP_MARGIN;
if (y + rect.height > vh - SNAP_THRESHOLD) y = vh - rect.height - SNAP_MARGIN;
panel.style.top = y + 'px';
panel.style.left = x + 'px';
panel.style.transform = 'none';
this._persist(panel.id, x / vw, y / vh);
};
DragManager.prototype._persist = function (id, xPct, yPct) {
try {
localStorage.setItem('panel-drag-' + id,
JSON.stringify({ xPct: xPct, yPct: yPct }));
} catch (_) { /* quota exceeded — silent */ }
};
DragManager.prototype.enable = function () { this.enabled = true; };
DragManager.prototype.disable = function () {
this.enabled = false;
if (this.state !== 'IDLE' && this.activePanel) {
this.activePanel.classList.remove('is-dragging');
this._reset();
}
};
DragManager.prototype.restorePositions = function () {
var panels = this._panels;
for (var i = 0; i < panels.length; i++) {
var panel = panels[i];
var raw = localStorage.getItem('panel-drag-' + panel.id);
if (!raw) continue;
try {
var pos = JSON.parse(raw);
var x = pos.xPct * window.innerWidth;
var y = pos.yPct * window.innerHeight;
panel.removeAttribute('data-position');
panel.dataset.dragged = 'true';
panel.style.position = 'fixed';
panel.style.top = y + 'px';
panel.style.left = x + 'px';
panel.style.right = 'auto';
panel.style.bottom = 'auto';
panel.style.transform = 'none';
} catch (_) {
localStorage.removeItem('panel-drag-' + panel.id);
}
}
};
DragManager.prototype.handleResize = function () {
var panels = document.querySelectorAll('.live-overlay[data-dragged="true"]');
for (var i = 0; i < panels.length; i++) {
var panel = panels[i];
var rect = panel.getBoundingClientRect();
var vw = window.innerWidth;
var vh = window.innerHeight;
var x = rect.left, y = rect.top, moved = false;
if (rect.right > vw) { x = vw - rect.width - SNAP_MARGIN; moved = true; }
if (rect.bottom > vh) { y = vh - rect.height - SNAP_MARGIN; moved = true; }
if (x < 0) { x = SNAP_MARGIN; moved = true; }
if (y < 0) { y = SNAP_MARGIN; moved = true; }
if (moved) {
panel.style.left = x + 'px';
panel.style.top = y + 'px';
this._persist(panel.id, x / vw, y / vh);
}
}
};
// Export
window.DragManager = DragManager;
})();
-170
View File
@@ -1,170 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>GeoFilter Builder — CoreScope</title>
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"/>
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
<style>
* { box-sizing: border-box; margin: 0; padding: 0; }
body { font-family: system-ui, sans-serif; background: #1a1a2e; color: #e0e0e0; height: 100vh; display: flex; flex-direction: column; }
header { padding: 12px 16px; background: #0f0f23; border-bottom: 1px solid #333; display: flex; align-items: center; gap: 16px; flex-wrap: wrap; }
header h1 { font-size: 1rem; font-weight: 600; color: #4a9eff; white-space: nowrap; }
.controls { display: flex; gap: 8px; flex-wrap: wrap; }
button { padding: 6px 14px; border: none; border-radius: 6px; cursor: pointer; font-size: 0.85rem; font-weight: 500; }
#btnUndo { background: #333; color: #ccc; }
#btnClear { background: #5a2020; color: #ffaaaa; }
#btnUndo:hover { background: #444; }
#btnClear:hover { background: #7a2020; }
.hint { font-size: 0.8rem; color: #888; margin-left: auto; }
#map { flex: 1; }
#output-panel { background: #0f0f23; border-top: 1px solid #333; padding: 12px 16px; display: flex; gap: 12px; align-items: flex-start; }
#output-panel label { font-size: 0.75rem; color: #888; white-space: nowrap; padding-top: 6px; }
#output { flex: 1; background: #111; border: 1px solid #333; border-radius: 6px; padding: 10px 12px; font-family: monospace; font-size: 0.78rem; color: #7ec8e3; white-space: pre; overflow-x: auto; min-height: 54px; max-height: 140px; overflow-y: auto; cursor: text; }
#output.empty { color: #555; font-style: italic; }
#btnCopy { padding: 6px 14px; background: #1a4a7a; color: #7ec8e3; border-radius: 6px; border: none; cursor: pointer; font-size: 0.85rem; white-space: nowrap; align-self: flex-end; }
#btnCopy:hover { background: #2a6aaa; }
#btnCopy.copied { background: #1a6a3a; color: #7effa0; }
#counter { font-size: 0.8rem; color: #888; padding-top: 6px; white-space: nowrap; }
.bufferRow { display: flex; align-items: center; gap: 8px; }
.bufferRow label { font-size: 0.85rem; color: #aaa; }
.bufferRow input { width: 60px; padding: 5px 8px; background: #222; border: 1px solid #444; border-radius: 6px; color: #eee; font-size: 0.85rem; }
#help-bar { background: #0f0f23; padding: 6px 16px; font-size: 0.75rem; color: #666; border-top: 1px solid #222; }
#help-bar a { color: #4a9eff; text-decoration: none; }
#help-bar a:hover { text-decoration: underline; }
#back-link { font-size: 0.8rem; color: #4a9eff; text-decoration: none; white-space: nowrap; }
#back-link:hover { text-decoration: underline; }
</style>
</head>
<body>
<header>
<a href="/" id="back-link">← CoreScope</a>
<h1>GeoFilter Builder</h1>
<div class="controls">
<button id="btnUndo">↩ Undo</button>
<button id="btnClear">✕ Clear</button>
</div>
<div class="bufferRow">
<label for="bufferKm">Buffer km:</label>
<!-- Extra margin (km) outside the polygon edge that still passes the filter -->
<input type="number" id="bufferKm" value="20" min="0" max="500"/>
</div>
<span class="hint">Click on the map to add polygon points</span>
</header>
<div id="map"></div>
<!-- Output panel: shows the geo_filter JSON block ready to paste into config.json -->
<div id="output-panel">
<label>config.json</label>
<div id="output" class="empty">Add at least 3 points to generate config…</div>
<div style="display:flex;flex-direction:column;gap:8px;align-items:flex-end">
<span id="counter">0 points</span>
<button id="btnCopy">Copy</button>
</div>
</div>
<!-- Instructions: paste the output into config.json as a top-level "geo_filter" key, then restart the server -->
<div id="help-bar">
Copy the JSON above → paste as a top-level key in <code>config.json</code> → restart the server.
Nodes with no GPS fix always pass through. Remove the <code>geo_filter</code> block to disable filtering.
&nbsp;·&nbsp; <a href="https://github.com/Kpa-clawbot/CoreScope/blob/master/docs/user-guide/geofilter.md" target="_blank">Documentation ↗</a>
</div>
<script>
const map = L.map('map').setView([50.5, 4.4], 8);
L.tileLayer('https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png', {
attribution: '© OpenStreetMap © CartoDB',
maxZoom: 19
}).addTo(map);
let points = [];
let markers = [];
let polygon = null;
let closingLine = null;
function latLonPair(latlng) {
return [parseFloat(latlng.lat.toFixed(6)), parseFloat(latlng.lng.toFixed(6))];
}
function render() {
// Remove existing polygon and closing line
if (polygon) { map.removeLayer(polygon); polygon = null; }
if (closingLine) { map.removeLayer(closingLine); closingLine = null; }
if (points.length >= 3) {
polygon = L.polygon(points, {
color: '#4a9eff', weight: 2, fillColor: '#4a9eff', fillOpacity: 0.12
}).addTo(map);
} else if (points.length === 2) {
closingLine = L.polyline(points, { color: '#4a9eff', weight: 2, dashArray: '5,5' }).addTo(map);
}
updateOutput();
}
function updateOutput() {
const el = document.getElementById('output');
const counter = document.getElementById('counter');
counter.textContent = points.length + ' point' + (points.length !== 1 ? 's' : '');
if (points.length < 3) {
el.textContent = 'Add at least 3 points to generate config…';
el.classList.add('empty');
return;
}
el.classList.remove('empty');
const bufferKm = parseFloat(document.getElementById('bufferKm').value) || 0;
// Output format: { "geo_filter": { "bufferKm": N, "polygon": [[lat,lon], ...] } }
// Paste this as a top-level key in config.json
const config = { bufferKm, polygon: points };
el.textContent = JSON.stringify({ geo_filter: config }, null, 2);
}
map.on('click', function(e) {
const pt = latLonPair(e.latlng);
points.push(pt);
const idx = points.length;
const marker = L.circleMarker(e.latlng, {
radius: 6, color: '#4a9eff', weight: 2, fillColor: '#4a9eff', fillOpacity: 0.9
}).addTo(map).bindTooltip(String(idx), { permanent: true, direction: 'top', offset: [0, -8], className: 'pt-label' });
markers.push(marker);
render();
});
document.getElementById('btnUndo').addEventListener('click', function() {
if (!points.length) return;
points.pop();
const m = markers.pop();
if (m) map.removeLayer(m);
render();
});
document.getElementById('btnClear').addEventListener('click', function() {
points = [];
markers.forEach(m => map.removeLayer(m));
markers = [];
render();
});
document.getElementById('bufferKm').addEventListener('input', updateOutput);
document.getElementById('btnCopy').addEventListener('click', function() {
if (points.length < 3) return;
const text = document.getElementById('output').textContent;
navigator.clipboard.writeText(text).then(() => {
const btn = document.getElementById('btnCopy');
btn.textContent = 'Copied!';
btn.classList.add('copied');
setTimeout(() => { btn.textContent = 'Copy'; btn.classList.remove('copied'); }, 2000);
});
});
</script>
</body>
</html>
+3 -8
View File
@@ -302,19 +302,14 @@
<button class="mnc-btn" data-action="packets" data-key="${mn.pubkey}">View packets </button>
</div>
</div>`;
} catch (err) {
const is404 = err && err.message && err.message.includes('404');
const statusIcon = is404 ? '📡' : '❓';
const statusMsg = is404
? 'Waiting for first advert — this node has been seen in channel messages but hasn\u2019t advertised yet'
: 'Could not load data';
} catch {
return `<div class="my-node-card silent" data-key="${mn.pubkey}" tabindex="0" role="button">
<div class="mnc-header">
<div class="mnc-status">${statusIcon}</div>
<div class="mnc-status"></div>
<div class="mnc-name">${escapeHtml(mn.name || truncate(mn.pubkey, 12))}</div>
<button class="mnc-remove" data-key="${mn.pubkey}" title="Remove" aria-label="Remove ${escapeAttr(mn.name || truncate(mn.pubkey, 12))} from My Mesh"></button>
</div>
<div class="mnc-status-text">${statusMsg}</div>
<div class="mnc-status-text">Could not load data</div>
</div>`;
}
}));
+1 -5
View File
@@ -81,13 +81,9 @@ window.HopDisplay = (function() {
const regionalConflicts = conflicts.filter(c => c.regional);
const badgeCount = regionalConflicts.length > 0 ? regionalConflicts.length : (globalFallback ? conflicts.length : 0);
const conflictData = escapeHtml(JSON.stringify({ h, conflicts, globalFallback }));
const conflictBadge = badgeCount > 1
const warnBadge = badgeCount > 1
? ` <button class="hop-conflict-btn" data-conflict='${conflictData}' onclick="event.preventDefault();event.stopPropagation();HopDisplay._showFromBtn(this)" title="${badgeCount} candidates — click for details">⚠${badgeCount}</button>`
: '';
const unreliableBadge = unreliable
? ' <button class="hop-unreliable-btn" aria-label="Unreliable name resolution" title="Unreliable name resolution — this hash\u2192name match is geographically inconsistent with the surrounding path hops. The repeater itself may be fine; this specific hop assignment is uncertain.">⚠️</button>'
: '';
const warnBadge = conflictBadge + unreliableBadge;
const cls = [
'hop',
-2
View File
@@ -95,7 +95,6 @@
<script src="table-sort.js?v=__BUST__"></script>
<script src="packet-filter.js?v=__BUST__"></script>
<script src="packet-helpers.js?v=__BUST__"></script>
<script src="channel-decrypt.js?v=__BUST__"></script>
<script src="channel-colors.js?v=__BUST__"></script>
<script src="channel-color-picker.js?v=__BUST__"></script>
<script src="packets.js?v=__BUST__"></script>
@@ -110,7 +109,6 @@
<script src="audio-v1-constellation.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="audio-v2-constellation.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="audio-lab.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="drag-manager.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="live.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="observers.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
<script src="observer-detail.js?v=__BUST__" onerror="console.error('Failed to load:', this.src)"></script>
+5 -57
View File
@@ -132,7 +132,7 @@
/* ---- Node Detail Panel ---- */
.live-node-detail {
top: 64px;
top: 60px;
right: 12px;
width: 320px;
max-height: calc(100vh - 140px);
@@ -325,14 +325,11 @@
}
.live-stats-row { flex-wrap: wrap; gap: 4px; }
.live-stat-pill { font-size: 11px; padding: 2px 7px; }
.live-toggles { font-size: 10px; gap: 6px; margin-left: 0; overflow-x: auto; flex-wrap: nowrap; -webkit-overflow-scrolling: touch; width: 100%; min-width: 0; }
.live-toggles { font-size: 10px; gap: 6px; margin-left: 0; }
.live-title { font-size: 12px; letter-spacing: 1px; }
/* #203 — bottom-sheet node detail on mobile */
.live-node-detail { width: 100%; right: 0; left: 0; top: auto; bottom: 0; max-height: 60dvh; border-radius: 16px 16px 0 0; overflow-y: auto; z-index: 1050; }
.live-node-detail { width: 100%; right: 0; left: 0; top: auto; bottom: 0; max-height: 60vh; border-radius: 16px 16px 0 0; overflow-y: auto; }
.live-node-detail.hidden { transform: translateY(100%); }
/* Close button was unreachable: panel-header collapsed to 8px on mobile, panel-content
scroll area started at y=8, overlapping the button's 36px tap target (y=642) */
.live-node-detail .panel-header { min-height: 44px; }
.feed-detail-card {
position: fixed !important;
right: 0 !important;
@@ -692,18 +689,6 @@
.live-feed { bottom: 68px; }
.feed-show-btn { bottom: 68px !important; }
/* Backdrop for mobile tap-outside-to-close (#797) */
.node-detail-backdrop {
display: none;
position: absolute;
inset: 0;
z-index: 1049;
background: rgba(0, 0, 0, 0.25);
}
@media (max-width: 640px) {
.node-detail-backdrop.active { display: block; }
}
/* Mobile VCR */
@media (max-width: 640px) {
/* Mobile VCR: two-row stacked layout */
@@ -820,8 +805,8 @@
/* Corner positions — applied via data-position attribute on .live-overlay panels */
.live-overlay[data-position="tl"] { top: 64px; left: 12px; bottom: auto; right: auto; }
.live-overlay[data-position="tr"] { top: 64px; right: 12px; bottom: auto; left: auto; }
.live-overlay[data-position="bl"] { bottom: 58px; left: 12px; top: auto; right: auto; }
.live-overlay[data-position="br"] { bottom: 58px; right: 12px; top: auto; left: auto; }
.live-overlay[data-position="bl"] { bottom: 12px; left: 12px; top: auto; right: auto; }
.live-overlay[data-position="br"] { bottom: 12px; right: 12px; top: auto; left: auto; }
/* Override hide animations for positioned panels — slide toward nearest edge */
.live-overlay[data-position="tl"].hidden,
@@ -864,40 +849,3 @@
}
}
/* ── Drag Manager (#608 M1) ── */
/* Panel header as drag handle — desktop pointer devices only */
@media (pointer: fine) {
.live-overlay .panel-header {
cursor: grab;
user-select: none;
touch-action: none;
}
.live-overlay.is-dragging .panel-header {
cursor: grabbing;
}
.live-overlay .panel-header:hover {
background: var(--bg-hover, rgba(255, 255, 255, 0.03));
}
}
/* Panel during drag */
.live-overlay.is-dragging {
opacity: 0.92;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.5);
will-change: transform;
transition: none !important;
}
/* Freely placed panel — no corner transition animations */
.live-overlay[data-dragged="true"] {
transition: none;
}
@media (prefers-reduced-motion: reduce) {
.live-overlay.is-dragging,
.live-overlay[data-dragged="true"] {
transition: none;
}
}
+6 -71
View File
@@ -30,7 +30,6 @@
let _lcdClockInterval = null;
let _rateCounterInterval = null;
let _pruneInterval = null;
let _feedTimestampInterval = null;
let activeNodeDetailKey = null;
// === VCR State Machine ===
@@ -849,7 +848,6 @@
<div class="panel-content" aria-live="polite" aria-relevant="additions" role="log"></div>
</div>
<button class="feed-show-btn hidden" id="feedShowBtn" title="Show feed">📋</button>
<div id="nodeDetailBackdrop" class="node-detail-backdrop"></div>
<div class="live-overlay live-node-detail hidden" id="liveNodeDetail">
<div class="panel-header">
<button class="panel-corner-btn" data-panel="liveNodeDetail" title="Move panel to next corner" aria-label="Move panel to next corner"></button>
@@ -1163,48 +1161,6 @@
// Populate role legend from shared roles.js
// Initialize panel corner positions (#608 M0)
initPanelPositions();
// Initialize DragManager for free-form panel dragging (#608 M1)
if (window.DragManager) {
var dragMgr = new DragManager();
var dragPanels = ['liveFeed', 'liveLegend', 'liveNodeDetail'];
for (var di = 0; di < dragPanels.length; di++) {
dragMgr.register(document.getElementById(dragPanels[di]));
}
dragMgr.restorePositions();
// Responsive gate: disable drag below medium breakpoint or on touch
var dragMql = window.matchMedia('(pointer: fine) and (min-width: 768px)');
function onDragMediaChange(e) {
if (!e.matches) {
// Revert dragged panels to corner positions
document.querySelectorAll('.live-overlay[data-dragged="true"]').forEach(function (p) {
delete p.dataset.dragged;
p.style.transform = '';
p.style.top = '';
p.style.left = '';
p.style.right = '';
p.style.bottom = '';
});
initPanelPositions();
dragMgr.disable();
} else {
dragMgr.enable();
dragMgr.restorePositions();
}
}
dragMql.addEventListener('change', onDragMediaChange);
// Initial check
if (!dragMql.matches) dragMgr.disable();
// Resize clamping (debounced)
var resizeTimer = null;
window.addEventListener('resize', function () {
clearTimeout(resizeTimer);
resizeTimer = setTimeout(function () { dragMgr.handleResize(); }, 200);
});
}
const roleLegendList = document.getElementById('roleLegendList');
if (roleLegendList) {
for (const role of (window.ROLE_SORT || ['repeater', 'companion', 'room', 'sensor', 'observer'])) {
@@ -1217,14 +1173,10 @@
// Node detail panel
const nodeDetailPanel = document.getElementById('liveNodeDetail');
const nodeDetailContent = document.getElementById('nodeDetailContent');
const nodeDetailBackdrop = document.getElementById('nodeDetailBackdrop');
function closeNodeDetail() {
document.getElementById('nodeDetailClose').addEventListener('click', () => {
activeNodeDetailKey = null;
nodeDetailPanel.classList.add('hidden');
nodeDetailBackdrop.classList.remove('active');
}
document.getElementById('nodeDetailClose').addEventListener('click', closeNodeDetail);
nodeDetailBackdrop.addEventListener('click', closeNodeDetail);
});
// Feed panel resize handle (#27)
const savedFeedWidth = localStorage.getItem('live-feed-width');
@@ -1382,13 +1334,6 @@
// Prune stale nodes every 60 seconds
_pruneInterval = setInterval(pruneStaleNodes, 60000);
// Refresh relative timestamps in feed every 10 seconds (#701)
_feedTimestampInterval = setInterval(function() {
document.querySelectorAll('.feed-time[data-ts]').forEach(function(el) {
el.innerHTML = formatLiveTimestampHtml(Number(el.dataset.ts));
});
}, 10000);
// Auto-hide nav with pin toggle (#62)
const topNav = document.querySelector('.top-nav');
if (topNav) { topNav.style.position = 'fixed'; topNav.style.width = '100%'; topNav.style.zIndex = '1100'; }
@@ -1456,7 +1401,6 @@
const panel = document.getElementById('liveNodeDetail');
const content = document.getElementById('nodeDetailContent');
panel.classList.remove('hidden');
document.getElementById('nodeDetailBackdrop').classList.add('active');
content.innerHTML = '<div style="padding:20px;color:var(--text-muted)">Loading…</div>';
try {
const [data, healthData] = await Promise.all([
@@ -1724,7 +1668,7 @@
<span class="feed-type" style="color:${color}">${typeName}</span>
${dotHtml1}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time" data-ts="${group.latestTs || Date.now()}">${formatLiveTimestampHtml(group.latestTs || Date.now())}</span>
<span class="feed-time">${formatLiveTimestampHtml(group.latestTs || Date.now())}</span>
`;
if (_ccChan1) item._ccChannel = _ccChan1; // channel color picker (#674)
item.addEventListener('click', () => showFeedCard(item, pkt, color));
@@ -2736,7 +2680,6 @@
const preview = text ? ' ' + (text.length > 35 ? text.slice(0, 35) + '…' : text) : '';
const hopStr = hops.length ? `<span class="feed-hops">${hops.length}⇢</span>` : '';
const obsBadge = pkt.observation_count > 1 ? `<span class="badge badge-obs" style="font-size:10px;margin-left:4px">👁 ${pkt.observation_count}</span>` : '';
const anomalyIcon = (pkt.decoded && pkt.decoded.anomaly) ? '<span title="Anomaly detected" style="margin-left:4px">⚠️</span>' : '';
var _ccPayload2 = (pkt.decoded || {}).payload || {};
var _ccChan = (typeName === 'GRP_TXT' || typeName === 'CHAN') ? (_ccPayload2.channel || null) : null;
var dotHtml = _ccChan ? _feedColorDot(_ccChan) : '';
@@ -2751,9 +2694,9 @@
item.innerHTML = `
<span class="feed-icon" style="color:${color}">${icon}</span>
<span class="feed-type" style="color:${color}">${typeName}</span>
${dotHtml}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}${anomalyIcon}
${dotHtml}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time" data-ts="${pkt._ts || Date.now()}">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
`;
if (_ccChan) item._ccChannel = _ccChan; // channel color picker (#674)
item.addEventListener('click', () => showFeedCard(item, pkt, color));
@@ -2799,13 +2742,6 @@
requestAnimationFrame(() => requestAnimationFrame(() => entry.element.classList.remove('live-feed-enter')));
// Re-add to DOM top (works even if it was trimmed out)
feed.prepend(entry.element);
// Update timestamp to latest observation (#701)
var _dedupTimeSpan = entry.element.querySelector('.feed-time');
if (_dedupTimeSpan) {
var _dedupNow = pkt._ts || Date.now();
_dedupTimeSpan.setAttribute('data-ts', _dedupNow);
_dedupTimeSpan.innerHTML = formatLiveTimestampHtml(_dedupNow);
}
entry.pkt.observation_count = entry.count;
return;
}
@@ -2836,7 +2772,7 @@
<span class="feed-type" style="color:${color}">${typeName}</span>
${dotHtml3}${transportBadge(pkt.route_type)}${hopStr}${obsBadge}
<span class="feed-text">${escapeHtml(preview)}</span>
<span class="feed-time" data-ts="${pkt._ts || Date.now()}">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
<span class="feed-time">${formatLiveTimestampHtml(pkt._ts || Date.now())}</span>
`;
if (_ccChan3) item._ccChannel = _ccChan3; // channel color picker (#674)
item.addEventListener('click', () => showFeedCard(item, pkt, color));
@@ -2913,7 +2849,6 @@
if (_lcdClockInterval) { clearInterval(_lcdClockInterval); _lcdClockInterval = null; }
if (_rateCounterInterval) { clearInterval(_rateCounterInterval); _rateCounterInterval = null; }
if (_pruneInterval) { clearInterval(_pruneInterval); _pruneInterval = null; }
if (_feedTimestampInterval) { clearInterval(_feedTimestampInterval); _feedTimestampInterval = null; }
if (_affinityInterval) { clearInterval(_affinityInterval); _affinityInterval = null; }
if (ws) { ws.onclose = null; ws.close(); ws = null; }
if (map) { map.remove(); map = null; }
+10 -44
View File
@@ -25,7 +25,7 @@
// Roles loaded from shared roles.js (ROLE_STYLE, ROLE_LABELS, ROLE_COLORS globals)
function makeMarkerIcon(role, isStale, isAlsoObserver) {
function makeMarkerIcon(role, isStale) {
const s = ROLE_STYLE[role] || ROLE_STYLE.companion;
const size = s.radius * 2 + 4;
const c = size / 2;
@@ -56,22 +56,7 @@
default: // circle
path = `<circle cx="${c}" cy="${c}" r="${c-2}" fill="${s.color}" stroke="#fff" stroke-width="2"/>`;
}
// If this node is also an observer, add a small star overlay
let obsOverlay = '';
if (isAlsoObserver) {
const starSize = 8;
const sx = size - starSize, sy = 0;
const scx = starSize / 2, scy = starSize / 2, so = starSize / 2 - 0.5, si = so * 0.4;
let starPts = '';
for (let i = 0; i < 5; i++) {
const aO = (i * 72 - 90) * Math.PI / 180;
const aI = ((i * 72) + 36 - 90) * Math.PI / 180;
starPts += `${scx + so * Math.cos(aO)},${scy + so * Math.sin(aO)} `;
starPts += `${scx + si * Math.cos(aI)},${scy + si * Math.sin(aI)} `;
}
obsOverlay = `<g transform="translate(${sx},${sy})"><polygon points="${starPts.trim()}" fill="${ROLE_COLORS.observer || '#f1c40f'}" stroke="#fff" stroke-width="0.8"/></g>`;
}
const svg = `<svg width="${size}" height="${size}" viewBox="0 0 ${size} ${size}" xmlns="http://www.w3.org/2000/svg">${path}${obsOverlay}</svg>`;
const svg = `<svg width="${size}" height="${size}" viewBox="0 0 ${size} ${size}" xmlns="http://www.w3.org/2000/svg">${path}</svg>`;
return L.divIcon({
html: svg,
className: 'meshcore-marker' + (isStale ? ' marker-stale' : ''),
@@ -81,16 +66,14 @@
});
}
function makeRepeaterLabelIcon(node, isStale, isAlsoObserver) {
function makeRepeaterLabelIcon(node, isStale) {
var s = ROLE_STYLE['repeater'] || ROLE_STYLE.companion;
var hs = node.hash_size || 1;
// Show the short mesh hash ID (first N bytes of pubkey, uppercased)
var shortHash = node.public_key ? node.public_key.slice(0, hs * 2).toUpperCase() : '??';
var bgColor = s.color;
// If this repeater is also an observer, show a star indicator inside the label
var obsIndicator = isAlsoObserver ? ' <span style="color:' + (ROLE_COLORS.observer || '#f1c40f') + ';font-size:13px;line-height:1;" title="Also an observer">★</span>' : '';
var html = '<div style="background:' + bgColor + ';color:#fff;font-weight:bold;font-size:11px;padding:2px 5px;border-radius:3px;border:2px solid #fff;box-shadow:0 1px 3px rgba(0,0,0,0.4);text-align:center;line-height:1.2;white-space:nowrap;">' +
shortHash + obsIndicator + '</div>';
shortHash + '</div>';
return L.divIcon({
html: html,
className: 'meshcore-marker meshcore-label-marker' + (isStale ? ' marker-stale' : ''),
@@ -564,8 +547,7 @@
const el = document.getElementById('mcRoleChecks');
if (!el) return;
el.innerHTML = '';
const nodePubkeys = new Set(nodes.map(n => (n.public_key || '').toLowerCase()));
const obsCount = observers.filter(o => o.lat && o.lon && !(o.id && nodePubkeys.has(o.id.toLowerCase()))).length;
const obsCount = observers.filter(o => o.lat && o.lon).length;
const roles = ['repeater', 'companion', 'room', 'sensor', 'observer'];
const shapeMap = { repeater: '◆', companion: '●', room: '■', sensor: '▲', observer: '★' };
@@ -656,7 +638,6 @@
var _renderingMarkers = false;
var _lastDeconflictZoom = null;
var _currentMarkerData = []; // stored marker data for zoom-only repositioning
var _observerByPubkey = new Map(); // observer id (pubkey) → observer object, rebuilt on each render
var _zoomResizeTimer = null;
function deconflictLabels(markers, mapRef) {
@@ -799,31 +780,19 @@
const allMarkers = [];
// Build a set of observer public keys for quick lookup
_observerByPubkey = new Map();
for (const obs of observers) {
if (obs.id) _observerByPubkey.set(obs.id.toLowerCase(), obs);
}
for (const node of filtered) {
const lastSeenTime = node.last_heard || node.last_seen;
const isStale = getNodeStatus(node.role || 'companion', lastSeenTime ? new Date(lastSeenTime).getTime() : 0) === 'stale';
const pk = (node.public_key || '').toLowerCase();
const isAlsoObserver = _observerByPubkey.has(pk);
const useLabel = node.role === 'repeater' && filters.hashLabels;
const icon = useLabel ? makeRepeaterLabelIcon(node, isStale, isAlsoObserver) : makeMarkerIcon(node.role || 'companion', isStale, isAlsoObserver);
const icon = useLabel ? makeRepeaterLabelIcon(node, isStale) : makeMarkerIcon(node.role || 'companion', isStale);
const latLng = L.latLng(node.lat, node.lon);
allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: (node.name || 'Unknown') + ' (' + (node.role || 'node') + (isAlsoObserver ? ' + observer' : '') + ')' });
allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: (node.name || 'Unknown') + ' (' + (node.role || 'node') + ')' });
}
// Add observer markers (skip observers already represented as a node marker)
// Build set of node pubkeys that are displayed on the map
const displayedNodePubkeys = new Set(filtered.map(n => (n.public_key || '').toLowerCase()));
// Add observer markers
if (filters.observer) {
for (const obs of observers) {
if (!obs.lat || !obs.lon) continue;
// Skip observers whose pubkey matches a displayed node — they're shown as combined markers
if (obs.id && displayedNodePubkeys.has(obs.id.toLowerCase())) continue;
const icon = makeMarkerIcon('observer');
const latLng = L.latLng(obs.lat, obs.lon);
allMarkers.push({ latLng, node: obs, icon, isLabel: false, popupFn: function() { return buildObserverPopup(obs); }, alt: (obs.name || obs.id || 'Unknown') + ' (observer)' });
@@ -940,9 +909,6 @@
const loc = (node.lat && node.lon) ? `${node.lat.toFixed(5)}, ${node.lon.toFixed(5)}` : '—';
const lastAdvert = node.last_seen ? timeAgo(node.last_seen) : '—';
const roleBadge = `<span style="display:inline-block;padding:2px 8px;border-radius:12px;font-size:11px;font-weight:600;background:${ROLE_COLORS[node.role] || '#4b5563'};color:#fff;">${(node.role || 'unknown').toUpperCase()}</span>`;
// Check if this node is also an observer (combined repeater+observer)
const matchingObs = node.public_key ? _observerByPubkey.get(node.public_key.toLowerCase()) : null;
const obsBadge = matchingObs ? ` <span style="display:inline-block;padding:2px 8px;border-radius:12px;font-size:11px;font-weight:600;background:${ROLE_COLORS.observer || '#f1c40f'};color:#fff;">OBSERVER</span>` : '';
const hs = node.hash_size || 1;
const hashPrefix = node.public_key ? node.public_key.slice(0, hs * 2).toUpperCase() : '—';
const hashPrefixRow = `<dt style="color:var(--text-muted);float:left;clear:left;width:80px;padding:2px 0;">Hash Prefix</dt>
@@ -951,7 +917,7 @@
return `
<div class="map-popup" style="font-family:var(--font);min-width:180px;">
<h3 style="font-weight:700;font-size:14px;margin:0 0 4px;">${safeEsc(node.name || 'Unknown')}</h3>
${roleBadge}${obsBadge}
${roleBadge}
<dl style="margin-top:8px;font-size:12px;">
${hashPrefixRow}
<dt style="color:var(--text-muted);float:left;clear:left;width:80px;padding:2px 0;">Key</dt>
@@ -965,7 +931,7 @@
</dl>
<div style="margin-top:8px;clear:both;">
<a href="#/nodes/${node.public_key}" style="color:var(--accent);font-size:12px;">View Node </a>
${node.public_key ? ` · <a href="javascript:void(0)" role="button" data-show-neighbors data-pubkey="${escapeHtml(node.public_key)}" data-name="${escapeHtml(node.name || 'Unknown')}" style="color:var(--accent);font-size:12px;cursor:pointer;">Show Neighbors</a>` : ''}
${node.public_key ? ` · <a href="#" data-show-neighbors data-pubkey="${escapeHtml(node.public_key)}" data-name="${escapeHtml(node.name || 'Unknown')}" style="color:var(--accent);font-size:12px;">Show Neighbors</a>` : ''}
</div>
</div>`;
}
+14 -145
View File
@@ -286,29 +286,11 @@
if (h) h.textContent = 'Neighbors (' + data.neighbors.length + ')';
}
var html = renderNeighborTable(data.neighbors, limit);
if (limit && data.neighbors.length > limit) {
html += '<div style="margin-top:6px;text-align:right"><button class="btn-link show-all-neighbors-btn" style="font-size:12px;cursor:pointer;background:none;border:none;color:var(--accent);padding:0">Show all ' + data.neighbors.length + ' neighbors </button></div>';
} else if (!limit && data.neighbors.length > 5) {
// Collapse toggle when expanded (#855)
html += '<div style="margin-top:6px;text-align:right"><button class="btn-link collapse-neighbors-btn" style="font-size:12px;cursor:pointer;background:none;border:none;color:var(--accent);padding:0">Show fewer ▲</button></div>';
if (limit && data.neighbors.length > limit && viewAllPubkey) {
html += '<div style="margin-top:6px;text-align:right"><a href="#/nodes/' + encodeURIComponent(viewAllPubkey) + '?section=node-neighbors" style="font-size:12px">View all ' + data.neighbors.length + ' neighbors </a></div>';
}
el.innerHTML = html;
// Wire "Show all neighbors" expand button (#855)
var expandBtn = el.querySelector('.show-all-neighbors-btn');
if (expandBtn) {
expandBtn.addEventListener('click', function() {
renderNeighborData(data, containerId, 0, headerSelector, null);
});
}
// Wire collapse button (#855)
var collapseBtn = el.querySelector('.collapse-neighbors-btn');
if (collapseBtn) {
collapseBtn.addEventListener('click', function() {
renderNeighborData(data, containerId, 5, headerSelector, null);
});
}
// Initialize TableSort on neighbor table
var neighborTable = el.querySelector('.neighbor-sort-table');
if (neighborTable && window.TableSort) {
@@ -337,10 +319,7 @@
directNode = routeParam || null;
if (directNode) {
// Full-screen single node view (desktop + mobile).
// Reached via the 🔍 Details link or a deep link to #/nodes/{pubkey}.
// Row clicks use history.replaceState (no hashchange → no re-init),
// so the split-panel UX on desktop is preserved.
// Full-screen single node view
app.innerHTML = `<div class="node-fullscreen">
<div class="node-full-header">
<button class="detail-back-btn node-back-btn" id="nodeBackBtn" aria-label="Back to nodes"></button>
@@ -373,7 +352,7 @@
app.innerHTML = `<div class="nodes-page">
<div class="nodes-topbar">
<input type="text" class="nodes-search" id="nodeSearch" placeholder="Search by name or pubkey prefix…" aria-label="Search nodes by name or pubkey prefix">
<input type="text" class="nodes-search" id="nodeSearch" placeholder="Search nodes by name…" aria-label="Search nodes by name">
<div class="nodes-counts" id="nodeCounts"></div>
</div>
<div id="nodesRegionFilter" class="region-filter-container"></div>
@@ -384,7 +363,7 @@
</div>`;
RegionFilter.init(document.getElementById('nodesRegionFilter'));
regionChangeHandler = RegionFilter.onChange(function () { _allNodes = null; _fleetSkew = null; loadNodes(); });
regionChangeHandler = RegionFilter.onChange(function () { _allNodes = null; loadNodes(); });
if (search) {
var _si = document.getElementById('nodeSearch');
@@ -398,7 +377,6 @@
}, 250));
loadNodes();
if (directNode) selectNode(directNode);
// Auto-refresh when ADVERT packets arrive via WebSocket (fixes #131)
wsHandler = debouncedOnWS(function (msgs) {
const advertMsgs = msgs.filter(isAdvertMessage);
@@ -431,7 +409,6 @@
if (needReload) {
_allNodes = null;
_fleetSkew = null;
invalidateApiCache('/nodes');
}
loadNodes(true);
@@ -516,8 +493,6 @@
<tr><td>Hash Prefix</td><td>${n.hash_size ? '<code style="font-family:var(--mono);font-weight:700">' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + '</code> (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' <span style="color:var(--status-yellow);cursor:help" title="Seen: ' + (Array.isArray(n.hash_sizes_seen) ? n.hash_sizes_seen : []).join(', ') + '-byte"> varies</span>' : ''}</td></tr>
</table>
<div class="node-full-card skew-detail-section" id="node-clock-skew" style="display:none"></div>
${observers.length ? `<div class="node-full-card" id="node-observers">
${(() => { const regions = [...new Set(observers.map(o => o.iata).filter(Boolean))]; return regions.length ? `<div style="margin-bottom:8px"><strong>Regions:</strong> ${regions.map(r => '<span class="badge" style="margin:0 2px">' + escapeHtml(r) + '</span>').join(' ')}</div>` : ''; })()}
<h4>Heard By (${observers.length} observer${observers.length > 1 ? 's' : ''})</h4>
@@ -559,10 +534,9 @@
</div>
<div class="node-full-card" id="node-packets">
${(() => { const validPackets = adverts.filter(p => p.hash && p.timestamp); return `
<h4>Recent Packets (${validPackets.length})</h4>
<h4>Recent Packets (${adverts.length})</h4>
<div class="node-activity-list">
${validPackets.length ? validPackets.map(p => {
${adverts.length ? adverts.map(p => {
let decoded; try { decoded = JSON.parse(p.decoded_json); } catch {}
const typeLabel = p.payload_type === 4 ? '📡 Advert' : p.payload_type === 5 ? '💬 Channel' : p.payload_type === 2 ? '✉️ DM' : '📦 Packet';
const detail = decoded?.text ? ': ' + escapeHtml(truncate(decoded.text, 50)) : decoded?.name ? ' — ' + escapeHtml(decoded.name) : '';
@@ -588,7 +562,6 @@
</div>`;
}).join('') : '<div class="text-muted">No recent packets</div>'}
</div>
`; })()}
</div>`;
// Map
@@ -651,10 +624,6 @@
headerSelector: '#fullNeighborsHeader'
});
// #690 — Clock Skew detail section (full-screen view)
loadClockSkewInto(document.getElementById('node-clock-skew'), n.public_key);
// Affinity debug panel — show if debugAffinity is enabled
(function loadAffinityDebug() {
var show = (window.CLIENT_CONFIG && window.CLIENT_CONFIG.debugAffinity) || localStorage.getItem('meshcore-affinity-debug') === 'true';
@@ -808,59 +777,6 @@
let _themeRefreshHandler = null;
let _allNodes = null; // cached full node list
let _fleetSkew = null; // cached clock skew map: pubkey → {severity, recentMedianSkewSec, medianSkewSec, ...}
/**
* Fetch per-node clock skew and render into the given container element.
* Shared between the full-screen detail page and the side panel (#813, #690).
* No-op if the container is missing, the API errors, or the response lacks severity.
*/
async function loadClockSkewInto(container, pubkey) {
if (!container) return;
try {
var cs = await api('/nodes/' + encodeURIComponent(pubkey) + '/clock-skew', { ttl: 30000 });
if (!cs || !cs.severity) return;
container.style.display = '';
var driftHtml = cs.driftPerDaySec ? '<div style="font-size:12px;color:var(--text-muted);margin-top:2px">Drift: ' + formatDrift(cs.driftPerDaySec) + '</div>' : '';
var sparkHtml = renderSkewSparkline(cs.samples, 200, 32);
var skewVal = window.currentSkewValue(cs);
var skewDisplay = cs.severity === 'no_clock'
? '<span style="font-size:18px;font-weight:700;color:var(--text-muted)">No Clock</span>'
: '<span style="font-size:18px;font-weight:700;font-family:var(--mono)">' + formatSkew(skewVal) + '</span>';
var bimodalWarning = '';
if (cs.severity === 'bimodal_clock') {
var totalRecent = cs.recentSampleCount || 0;
bimodalWarning = '<div style="font-size:12px;color:var(--status-amber-text);margin-top:4px">⚠️ ' + (cs.recentBadSampleCount || '?') + ' of last ' + (totalRecent || '?') + ' adverts had nonsense timestamps (likely RTC reset)</div>';
}
container.innerHTML =
'<h4 style="margin:0 0 6px">⏰ Clock Skew</h4>' +
'<div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap">' +
skewDisplay +
renderSkewBadge(cs.severity, skewVal, cs) +
(cs.calibrated ? ' <span style="font-size:10px;color:var(--text-muted)" title="Observer-calibrated">✓ calibrated</span>' : '') +
'</div>' +
driftHtml +
(sparkHtml ? '<div class="skew-sparkline-wrap" style="margin-top:8px">' + sparkHtml + '<div style="font-size:10px;color:var(--text-muted)">Skew over time (' + (cs.samples || []).length + ' samples)</div></div>' : '') +
bimodalWarning;
} catch (e) {
// Non-fatal — section stays hidden
}
}
/** Fetch fleet clock skew once, return map keyed by pubkey */
async function getFleetSkew() {
if (_fleetSkew) return _fleetSkew;
try {
const data = await api('/nodes/clock-skew', { ttl: 30000 });
_fleetSkew = {};
(Array.isArray(data) ? data : []).forEach(function(cs) {
if (cs && cs.pubkey) _fleetSkew[cs.pubkey] = cs;
});
} catch (e) {
_fleetSkew = {};
}
return _fleetSkew;
}
// Build a map of lowercased name → count of distinct pubkeys sharing that name
function buildDupNameMap(allNodes) {
@@ -890,10 +806,7 @@
const params = new URLSearchParams({ limit: '5000' });
const rp = RegionFilter.getRegionParam();
if (rp) params.set('region', rp);
const [data] = await Promise.all([
api('/nodes?' + params, { ttl: CLIENT_TTL.nodeList }),
getFleetSkew() // pre-fetch clock skew in parallel
]);
const data = await api('/nodes?' + params, { ttl: CLIENT_TTL.nodeList });
_allNodes = data.nodes || [];
counts = data.counts || {};
}
@@ -902,7 +815,8 @@
let filtered = _allNodes;
if (activeTab !== 'all') filtered = filtered.filter(n => (n.role || '').toLowerCase() === activeTab);
if (search) {
filtered = filtered.filter(n => window._nodesMatchesSearch(n, search));
const q = search.toLowerCase();
filtered = filtered.filter(n => (n.name || '').toLowerCase().includes(q) || (n.public_key || '').toLowerCase().includes(q));
}
if (lastHeard) {
const ms = { '1h': 3600000, '2h': 7200000, '6h': 21600000, '12h': 43200000, '24h': 86400000, '48h': 172800000, '3d': 259200000, '7d': 604800000, '14d': 1209600000, '30d': 2592000000 }[lastHeard];
@@ -1065,7 +979,6 @@
panel.classList.add('empty');
panel.innerHTML = '<span>Select a node to view details</span>';
selectedKey = null;
history.replaceState(null, '', '#/nodes');
renderRows();
}
}
@@ -1073,25 +986,11 @@
// #630: Close button for node detail panel (important for mobile full-screen overlay)
document.getElementById('nodesRight').addEventListener('click', function(e) {
// #778/#856: Analytics link — force hashchange via replaceState + assign.
// (Details button is handled separately via .node-detail-btn click listener)
var link = e.target.closest('a.btn-primary[href^="#/nodes/"]');
if (link) {
e.preventDefault();
var href = link.getAttribute('href');
if (href.indexOf('/analytics') !== -1) {
// Analytics link — different page, force hashchange via replaceState + assign
history.replaceState(null, '', '#/');
location.hash = href.substring(1);
}
return;
}
if (e.target.closest('.panel-close-btn')) {
const panel = document.getElementById('nodesRight');
panel.classList.add('empty');
panel.innerHTML = '<span>Select a node to view details</span>';
selectedKey = null;
history.replaceState(null, '', '#/nodes');
renderRows();
}
});
@@ -1130,10 +1029,8 @@
const lastSeenTime = n.last_heard || n.last_seen;
const status = getNodeStatus(n.role || 'companion', lastSeenTime ? new Date(lastSeenTime).getTime() : 0);
const lastSeenClass = status === 'active' ? 'last-seen-active' : 'last-seen-stale';
const cs = _fleetSkew && _fleetSkew[n.public_key];
const skewBadgeHtml = cs && cs.severity && cs.severity !== 'ok' ? renderSkewBadge(cs.severity, window.currentSkewValue(cs), cs) : '';
return `<tr data-key="${n.public_key}" data-action="select" data-value="${n.public_key}" tabindex="0" role="row" class="${selectedKey === n.public_key ? 'selected' : ''}${isClaimed ? ' claimed-row' : ''}">
<td>${favStar(n.public_key, 'node-fav')}${isClaimed ? '<span class="claimed-badge" title="My Mesh">★</span> ' : ''}<strong>${n.name || '(unnamed)'}</strong>${dupNameBadge(n.name, n.public_key, dupMap)}${skewBadgeHtml}</td>
<td>${favStar(n.public_key, 'node-fav')}${isClaimed ? '<span class="claimed-badge" title="My Mesh">★</span> ' : ''}<strong>${n.name || '(unnamed)'}</strong>${dupNameBadge(n.name, n.public_key, dupMap)}</td>
<td class="mono col-pubkey">${truncate(n.public_key, 16)}</td>
<td><span class="badge" style="background:${roleColor}20;color:${roleColor}">${n.role}</span></td>
<td class="${lastSeenClass}">${renderNodeTimestampHtml(n.last_heard || n.last_seen)}</td>
@@ -1151,7 +1048,6 @@
return;
}
selectedKey = pubkey;
history.replaceState(null, '', '#/nodes/' + encodeURIComponent(pubkey));
renderRows();
const panel = document.getElementById('nodesRight');
panel.classList.remove('empty');
@@ -1190,7 +1086,7 @@
<div class="node-detail">
<div class="node-detail-name">${escapeHtml(n.name || '(unnamed)')}${dupBadge}</div>
<div class="node-detail-role">${renderNodeBadges(n, roleColor)}
<button class="btn-primary node-detail-btn" data-pubkey="${encodeURIComponent(n.public_key)}" aria-label="View details for ${escapeHtml(n.name || n.public_key)}" style="font-size:11px;padding:2px 8px;margin-left:8px;cursor:pointer">🔍 Details</button>
<a href="#/nodes/${encodeURIComponent(n.public_key)}" class="btn-primary" style="display:inline-block;text-decoration:none;font-size:11px;padding:2px 8px;margin-left:8px">🔍 Details</a>
<a href="#/nodes/${encodeURIComponent(n.public_key)}/analytics" class="btn-primary" style="display:inline-block;margin-left:4px;text-decoration:none;font-size:11px;padding:2px 8px">📊 Analytics</a>
</div>
${renderStatusExplanation(n)}
@@ -1217,8 +1113,6 @@
</dl>
</div>
<div class="node-detail-section skew-detail-section" id="node-clock-skew" style="display:none"></div>
${observers.length ? `<div class="node-detail-section">
${(() => { const regions = [...new Set(observers.map(o => o.iata).filter(Boolean))]; return regions.length ? `<div style="margin-bottom:6px;font-size:12px"><strong>Regions:</strong> ${regions.join(', ')}</div>` : ''; })()}
<h4>Heard By (${observers.length} observer${observers.length > 1 ? 's' : ''})</h4>
@@ -1241,10 +1135,9 @@
</div>
<div class="node-detail-section">
${(() => { const validPackets = adverts.filter(a => a.hash && a.timestamp); return `
<h4>Recent Packets (${validPackets.length})</h4>
<h4>Recent Packets (${adverts.length})</h4>
<div id="advertTimeline">
${validPackets.length ? validPackets.map(a => {
${adverts.length ? adverts.map(a => {
let decoded;
try { decoded = JSON.parse(a.decoded_json); } catch {}
const pType = PAYLOAD_TYPES[a.payload_type] || 'Packet';
@@ -1263,7 +1156,6 @@
</div>`;
}).join('') : '<div class="text-muted" style="padding:8px">No recent packets</div>'}
</div>
`; })()}
</div>
</div>`;
@@ -1307,15 +1199,6 @@
} catch {}
}
// #856: Wire "Details" button to navigate to full-screen node view
var detailBtn = panel.querySelector('.node-detail-btn');
if (detailBtn) {
detailBtn.addEventListener('click', function() {
var pk = detailBtn.getAttribute('data-pubkey');
location.hash = '#/nodes/' + pk;
});
}
// Fetch neighbors for this node (condensed panel — top 5)
fetchAndRenderNeighbors(n.public_key, 'panelNeighborsContent', {
limit: 5,
@@ -1323,10 +1206,6 @@
viewAllPubkey: n.public_key
});
// #813 — Clock Skew section in side panel (mirrors full-screen view)
loadClockSkewInto(document.getElementById('node-clock-skew'), n.public_key);
// Fetch paths through this node
api('/nodes/' + encodeURIComponent(n.public_key) + '/paths', { ttl: CLIENT_TTL.nodeDetail }).then(pathData => {
const el = document.getElementById('pathsContent');
@@ -1425,14 +1304,4 @@
window._nodesRenderNodeTimestampText = renderNodeTimestampText;
window._nodesGetStatusInfo = getStatusInfo;
window._nodesGetStatusTooltip = getStatusTooltip;
// #862: Expose search filter logic for testing
window._nodesMatchesSearch = function(node, query) {
if (!query) return true;
var q = query.toLowerCase();
var isHex = /^[0-9a-f]+$/i.test(q);
if ((node.name || '').toLowerCase().includes(q)) return true;
if (isHex && (node.public_key || '').toLowerCase().startsWith(q)) return true;
return false;
};
})();
+40 -236
View File
@@ -45,11 +45,6 @@
var parts = [];
if (timeWindowMin && timeWindowMin !== DEFAULT_TIME_WINDOW) parts.push('timeWindow=' + timeWindowMin);
if (regionParam) parts.push('region=' + encodeURIComponent(regionParam));
if (filters.hash) parts.push('hash=' + encodeURIComponent(filters.hash));
if (filters.node) parts.push('node=' + encodeURIComponent(filters.node));
if (filters.observer) parts.push('observer=' + encodeURIComponent(filters.observer));
if (filters.channel) parts.push('channel=' + encodeURIComponent(filters.channel));
if (filters._filterExpr) parts.push('filter=' + encodeURIComponent(filters._filterExpr));
return parts.length ? '?' + parts.join('&') : '';
}
window.buildPacketsQuery = buildPacketsQuery;
@@ -347,16 +342,6 @@
}
var _urlRegion = _initUrlParams.get('region');
if (_urlRegion) _pendingUrlRegion = _urlRegion;
var _urlHash = _initUrlParams.get('hash');
if (_urlHash) filters.hash = _urlHash;
var _urlNode = _initUrlParams.get('node');
if (_urlNode) { filters.node = _urlNode; filters.nodeName = _urlNode.slice(0, 8); }
var _urlObserver = _initUrlParams.get('observer');
if (_urlObserver) filters.observer = _urlObserver;
var _urlChannel = _initUrlParams.get('channel');
if (_urlChannel) filters.channel = _urlChannel;
var _urlFilterExpr = _initUrlParams.get('filter');
if (_urlFilterExpr) filters._filterExpr = _urlFilterExpr;
app.innerHTML = `<div class="split-layout detail-collapsed">
<div class="panel-left" id="pktLeft" aria-live="polite" aria-relevant="additions removals"></div>
@@ -387,7 +372,7 @@
const obs = data.observations.find(o => String(o.id) === String(obsTarget));
if (obs) {
expandedHashes.add(h);
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, direction: obs.direction, timestamp: obs.timestamp, first_seen: obs.timestamp};
const obsPacket = {...data.packet, observer_id: obs.observer_id, observer_name: obs.observer_name, snr: obs.snr, rssi: obs.rssi, path_json: obs.path_json, resolved_path: obs.resolved_path, timestamp: obs.timestamp, first_seen: obs.timestamp};
clearParsedCache(obsPacket);
selectPacket(obs.id, h, {packet: obsPacket, breakdown: data.breakdown, observations: data.observations}, obs.id);
} else {
@@ -625,7 +610,6 @@
if (filters.hash) params.set('hash', filters.hash);
if (filters.node) params.set('node', filters.node);
if (filters.observer) params.set('observer', filters.observer);
if (filters.channel) params.set('channel', filters.channel);
if (groupByHash) {
params.set('groupByHash', 'true');
} else {
@@ -754,11 +738,6 @@
<button class="multi-select-trigger" id="typeTrigger" title="Filter by packet type">All Types </button>
<div class="multi-select-menu" id="typeMenu"></div>
</div>
<div class="filter-group" style="display:inline-flex;align-items:center;gap:4px">
<select id="fChannel" class="filter-select" aria-label="Filter by channel" title="Filter Channel Messages (GRP_TXT) by channel">
<option value="">All Channels</option>
</select>
</div>
</div>
<div class="filter-group">
<button class="btn ${groupByHash ? 'active' : ''}" id="fGroup" title="Collapse duplicate observations of the same packet into expandable groups">Group by Hash</button>
@@ -818,12 +797,6 @@
var pfError = document.getElementById('packetFilterError');
var pfCount = document.getElementById('packetFilterCount');
if (!pfInput || !window.PacketFilter) return;
// Restore Wireshark filter expression from URL
if (filters._filterExpr) {
pfInput.value = filters._filterExpr;
var _restored = PacketFilter.compile(filters._filterExpr);
if (!_restored.error) { pfInput.classList.add('filter-active'); filters._packetFilter = _restored.filter; }
}
var pfTimer = null;
pfInput.addEventListener('input', function() {
clearTimeout(pfTimer);
@@ -834,8 +807,6 @@
pfError.style.display = 'none';
pfCount.style.display = 'none';
filters._packetFilter = null;
filters._filterExpr = undefined;
updatePacketsUrl();
renderTableRows();
return;
}
@@ -847,16 +818,12 @@
pfError.style.display = 'block';
pfCount.style.display = 'none';
filters._packetFilter = null;
filters._filterExpr = undefined;
updatePacketsUrl();
renderTableRows();
} else {
pfInput.classList.remove('filter-error');
pfInput.classList.add('filter-active');
pfError.style.display = 'none';
filters._packetFilter = compiled.filter;
filters._filterExpr = expr;
updatePacketsUrl();
renderTableRows();
}
}, 300);
@@ -901,7 +868,6 @@
if (filters.observer) localStorage.setItem('meshcore-observer-filter', filters.observer); else localStorage.removeItem('meshcore-observer-filter');
buildObserverMenu();
updateObsTrigger();
updatePacketsUrl();
renderTableRows();
});
@@ -947,63 +913,6 @@
renderTableRows();
});
// --- Channel filter (#812) ---
// Server-side filter: /api/packets?channel=<hash>. Triggers loadPackets()
// (not just renderTableRows) so the filter applies before pagination.
const channelSel = document.getElementById('fChannel');
if (channelSel) {
if (filters.channel) {
// Pre-seed an option so the current filter shows as selected even
// before the channels list arrives. Replaced when populateChannels resolves.
const opt = document.createElement('option');
opt.value = filters.channel;
opt.textContent = filters.channel;
opt.selected = true;
channelSel.appendChild(opt);
}
api('/channels').then(data => {
const channels = (data && data.channels) || [];
// Build options via DOM API: channel names are network-supplied
// and must NOT be interpolated into innerHTML (XSS, #812).
// Sort alphabetically (case-insensitive) for predictable picker order;
// the API returns last-activity order which is unstable for a dropdown.
const sorted = channels.slice().sort((a, b) => {
const an = (a.name || a.hash || '').toLowerCase();
const bn = (b.name || b.hash || '').toLowerCase();
return an < bn ? -1 : an > bn ? 1 : 0;
});
channelSel.textContent = '';
const allOpt = document.createElement('option');
allOpt.value = '';
allOpt.textContent = 'All Channels';
channelSel.appendChild(allOpt);
let matched = false;
for (const ch of sorted) {
const v = ch.hash || ch.name || '';
if (!v) continue;
const opt = document.createElement('option');
opt.value = v;
opt.textContent = ch.name || v;
if (v === filters.channel) { opt.selected = true; matched = true; }
channelSel.appendChild(opt);
}
// If current filter isn't in the list (encrypted hash, stale, or
// race with cache), keep it as a selected option so the UI reflects state.
if (filters.channel && !matched) {
const opt = document.createElement('option');
opt.value = filters.channel;
opt.textContent = filters.channel;
opt.selected = true;
channelSel.appendChild(opt);
}
}).catch(() => {});
channelSel.addEventListener('change', (e) => {
filters.channel = e.target.value || undefined;
updatePacketsUrl();
loadPackets();
});
}
// Close multi-select menus on outside click
bindDocumentHandler('menu', 'click', (e) => {
const obsWrap = document.getElementById('observerFilterWrap');
@@ -1021,7 +930,7 @@
// Filter event listeners
document.getElementById('fHash').value = filters.hash || '';
document.getElementById('fHash').addEventListener('input', debounce((e) => { filters.hash = e.target.value || undefined; updatePacketsUrl(); loadPackets(); }, 300));
document.getElementById('fHash').addEventListener('input', debounce((e) => { filters.hash = e.target.value || undefined; loadPackets(); }, 300));
// Time window dropdown — restore from localStorage and bind change
const fTimeWindow = document.getElementById('fTimeWindow');
@@ -1156,7 +1065,7 @@
if (!q) {
fNodeDrop.classList.add('hidden');
fNode.setAttribute('aria-expanded', 'false');
if (filters.node) { filters.node = undefined; filters.nodeName = undefined; updatePacketsUrl(); loadPackets(); }
if (filters.node) { filters.node = undefined; filters.nodeName = undefined; loadPackets(); }
return;
}
try {
@@ -1165,7 +1074,7 @@
const nodes = data.nodes || [];
if (nodes.length === 0) { fNodeDrop.classList.add('hidden'); fNode.setAttribute('aria-expanded', 'false'); return; }
fNodeDrop.innerHTML = nodes.map((n, i) =>
`<div class="node-filter-option" id="fNodeOpt-${i}" role="option" data-key="${n.public_key}" data-name="${escapeHtml(n.name || n.public_key.slice(0,8))}">${escapeHtml(n.name || n.public_key.slice(0,8))} <span style="color:var(--text-muted);font-size:0.8em">${n.public_key.slice(0,8)}</span></div>`
`<div class="node-filter-option" id="fNodeOpt-${i}" role="option" data-key="${n.public_key}" data-name="${escapeHtml(n.name || n.public_key.slice(0,8))}">${escapeHtml(n.name || n.public_key.slice(0,8))} <span style="color:var(--muted);font-size:0.8em">${n.public_key.slice(0,8)}</span></div>`
).join('');
fNodeDrop.classList.remove('hidden');
fNode.setAttribute('aria-expanded', 'true');
@@ -1185,7 +1094,6 @@
fNode.setAttribute('aria-expanded', 'false');
fNode.setAttribute('aria-activedescendant', '');
nodeActiveIdx = -1;
updatePacketsUrl();
loadPackets();
}
@@ -1246,7 +1154,7 @@
const child = group?._children?.find(c => String(c.id) === String(value));
if (child) {
const parentData = group._fetchedData;
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, direction: child.direction, timestamp: child.timestamp, first_seen: child.timestamp} : child;
const obsPacket = parentData ? {...parentData.packet, observer_id: child.observer_id, observer_name: child.observer_name, snr: child.snr, rssi: child.rssi, path_json: child.path_json, resolved_path: child.resolved_path, timestamp: child.timestamp, first_seen: child.timestamp} : child;
if (parentData) { clearParsedCache(obsPacket); }
selectPacket(child.id, parentHash, {packet: obsPacket, breakdown: parentData?.breakdown, observations: parentData?.observations}, child.id);
}
@@ -1797,49 +1705,19 @@
panel.innerHTML = isMobileNow ? '' : '<div class="panel-resize-handle" id="pktResizeHandle"></div>' + PANEL_CLOSE_HTML;
const content = document.createElement('div');
panel.appendChild(content);
await renderDetail(content, data, selectedObservationId);
await renderDetail(content, data);
if (!isMobileNow) initPanelResize();
} catch (e) {
panel.innerHTML = `<div class="text-muted">Error: ${e.message}</div>`;
}
}
async function renderDetail(panel, data, chosenObsId) {
async function renderDetail(panel, data) {
const pkt = data.packet;
const breakdown = data.breakdown || {};
const ranges = breakdown.ranges || [];
const observations = data.observations || [];
// Per-observation rendering (issue #849):
// When opened from a packet row (no specific observer), default to first observation.
// When opened from an observation child row, use that observation.
// Clicking a different observation row in the detail re-renders with that observation.
let currentObs = null;
const targetObsId = chosenObsId || selectedObservationId;
if (targetObsId && observations.length) {
currentObs = observations.find(o => String(o.id) === String(targetObsId));
}
if (!currentObs && observations.length) {
currentObs = observations[0]; // fall back to first observation
}
// If we have a current observation, build pkt fields from it so summary is per-observation
const effectivePkt = currentObs ? clearParsedCache({...pkt, ...currentObs, _isObservation: true}) : pkt;
const decoded = getParsedDecoded(effectivePkt) || {};
const pathHops = getParsedPath(effectivePkt) || [];
// Cross-check: hop count from raw_hex path_len byte vs path_json length
const obsRawHex = effectivePkt.raw_hex || pkt.raw_hex || '';
let rawHopCount = null;
if (obsRawHex.length >= 4) {
// path_len byte position depends on route type
const plOff = getPathLenOffset(pkt.route_type);
const plByte = parseInt(obsRawHex.slice(plOff * 2, plOff * 2 + 2), 16);
if (!isNaN(plByte)) rawHopCount = plByte & 0x3F;
}
if (rawHopCount != null && pathHops.length !== rawHopCount) {
console.warn(`[CoreScope] Hop count inconsistency for packet ${pkt.hash}: path_json has ${pathHops.length} hops but raw_hex path_len has ${rawHopCount}. Trusting raw_hex.`);
}
const decoded = getParsedDecoded(pkt) || {};
const pathHops = getParsedPath(pkt) || [];
// Resolve sender GPS — from packet directly, or from known node in DB
let senderLat = decoded.lat != null ? decoded.lat : (decoded.latitude || null);
@@ -1883,16 +1761,15 @@
}
// Parse hash size from path byte
const plOff = getPathLenOffset(pkt.route_type);
const rawPathByte = pkt.raw_hex ? parseInt(pkt.raw_hex.slice(plOff * 2, plOff * 2 + 2), 16) : NaN;
const rawPathByte = pkt.raw_hex ? parseInt(pkt.raw_hex.slice(2, 4), 16) : NaN;
const hashSize = (isNaN(rawPathByte) || (rawPathByte & 0x3F) === 0) ? null : ((rawPathByte >> 6) + 1);
const size = effectivePkt.raw_hex ? Math.floor(effectivePkt.raw_hex.length / 2) : (pkt.raw_hex ? Math.floor(pkt.raw_hex.length / 2) : 0);
const size = pkt.raw_hex ? Math.floor(pkt.raw_hex.length / 2) : 0;
const typeName = payloadTypeName(pkt.payload_type);
const snr = effectivePkt.snr ?? decoded.SNR ?? decoded.snr ?? null;
const rssi = effectivePkt.rssi ?? decoded.RSSI ?? decoded.rssi ?? null;
const hasRawHex = !!(effectivePkt.raw_hex || pkt.raw_hex);
const snr = pkt.snr ?? decoded.SNR ?? decoded.snr ?? null;
const rssi = pkt.rssi ?? decoded.RSSI ?? decoded.rssi ?? null;
const hasRawHex = !!pkt.raw_hex;
// Build message preview
let messageHtml = '';
@@ -1903,16 +1780,17 @@
const meta = [chLabel, hopLabel, snrLabel].filter(Boolean).join(' · ');
messageHtml = `<div class="detail-message" style="padding:12px;margin:8px 0;background:var(--card-bg);border-radius:8px;border-left:3px solid var(--accent)">
<div style="font-size:1.1em">${escapeHtml(decoded.text)}</div>
${meta ? `<div style="font-size:0.85em;color:var(--text-muted);margin-top:4px">${meta}</div>` : ''}
${meta ? `<div style="font-size:0.85em;color:var(--muted);margin-top:4px">${meta}</div>` : ''}
</div>`;
} else if (decoded.type === 'GRP_TXT' && decoded.channelHash != null) {
const hashHex = decoded.channelHashHex || decoded.channelHash.toString(16).padStart(2, '0').toUpperCase();
const statusLabel = decoded.decryptionStatus === 'no_key' ? 'no key' : 'decryption failed';
messageHtml = `<div class="detail-message" style="padding:12px;margin:8px 0;background:var(--card-bg);border-radius:8px;border-left:3px solid var(--warning, #f0ad4e)">
<div style="font-size:1.1em">🔒 Channel Hash: 0x${hashHex} <span style="color:var(--text-muted)">(${statusLabel})</span></div>
<div style="font-size:1.1em">🔒 Channel Hash: 0x${hashHex} <span style="color:var(--muted)">(${statusLabel})</span></div>
</div>`;
}
const observations = data.observations || [];
const obsCount = data.observation_count || observations.length || 1;
const uniqueObservers = new Set(observations.map(o => o.observer_id)).size;
@@ -1971,32 +1849,20 @@
}
}
const anomalyBanner = decoded.anomaly
? `<div class="anomaly-banner" style="background:var(--warning, #f0ad4e); color:#000; padding:8px 12px; border-radius:4px; margin-bottom:8px; font-weight:600;">⚠️ Anomaly: ${escapeHtml(decoded.anomaly)}</div>`
: '';
// Hop count display: trust raw_hex (firmware truth) over path_json
const displayHopCount = rawHopCount != null ? rawHopCount : pathHops.length;
const obsIndicator = currentObs && observations.length > 1
? `<span style="font-size:0.8em;color:var(--text-muted);margin-left:6px">(observation ${observations.indexOf(currentObs) + 1} of ${observations.length})</span>`
: '';
panel.innerHTML = `
${anomalyBanner}
<div class="detail-title">${hasRawHex ? `Packet Byte Breakdown (${size} bytes)` : typeName + ' Packet'}</div>
<div class="detail-hash">${pkt.hash || 'Packet #' + pkt.id}${obsIndicator}</div>
<div class="detail-hash">${pkt.hash || 'Packet #' + pkt.id}</div>
${messageHtml}
<dl class="detail-meta">
<dt>Observer</dt><dd>${obsName(effectivePkt.observer_id)}</dd>
<dt>Observer</dt><dd>${obsName(pkt.observer_id)}</dd>
<dt>Location</dt><dd>${locationHtml}</dd>
<dt>SNR / RSSI</dt><dd>${snr != null ? snr + ' dB' : '—'} / ${rssi != null ? rssi + ' dBm' : '—'}</dd>
<dt>Route Type</dt><dd>${routeTypeName(pkt.route_type)}</dd>
<dt>Payload Type</dt><dd><span class="badge badge-${payloadTypeColor(pkt.payload_type)}">${typeName}</span></dd>
${hashSize ? `<dt>Hash Size</dt><dd>${hashSize} byte${hashSize !== 1 ? 's' : ''}</dd>` : ''}
<dt>Timestamp</dt><dd>${renderTimestampCell(effectivePkt.timestamp)}</dd>
<dt>Timestamp</dt><dd>${renderTimestampCell(pkt.timestamp)}</dd>
<dt>Propagation</dt><dd>${propagationHtml}</dd>
<dt>Path</dt><dd>${displayHopCount > 0 ? `<span class="badge badge-info">${displayHopCount} hop${displayHopCount !== 1 ? 's' : ''}</span> ` + renderPath(pathHops, effectivePkt.observer_id) : ' (direct)'}</dd>
${effectivePkt.direction ? `<dt>Direction</dt><dd>${escapeHtml(effectivePkt.direction)}</dd>` : ''}
<dt>Path</dt><dd>${pathHops.length ? renderPath(pathHops, pkt.observer_id) : ''}</dd>
</dl>
<div class="detail-actions">
<button class="copy-link-btn" data-packet-hash="${pkt.hash || ''}" data-packet-id="${pkt.id}" title="Copy link to this packet">🔗 Copy Link</button>
@@ -2006,59 +1872,11 @@
</div>
${hasRawHex ? `<div class="hex-legend">${buildHexLegend(ranges)}</div>
<div class="hex-dump">${createColoredHexDump(effectivePkt.raw_hex || pkt.raw_hex, ranges)}</div>` : ''}
<div class="hex-dump">${createColoredHexDump(pkt.raw_hex, ranges)}</div>` : ''}
${hasRawHex ? buildFieldTable(effectivePkt.raw_hex ? effectivePkt : pkt, decoded, pathHops, ranges) : buildDecodedTable(decoded)}
${observations.length > 1 ? `
<div class="detail-observations" style="margin-top:16px">
<div style="font-weight:600;margin-bottom:6px">Observations (${observations.length})</div>
<table class="detail-obs-table" style="width:100%;border-collapse:collapse;font-size:0.9em">
<thead><tr style="border-bottom:1px solid var(--border)">
<th style="padding:4px 6px;text-align:left">Observer</th>
<th style="padding:4px 6px;text-align:left">Hops</th>
<th style="padding:4px 6px;text-align:left">SNR</th>
<th style="padding:4px 6px;text-align:left">RSSI</th>
<th style="padding:4px 6px;text-align:left">Time</th>
</tr></thead>
<tbody>${observations.map(o => {
const oPath = getParsedPath(o);
const isCurrent = currentObs && String(o.id) === String(currentObs.id);
return `<tr class="detail-obs-row${isCurrent ? ' observation-current' : ''}" data-obs-id="${o.id}" style="cursor:pointer;${isCurrent ? 'background:var(--accent-bg, rgba(0,122,255,0.1))' : ''}" title="Click to view this observation">
<td style="padding:4px 6px">${obsName(o.observer_id)}</td>
<td style="padding:4px 6px">${oPath.length}</td>
<td style="padding:4px 6px">${o.snr != null ? o.snr + ' dB' : '—'}</td>
<td style="padding:4px 6px">${o.rssi != null ? o.rssi + ' dBm' : '—'}</td>
<td style="padding:4px 6px">${renderTimestampCell(o.timestamp)}</td>
</tr>`;
}).join('')}</tbody>
</table>
</div>` : ''}
${observations.length > 1 ? (() => {
// Cross-observer aggregate (Option B): show longest observed path across all observers
const aggregatePath = getParsedPath(pkt) || [];
return `<div class="detail-aggregate" style="margin-top:12px;padding:10px;background:var(--card-bg);border-radius:6px;border:1px solid var(--border);font-size:0.9em">
<div style="font-weight:600;margin-bottom:4px;color:var(--text-muted)">Cross-observer aggregate</div>
<div>Longest observed path: ${aggregatePath.length ? `${aggregatePath.length} hops — ${renderPath(aggregatePath, pkt.observer_id)}` : '— (direct)'}</div>
<div style="font-size:0.8em;color:var(--text-muted);margin-top:2px">Longest path seen across all ${uniqueObservers} observer${uniqueObservers !== 1 ? 's' : ''}</div>
</div>`;
})() : ''}
${hasRawHex ? buildFieldTable(pkt, decoded, pathHops, ranges) : buildDecodedTable(decoded)}
`;
// Wire up observation row click handlers — re-render detail with clicked observation
panel.querySelectorAll('.detail-obs-row').forEach(row => {
row.addEventListener('click', () => {
const obsId = row.dataset.obsId;
selectedObservationId = obsId;
// Update URL hash to reflect selected observation (deep linking)
const pktHash = pkt.hash || pkt.id;
const obsParam = obsId ? `?obs=${obsId}` : '';
history.replaceState(null, '', `#/packets/${pktHash}${obsParam}`);
renderDetail(panel, data, obsId);
});
});
// Wire up copy link button
const copyLinkBtn = panel.querySelector('.copy-link-btn');
if (copyLinkBtn) {
@@ -2163,43 +1981,38 @@
// Header section
rows += sectionRow('Header', 'section-header');
rows += fieldRow(0, 'Header Byte', '0x' + (buf.slice(0, 2) || '??'), `Route: ${routeTypeName(pkt.route_type)}, Payload: ${payloadTypeName(pkt.payload_type)}`);
const pathByte0 = parseInt(buf.slice(2, 4), 16);
const hashSizeVal = isNaN(pathByte0) ? '?' : ((pathByte0 >> 6) + 1);
const hashCountVal = isNaN(pathByte0) ? '?' : (pathByte0 & 0x3F);
rows += fieldRow(1, 'Path Length', '0x' + (buf.slice(2, 4) || '??'), hashCountVal === 0 ? `hash_count=0 (direct advert)` : `hash_size=${hashSizeVal} byte${hashSizeVal !== 1 ? 's' : ''}, hash_count=${hashCountVal}`);
// Transport codes come BEFORE path length for transport routes (bytes 1-4)
let off = 1;
if (isTransportRoute(pkt.route_type)) {
// Transport codes
let off = 2;
if (pkt.route_type === 0 || pkt.route_type === 3) {
rows += sectionRow('Transport Codes', 'section-transport');
rows += fieldRow(off, 'Next Hop', buf.slice(off * 2, (off + 2) * 2), '');
rows += fieldRow(off + 2, 'Last Hop', buf.slice((off + 2) * 2, (off + 4) * 2), '');
off += 4;
}
// Path length byte is at current offset (byte 1 for non-transport, byte 5 for transport)
const pathLenOffset = off;
const pathByte0 = parseInt(buf.slice(off * 2, off * 2 + 2), 16);
const hashSizeVal = isNaN(pathByte0) ? '?' : ((pathByte0 >> 6) + 1);
const hashCountVal = isNaN(pathByte0) ? '?' : (pathByte0 & 0x3F);
rows += fieldRow(off, 'Path Length', '0x' + (buf.slice(off * 2, off * 2 + 2) || '??'), hashCountVal === 0 ? `hash_count=0 (direct advert)` : `hash_size=${hashSizeVal} byte${hashSizeVal !== 1 ? 's' : ''}, hash_count=${hashCountVal}`);
off += 1;
// Path — derive hop count from path_len byte (firmware truth), not aggregated _parsedPath
const hashSize = isNaN(pathByte0) ? 1 : ((pathByte0 >> 6) + 1);
if (typeof hashCountVal === 'number' && hashCountVal > 0) {
rows += sectionRow('Path (' + hashCountVal + ' hops)', 'section-path');
for (let i = 0; i < hashCountVal; i++) {
const hopOff = off + i * hashSize;
const hex = buf.slice(hopOff * 2, (hopOff + hashSize) * 2).toUpperCase();
const hopHtml = HopDisplay.renderHop(hex, hopNameCache[hex]);
// Path
if (pathHops.length > 0) {
rows += sectionRow('Path (' + pathHops.length + ' hops)', 'section-path');
const pathByte = parseInt(buf.slice(2, 4), 16);
const hashSize = (pathByte >> 6) + 1;
for (let i = 0; i < pathHops.length; i++) {
const hopHtml = HopDisplay.renderHop(pathHops[i], hopNameCache[pathHops[i]]);
const label = `Hop ${i}${hopHtml}`;
rows += fieldRow(hopOff, label, hex, '');
rows += fieldRow(off + i * hashSize, label, pathHops[i], '');
}
off += hashSize * hashCountVal;
off += hashSize * pathHops.length;
}
// Payload
rows += sectionRow('Payload — ' + payloadTypeName(pkt.payload_type), 'section-payload');
if (decoded.type === 'ADVERT') {
if (hashCountVal !== 0) rows += fieldRow(pathLenOffset, 'Advertised Hash Size', hashSizeVal + ' byte' + (hashSizeVal !== 1 ? 's' : ''), 'From path byte 0x' + (buf.slice(pathLenOffset * 2, pathLenOffset * 2 + 2) || '??') + ' — bits 7-6 = ' + (hashSizeVal - 1));
if (hashCountVal !== 0) rows += fieldRow(1, 'Advertised Hash Size', hashSizeVal + ' byte' + (hashSizeVal !== 1 ? 's' : ''), 'From path byte 0x' + (buf.slice(2, 4) || '??') + ' — bits 7-6 = ' + (hashSizeVal - 1));
rows += fieldRow(off, 'Public Key (32B)', truncate(decoded.pubKey || '', 24), '');
rows += fieldRow(off + 32, 'Timestamp (4B)', decoded.timestampISO || '', 'Unix: ' + (decoded.timestamp || ''));
rows += fieldRow(off + 36, 'Signature (64B)', truncate(decoded.signature || '', 24), '');
@@ -2240,10 +2053,6 @@
rows += fieldRow(off, 'Raw', truncate(buf.slice(off * 2), 40), '');
}
if (decoded.anomaly) {
rows += `<tr class="anomaly-row" style="background:var(--warning, #f0ad4e); color:#000; font-weight:600;"><td colspan="2">⚠️ Anomaly</td><td colspan="2">${escapeHtml(decoded.anomaly)}</td></tr>`;
}
return `<table class="field-table">
<thead><tr><th scope="col">Offset</th><th scope="col">Field</th><th scope="col">Value</th><th scope="col">Description</th></tr></thead>
<tbody>${rows}</tbody>
@@ -2335,11 +2144,6 @@
let html = '<div class="byop-decoded">';
// Anomaly banner
if (d.anomaly) {
html += '<div class="anomaly-banner" style="background:var(--warning, #f0ad4e); color:#000; padding:8px 12px; border-radius:4px; margin-bottom:8px; font-weight:600;">⚠️ Anomaly: ' + escapeHtml(d.anomaly) + '</div>';
}
// Header section
html += '<div class="byop-section">'
+ '<div class="byop-section-title">Header</div>'
+1 -2
View File
@@ -91,9 +91,8 @@
const ps = server.packetStore;
html += `<h3>In-Memory Packet Store</h3><div style="display:flex;gap:16px;flex-wrap:wrap;margin:8px 0;">
<div class="perf-card"><div class="perf-num">${ps.inMemory.toLocaleString()}</div><div class="perf-label">Packets in RAM</div></div>
<div class="perf-card"><div class="perf-num">${ps.trackedMB}MB</div><div class="perf-label">Tracked Memory</div></div>
<div class="perf-card"><div class="perf-num">${ps.estimatedMB}MB</div><div class="perf-label">Memory Used</div></div>
<div class="perf-card"><div class="perf-num">${ps.maxMB}MB</div><div class="perf-label">Memory Limit</div></div>
<div class="perf-card"><div class="perf-num">${ps.estimatedMB}MB</div><div class="perf-label">Heap (debug)</div></div>
<div class="perf-card"><div class="perf-num">${ps.queries.toLocaleString()}</div><div class="perf-label">Queries Served</div></div>
<div class="perf-card"><div class="perf-num">${ps.inserts.toLocaleString()}</div><div class="perf-label">Live Inserts</div></div>
<div class="perf-card"><div class="perf-num">${ps.evicted.toLocaleString()}</div><div class="perf-label">Evicted</div></div>
-79
View File
@@ -394,83 +394,4 @@
});
return html;
};
// #690 — Clock Skew shared helpers
var SKEW_SEVERITY_COLORS = {
ok: 'var(--status-green)',
warning: 'var(--status-yellow)',
critical: 'var(--status-orange)',
absurd: 'var(--status-purple)',
bimodal_clock: 'var(--status-amber)',
no_clock: 'var(--text-muted)'
};
var SKEW_SEVERITY_LABELS = {
ok: 'OK', warning: 'Warning', critical: 'Critical', absurd: 'Absurd', bimodal_clock: 'Bimodal', no_clock: 'No Clock'
};
var SKEW_SEVERITY_ORDER = { no_clock: 0, bimodal_clock: 1, absurd: 2, critical: 3, warning: 4, ok: 5 };
window.SKEW_SEVERITY_COLORS = SKEW_SEVERITY_COLORS;
window.SKEW_SEVERITY_LABELS = SKEW_SEVERITY_LABELS;
window.SKEW_SEVERITY_ORDER = SKEW_SEVERITY_ORDER;
/** Format skew seconds into human-readable string like "+2m 34s" or "-15h 22m" */
window.formatSkew = function(sec) {
if (sec == null) return '—';
var abs = Math.abs(sec);
var sign = sec >= 0 ? '+' : '-';
if (abs < 60) return sign + Math.round(abs) + 's';
if (abs < 3600) return sign + Math.floor(abs / 60) + 'm ' + Math.round(abs % 60) + 's';
if (abs < 86400) return sign + Math.floor(abs / 3600) + 'h ' + Math.round((abs % 3600) / 60) + 'm';
return sign + Math.floor(abs / 86400) + 'd ' + Math.round((abs % 86400) / 3600) + 'h';
};
/** Format drift rate as "+X.Xs/day" or "—" if falsy */
window.formatDrift = function(secPerDay) {
if (!secPerDay) return '—';
return (secPerDay >= 0 ? '+' : '') + secPerDay.toFixed(1) + ' s/day';
};
/** Pick the skew value that drives current-health UI: prefer the
* recent-window median (#789, current health) over the all-time median
* (poisoned by historical bad samples). Falls back gracefully if the
* field isn't present (older API responses). */
window.currentSkewValue = function(cs) {
if (!cs) return null;
return cs.recentMedianSkewSec != null ? cs.recentMedianSkewSec : cs.medianSkewSec;
};
/** Render a clock skew badge HTML */
window.renderSkewBadge = function(severity, skewSec, cs) {
if (!severity) return '';
var cls = 'skew-badge skew-badge--' + severity;
if (severity === 'no_clock') {
return '<span class="' + cls + '" title="Uninitialized RTC — no valid clock">🚫 No Clock</span>';
}
if (severity === 'bimodal_clock' && cs) {
var badPct = cs.goodFraction != null ? Math.round((1 - cs.goodFraction) * 100) : '?';
var label = '⏰ ' + window.formatSkew(skewSec);
return '<span class="' + cls + '" title="Clock skew: ' + window.formatSkew(skewSec) + ' (bimodal: ' + badPct + '% of recent adverts have nonsense timestamps)">' + label + '</span>';
}
var label = severity === 'ok' ? '⏰' : '⏰ ' + window.formatSkew(skewSec);
return '<span class="' + cls + '" title="Clock skew: ' + window.formatSkew(skewSec) + ' (' + (SKEW_SEVERITY_LABELS[severity] || severity) + ')">' + label + '</span>';
};
/** Render a skew sparkline SVG (inline, word-sized) */
window.renderSkewSparkline = function(samples, w, h) {
w = w || 120; h = h || 24;
if (!samples || samples.length < 2) return '';
var values = samples.map(function(s) { return s.skew; });
var max = Math.max.apply(null, values.map(function(v) { return Math.abs(v); }).concat([1]));
var pts = values.map(function(v, i) {
var x = i * (w / Math.max(values.length - 1, 1));
var y = h / 2 - (v / max) * (h / 2 - 2);
return x.toFixed(1) + ',' + y.toFixed(1);
}).join(' ');
// Zero line
var zeroY = h / 2;
return '<svg viewBox="0 0 ' + w + ' ' + h + '" style="width:' + w + 'px;height:' + h + 'px" role="img" aria-label="Clock skew sparkline">' +
'<title>Clock skew over time</title>' +
'<line x1="0" y1="' + zeroY + '" x2="' + w + '" y2="' + zeroY + '" stroke="var(--border)" stroke-width="0.5" stroke-dasharray="2"/>' +
'<polyline points="' + pts + '" fill="none" stroke="var(--accent)" stroke-width="1.5"/></svg>';
};
})();
+2 -101
View File
@@ -11,12 +11,6 @@
--status-green: #22c55e;
--status-yellow: #eab308;
--status-red: #ef4444;
--status-orange: #f97316;
--status-purple: #a855f7;
--status-amber: #f59e0b;
--status-amber-light: #fef3c7;
--status-amber-text: #92400e;
--role-observer: #8b5cf6;
--accent-hover: #6db3ff;
--text: #1a1a2e;
--text-muted: #5b6370;
@@ -47,11 +41,6 @@
--status-green: #22c55e;
--status-yellow: #eab308;
--status-red: #ef4444;
--status-orange: #f97316;
--status-purple: #a855f7;
--status-amber: #f59e0b;
--status-amber-light: #422006;
--status-amber-text: #fcd34d;
--surface-0: #0f0f23;
--surface-1: #1a1a2e;
--surface-2: #232340;
@@ -76,11 +65,6 @@
--status-green: #22c55e;
--status-yellow: #eab308;
--status-red: #ef4444;
--status-orange: #f97316;
--status-purple: #a855f7;
--status-amber: #f59e0b;
--status-amber-light: #422006;
--status-amber-text: #fcd34d;
--surface-0: #0f0f23;
--surface-1: #1a1a2e;
--surface-2: #232340;
@@ -354,9 +338,6 @@ a:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible
}
.detail-meta dt { color: var(--text-muted); font-size: 11px; text-transform: uppercase; letter-spacing: .3px; }
.detail-meta dd { font-weight: 500; margin-bottom: 4px; }
.observation-current { background: var(--accent-bg, rgba(0,122,255,0.1)); font-weight: 600; }
.detail-obs-row:hover { background: var(--hover-bg, rgba(255,255,255,0.05)); }
.detail-obs-table th { font-size: 0.8em; text-transform: uppercase; color: var(--text-muted); }
/* === Hex Dump === */
.hex-dump {
@@ -482,14 +463,6 @@ fieldset.mc-section legend.mc-label { padding: 0; }
.ch-sidebar-title {
display: flex; align-items: center; gap: 8px; font-size: 16px; font-weight: 700; margin-bottom: 8px;
}
.ch-encrypted-toggle {
display: flex; align-items: center; gap: 4px; font-size: 11px; color: var(--text-muted);
cursor: pointer; user-select: none; margin-bottom: 4px;
}
.ch-encrypted-toggle input { margin: 0; cursor: pointer; }
.ch-toggle-label { white-space: nowrap; }
.ch-item.ch-encrypted { opacity: 0.55; }
.ch-item.ch-encrypted .ch-item-name { font-style: italic; }
.ch-icon { font-size: 20px; }
.ch-sidebar-controls { display: flex; align-items: center; gap: 6px; }
.ch-region-select {
@@ -522,9 +495,6 @@ button.ch-item.selected { background: var(--selected-bg); }
.ch-item-top { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 2px; }
.ch-item-name { font-weight: 600; font-size: 14px; }
.ch-item-time { font-size: 11px; color: var(--text-muted); white-space: nowrap; }
.ch-remove-btn { background: none; border: none; color: var(--text-muted); cursor: pointer; font-size: 13px; padding: 0 2px; margin-left: 4px; opacity: 0; transition: opacity 0.15s; line-height: 1; }
button.ch-item:hover .ch-remove-btn { opacity: 0.6; }
.ch-remove-btn:hover { opacity: 1 !important; color: var(--danger, #dc2626); }
.ch-item-preview { font-size: 12px; color: var(--text-muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.ch-main { flex: 1; display: flex; flex-direction: column; overflow: hidden; position: relative; }
@@ -709,9 +679,7 @@ button.ch-item:hover .ch-remove-btn { opacity: 0.6; }
.advert-dot {
width: 10px; height: 10px; border-radius: 50%; flex-shrink: 0; margin-top: 4px;
}
/* #829: explicit color so text stays readable when inherited color matches card-bg */
.advert-info { font-size: 12px; line-height: 1.5; color: var(--text); }
.advert-info a { color: var(--accent); }
.advert-info { font-size: 12px; line-height: 1.5; }
/* === Traces Page === */
.traces-page { padding: 16px; max-width: var(--trace-max-width, 95vw); margin: 0 auto; }
@@ -1129,49 +1097,6 @@ button.ch-item.ch-item-encrypted:hover { opacity: 0.7; }
button.ch-item.ch-item-encrypted.selected { opacity: 0.8; }
button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
/* Channel key input (#725 M2) */
.ch-key-input {
flex: 1;
min-width: 0;
box-sizing: border-box;
padding: 6px 8px;
border: 1px solid var(--border);
border-radius: 6px 0 0 6px;
background: var(--card-bg);
color: var(--text);
font-size: 12px;
font-family: inherit;
}
.ch-key-input:focus {
outline: 2px solid var(--accent, #3b82f6);
outline-offset: -1px;
border-color: var(--accent, #3b82f6);
}
.ch-key-input::placeholder { color: var(--text-muted); }
.ch-key-input-wrap { margin-bottom: 4px; }
.ch-wrong-key { color: var(--danger, #ef4444); font-weight: 500; }
/* Add channel form (#759) */
.ch-add-form { margin: 0; }
.ch-add-label { display: block; font-weight: 600; font-size: 13px; color: var(--text); margin-bottom: 4px; }
.ch-key-input, .ch-add-btn { height: 32px; box-sizing: border-box; }
.ch-add-row { display: flex; align-items: stretch; }
.ch-add-btn {
width: 32px; height: 32px; flex-shrink: 0;
border: 1px solid var(--accent, #3b82f6); border-left: none;
border-radius: 0 6px 6px 0;
background: var(--accent, #3b82f6); color: #fff;
font-size: 18px; font-weight: 700; line-height: 1;
cursor: pointer; display: flex; align-items: center; justify-content: center;
}
.ch-add-btn:hover { opacity: 0.85; }
.ch-add-hint { font-size: 11px; color: var(--text-muted); margin-top: 4px; line-height: 1.3; }
.ch-add-status { font-size: 12px; margin-top: 4px; padding: 4px 6px; border-radius: 4px; }
.ch-add-status--loading { color: var(--text-muted); }
.ch-add-status--success { color: var(--success, #22c55e); }
.ch-add-status--warn { color: var(--warning, #eab308); }
.ch-add-status--error { color: var(--danger, #ef4444); }
/* Touch-friendly tappable elements */
.ch-tappable {
cursor: pointer;
@@ -1437,9 +1362,7 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); }
.hop-conflict-name { font-weight: 600; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.hop-conflict-dist { font-size: 11px; color: var(--text-muted); font-family: var(--mono); white-space: nowrap; }
.hop-conflict-pk { font-size: 10px; color: var(--text-muted); font-family: var(--mono); }
.hop-unreliable { opacity: 0.85; }
.hop-unreliable-btn { background: none; border: none; color: var(--status-yellow, #f59e0b); font-size: 13px;
cursor: help; vertical-align: middle; margin-left: 2px; padding: 0 2px; line-height: 1; }
.hop-unreliable { opacity: 0.5; text-decoration: line-through; }
.hop-global-fallback { border-bottom: 1px dashed var(--status-red); }
.hop-current { font-weight: 700 !important; color: var(--accent) !important; }
@@ -2288,25 +2211,3 @@ th[data-sort-key] { cursor: pointer; user-select: none; }
th[data-sort-key]:hover { background: var(--hover-bg, rgba(255,255,255,0.05)); }
th.sort-active { color: var(--accent, #60a5fa); }
.sort-arrow { font-size: 0.75em; opacity: 0.8; }
/* #690 — Clock Skew badges & fleet table */
.skew-badge { display: inline-block; font-size: 10px; padding: 1px 5px; border-radius: 3px; margin-left: 4px; font-weight: 600; white-space: nowrap; }
.skew-badge--ok { background: var(--status-green); color: #fff; }
.skew-badge--warning { background: var(--status-yellow); color: #000; }
.skew-badge--critical { background: var(--status-orange); color: #fff; }
.skew-badge--absurd { background: var(--status-purple); color: #fff; }
.skew-badge--no_clock { background: var(--text-muted); color: #fff; }
.skew-badge--bimodal_clock { background: var(--status-amber-light); color: var(--status-amber-text); border: 1px solid var(--status-amber); }
.skew-detail-section { padding: 10px 16px; margin-bottom: 8px; }
.skew-sparkline-wrap { margin-top: 6px; }
.skew-sparkline-wrap svg { display: block; }
.clock-fleet-row--warning { background: color-mix(in srgb, var(--status-yellow) 10%, transparent); }
.clock-fleet-row--critical { background: color-mix(in srgb, var(--status-orange) 10%, transparent); }
.clock-fleet-row--absurd { background: color-mix(in srgb, var(--status-purple) 10%, transparent); }
.clock-fleet-row--no_clock { background: color-mix(in srgb, var(--text-muted) 10%, transparent); }
.clock-filter-btn { font-size: 12px; padding: 3px 8px; border: 1px solid var(--border); border-radius: 4px; background: var(--card-bg, #fff); color: var(--text); cursor: pointer; margin-right: 4px; }
.clock-filter-btn.active { background: var(--accent); color: #fff; border-color: var(--accent); }
-45
View File
@@ -1,45 +0,0 @@
# CoreScope QA artifacts
Project-specific assets for the [`qa-suite`](https://github.com/Kpa-clawbot/ai-sdlc/tree/master/skills/qa-suite) skill.
## Layout
```
qa/
├── README.md ← this file
├── plans/
│ └── <release>.md ← per-release test plans (one file per RC)
└── scripts/
└── api-contract-diff.sh ← CoreScope-tuned API contract diff
```
## How to run
```
qa staging # use the latest plans/v*-rc.md against staging
qa pr 806 # use plans/pr-806.md if it exists, else latest plans/v*-rc.md
qa v3.6.0-rc # use plans/v3.6.0-rc.md
```
The parent agent loads the qa-suite skill, which reads:
1. The plan file from `qa/plans/`
2. Bundled scripts from `qa/scripts/`
3. The reusable engine + qa-engineer persona from the skill itself
## Adding a new plan
For each release candidate, copy the latest `plans/v*-rc.md` to `plans/<new-tag>.md` and update:
- The commit-range header (`vN.M..master`)
- Any new sections for new features in the release
- The "Test data" section if new fixture types are needed
- The GO criteria (which sections are blockers)
## Adding a new script
Custom scripts go in `qa/scripts/` with `mode=auto: <script-name>` referenced from the plan. The qa-engineer subagent runs them with two args: `BASELINE_URL TARGET_URL`.
Authoring rules from the qa-suite skill:
- 4-way error classification: `curl-failed` / `parse-empty` / `shape-diff` / field-missing
- Distinguish HTTP errors from jq parse failures
- Don't silence stderr — script bugs must surface
- Exit code = number of failures
-108
View File
@@ -1,108 +0,0 @@
# Plan: v3.6.0-rc
Targets the changes between v3.5.1 and v3.6.0 candidate (~34 commits).
## Test data
The qa-engineer should pick concrete test fixtures at run time and include them in the report:
- **Pivot node pubkey**: pick the top-result from `/api/nodes?limit=20&sort=advert_count` that has `role=repeater` AND a non-zero `totalPaths` from `/api/nodes/{pk}/paths`. Used for sections 5.1, 8.1, 8.2.
- **Multi-role pubkey** (section 8.6): pick a node whose pubkey appears in BOTH `/api/observers` and `/api/nodes?role=repeater`. If none → mark 8.6 `needs-human`.
- **Sample packet hash**: `/api/packets?limit=1``.packets[0].hash`. Used for sections 3.x.
- **Channel sample**: pick a channel name from `/api/channels` (if endpoint exists) or scrape `/#channels` page.
Record every fixture used in the final report so failures are reproducible.
## Sections
### 1. Memory & Load
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 1.1 | Container with **3 GB** limit starts on heaviest available DB | No OOM, steady state under limit. Note: 1 GB cap is unrealistic without `GOMEMLIMIT` and bounded cold-load — see #836 | #806/#836 | human |
| 1.2 | Hit `/debug/pprof/heap` after Load completes; run `pprof-snapshot.sh` | `unmarshalResolvedPath` absent from top-15 inuse_space; `Load()`-attributed inuse_space ≤ 250 MB on staging-sized DB (~1.5M obs); total heap < 1 GB | #806 | auto: pprof-snapshot.sh |
| 1.3 | Set tight `MaxLoadMemMB`, restart | Load stops gracefully at budget; server still serves `/api/stats` 200 | #790 | human |
| 1.4 | Watch `processRSSMB` (from `/api/stats`) vs procfs RSS over ingest+eviction cycles | `processRSSMB` tracks `cat /proc/$(pidof corescope)/status | awk '/VmRSS/{print $2}'` (kB → MB) within ±20% across one full eviction cycle. Note: `storeDataMB` (formerly `trackedMB`) is the in-store packet byte estimate and is expected to be a **subset** of RSS, not equal to it. | #751, #832 | human |
| 1.5 | Run 30 min under live ingest | Sawtooth heap pattern (≥1 eviction-driven dip), not monotonic ramp | #806/#807 | human |
### 2. API contract
Run `scripts/api-contract-diff.sh BASELINE_URL TARGET_URL` once. Report the script's exit code; nonzero = failures.
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 2.1 | api-contract-diff baseline vs target | Exit code 0; all endpoints carry `resolved_path` where expected | #806 | auto: api-contract-diff.sh |
| 2.2 | WebSocket `/ws` carries `resolved_path` on broadcasts | Run JS hook in browser console: `(function(){let n=0,r=0; const W=WebSocket; window.WebSocket=function(...a){const s=new W(...a); s.addEventListener('message',e=>{n++; try{const m=JSON.parse(e.data); if(m && (m.resolved_path !== undefined || (m.observations||[]).some(o=>o.resolved_path!==undefined))) r++;}catch{}}); return s;}; window.__wsCount=()=>({n,r});})()` then navigate to `/`, wait 30s, eval `__wsCount()``r` must be ≥ 1 if `n` ≥ 1 | #806 | browser |
### 3. Decoder & hashing
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 3.1 | Recompute content hashes for sample of recent packets vs stored | All match (hash uses payload-type bits only) | #787 | human |
| 3.2 | Inspect a TRACE packet detail panel | path_json length matches path_sz from flags byte | #732 | browser |
| 3.3 | Check `hash_size` on transport-route packet AND zero-hop advert | Correct hash_size detected | #747 | browser |
| 3.4 | Field-table column offsets for transport-route packet | Snapshot of detail panel: each field row has nonzero `offset`/`length` cells AND offsets monotonically increase | #766 | browser |
| 3.5 | Corrupt advert ingest log check | Rejected, counted, no DB entry | #794 | human |
| 3.6 | Public channel packet rendering | No empty/garbled decode | #761 | browser |
### 4. Channels (#725)
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 4.1 | Channel list — full message history loads from DB | Past messages persist across reload | #726 | browser |
| 4.2 | Add custom channel via UI (then remove it as teardown) | Channel appears, encrypted msgs decrypt; teardown removes it cleanly. STAGING ONLY. | #733 | browser |
| 4.3 | PSK channel add + channel removal (already a self-teardown) | Both work, UI state correct after | #750 | browser |
| 4.4 | Deep link to encrypted channel without key | Lock message shows | #783 | browser |
| 4.5 | Undecryptable msgs hidden by default + toggle | Hidden default; toggle shows | #728 | browser |
| 4.6 | Add-channel button + hint + status feedback | All present | #760 | browser |
| 4.7 | Filter packets by channel | Functional: filter applies, packet count drops; performance: response time ≤ 500 ms for `/api/packets?channel=<name>&limit=100` (timed via `curl -w '%{time_total}'`) | #762/#763 | browser+auto |
### 5. Clock skew (#690)
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 5.1 | Node detail clock-skew badge + sparkline | Both render | #746/#752 | browser |
| 5.2 | Analytics fleet clock-skew page | Renders, epoch-0 filtered | #769 | browser |
| 5.3 | Outlier sample doesn't poison median | Sanity caps respected; severity uses `recentMedianSkewSec` (#789), not all-time `medianSkewSec` | #769/#789 | human |
| 5.4 | Roles page clock-skew indicator | Renders | #752 | browser |
### 6. Observers
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 6.1 | Observer with no packets in N days disappears after retention sweep | Removed | #764 | human |
| 6.2 | Analytics observer-graph (M1+M2) | Renders (`#observerGraph` element present at `public/analytics.js:2048-2051`) | #774 | browser |
### 7. Multi-byte hash adopters
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 7.1 | Hash Usage Matrix collision details for all hash sizes | Click cell → colliding pubkeys shown | #758 | browser |
| 7.2 | Multi-byte adopter table includes all node types | Repeaters, room servers, sensors all present | #767 | browser |
| 7.3 | Role column reflects multi-byte adoption + advert precedence | For 3 sample multi-byte adopter pubkeys (from #758 matrix), the Role column on `/#nodes` matches the role inferred from their latest advert flags via `/api/nodes/{pk}/health` | #767 | browser |
### 8. Frontend nav & deep linking
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 8.1 | Click node on map/list — URL hash updates, panel opens | Hash matches | #739 | browser |
| 8.2 | Open saved deep-link to a node | Full-screen detail view opens (post-#823: desktop deep links match the Details-link path) | #739/#823 | browser |
| 8.3 | Packets page filter URL hash | Reload preserves filters | #740 | browser |
| 8.4 | Details/Analytics links in node detail panel | Navigate without router glitch | #779/#785 | browser |
| 8.5 | Neighbor graph slider | Persists across reloads, default 0.7 | #776 | browser |
| 8.6 | Repeater that's also observer | Single map marker | #745 | browser |
| 8.7 | Side-panel "Recent Packets" — click any entry, lands on packet detail (no 404), entry text is readable in current theme | DB-fallback works (#827); `.advert-info` has explicit color (#829) | #827/#829 | browser |
### 9. Geofilter & customizer
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 9.1 | Customize → "Open geofilter builder" link | Opens app-served builder | #735 | browser |
| 9.2 | Build a filter, save, reload (STAGING ONLY; teardown: delete the saved filter) | Persists across reload; teardown removes it | #735 | browser |
| 9.3 | Geofilter docs page | Renders, content matches behavior | #734 | browser |
### 10. Node blacklist
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 10.1 | Add node pubkey to nodeBlacklist config; restart | Hidden from listings/map/neighbor graph | #742 | auto: blacklist-test.sh |
| 10.2 | Packets still in DB | Yes (filter not delete) | #742 | auto: blacklist-test.sh |
`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Optional: `TARGET_DB_PATH` or `ADMIN_API_TOKEN` for §10.2 probe; `TARGET_SSH_KEY` (default `/root/.ssh/id_ed25519`). Mandatory teardown removes the pubkey and verifies the node returns to listings.
### 11. Deploy/ops
| # | Step | Pass criteria | Source | Mode |
|---|---|---|---|---|
| 11.1 | Force-redeploy staging | Container removed cleanly even if `docker run`, not compose. Playwright E2E `Desktop: deep link #/nodes/{pubkey} opens full-screen detail view` passes (updated #833 — was asserting old pre-#823 split-panel behavior). | fa348ef/#833 | human |
## GO criteria
- Sections 1.2, 2, 3 must all pass — release blockers
- Section 4 (channels) — any visible regression must be fixed before tag
- Other sections: file follow-up issues; decide per-item whether to tag with known issues
-134
View File
@@ -1,134 +0,0 @@
#!/usr/bin/env bash
# api-contract-diff.sh — diff CoreScope API endpoints between two deployments.
# Usage: api-contract-diff.sh BASELINE_URL TARGET_URL [-k AUTH_HEADER]
#
# Compares JSON shape (recursive key set) per endpoint and asserts presence of
# `resolved_path` where contract requires it. Prints a per-endpoint result line
# (✅/❌) and a summary. Exit code = number of failures.
#
# Distinguishes:
# curl-failed → HTTP error or network timeout (real outage)
# parse-empty → curl succeeded but response shape unexpected (probable
# contract drift in this script or in the API)
# shape-diff → recursive key set differs between baseline and target
# rp-missing → resolved_path absent on target where it was promised
#
# PUBLIC repo: do not commit URLs or keys here. Caller passes them.
set -uo pipefail
OLD="${1:-}"; NEW="${2:-}"
[[ -z "$OLD" || -z "$NEW" ]] && { echo "usage: $0 BASELINE_URL TARGET_URL [-k AUTH_HEADER]" >&2; exit 2; }
shift 2 || true
AUTH=""
while [[ $# -gt 0 ]]; do
case "$1" in
-k) AUTH="$2"; shift 2 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
done
TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT
# Wrapper: fetch URL, return body on stdout, exit 1 on HTTP error / timeout.
fetch() {
local url="$1" out="$2"
local code
code=$(curl -s -m 30 -o "$out" -w "%{http_code}" ${AUTH:+-H "$AUTH"} "$url" 2>/dev/null) || code="000"
if [[ "$code" != "2"* ]]; then
echo " HTTP $code"
return 1
fi
return 0
}
# Seed lookups from TARGET (so the picked IDs are guaranteed present there).
seed_packets="$TMP/seed_packets.json"
seed_observers="$TMP/seed_observers.json"
seed_nodes="$TMP/seed_nodes.json"
if ! fetch "$NEW/api/packets?limit=1" "$seed_packets"; then echo "seed /api/packets failed" >&2; fi
if ! fetch "$NEW/api/observers" "$seed_observers"; then echo "seed /api/observers failed" >&2; fi
if ! fetch "$NEW/api/nodes?limit=1" "$seed_nodes"; then echo "seed /api/nodes failed" >&2; fi
HASH=$(jq -r '.packets[0].hash // empty' "$seed_packets" 2>/dev/null || true)
OBSID=$(jq -r '.observers[0].id // empty' "$seed_observers" 2>/dev/null || true)
NODEPK=$(jq -r '.nodes[0].public_key // empty' "$seed_nodes" 2>/dev/null || true)
[[ -z "$HASH" ]] && echo "warn: no packet hash from /api/packets — packet-detail endpoints will be skipped" >&2
[[ -z "$OBSID" ]] && echo "warn: no observer id from /api/observers — observer-detail endpoints will be skipped" >&2
[[ -z "$NODEPK" ]] && echo "warn: no node pubkey from /api/nodes — node-detail endpoints will be skipped" >&2
# Endpoints to diff: path | jq filter (selects subobject to compare) | RP-required(yes/no)
declare -a ENDPOINTS
ENDPOINTS+=("/api/packets?limit=20|.packets[0]|yes")
ENDPOINTS+=("/api/packets?limit=20&expandObservations=true|.packets[0]|yes")
ENDPOINTS+=("/api/observers|.observers[0]|no")
[[ -n "$HASH" ]] && ENDPOINTS+=("/api/packets/$HASH|.|yes")
[[ -n "$OBSID" ]] && ENDPOINTS+=("/api/observers/$OBSID|.|no")
[[ -n "$OBSID" ]] && ENDPOINTS+=("/api/observers/$OBSID/analytics|.|no")
[[ -n "$NODEPK" ]] && ENDPOINTS+=("/api/nodes/$NODEPK/health|.recentPackets[0]|yes")
[[ -n "$NODEPK" ]] && ENDPOINTS+=("/api/nodes/$NODEPK/paths|.|no")
# Strip volatile fields (timestamps + counters) from a JSON value.
STRIP='walk(if type=="object" then del(.timestamp, .first_seen, .last_seen, .last_heard, .updated_at, .server_time, .packet_count, .packetsLastHour, .uptime_secs, .battery_mv, .noise_floor, .observation_count, .advert_count) else . end)'
fails=0
for ep in "${ENDPOINTS[@]}"; do
IFS='|' read -r path filter need_rp <<<"$ep"
echo "=== $path (resolved_path required: $need_rp) ==="
oldfile="$TMP/old.json"; newfile="$TMP/new.json"
if ! fetch "$OLD$path" "$oldfile"; then echo " ❌ baseline curl-failed"; fails=$((fails+1)); continue; fi
if ! fetch "$NEW$path" "$newfile"; then echo " ❌ target curl-failed"; fails=$((fails+1)); continue; fi
# Selector + strip on each side. jq stderr is preserved so script bugs surface.
oldj=$(jq "$filter | $STRIP" "$oldfile")
jq_old_rc=$?
newj=$(jq "$filter | $STRIP" "$newfile")
jq_new_rc=$?
if [[ $jq_old_rc -ne 0 ]]; then
echo " ❌ baseline jq-error (filter='$filter') — likely script bug or API shape changed"
fails=$((fails+1)); continue
fi
if [[ $jq_new_rc -ne 0 ]]; then
echo " ❌ target jq-error (filter='$filter') — likely script bug or API shape changed"
fails=$((fails+1)); continue
fi
if [[ -z "$oldj" || "$oldj" == "null" ]]; then
echo " ❌ baseline parse-empty (filter returned empty/null; check API shape)"
fails=$((fails+1)); continue
fi
if [[ -z "$newj" || "$newj" == "null" ]]; then
echo " ❌ target parse-empty (filter returned empty/null; check API shape)"
fails=$((fails+1)); continue
fi
# Recursive key-set diff. Canonicalize array indices (numbers) → "[]" so two
# different sample responses with different array lengths don't false-positive.
KEYS_FILTER='[paths(scalars or type=="null" or (type=="array" and length==0) or (type=="object" and length==0)) | map(if type=="number" then "[]" else . end) | join(".")] | unique | .[]'
oldkeys=$(echo "$oldj" | jq -r "$KEYS_FILTER" | sort -u)
newkeys=$(echo "$newj" | jq -r "$KEYS_FILTER" | sort -u)
if ! diff <(echo "$oldkeys") <(echo "$newkeys") >/dev/null; then
echo " ❌ shape-diff (key set differs):"
diff <(echo "$oldkeys") <(echo "$newkeys") | sed 's/^/ /'
fails=$((fails+1))
continue
fi
# If RP expected, assert present on target (any value, may be null).
if [[ "$need_rp" == "yes" ]]; then
if ! echo "$newj" | jq -e '.. | objects | select(has("resolved_path")) | .resolved_path' >/dev/null 2>&1; then
echo " ❌ rp-missing (resolved_path not present anywhere in selector)"
fails=$((fails+1))
continue
fi
fi
echo " ✅ ok"
done
echo
echo "failures: $fails / ${#ENDPOINTS[@]}"
exit $fails
-271
View File
@@ -1,271 +0,0 @@
#!/usr/bin/env bash
# blacklist-test.sh — verify nodeBlacklist hides a pubkey from API surface
# while retaining its packets in the DB. Implements QA plan §10.1 + §10.2.
#
# Usage:
# blacklist-test.sh BASELINE_URL TARGET_URL
#
# BASELINE_URL is currently unused for assertions but kept as a positional
# arg for parity with other qa-suite scripts (always called with two URLs).
#
# Required env (target host control + test data):
# TEST_NODE_PUBKEY — hex pubkey of a real, currently-visible node on TARGET_URL
# TARGET_SSH_HOST — e.g. runner@example
# TARGET_SSH_KEY — path to ssh private key (default: /root/.ssh/id_ed25519)
# TARGET_CONFIG_PATH — absolute path to config.json on the target
# TARGET_CONTAINER — docker container name on the target
# Optional env:
# TARGET_DB_PATH — sqlite db path on the target (for §10.2 sqlite probe)
# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of ssh+sqlite
# (read from env, not argv — never appears in ps)
# CURL_TIMEOUT — per-request curl timeout, seconds (default 60)
# RESTART_WAIT_S — max wait for /api/stats after restart (default 120)
#
# Distinguishes:
# ssh-failed → cannot reach/control target
# restart-stuck → /api/stats not 200 within RESTART_WAIT_S
# hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail)
# retain-failed → blacklisted pubkey absent from DB (§10.2 fail)
# teardown-failed→ post-test removal did not restore listing
#
# Exit code = number of failures (0 = pass).
# PUBLIC repo: zero PII — no real pubkeys, IPs, or hostnames as defaults.
set -uo pipefail
BASELINE_URL="${1:-}"
TARGET_URL="${2:-}"
if [[ -z "$BASELINE_URL" || -z "$TARGET_URL" ]]; then
echo "usage: $0 BASELINE_URL TARGET_URL (TEST_NODE_PUBKEY+TARGET_* via env)" >&2
exit 2
fi
TEST_PUBKEY="${TEST_NODE_PUBKEY:-}"
TARGET_SSH_HOST="${TARGET_SSH_HOST:-}"
TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}"
TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}"
TARGET_CONTAINER="${TARGET_CONTAINER:-}"
TARGET_DB_PATH="${TARGET_DB_PATH:-}"
ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}"
if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then
echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2
exit 2
fi
# Hard input validation — these strings are interpolated into remote shell/SQL.
# Pubkey must be hex (MeshCore pubkeys are hex-encoded ed25519 prefixes).
if ! [[ "$TEST_PUBKEY" =~ ^[0-9a-fA-F]+$ ]]; then
echo "error: TEST_NODE_PUBKEY must be hex (got: redacted)" >&2
exit 2
fi
# Container name must match docker's allowed chars: [a-zA-Z0-9][a-zA-Z0-9_.-]*
if ! [[ "$TARGET_CONTAINER" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then
echo "error: TARGET_CONTAINER has illegal chars" >&2
exit 2
fi
# Config path must be an absolute, sane path (no spaces, quotes, $, ;, etc.).
if ! [[ "$TARGET_CONFIG_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then
echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2
exit 2
fi
if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then
echo "error: TARGET_DB_PATH must be a sane absolute path" >&2
exit 2
fi
CURL_TIMEOUT="${CURL_TIMEOUT:-60}"
RESTART_WAIT_S="${RESTART_WAIT_S:-120}"
SSH_OPTS=(-i "$TARGET_SSH_KEY" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o BatchMode=yes)
ssh_t() { ssh "${SSH_OPTS[@]}" "$TARGET_SSH_HOST" "$@"; }
TMP=$(mktemp -d)
fails=0
TEARDOWN_DONE=0
# -----------------------------------------------------------------------------
# Teardown — MANDATORY in all exit paths.
# -----------------------------------------------------------------------------
teardown() {
local rc=$?
if [[ "$TEARDOWN_DONE" == "1" ]]; then rm -rf "$TMP"; exit "$rc"; fi
TEARDOWN_DONE=1
echo "=== teardown: removing $TEST_PUBKEY from nodeBlacklist ==="
if remove_from_blacklist && restart_target && wait_for_stats; then
if node_visible; then
echo " ✅ teardown ok — node returned to listings"
else
echo " ❌ teardown-failed: node still hidden after removal"
rc=$((rc + 1))
fi
else
echo " ❌ teardown-failed: could not restore config / restart / stats"
rc=$((rc + 1))
fi
rm -rf "$TMP"
exit "$rc"
}
trap teardown EXIT INT TERM
# -----------------------------------------------------------------------------
# Helpers
# -----------------------------------------------------------------------------
fetch_code() {
local url="$1" out="$2"
curl -s -m "$CURL_TIMEOUT" -o "$out" -w "%{http_code}" "$url" 2>/dev/null || echo "000"
}
wait_for_stats() {
local deadline code
echo " waiting up to ${RESTART_WAIT_S}s for $TARGET_URL/api/stats ..."
deadline=$(( $(date +%s) + RESTART_WAIT_S ))
while (( $(date +%s) < deadline )); do
code=$(fetch_code "$TARGET_URL/api/stats" "$TMP/stats.json")
if [[ "$code" == "200" ]]; then echo " stats OK"; return 0; fi
sleep 3
done
echo " ❌ restart-stuck: /api/stats never returned 200"
return 1
}
restart_target() {
echo " restarting container $TARGET_CONTAINER ..."
# TARGET_CONTAINER is validated above; still quote defensively.
if ! ssh_t "docker restart $(printf %q "$TARGET_CONTAINER")" >/dev/null; then
echo " ❌ ssh-failed: docker restart failed"
return 1
fi
return 0
}
# Mutate config.json on target. Values pass via env (printf %q + single-quoted
# heredoc) so $TEST_PUBKEY etc. never enter the remote shell as code.
set_blacklist_state() {
local mode="$1" # add | remove
ssh_t "CFG=$(printf %q "$TARGET_CONFIG_PATH") PK=$(printf %q "$TEST_PUBKEY") MODE=$(printf %q "$mode") bash -s" <<'REMOTE'
set -euo pipefail
TMP="$(mktemp)"
trap 'rm -f "$TMP"' EXIT
if command -v jq >/dev/null; then
if [ "$MODE" = "add" ]; then
jq --arg pk "$PK" '.nodeBlacklist = ((.nodeBlacklist // []) + [$pk] | unique)' "$CFG" > "$TMP"
else
jq --arg pk "$PK" '.nodeBlacklist = ((.nodeBlacklist // []) - [$pk])' "$CFG" > "$TMP"
fi
else
python3 - "$CFG" "$PK" "$MODE" "$TMP" <<'PY'
import json, sys
cfg, pk, mode, out = sys.argv[1:]
with open(cfg) as f: d = json.load(f)
bl = list(dict.fromkeys(d.get("nodeBlacklist") or []))
if mode == "add":
if pk not in bl: bl.append(pk)
else:
bl = [x for x in bl if x != pk]
d["nodeBlacklist"] = bl
with open(out, "w") as f: json.dump(d, f, indent=2)
PY
fi
# Preserve mode and ownership; mv across same FS is atomic.
chmod --reference="$CFG" "$TMP" 2>/dev/null || true
chown --reference="$CFG" "$TMP" 2>/dev/null || true
mv "$TMP" "$CFG"
trap - EXIT
REMOTE
local rc=$?
if (( rc != 0 )); then
echo " ❌ ssh-failed: could not edit $TARGET_CONFIG_PATH ($mode)"
return 1
fi
return 0
}
add_to_blacklist() { set_blacklist_state add; }
remove_from_blacklist() { set_blacklist_state remove; }
node_visible() {
# Returns 0 if the pubkey is currently visible via API.
local code
code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/node.json")
if [[ "$code" == "200" ]]; then return 0; fi
fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/nodes.json" >/dev/null
if grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/nodes.json" 2>/dev/null; then
return 0
fi
return 1
}
# -----------------------------------------------------------------------------
# §10.1 — hide
# -----------------------------------------------------------------------------
echo "=== §10.1 add $TEST_PUBKEY to nodeBlacklist ==="
if ! add_to_blacklist; then fails=$((fails+1)); exit "$fails"; fi
if ! restart_target; then fails=$((fails+1)); exit "$fails"; fi
if ! wait_for_stats; then fails=$((fails+1)); exit "$fails"; fi
detail_code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/detail.json")
list_code=$(fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/list.json")
in_list=0
if [[ "$list_code" == "200" ]] && grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/list.json"; then
in_list=1
fi
if [[ "$detail_code" == "404" || "$in_list" == "0" ]]; then
echo " ✅ hide ok: detail=$detail_code in_list=$in_list"
else
echo " ❌ hide-failed: detail=$detail_code in_list=$in_list — pubkey still surfaced"
fails=$((fails+1))
fi
topo_code=$(fetch_code "$TARGET_URL/api/topology" "$TMP/topo.json")
if [[ "$topo_code" != "200" ]]; then
echo " ⚠️ /api/topology HTTP $topo_code — skipping topology assertion"
elif grep -qF -- "$TEST_PUBKEY" "$TMP/topo.json"; then
echo " ❌ hide-failed: /api/topology references blacklisted pubkey"
fails=$((fails+1))
else
echo " ✅ topology clean"
fi
# -----------------------------------------------------------------------------
# §10.2 — DB retain
# -----------------------------------------------------------------------------
echo "=== §10.2 verify packets retained in DB ==="
count=""
if [[ -n "$ADMIN_API_TOKEN" ]]; then
# Read auth header from stdin so the token never enters argv (ps-safe).
code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \
curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \
"$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000")
if [[ "$code" == "200" ]]; then
count=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "")
fi
fi
if [[ -z "$count" ]]; then
if [[ -z "$TARGET_DB_PATH" ]]; then
echo " ❌ retain-failed: TARGET_DB_PATH unset and no ADMIN_API_TOKEN — cannot probe"
fails=$((fails+1))
else
# TEST_PUBKEY is hex-validated → safe to inline single-quoted in SQL.
# Container/db path also validated; printf %q for defense in depth.
q="SELECT COUNT(*) FROM transmissions WHERE from_node = '$TEST_PUBKEY';"
qq=$(printf %q "$q")
if ! count=$(ssh_t "docker exec $(printf %q "$TARGET_CONTAINER") sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null); then
count=$(ssh_t "sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null || echo "")
fi
fi
fi
if [[ -z "$count" ]]; then
echo " ❌ retain-failed: could not read transmissions count"
fails=$((fails+1))
elif [[ "$count" =~ ^[0-9]+$ ]] && (( count > 0 )); then
echo " ✅ DB retains $count packets from $TEST_PUBKEY"
else
echo " ❌ retain-failed: count=$count (expected > 0)"
fails=$((fails+1))
fi
echo "=== summary: $fails failure(s) before teardown ==="
# trap handles teardown + exit
exit "$fails"
-136
View File
@@ -1,136 +0,0 @@
// migrate-fixture-hashes recomputes content hashes in a fixture DB using the
// current ComputeContentHash formula. Run once; idempotent.
package main
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"log"
"os"
_ "modernc.org/sqlite"
)
func computeContentHash(rawHex string) string {
buf, err := hex.DecodeString(rawHex)
if err != nil || len(buf) < 2 {
if len(rawHex) >= 16 {
return rawHex[:16]
}
return rawHex
}
headerByte := buf[0]
offset := 1
routeType := int(headerByte & 0x03)
if routeType == 2 || routeType == 3 { // transport
offset += 4
}
if offset >= len(buf) {
if len(rawHex) >= 16 {
return rawHex[:16]
}
return rawHex
}
pathByte := buf[offset]
offset++
hashSize := int((pathByte>>6)&0x3) + 1
hashCount := int(pathByte & 0x3F)
pathBytes := hashSize * hashCount
payloadStart := offset + pathBytes
if payloadStart > len(buf) {
if len(rawHex) >= 16 {
return rawHex[:16]
}
return rawHex
}
payload := buf[payloadStart:]
payloadType := (headerByte >> 2) & 0x0F
toHash := []byte{payloadType}
// TRACE = payload type 7
if int(payloadType) == 7 {
toHash = append(toHash, pathByte, 0x00)
}
toHash = append(toHash, payload...)
h := sha256.Sum256(toHash)
return hex.EncodeToString(h[:])[:16]
}
func main() {
if len(os.Args) < 2 {
fmt.Fprintf(os.Stderr, "usage: %s <db-path>\n", os.Args[0])
os.Exit(1)
}
dbPath := os.Args[1]
db, err := sql.Open("sqlite", dbPath)
if err != nil {
log.Fatal(err)
}
defer db.Close()
rows, err := db.Query("SELECT id, raw_hex, hash FROM transmissions")
if err != nil {
log.Fatal(err)
}
type update struct {
id int
newHash string
}
var updates []update
for rows.Next() {
var id int
var rawHex, oldHash string
if err := rows.Scan(&id, &rawHex, &oldHash); err != nil {
log.Printf("scan: %v", err)
continue
}
newHash := computeContentHash(rawHex)
if newHash != oldHash {
updates = append(updates, update{id, newHash})
}
}
rows.Close()
if len(updates) == 0 {
fmt.Println("All hashes already match current formula.")
return
}
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
stmt, err := tx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?")
if err != nil {
log.Fatal(err)
}
merged := 0
for _, u := range updates {
if _, err := stmt.Exec(u.newHash, u.id); err != nil {
// UNIQUE constraint = duplicate (same content, different old hash).
// Move observations to the surviving tx, then delete the dup.
log.Printf("update id %d: %v — merging duplicate", u.id, err)
// Find surviving tx id
var survID int
if err2 := tx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil {
tx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.id)
tx.Exec("DELETE FROM transmissions WHERE id = ?", u.id)
merged++
}
}
}
stmt.Close()
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
fmt.Printf("Migrated %d hashes, merged %d duplicates.\n", len(updates)-merged, merged)
}
-64
View File
@@ -1,64 +0,0 @@
/**
* Tests for #759 Add channel UX: button, hint, status feedback.
* Validates the HTML structure rendered by channels.js init.
*/
'use strict';
const fs = require('fs');
let passed = 0;
let failed = 0;
function assert(cond, msg) {
if (cond) { passed++; console.log(' ✓ ' + msg); }
else { failed++; console.error(' ✗ ' + msg); }
}
function assertIncludes(html, substr, msg) {
assert(html.includes(substr), msg);
}
// Read the channels.js source to extract the HTML template
const src = fs.readFileSync(__dirname + '/public/channels.js', 'utf8');
// Extract the sidebar HTML from the template literal
const htmlMatch = src.match(/app\.innerHTML\s*=\s*`([\s\S]*?)`;/);
const html = htmlMatch ? htmlMatch[1] : '';
console.log('Test: Add channel UX (#759)');
// 1. Button renders in the form
assertIncludes(html, 'class="ch-add-btn"', 'Add button has ch-add-btn class');
assertIncludes(html, 'type="submit"', 'Button is type=submit');
assertIncludes(html, '>+</button>', 'Button shows + text');
// 2. Form has proper structure
assertIncludes(html, 'class="ch-add-form"', 'Form has ch-add-form class');
assertIncludes(html, 'class="ch-add-row"', 'Row wrapper present');
assert(!html.includes('class="ch-add-label"'), 'Label removed (redundant with hint)');
// 3. Hint text present
assertIncludes(html, 'class="ch-add-hint"', 'Hint div present');
assertIncludes(html, 'e.g. #LongFast or 32-char hex key', 'Hint text correct');
// 4. Status div present
assertIncludes(html, 'id="chAddStatus"', 'Status div has correct id');
assertIncludes(html, 'class="ch-add-status"', 'Status div has correct class');
assertIncludes(html, 'style="display:none"', 'Status div hidden by default');
// 5. showAddStatus function exists in source
assert(src.includes('function showAddStatus('), 'showAddStatus function defined');
assert(src.includes("'success'"), 'Success status type referenced');
assert(src.includes("'error'"), 'Error status type referenced');
// 6. CSS classes exist
const css = fs.readFileSync(__dirname + '/public/style.css', 'utf8');
assert(css.includes('.ch-add-form'), 'CSS: .ch-add-form defined');
assert(css.includes('.ch-add-btn'), 'CSS: .ch-add-btn defined');
assert(css.includes('.ch-add-hint'), 'CSS: .ch-add-hint defined');
assert(css.includes('.ch-add-status'), 'CSS: .ch-add-status defined');
assert(css.includes('.ch-add-row'), 'CSS: .ch-add-row defined');
// .ch-add-label CSS kept for backward compat but label removed from HTML
console.log('\n' + passed + ' passed, ' + failed + ' failed');
process.exit(failed > 0 ? 1 : 0);
-157
View File
@@ -1,157 +0,0 @@
/**
* Tests for #725 M3 (PSK hex key), M4 (channel removal), M5 (message caching).
* Runs in Node.js via vm.createContext to simulate browser environment.
*/
'use strict';
const vm = require('vm');
const fs = require('fs');
const { subtle } = require('crypto').webcrypto;
let passed = 0;
let failed = 0;
function assert(cond, msg) {
if (cond) { passed++; console.log(' ✓ ' + msg); }
else { failed++; console.error(' ✗ ' + msg); }
}
// Build a minimal browser-like sandbox
function createSandbox() {
const storage = {};
const localStorage = {
getItem: (k) => storage[k] !== undefined ? storage[k] : null,
setItem: (k, v) => { storage[k] = String(v); },
removeItem: (k) => { delete storage[k]; },
_data: storage
};
const ctx = {
window: {},
crypto: { subtle },
TextEncoder: TextEncoder,
TextDecoder: TextDecoder,
Uint8Array,
localStorage,
console,
Date,
JSON,
parseInt,
Math,
String,
Number,
Object,
Array,
RegExp,
Error,
Promise,
setTimeout,
btoa: (s) => Buffer.from(s, 'binary').toString('base64'),
atob: (s) => Buffer.from(s, 'base64').toString('binary'),
};
ctx.window = ctx;
ctx.self = ctx;
return ctx;
}
async function runTests() {
console.log('\n=== M3: PSK hex key detection ===');
// Load channel-decrypt.js in sandbox
const cdSrc = fs.readFileSync(__dirname + '/public/channel-decrypt.js', 'utf8');
const sandbox = createSandbox();
const context = vm.createContext(sandbox);
vm.runInContext(cdSrc, context);
const CD = sandbox.window.ChannelDecrypt;
// Test: isHexKey detection (via channels.js logic)
// We test the pattern directly since isHexKey is inside channels.js IIFE
const isHexKey = (val) => /^[0-9a-fA-F]{32}$/.test(val);
assert(isHexKey('0123456789abcdef0123456789abcdef'), 'Valid 32-char hex detected');
assert(isHexKey('AABBCCDD11223344AABBCCDD11223344'), 'Valid uppercase hex detected');
assert(!isHexKey('#LongFast'), 'Hashtag name NOT detected as hex');
assert(!isHexKey('0123456789abcdef'), 'Short hex (16 chars) NOT detected');
assert(!isHexKey('0123456789abcdef0123456789abcdefXX'), 'Too long NOT detected');
assert(!isHexKey('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'), 'Non-hex chars NOT detected');
// Test: PSK decrypt with known key bytes
console.log('\n=== M3: PSK decrypt produces correct plaintext ===');
// Derive a key from #LongFast for testing
const keyBytes = await CD.deriveKey('#LongFast');
assert(keyBytes.length === 16, 'Derived key is 16 bytes');
const keyHex = CD.bytesToHex(keyBytes);
assert(keyHex.length === 32, 'Key hex is 32 chars');
// Round-trip: hex → bytes → hex
const roundTrip = CD.bytesToHex(CD.hexToBytes(keyHex));
assert(roundTrip === keyHex, 'Hex round-trip preserves key');
// Channel hash computation works
const hashByte = await CD.computeChannelHash(keyBytes);
assert(typeof hashByte === 'number' && hashByte >= 0 && hashByte <= 255, 'Channel hash byte is valid (0-255)');
// PSK key (raw hex) stored and retrieved correctly
const pskHex = 'aabbccdd11223344aabbccdd11223344';
CD.storeKey('psk:aabbccdd', pskHex);
const keys = CD.getStoredKeys();
assert(keys['psk:aabbccdd'] === pskHex, 'PSK key stored and retrieved correctly');
console.log('\n=== M4: Channel removal clears key + cache ===');
// Store a key and some cached messages
CD.storeKey('#TestChannel', 'deadbeefdeadbeefdeadbeefdeadbeef');
CD.setCache('#TestChannel', [{ sender: 'A', text: 'hello', timestamp: '2026-01-01T00:00:00Z', packetHash: 'h1' }], '2026-01-01T00:00:00Z', 1);
// Verify they exist
var storedKeys = CD.getStoredKeys();
assert(storedKeys['#TestChannel'] === 'deadbeefdeadbeefdeadbeefdeadbeef', 'Key exists before removal');
var cachedBefore = CD.getCache('#TestChannel');
assert(cachedBefore && cachedBefore.messages.length === 1, 'Cache exists before removal');
// Remove the key (also clears cache)
CD.removeKey('#TestChannel');
var storedAfter = CD.getStoredKeys();
assert(!storedAfter['#TestChannel'], 'Key cleared after removal');
var cachedAfter = CD.getCache('#TestChannel');
assert(!cachedAfter, 'Cache cleared after removal');
console.log('\n=== M5: Cache operations ===');
// Test: setCache with count and size limit
var bigMessages = [];
for (var i = 0; i < 1200; i++) {
bigMessages.push({ sender: 'S', text: 'msg' + i, timestamp: '2026-01-01T00:00:' + String(i).padStart(2, '0') + 'Z', packetHash: 'h' + i });
}
CD.setCache('bigchannel', bigMessages, '2026-01-01T00:20:00Z', 1200);
var bigCached = CD.getCache('bigchannel');
assert(bigCached.messages.length <= 1000, 'Cache enforces 1000 message limit (got ' + bigCached.messages.length + ')');
assert(bigCached.count === 1200, 'Cache stores total count');
assert(bigCached.lastTimestamp === '2026-01-01T00:20:00Z', 'Cache stores lastTimestamp');
// Should keep most recent 1000
assert(bigCached.messages[0].packetHash === 'h200', 'Cache keeps most recent 1000 (first is h200)');
// Test: cache hit (delta fetch scenario)
CD.setCache('deltatest', [
{ sender: 'A', text: 'old', timestamp: '2026-01-01T00:00:00Z', packetHash: 'p1' }
], '2026-01-01T00:00:00Z', 1);
var deltaCache = CD.getCache('deltatest');
assert(deltaCache.messages.length === 1, 'Delta cache has 1 message');
assert(deltaCache.lastTimestamp === '2026-01-01T00:00:00Z', 'Delta cache lastTimestamp correct');
assert(deltaCache.count === 1, 'Delta cache count correct');
// Test: clearChannelCache
CD.setCache('clearthis', [{ sender: 'X', text: 'y' }], 'ts', 1);
assert(CD.getCache('clearthis') !== null, 'Cache exists before clear');
CD.clearChannelCache('clearthis');
assert(CD.getCache('clearthis') === null, 'Cache cleared by clearChannelCache');
console.log('\n=== Results ===');
console.log('Passed: ' + passed + ', Failed: ' + failed);
process.exit(failed > 0 ? 1 : 0);
}
runTests().catch(e => { console.error(e); process.exit(1); });
-454
View File
@@ -1,454 +0,0 @@
/* test-drag-manager.js — Unit tests for DragManager (#608 M1) */
'use strict';
const vm = require('vm');
const fs = require('fs');
const path = require('path');
const assert = require('assert');
// Minimal DOM shim
function makePanel(id) {
const listeners = {};
const style = {};
const dataset = {};
const classList = {
_set: new Set(),
add(c) { this._set.add(c); },
remove(c) { this._set.delete(c); },
contains(c) { return this._set.has(c); }
};
let attrs = {};
const header = {
_listeners: {},
addEventListener(ev, fn) {
if (!this._listeners[ev]) this._listeners[ev] = [];
this._listeners[ev].push(fn);
},
setPointerCapture() {},
releasePointerCapture() {},
_fire(ev, data) {
(this._listeners[ev] || []).forEach(fn => fn(data));
}
};
return {
id: id,
style: style,
dataset: dataset,
classList: classList,
querySelector(sel) {
if (sel === '.panel-header') return header;
return null;
},
getAttribute(k) { return attrs[k] || null; },
setAttribute(k, v) { attrs[k] = v; },
removeAttribute(k) { delete attrs[k]; },
getBoundingClientRect() {
return {
left: parseFloat(style.left) || 0,
top: parseFloat(style.top) || 0,
right: (parseFloat(style.left) || 0) + 300,
bottom: (parseFloat(style.top) || 0) + 200,
width: 300,
height: 200
};
},
_header: header
};
}
// Mock globals
const storage = {};
const mockWindow = {
innerWidth: 1920,
innerHeight: 1080,
DragManager: null,
matchMedia() { return { matches: true, addEventListener() {} }; },
addEventListener() {}
};
const mockDocument = {
addEventListener(ev, fn) {
if (!mockDocument._listeners) mockDocument._listeners = {};
if (!mockDocument._listeners[ev]) mockDocument._listeners[ev] = [];
mockDocument._listeners[ev].push(fn);
},
removeEventListener(ev, fn) {
if (mockDocument._listeners && mockDocument._listeners[ev]) {
mockDocument._listeners[ev] = mockDocument._listeners[ev].filter(f => f !== fn);
}
},
querySelectorAll() { return []; }
};
const mockLocalStorage = {
_data: {},
getItem(k) { return this._data[k] || null; },
setItem(k, v) { this._data[k] = v; },
removeItem(k) { delete this._data[k]; },
clear() { this._data = {}; }
};
// Load DragManager
const src = fs.readFileSync(path.join(__dirname, 'public', 'drag-manager.js'), 'utf8');
const ctx = vm.createContext({
window: mockWindow,
document: mockDocument,
localStorage: mockLocalStorage,
Math: Math,
JSON: JSON,
console: console,
setTimeout: setTimeout,
clearTimeout: clearTimeout,
parseFloat: parseFloat
});
vm.runInContext(src, ctx);
const DragManager = ctx.window.DragManager;
let passed = 0;
let failed = 0;
function test(name, fn) {
try {
mockLocalStorage.clear();
fn();
passed++;
console.log(' ✓ ' + name);
} catch (e) {
failed++;
console.log(' ✗ ' + name + ': ' + e.message);
}
}
console.log('DragManager tests:');
test('constructor initializes IDLE state', () => {
const dm = new DragManager();
assert.strictEqual(dm.state, 'IDLE');
assert.strictEqual(dm.enabled, true);
});
test('register adds panel', () => {
const dm = new DragManager();
const panel = makePanel('testPanel');
dm.register(panel);
assert.strictEqual(dm._panels.length, 1);
});
test('register ignores null panel', () => {
const dm = new DragManager();
dm.register(null);
assert.strictEqual(dm._panels.length, 0);
});
test('pointerdown transitions to PENDING', () => {
const dm = new DragManager();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
assert.strictEqual(dm.state, 'PENDING');
assert.strictEqual(dm.activePanel, panel);
});
test('pointerdown ignores non-left button', () => {
const dm = new DragManager();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 2, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
assert.strictEqual(dm.state, 'IDLE');
});
test('pointerdown ignores button clicks', () => {
const dm = new DragManager();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest(sel) { return sel === 'button' ? {} : null; } }
});
assert.strictEqual(dm.state, 'IDLE');
});
test('pointermove within dead zone stays PENDING', () => {
const dm = new DragManager();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 103, clientY: 102 });
assert.strictEqual(dm.state, 'PENDING');
assert.ok(!panel.classList.contains('is-dragging'));
});
test('pointermove beyond dead zone transitions to DRAGGING', () => {
const dm = new DragManager();
const panel = makePanel('p1');
panel.setAttribute('data-position', 'bl');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 110, clientY: 110 });
assert.strictEqual(dm.state, 'DRAGGING');
assert.ok(panel.classList.contains('is-dragging'));
assert.strictEqual(panel.getAttribute('data-position'), null); // removed
assert.strictEqual(panel.dataset.dragged, 'true');
});
test('pointerup after drag finalizes position', () => {
const dm = new DragManager();
const panel = makePanel('p1');
panel.setAttribute('data-position', 'bl');
ctx.window.innerWidth = 1920;
ctx.window.innerHeight = 1080;
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 200, clientY: 300 });
panel._header._fire('pointerup', { pointerId: 1 });
assert.strictEqual(dm.state, 'IDLE');
assert.ok(!panel.classList.contains('is-dragging'));
// Should have persisted
const saved = JSON.parse(mockLocalStorage.getItem('panel-drag-p1'));
assert.ok(saved.xPct >= 0);
assert.ok(saved.yPct >= 0);
});
test('pointerup from PENDING (click) does not finalize', () => {
const dm = new DragManager();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointerup', { pointerId: 1 });
assert.strictEqual(dm.state, 'IDLE');
assert.strictEqual(mockLocalStorage.getItem('panel-drag-p1'), null);
});
test('disable prevents drag', () => {
const dm = new DragManager();
dm.disable();
const panel = makePanel('p1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
assert.strictEqual(dm.state, 'IDLE');
});
test('snap-to-edge works within threshold', () => {
const dm = new DragManager();
const panel = makePanel('p1');
panel.setAttribute('data-position', 'tl');
ctx.window.innerWidth = 1920;
ctx.window.innerHeight = 1080;
dm.register(panel);
// Simulate drag to near top-left edge
panel._header._fire('pointerdown', {
button: 0, clientX: 500, clientY: 500,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 510, clientY: 510 }); // trigger DRAGGING
// Panel is now detached; set its position near edge
panel.style.left = '5px';
panel.style.top = '10px';
panel._header._fire('pointerup', { pointerId: 1 });
// Should have snapped to margin (12px)
assert.strictEqual(panel.style.left, '12px');
assert.strictEqual(panel.style.top, '12px');
});
test('restorePositions applies saved viewport percentages', () => {
const dm = new DragManager();
const panel = makePanel('p2');
panel.setAttribute('data-position', 'br');
ctx.window.innerWidth = 1000;
ctx.window.innerHeight = 800;
dm.register(panel);
mockLocalStorage.setItem('panel-drag-p2', JSON.stringify({ xPct: 0.5, yPct: 0.25 }));
dm.restorePositions();
assert.strictEqual(panel.style.left, '500px');
assert.strictEqual(panel.style.top, '200px');
assert.strictEqual(panel.dataset.dragged, 'true');
assert.strictEqual(panel.getAttribute('data-position'), null);
});
test('handleResize clamps panels inside viewport', () => {
const dm = new DragManager();
const panel = makePanel('p3');
dm.register(panel);
// Simulate a dragged panel that's now off-screen
panel.dataset.dragged = 'true';
panel.style.left = '1800px';
panel.style.top = '900px';
ctx.window.innerWidth = 1000;
ctx.window.innerHeight = 600;
// Need querySelectorAll to return this panel
const origQSA = mockDocument.querySelectorAll;
mockDocument.querySelectorAll = function (sel) {
if (sel === '.live-overlay[data-dragged="true"]') return [panel];
return [];
};
dm.handleResize();
mockDocument.querySelectorAll = origQSA;
// Should be clamped
const left = parseFloat(panel.style.left);
const top = parseFloat(panel.style.top);
assert.ok(left + 300 <= 1000, 'left clamped: ' + left);
assert.ok(top + 200 <= 600, 'top clamped: ' + top);
});
test('Escape during drag reverts to corner position', () => {
const dm = new DragManager();
const panel = makePanel('esc1');
panel.setAttribute('data-position', 'bl');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 120, clientY: 120 }); // trigger DRAGGING
assert.strictEqual(dm.state, 'DRAGGING');
// Simulate Escape
dm._handleKeyDown({ key: 'Escape' });
assert.strictEqual(dm.state, 'IDLE');
assert.ok(!panel.classList.contains('is-dragging'));
assert.strictEqual(panel.dataset.dragged, undefined);
assert.strictEqual(panel.style.transform, '');
});
test('Escape during drag reverts to saved position', () => {
const dm = new DragManager();
const panel = makePanel('esc2');
ctx.window.innerWidth = 1000;
ctx.window.innerHeight = 800;
dm.register(panel);
// Pre-save a dragged position
mockLocalStorage.setItem('panel-drag-esc2', JSON.stringify({ xPct: 0.3, yPct: 0.4 }));
dm.restorePositions();
assert.strictEqual(panel.dataset.dragged, 'true');
panel._header._fire('pointerdown', {
button: 0, clientX: 400, clientY: 400,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 500, clientY: 500 });
assert.strictEqual(dm.state, 'DRAGGING');
dm._handleKeyDown({ key: 'Escape' });
assert.strictEqual(dm.state, 'IDLE');
assert.strictEqual(panel.style.transform, 'none');
});
test('pointercancel during drag finalizes position', () => {
const dm = new DragManager();
const panel = makePanel('pc1');
panel.setAttribute('data-position', 'tl');
ctx.window.innerWidth = 1920;
ctx.window.innerHeight = 1080;
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 200, clientY: 200 });
assert.strictEqual(dm.state, 'DRAGGING');
panel._header._fire('pointercancel', {});
assert.strictEqual(dm.state, 'IDLE');
assert.ok(!panel.classList.contains('is-dragging'));
});
test('z-index increments on drag', () => {
const dm = new DragManager();
const p1 = makePanel('z1');
const p2 = makePanel('z2');
dm.register(p1);
dm.register(p2);
// Drag p1
p1._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
p1._header._fire('pointermove', { clientX: 110, clientY: 110 });
const z1 = parseInt(p1.style.zIndex);
p1._header._fire('pointerup', { pointerId: 1 });
// Drag p2
p2._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
p2._header._fire('pointermove', { clientX: 110, clientY: 110 });
const z2 = parseInt(p2.style.zIndex);
p2._header._fire('pointerup', { pointerId: 1 });
assert.ok(z2 > z1, 'z2 (' + z2 + ') should be greater than z1 (' + z1 + ')');
assert.ok(z1 >= 1001, 'z1 should be >= 1001');
});
test('disable mid-drag resets state', () => {
const dm = new DragManager();
const panel = makePanel('dis1');
dm.register(panel);
panel._header._fire('pointerdown', {
button: 0, clientX: 100, clientY: 100,
preventDefault() {},
target: { closest() { return null; } }
});
panel._header._fire('pointermove', { clientX: 120, clientY: 120 });
assert.strictEqual(dm.state, 'DRAGGING');
dm.disable();
assert.strictEqual(dm.state, 'IDLE');
assert.ok(!panel.classList.contains('is-dragging'));
assert.strictEqual(dm.enabled, false);
});
console.log('\n' + passed + ' passed, ' + failed + ' failed');
if (failed > 0) process.exit(1);
+4 -96
View File
@@ -231,30 +231,6 @@ async function run() {
assert(hasStatus, 'No status indicator found in node detail');
});
// Test: Node side panel Details link navigates to full detail page (#778)
await test('Node side panel Details link navigates', async () => {
await page.goto(`${BASE}/#/nodes`, { waitUntil: 'domcontentloaded' });
await page.waitForSelector('table tbody tr');
// Click first row to open side panel
const firstRow = await page.$('table tbody tr');
assert(firstRow, 'No node rows found');
await firstRow.click();
await page.waitForSelector('.node-detail');
// Find the Details link in the side panel
const detailsLink = await page.$('#nodesRight a.btn-primary[href^="#/nodes/"]');
assert(detailsLink, 'Details link not found in side panel');
const href = await detailsLink.getAttribute('href');
// Click the Details link — this should navigate to the full detail page
await detailsLink.click();
// Wait for navigation — the full detail page has sections like neighbors/packets
await page.waitForFunction((expectedHash) => {
return location.hash === expectedHash;
}, href, { timeout: 5000 });
// Verify we're on the full detail page (should have section tabs or detail content)
const hash = await page.evaluate(() => location.hash);
assert(hash === href, `Expected hash "${href}" but got "${hash}"`);
});
// Test: Nodes page has WebSocket auto-update listener (#131)
await test('Nodes page has WebSocket auto-update', async () => {
await page.goto(`${BASE}/#/nodes`, { waitUntil: 'domcontentloaded' });
@@ -422,13 +398,8 @@ async function run() {
}
}, { timeout: 10000 });
// Force a full page reload to reset module-level state (savedTimeWindowMin is
// read from localStorage once at IIFE time). Navigating from /#/packets to /#/packets
// is a hash-only change — no reload, so the IIFE never re-reads localStorage.
// Going to / first forces a fresh page load, then the hash change to /#/packets
// calls init() with the freshly-read savedTimeWindowMin = 60.
await page.goto(`${BASE}/`, { waitUntil: 'load' });
await page.goto(`${BASE}/#/packets`, { waitUntil: 'load' });
// Full reload on the packets page — scripts re-execute, IIFE reads localStorage
await page.reload({ waitUntil: 'load' });
await page.waitForSelector('#fTimeWindow', { timeout: 10000 });
const timeWindowValue = await page.$eval('#fTimeWindow', (el) => el.value);
assert(timeWindowValue === '60', `Expected time window dropdown to restore 60, got ${timeWindowValue}`);
@@ -1499,18 +1470,13 @@ async function run() {
// ─── Neighbor section tests ───────────────────────────────────────────────
await test('Node detail: neighbors section exists with correct columns', async () => {
// Full-screen node view (with #node-neighbors) is mobile-only since #676 fix.
await page.setViewportSize({ width: 390, height: 844 });
// Navigate to a node detail page (use the first node in the list)
await page.goto(BASE + '/#/nodes');
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
// Get the first node's pubkey from the row's data-key attribute
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
// Use evaluate to change hash (reliable same-document navigation)
await page.evaluate((pk) => { location.hash = '#/nodes/' + pk; }, pubkey);
// Wait for the full-node view to render (async API fetch populates body)
await page.waitForSelector('.node-fullscreen', { timeout: 10000 });
await page.waitForSelector('#node-neighbors', { timeout: 25000 });
await page.goto(BASE + '/#/nodes/' + pubkey);
await page.waitForSelector('#node-neighbors', { timeout: 10000 });
// Check the section exists
const header = await page.$eval('#fullNeighborsHeader', el => el.textContent);
assert(header.startsWith('Neighbors'), 'Header should start with "Neighbors", got: ' + header);
@@ -1534,7 +1500,6 @@ async function run() {
const text = await page.$eval('#fullNeighborsContent', el => el.textContent);
assert(text.includes('No neighbor data') || text.includes('Could not load'), 'Should show empty or error state');
}
await page.setViewportSize({ width: 1280, height: 800 });
});
@@ -1688,33 +1653,6 @@ async function run() {
assert(url.includes('tab=room'), `URL should contain tab=room after click, got: ${url}`);
});
// Test: clicking a node on desktop updates URL hash (#676)
await test('Desktop: clicking a node updates URL to #/nodes/{pubkey}', async () => {
await page.setViewportSize({ width: 1280, height: 800 });
await page.goto(BASE + '#/nodes', { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
await page.click('#nodesBody tr[data-key]');
await page.waitForTimeout(300);
const url = page.url();
assert(url.includes(encodeURIComponent(pubkey)), `URL should contain pubkey after click, got: ${url}`);
assert(!url.includes('node-fullscreen') || await page.$('#nodesRight:not(.empty)'), 'Split panel should be visible on desktop');
});
// Test: loading #/nodes/{pubkey} on desktop opens full-screen detail view (#823)
// Updated from #676's earlier "split panel on desktop" assertion. The Details
// link now opens the full-screen single-node view on desktop too — see PR #824.
await test('Desktop: deep link #/nodes/{pubkey} opens full-screen detail view', async () => {
await page.setViewportSize({ width: 1280, height: 800 });
await page.goto(BASE + '#/nodes', { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 10000 });
const pubkey = await page.$eval('#nodesBody tr[data-key]', el => el.dataset.key);
await page.goto(BASE + '#/nodes/' + encodeURIComponent(pubkey), { waitUntil: 'domcontentloaded' });
await page.waitForTimeout(500);
const hasFullScreen = await page.$('.node-fullscreen');
assert(hasFullScreen, 'Full-screen detail view should be open on desktop deep link (#823)');
});
// Test: packets timeWindow deep link
await test('Packets timeWindow deep link restores dropdown', async () => {
await page.goto(BASE + '#/packets?timeWindow=60', { waitUntil: 'domcontentloaded' });
@@ -1725,36 +1663,6 @@ async function run() {
assert(url.includes('timeWindow=60'), `URL should still contain timeWindow=60, got: ${url}`);
});
// Test: hash filter updates URL and is restored (#682)
await test('Packets hash filter updates URL and restores on reload', async () => {
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#fHash', { timeout: 8000 });
await page.fill('#fHash', 'abc123');
await page.waitForTimeout(500);
const url = page.url();
assert(url.includes('hash=abc123'), `URL should contain hash=abc123, got: ${url}`);
// Reload and check input restored
await page.goto(url, { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#fHash', { timeout: 8000 });
const val = await page.$eval('#fHash', el => el.value);
assert(val === 'abc123', `fHash should be restored to abc123, got: ${val}`);
});
// Test: Wireshark filter expression updates URL and is restored (#682)
await test('Packets filter expression updates URL and restores on reload', async () => {
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#packetFilterInput', { timeout: 8000 });
await page.fill('#packetFilterInput', 'type == ADVERT');
await page.waitForTimeout(500);
const url = page.url();
assert(url.includes('filter=') && url.includes('ADVERT'), `URL should contain filter=type%3D%3DADVERT, got: ${url}`);
// Reload and check expression restored
await page.goto(url, { waitUntil: 'domcontentloaded' });
await page.waitForSelector('#packetFilterInput', { timeout: 8000 });
const val = await page.$eval('#packetFilterInput', el => el.value);
assert(val === 'type == ADVERT', `packetFilterInput should be restored, got: ${val}`);
});
// Test: timeWindow change updates URL
await test('Packets timeWindow change updates URL', async () => {
await page.goto(BASE + '#/packets', { waitUntil: 'domcontentloaded' });

Some files were not shown because too many files have changed in this diff Show More