mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-07 22:19:45 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 you
|
4f265b312d | ||
|
you
|
5a959093fe | ||
|
you
|
d259076285 | ||
|
you
|
6dc4a21a1f | ||
|
you
|
507ed19d0e | ||
|
you
|
0c93c2f548 |
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"725 passed","color":"brightgreen"}
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"45 passed","color":"brightgreen"}
|
||||
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"36.05%","color":"red"}
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"39.68%","color":"red"}
|
||||
-286
@@ -1,286 +0,0 @@
|
||||
{
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2022,
|
||||
"sourceType": "script"
|
||||
},
|
||||
"env": {
|
||||
"browser": true,
|
||||
"es2022": true
|
||||
},
|
||||
"globals": {
|
||||
"AreaFilter": "readonly",
|
||||
"CACHE_INVALIDATE_MS": "readonly",
|
||||
"CLIENT_CONFIG": "readonly",
|
||||
"CLIENT_TTL": "readonly",
|
||||
"ChannelColorPicker": "readonly",
|
||||
"ChannelColors": "readonly",
|
||||
"ChannelDecrypt": "readonly",
|
||||
"ChannelQR": "readonly",
|
||||
"Chart": "readonly",
|
||||
"DIST_THRESHOLDS": "readonly",
|
||||
"DragManager": "readonly",
|
||||
"EXTERNAL_URLS": "readonly",
|
||||
"FAV_KEY": "readonly",
|
||||
"FilterUX": "readonly",
|
||||
"GestureHints": "readonly",
|
||||
"HEALTH_THRESHOLDS": "readonly",
|
||||
"HashColor": "readonly",
|
||||
"HopDisplay": "readonly",
|
||||
"HopResolver": "readonly",
|
||||
"IATA_CITIES": "readonly",
|
||||
"IATA_COORDS_GEO": "readonly",
|
||||
"L": "readonly",
|
||||
"LIMITS": "readonly",
|
||||
"Logo": "readonly",
|
||||
"MAX_HOP_DIST": "readonly",
|
||||
"MeshAudio": "readonly",
|
||||
"MeshConfigReady": "readonly",
|
||||
"PAYLOAD_COLORS": "readonly",
|
||||
"PAYLOAD_TYPES": "readonly",
|
||||
"PERF_SLOW_MS": "readonly",
|
||||
"PROPAGATION_BUFFER_MS": "readonly",
|
||||
"PULL_THRESHOLD_PX": "readonly",
|
||||
"PacketFilter": "readonly",
|
||||
"PathInspector": "readonly",
|
||||
"PrefixReserved": "readonly",
|
||||
"QRCode": "readonly",
|
||||
"ROLE_COLORS": "readonly",
|
||||
"ROLE_EMOJI": "readonly",
|
||||
"ROLE_LABELS": "readonly",
|
||||
"ROLE_SHAPES": "readonly",
|
||||
"ROLE_SORT": "readonly",
|
||||
"ROLE_STYLE": "readonly",
|
||||
"ROUTE_TYPES": "readonly",
|
||||
"RegionFilter": "readonly",
|
||||
"SITE_CONFIG": "readonly",
|
||||
"SKEW_SEVERITY_COLORS": "readonly",
|
||||
"SKEW_SEVERITY_LABELS": "readonly",
|
||||
"SKEW_SEVERITY_ORDER": "readonly",
|
||||
"SNR_THRESHOLDS": "readonly",
|
||||
"SlideOver": "readonly",
|
||||
"TILE_DARK": "readonly",
|
||||
"TILE_LIGHT": "readonly",
|
||||
"MC_TILE_PROVIDERS": "readonly",
|
||||
"MC_setDarkTileProvider": "readonly",
|
||||
"MC_getDarkTileProvider": "readonly",
|
||||
"MC_setServerDefaultTileProvider": "readonly",
|
||||
"MC_applyTileFilter": "readonly",
|
||||
"MC_DARK_TILE_DEFAULT": "readonly",
|
||||
"TYPE_COLORS": "readonly",
|
||||
"TableResponsive": "readonly",
|
||||
"TableSort": "readonly",
|
||||
"TouchGestures": "readonly",
|
||||
"TracesHelpers": "readonly",
|
||||
"URLState": "readonly",
|
||||
"WS_RECONNECT_MS": "readonly",
|
||||
"_SITE_CONFIG_ORIGINAL_HOME": "readonly",
|
||||
"__PERF_LOG_RENDER": "readonly",
|
||||
"__bottomNavInitDone": "readonly",
|
||||
"__corescopeLogo": "readonly",
|
||||
"__dirname": "readonly",
|
||||
"__filename": "readonly",
|
||||
"__gestureHints1065Init": "readonly",
|
||||
"__liveMQLBindCount": "readonly",
|
||||
"__meshcoreMapInternals": "readonly",
|
||||
"__navDrawer": "readonly",
|
||||
"__navDrawerPointerBindCount": "readonly",
|
||||
"__pathOverflowWired": "readonly",
|
||||
"__scrollLock": "readonly",
|
||||
"__touchGestures1062InitCount": "readonly",
|
||||
"_analyticsChannelTbodyHtml": "readonly",
|
||||
"_analyticsChannelTheadHtml": "readonly",
|
||||
"_analyticsDecorateChannels": "readonly",
|
||||
"_analyticsHashStatCardsHtml": "readonly",
|
||||
"_analyticsLoadChannelSort": "readonly",
|
||||
"_analyticsRenderCollisionsFromServer": "readonly",
|
||||
"_analyticsRenderMultiByteAdopters": "readonly",
|
||||
"_analyticsRenderMultiByteCapability": "readonly",
|
||||
"_analyticsRfNFColumnChart": "readonly",
|
||||
"_analyticsSaveChannelSort": "readonly",
|
||||
"_analyticsSortChannels": "readonly",
|
||||
"_apiCache": "readonly",
|
||||
"_apiPerf": "readonly",
|
||||
"_channelsBeginMessageRequestForTest": "readonly",
|
||||
"_channelsGetStateForTest": "readonly",
|
||||
"_channelsHandleWSBatchForTest": "readonly",
|
||||
"_channelsIsStaleMessageRequestForTest": "readonly",
|
||||
"_channelsLoadChannelsForTest": "readonly",
|
||||
"_channelsProcessWSBatchForTest": "readonly",
|
||||
"_channelsReconcileSelectionForTest": "readonly",
|
||||
"_channelsRefreshMessagesForTest": "readonly",
|
||||
"_channelsSelectChannelForTest": "readonly",
|
||||
"_channelsSetObserverRegionsForTest": "readonly",
|
||||
"_channelsSetStateForTest": "readonly",
|
||||
"_channelsShouldProcessWSMessageForRegion": "readonly",
|
||||
"_customizerV2": "readonly",
|
||||
"_ensurePullIndicator": "readonly",
|
||||
"_inflight": "readonly",
|
||||
"_isTouchDevice": "readonly",
|
||||
"_liveAddFeedItem": "readonly",
|
||||
"_liveBufferPacket": "readonly",
|
||||
"_liveBuildClickablePathPopupHtml": "readonly",
|
||||
"_liveBuildObserverIataMap": "readonly",
|
||||
"_liveClickablePaths": "readonly",
|
||||
"_liveDbPacketToLive": "readonly",
|
||||
"_liveExpandToBufferEntries": "readonly",
|
||||
"_liveExpandToBufferEntriesAsync": "readonly",
|
||||
"_liveFormatLiveTimestampHtml": "readonly",
|
||||
"_liveGetFavoritePubkeys": "readonly",
|
||||
"_liveGetNodeFilterKeys": "readonly",
|
||||
"_liveGetObserverIataMap": "readonly",
|
||||
"_liveIsNodeFavorited": "readonly",
|
||||
"_liveNodeActivity": "readonly",
|
||||
"_liveNodeData": "readonly",
|
||||
"_liveNodeMarkers": "readonly",
|
||||
"_livePacketInvolvesFavorite": "readonly",
|
||||
"_livePacketInvolvesFilterNode": "readonly",
|
||||
"_livePacketMatchesRegion": "readonly",
|
||||
"_livePruneClickablePaths": "readonly",
|
||||
"_livePruneStaleNodes": "readonly",
|
||||
"_liveRebuildFeedList": "readonly",
|
||||
"_liveResolveHopPositions": "readonly",
|
||||
"_liveSEG_MAP": "readonly",
|
||||
"_liveSetMarkerColor": "readonly",
|
||||
"_liveSetMarkerSize": "readonly",
|
||||
"_liveSetNodeFilter": "readonly",
|
||||
"_liveSetObserverIataMap": "readonly",
|
||||
"_liveSpeedLabel": "readonly",
|
||||
"_liveVCR": "readonly",
|
||||
"_liveVcrPause": "readonly",
|
||||
"_liveVcrResumeLive": "readonly",
|
||||
"_liveVcrSetMode": "readonly",
|
||||
"_liveVcrSpeedCycle": "readonly",
|
||||
"_live_packetTimestamp": "readonly",
|
||||
"_mapGetNeighborPubkeys": "readonly",
|
||||
"_mapSelectRefNode": "readonly",
|
||||
"_meshAudioVoices": "readonly",
|
||||
"_meshcoreHeatLayer": "readonly",
|
||||
"_meshcoreLiveHeatLayer": "readonly",
|
||||
"_nodesGetAllNodes": "readonly",
|
||||
"_nodesGetSortState": "readonly",
|
||||
"_nodesGetStatusInfo": "readonly",
|
||||
"_nodesGetStatusTooltip": "readonly",
|
||||
"_nodesIsAdvertMessage": "readonly",
|
||||
"_nodesMatchesSearch": "readonly",
|
||||
"_nodesRenderNodeTimestampHtml": "readonly",
|
||||
"_nodesRenderNodeTimestampText": "readonly",
|
||||
"_nodesSetAllNodes": "readonly",
|
||||
"_nodesSetSortState": "readonly",
|
||||
"_nodesSortArrow": "readonly",
|
||||
"_nodesSortNodes": "readonly",
|
||||
"_nodesSyncClaimedToFavorites": "readonly",
|
||||
"_nodesToggleSort": "readonly",
|
||||
"_packetsTestAPI": "readonly",
|
||||
"_panelCorner": "readonly",
|
||||
"_pendingPathInspectorRoute": "readonly",
|
||||
"_perfWriteSourcesPrev": "readonly",
|
||||
"_pullIndicator": "readonly",
|
||||
"_pullToast": "readonly",
|
||||
"_pullToastTimer": "readonly",
|
||||
"_reducedMotionMQL": "readonly",
|
||||
"_showPullToast": "readonly",
|
||||
"_themeRefreshTimer": "readonly",
|
||||
"_vcrFormatTime": "readonly",
|
||||
"addEventListener": "readonly",
|
||||
"api": "readonly",
|
||||
"apiPerf": "readonly",
|
||||
"bindFavStars": "readonly",
|
||||
"buildHexLegend": "readonly",
|
||||
"buildNodesQuery": "readonly",
|
||||
"buildPacketsQuery": "readonly",
|
||||
"clearParsedCache": "readonly",
|
||||
"closeMoreMenu": "readonly",
|
||||
"closeNav": "readonly",
|
||||
"comparePacketSets": "readonly",
|
||||
"computeBreakdownRanges": "readonly",
|
||||
"computeOverlapStats": "readonly",
|
||||
"connectWS": "readonly",
|
||||
"copyToClipboard": "readonly",
|
||||
"createColoredHexDump": "readonly",
|
||||
"currentPage": "readonly",
|
||||
"currentSkewValue": "readonly",
|
||||
"debounce": "readonly",
|
||||
"debouncedOnWS": "readonly",
|
||||
"destroy": "readonly",
|
||||
"devicePixelRatio": "readonly",
|
||||
"dispatchEvent": "readonly",
|
||||
"drawPacketRoute": "readonly",
|
||||
"escapeHtml": "readonly",
|
||||
"exports": "readonly",
|
||||
"favStar": "readonly",
|
||||
"filterPacketsByRoute": "readonly",
|
||||
"formatAbsoluteTimestamp": "readonly",
|
||||
"formatChartAxisLabel": "readonly",
|
||||
"formatDistance": "readonly",
|
||||
"formatDistanceRound": "readonly",
|
||||
"formatDrift": "readonly",
|
||||
"formatEngineBadge": "readonly",
|
||||
"formatHex": "readonly",
|
||||
"formatIsoLike": "readonly",
|
||||
"formatSkew": "readonly",
|
||||
"formatTimestamp": "readonly",
|
||||
"formatTimestampCustom": "readonly",
|
||||
"formatTimestampWithTooltip": "readonly",
|
||||
"formatVersionBadge": "readonly",
|
||||
"getDistanceUnit": "readonly",
|
||||
"getFavorites": "readonly",
|
||||
"getHashParams": "readonly",
|
||||
"getHealthThresholds": "readonly",
|
||||
"getNodeStatus": "readonly",
|
||||
"getParsedDecoded": "readonly",
|
||||
"getParsedPath": "readonly",
|
||||
"getPathLenOffset": "readonly",
|
||||
"getResolvedPath": "readonly",
|
||||
"getTileUrl": "readonly",
|
||||
"getTimestampCustomFormat": "readonly",
|
||||
"getTimestampFormatPreset": "readonly",
|
||||
"getTimestampMode": "readonly",
|
||||
"getTimestampTimezone": "readonly",
|
||||
"global": "readonly",
|
||||
"initGeoFilterOverlay": "readonly",
|
||||
"initTabBar": "readonly",
|
||||
"invalidateApiCache": "readonly",
|
||||
"isFavorite": "readonly",
|
||||
"isTransportRoute": "readonly",
|
||||
"makeColumnsResizable": "readonly",
|
||||
"makeRoleMarkerSVG": "readonly",
|
||||
"miniMarkdown": "readonly",
|
||||
"module": "readonly",
|
||||
"navigate": "readonly",
|
||||
"observerSkewSeverity": "readonly",
|
||||
"offWS": "readonly",
|
||||
"onWS": "readonly",
|
||||
"pad2": "readonly",
|
||||
"pad3": "readonly",
|
||||
"pages": "readonly",
|
||||
"payloadTypeColor": "readonly",
|
||||
"payloadTypeName": "readonly",
|
||||
"process": "readonly",
|
||||
"pullReconnect": "readonly",
|
||||
"qrcode": "readonly",
|
||||
"registerPage": "readonly",
|
||||
"renderSkewBadge": "readonly",
|
||||
"renderSkewSparkline": "readonly",
|
||||
"require": "readonly",
|
||||
"routeLayer": "readonly",
|
||||
"routeTypeName": "readonly",
|
||||
"setupPullToReconnect": "readonly",
|
||||
"syncBadgeColors": "readonly",
|
||||
"timeAgo": "readonly",
|
||||
"toggleFavorite": "readonly",
|
||||
"transportBadge": "readonly",
|
||||
"truncate": "readonly",
|
||||
"ws": "readonly",
|
||||
"wsListeners": "readonly"
|
||||
},
|
||||
"rules": {
|
||||
"no-undef": "error",
|
||||
"no-unused-vars": [
|
||||
"warn",
|
||||
{
|
||||
"argsIgnorePattern": "^_",
|
||||
"varsIgnorePattern": "^_"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+45
-300
@@ -3,18 +3,13 @@ name: CI/CD Pipeline
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
branches: [master]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
@@ -23,8 +18,8 @@ env:
|
||||
STAGING_CONTAINER: corescope-staging-go
|
||||
|
||||
# Pipeline (sequential, fail-fast):
|
||||
# go-test → e2e-test → build-and-publish → deploy → publish-badges
|
||||
# PRs stop after build-and-publish (no GHCR push). Master continues to deploy + badges.
|
||||
# go-test → e2e-test → build → deploy → publish
|
||||
# PRs stop after build. Master continues to deploy + publish.
|
||||
|
||||
jobs:
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
@@ -55,9 +50,7 @@ jobs:
|
||||
set -e -o pipefail
|
||||
cd cmd/server
|
||||
go build .
|
||||
# -race gates PR #1208's atomic.Pointer migration: the race-detector
|
||||
# is what makes path_inspect_atomic_race_test.go actually assert.
|
||||
go test -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log
|
||||
go test -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log
|
||||
echo "--- Go Server Coverage ---"
|
||||
go tool cover -func=server-coverage.out | tail -1
|
||||
|
||||
@@ -70,68 +63,6 @@ jobs:
|
||||
echo "--- Go Ingestor Coverage ---"
|
||||
go tool cover -func=ingestor-coverage.out | tail -1
|
||||
|
||||
- name: Build and test channel library + decrypt CLI
|
||||
run: |
|
||||
set -e -o pipefail
|
||||
cd internal/channel
|
||||
go test ./...
|
||||
echo "--- Channel library tests passed ---"
|
||||
cd ../../cmd/decrypt
|
||||
CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt .
|
||||
go test ./...
|
||||
echo "--- Decrypt CLI tests passed ---"
|
||||
|
||||
- name: Lint CSS variables (issue #1128)
|
||||
run: |
|
||||
set -e
|
||||
node scripts/check-css-vars.js
|
||||
node scripts/test-check-css-vars.js
|
||||
|
||||
- name: Run JS unit tests (packet-filter)
|
||||
run: |
|
||||
set -e
|
||||
node test-packet-filter.js
|
||||
node test-packet-filter-time.js
|
||||
node test-channel-decrypt-insecure-context.js
|
||||
node test-live-region-filter.js
|
||||
node test-issue-1136-observer-iata-map.js
|
||||
node test-channel-qr.js
|
||||
node test-channel-qr-wiring.js
|
||||
node test-channel-modal-ux.js
|
||||
node test-channel-issue-1087.js
|
||||
node test-issue-1409-no-encrypted-flood.js
|
||||
node test-channel-issue-1101.js
|
||||
node test-observer-iata-1188.js
|
||||
node test-pull-to-reconnect-1091.js
|
||||
node test-channel-fluid-layout.js
|
||||
node test-issue-1279-p2-code-filter.js
|
||||
node test-area-filter.js
|
||||
node test-issue-1293-marker-shapes.js
|
||||
node test-issue-1356-map-a11y.js
|
||||
node test-issue-1360-pill-letter-count.js
|
||||
node test-issue-1364-pill-no-clamp.js
|
||||
node test-issue-1375-scope-stats-fetch.js
|
||||
node test-issue-1361-cb-presets.js
|
||||
node test-issue-1407-cb-preset-propagation.js
|
||||
node test-issue-1412-customizer-no-override.js
|
||||
node test-issue-1418-raw-hex-extraction.js
|
||||
node test-issue-1418-edge-weights.js
|
||||
node test-issue-1418-cb-preset-ramp.js
|
||||
node test-issue-1418-spider-fan.js
|
||||
node test-issue-1418-deeplink-hops-channels.js
|
||||
node test-issue-1418-polish-review.js
|
||||
node test-issue-1420-tile-providers.js
|
||||
node test-issue-1438-marker-css-vars.js
|
||||
node test-live.js
|
||||
|
||||
- name: 🧹 Frontend lint (eslint no-undef) — issue #1342
|
||||
run: |
|
||||
set -e
|
||||
# Use eslint@8 (legacy .eslintrc.json). Don't migrate to flat-config / eslint@9.
|
||||
# --no-save: avoid touching package.json / no committed node_modules.
|
||||
npm install --no-save --no-audit --no-fund eslint@8
|
||||
npx eslint public/*.js
|
||||
|
||||
- name: Verify proto syntax
|
||||
run: |
|
||||
set -e
|
||||
@@ -188,7 +119,7 @@ jobs:
|
||||
e2e-test:
|
||||
name: "🎭 Playwright E2E Tests"
|
||||
needs: [go-test]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, Linux]
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -198,6 +129,13 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Free disk space
|
||||
run: |
|
||||
# Prune old runner diagnostic logs (can accumulate 50MB+)
|
||||
find ~/actions-runner/_diag/ -name '*.log' -mtime +3 -delete 2>/dev/null || true
|
||||
# Show available disk space
|
||||
df -h / | tail -1
|
||||
|
||||
- name: Set up Node.js 22
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
@@ -218,12 +156,6 @@ jobs:
|
||||
go build -o ../../corescope-server .
|
||||
echo "Go server built successfully"
|
||||
|
||||
- name: Build Go migrate tool
|
||||
run: |
|
||||
cd cmd/migrate
|
||||
go build -o ../../corescope-migrate .
|
||||
echo "Go migrate tool built successfully"
|
||||
|
||||
- name: Install npm dependencies
|
||||
run: npm ci --production=false
|
||||
|
||||
@@ -235,18 +167,6 @@ jobs:
|
||||
- name: Instrument frontend JS for coverage
|
||||
run: sh scripts/instrument-frontend.sh
|
||||
|
||||
- name: Freshen fixture timestamps
|
||||
run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db
|
||||
|
||||
- name: Migrate fixture DB to current schema (#1287)
|
||||
# Server now ASSERTs schema is migrated and refuses to start
|
||||
# otherwise (cmd/server/main.go: dbschema.AssertReady). In prod
|
||||
# the ingestor owns dbschema.Apply, but CI starts only the
|
||||
# server against the committed e2e fixture — so we run the
|
||||
# standalone migrate tool here to bring the fixture up to the
|
||||
# required shape before the server boots.
|
||||
run: ./corescope-migrate -db test-fixtures/e2e-fixture.db
|
||||
|
||||
- name: Start Go server with fixture DB
|
||||
run: |
|
||||
fuser -k 13581/tcp 2>/dev/null || true
|
||||
@@ -254,7 +174,7 @@ jobs:
|
||||
./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented &
|
||||
echo $! > .server.pid
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then
|
||||
if curl -sf http://localhost:13581/api/stats > /dev/null 2>&1; then
|
||||
echo "Server ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
@@ -268,71 +188,6 @@ jobs:
|
||||
- name: Run Playwright E2E tests (fail-fast)
|
||||
run: |
|
||||
BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1185-scroll-discriminator-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gesture-hints-1065-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-touch-gestures-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-fluid-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1109-hamburger-dropdown-visible-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-layout-1178-1179-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1205-live-controls-anchor-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-mql-leak-1180-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1204-live-panel-structure-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-live-1297-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-lab-1297-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-qr-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-color-picker-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-customize-theme-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-customize-branding-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
|
||||
- name: Collect frontend coverage (parallel)
|
||||
if: success() && github.event_name == 'push'
|
||||
@@ -342,13 +197,7 @@ jobs:
|
||||
- name: Generate frontend coverage badges
|
||||
if: success()
|
||||
run: |
|
||||
# Aggregate per-suite PASS/FAIL across every test-*-e2e.js summary.
|
||||
# The previous regex (grep -oP '[0-9]+(?=/)' | tail -1) caught a
|
||||
# stray digits-before-slash like the '2' in '2/3 tests passed' from
|
||||
# some sub-output and stamped the badge as '2 passed'. See #1296.
|
||||
eval "$(bash scripts/aggregate-e2e-pass.sh e2e-output.txt)"
|
||||
E2E_PASS=${PASS:-0}
|
||||
E2E_FAIL=${FAIL:-0}
|
||||
E2E_PASS=$(grep -oP '[0-9]+(?=/)' e2e-output.txt | tail -1 || echo "0")
|
||||
|
||||
mkdir -p .badges
|
||||
if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then
|
||||
@@ -361,14 +210,7 @@ jobs:
|
||||
echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json
|
||||
echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
if [ "${E2E_FAIL:-0}" -gt 0 ]; then
|
||||
E2E_MSG="${E2E_PASS:-0} passed, ${E2E_FAIL} failed"
|
||||
E2E_COLOR="red"
|
||||
else
|
||||
E2E_MSG="${E2E_PASS:-0} passed"
|
||||
E2E_COLOR="brightgreen"
|
||||
fi
|
||||
echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json
|
||||
echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_PASS:-0} passed\",\"color\":\"brightgreen\"}" > .badges/e2e-tests.json
|
||||
|
||||
- name: Stop test server
|
||||
if: always()
|
||||
@@ -389,150 +231,54 @@ jobs:
|
||||
include-hidden-files: true
|
||||
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
# 3. Build & Publish Docker Image
|
||||
# 3. Build Docker Image
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
build-and-publish:
|
||||
name: "🏗️ Build & Publish Docker Image"
|
||||
build:
|
||||
name: "🏗️ Build Docker Image"
|
||||
needs: [e2e-test]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, Linux]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Compute build metadata
|
||||
id: meta
|
||||
run: |
|
||||
BUILD_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
GIT_COMMIT="${GITHUB_SHA::7}"
|
||||
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||
APP_VERSION="${GITHUB_REF#refs/tags/}"
|
||||
else
|
||||
APP_VERSION="edge"
|
||||
fi
|
||||
echo "build_time=$BUILD_TIME" >> "$GITHUB_OUTPUT"
|
||||
echo "git_commit=$GIT_COMMIT" >> "$GITHUB_OUTPUT"
|
||||
echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Build: version=$APP_VERSION commit=$GIT_COMMIT time=$BUILD_TIME"
|
||||
- name: Set up Node.js 22
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- name: Build Go Docker image (local staging)
|
||||
- name: Free disk space
|
||||
run: |
|
||||
GIT_COMMIT="${{ steps.meta.outputs.git_commit }}" \
|
||||
APP_VERSION="${{ steps.meta.outputs.app_version }}" \
|
||||
BUILD_TIME="${{ steps.meta.outputs.build_time }}" \
|
||||
docker system prune -af 2>/dev/null || true
|
||||
docker builder prune -af 2>/dev/null || true
|
||||
df -h /
|
||||
|
||||
- name: Build Go Docker image
|
||||
run: |
|
||||
echo "${GITHUB_SHA::7}" > .git-commit
|
||||
APP_VERSION=$(node -p "require('./package.json').version") \
|
||||
GIT_COMMIT="${GITHUB_SHA::7}" \
|
||||
APP_VERSION=$(grep -oP 'APP_VERSION:-\K[^}]+' docker-compose.yml | head -1 || echo "3.0.0")
|
||||
GIT_COMMIT=$(git rev-parse --short HEAD)
|
||||
BUILD_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
export APP_VERSION GIT_COMMIT BUILD_TIME
|
||||
docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE"
|
||||
echo "Built Go staging image ✅"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Set up QEMU (arm64 runtime stage)
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
if: github.event_name == 'push'
|
||||
id: docker-meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/kpa-clawbot/corescope
|
||||
tags: |
|
||||
type=semver,pattern=v{{version}}
|
||||
type=semver,pattern=v{{major}}.{{minor}}
|
||||
type=semver,pattern=v{{major}}
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
type=edge,branch=master
|
||||
|
||||
- name: Build and push to GHCR
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.docker-meta.outputs.tags }}
|
||||
labels: ${{ steps.docker-meta.outputs.labels }}
|
||||
build-args: |
|
||||
APP_VERSION=${{ steps.meta.outputs.app_version }}
|
||||
GIT_COMMIT=${{ steps.meta.outputs.git_commit }}
|
||||
BUILD_TIME=${{ steps.meta.outputs.build_time }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
# 4. Release Artifacts (tags only)
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
release-artifacts:
|
||||
name: "📦 Release Artifacts"
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: [go-test]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Go 1.22
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.22'
|
||||
|
||||
- name: Build corescope-decrypt (static, linux/amd64)
|
||||
run: |
|
||||
cd cmd/decrypt
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 .
|
||||
|
||||
- name: Build corescope-decrypt (static, linux/arm64)
|
||||
run: |
|
||||
cd cmd/decrypt
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 .
|
||||
|
||||
- name: Upload release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
corescope-decrypt-linux-amd64
|
||||
corescope-decrypt-linux-arm64
|
||||
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
# 4b. Deploy Staging (master only)
|
||||
# 4. Deploy Staging (master only)
|
||||
# ───────────────────────────────────────────────────────────────
|
||||
deploy:
|
||||
name: "🚀 Deploy Staging"
|
||||
if: |
|
||||
(github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
&& github.ref == 'refs/heads/master'
|
||||
needs: [build-and-publish]
|
||||
runs-on: [self-hosted, meshcore-runner-2]
|
||||
if: github.event_name == 'push'
|
||||
needs: [build]
|
||||
runs-on: [self-hosted, Linux]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Pull latest image from GHCR
|
||||
run: |
|
||||
# Try to pull the edge image from GHCR and tag for docker-compose compatibility
|
||||
if docker pull ghcr.io/kpa-clawbot/corescope:edge; then
|
||||
docker tag ghcr.io/kpa-clawbot/corescope:edge corescope-go:latest
|
||||
echo "Pulled and tagged GHCR edge image ✅"
|
||||
else
|
||||
echo "⚠️ GHCR pull failed — falling back to locally built image"
|
||||
fi
|
||||
|
||||
- name: Deploy staging
|
||||
run: |
|
||||
# Force-remove the staging container regardless of how it was created
|
||||
# (compose-managed OR manually created via docker run)
|
||||
docker stop corescope-staging-go 2>/dev/null || true
|
||||
docker rm -f corescope-staging-go 2>/dev/null || true
|
||||
# Stop old container and release memory
|
||||
docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --timeout 30 2>/dev/null || true
|
||||
|
||||
# Wait for container to be fully gone and OS to reclaim memory (3GB limit)
|
||||
@@ -574,11 +320,10 @@ jobs:
|
||||
|
||||
- name: Smoke test staging API
|
||||
run: |
|
||||
PORT="${STAGING_GO_HTTP_PORT:-80}"
|
||||
if curl -sf "http://localhost:${PORT}/api/stats" | grep -q engine; then
|
||||
if curl -sf http://localhost:82/api/stats | grep -q engine; then
|
||||
echo "Staging verified — engine field present ✅"
|
||||
else
|
||||
echo "Staging /api/stats did not return engine field (port ${PORT})"
|
||||
echo "Staging /api/stats did not return engine field"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -600,7 +345,7 @@ jobs:
|
||||
name: "📝 Publish Badges & Summary"
|
||||
if: github.event_name == 'push'
|
||||
needs: [deploy]
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, Linux]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
@@ -31,5 +31,3 @@ cmd/ingestor/ingestor.exe
|
||||
!test-fixtures/e2e-fixture.db
|
||||
corescope-server
|
||||
cmd/server/server
|
||||
# Local-only planning and design files
|
||||
docs/superpowers/
|
||||
|
||||
@@ -43,17 +43,6 @@ scripts/ — Tooling (coverage collector, fixture capture, frontend in
|
||||
2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket
|
||||
3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side
|
||||
|
||||
### Read/Write Separation Invariant (#1283)
|
||||
- **All DB writes live in `cmd/ingestor/`.** INSERT / UPDATE / DELETE / VACUUM /
|
||||
schema migrations / retention all run in the ingestor process.
|
||||
- **`cmd/server/` is read-only.** It opens SQLite with `mode=ro` and must not
|
||||
acquire a write lock. Adding a write-side helper (e.g. a `cachedRW`-style
|
||||
RW connection) regresses this invariant and races the ingestor → SQLITE_BUSY.
|
||||
- Enforcement: `cmd/server/readonly_invariant_test.go` reflect-asserts that
|
||||
`PruneOldPackets`, `PruneOldMetrics`, and `RemoveStaleObservers` are NOT
|
||||
methods on the server's `*DB`. If you need a new write, add it to
|
||||
`cmd/ingestor/`.
|
||||
|
||||
### What's Deprecated (DO NOT TOUCH)
|
||||
The following were part of the old Node.js backend and have been removed:
|
||||
- `server.js`, `db.js`, `decoder.js`, `server-helpers.js`, `packet-store.js`, `iata-coords.js`
|
||||
@@ -381,7 +370,6 @@ Existing patterns: `#/nodes/{pubkey}?section=node-neighbors`, `#/analytics?tab=c
|
||||
|
||||
## What NOT to Do
|
||||
- **Don't check in private information** — no names, API keys, tokens, passwords, IP addresses, personal data, or any identifying information. This is a PUBLIC repo.
|
||||
- **Don't introduce new `map[string]interface{}` in API response builders, handler returns, or internal data structures that cross domain boundaries.** Use a named Go struct with explicit JSON tags. CoreScope already carries 694 occurrences (see #1383); the count must monotonically decrease. If your change adds even one new occurrence in a touched file, the PR is wrong-shaped — fix the design, don't paper over with `interface{}`. Exempt: third-party library boundaries that genuinely return `interface{}`, and ad-hoc test fixture assertions.
|
||||
- Don't add npm dependencies without asking
|
||||
- Don't create a build step
|
||||
- Don't add framework abstractions (React, Vue, etc.)
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### 📝 Documentation Corrections
|
||||
- **PR #1324 historical record correction** (#1387) — the merged PR #1324 body referenced four tests that do NOT exist in master: `TestMultibyteCapPersistRoundTrip`, `TestMultibyteCapPersistSkipsUnknown`, `TestMaybePersistCoalesces`, and a `TryLock` coalescing test. The actual tests that landed are `TestRunMultibyteCapPersist_AppliesSnapshot` and `TestRunMultibyteCapPersist_NoSnapshot_NoOp`. See issue #1386 for the corrective test additions (round-trip, unknown-key skip, coalescing).
|
||||
|
||||
## [3.7.2] — 2026-05-06
|
||||
|
||||
Hotfix release branched from `v3.7.1`. Cherry-picks PR #1121 only — no other changes.
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
- **Ingestor: backfill infinite loop on `path_json='[]'` rows** (#1119, #1121) — `BackfillPathJSONAsync` re-selected observations whose `path_json` was already `'[]'`, rewrote them to `'[]'`, and looped forever. The migration marker was never recorded and the ingestor sustained 2–3 MB/s WAL writes at idle (~76% CPU in `sqlite.Exec`). Fix: drop `'[]'` from the WHERE clause so the loop terminates after one full pass and the `backfill_path_json_from_raw_hex_v1` marker is written.
|
||||
|
||||
## [2.5.0] "Digital Rain" — 2026-03-22
|
||||
|
||||
### ✨ Matrix Mode — Full Cyberpunk Map Theme
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
# Deploy CoreScope
|
||||
|
||||
Pre-built images are published to GHCR for `linux/amd64` and `linux/arm64` (Raspberry Pi 4/5).
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Docker run
|
||||
|
||||
```bash
|
||||
docker run -d --name corescope \
|
||||
-p 80:80 \
|
||||
-v corescope-data:/app/data \
|
||||
-e DISABLE_CADDY=true \
|
||||
ghcr.io/kpa-clawbot/corescope:latest
|
||||
```
|
||||
|
||||
Open `http://localhost` — done.
|
||||
|
||||
### Docker Compose
|
||||
|
||||
```bash
|
||||
curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \
|
||||
-o docker-compose.yml
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Image Tags
|
||||
|
||||
| Tag | Description |
|
||||
|-----|-------------|
|
||||
| `v3.4.1` | Pinned release (recommended for production) |
|
||||
| `v3.4` | Latest patch in v3.4.x |
|
||||
| `v3` | Latest minor+patch in v3.x |
|
||||
| `latest` | Latest release tag |
|
||||
| `edge` | Built from master — unstable, for testing |
|
||||
|
||||
## Configuration
|
||||
|
||||
Settings can be overridden via environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `DISABLE_CADDY` | `false` | Skip internal Caddy (set `true` behind a reverse proxy) |
|
||||
| `DISABLE_MOSQUITTO` | `false` | Skip internal MQTT broker (use external) |
|
||||
| `HTTP_PORT` | `80` | Host port mapping |
|
||||
| `DATA_DIR` | `./data` | Host path for persistent data |
|
||||
|
||||
For advanced configuration, mount a `config.json` into `/app/data/config.json`. See `config.example.json` in the repo.
|
||||
|
||||
## Updating
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Data
|
||||
|
||||
All persistent data lives in `/app/data`:
|
||||
- `meshcore.db` — SQLite database (packets, nodes)
|
||||
- `config.json` — custom config (optional)
|
||||
- `theme.json` — custom theme (optional)
|
||||
|
||||
**Backup:** `cp data/meshcore.db ~/backup/`
|
||||
|
||||
## TLS
|
||||
|
||||
Option A — **External reverse proxy** (recommended): Run with `DISABLE_CADDY=true`, put nginx/traefik/Cloudflare in front.
|
||||
|
||||
Option B — **Built-in Caddy**: Mount a custom Caddyfile at `/etc/caddy/Caddyfile` and expose ports 80+443.
|
||||
|
||||
---
|
||||
|
||||
## Migrating from manage.sh (existing admins)
|
||||
|
||||
If you're currently deploying with `manage.sh` (git clone + local build), you have two options going forward:
|
||||
|
||||
### Option A: Keep using manage.sh (no changes needed)
|
||||
|
||||
`manage.sh update` continues to work exactly as before — it fetches the latest tag, builds locally, and restarts. Nothing breaks.
|
||||
|
||||
```bash
|
||||
./manage.sh update # latest release
|
||||
./manage.sh update v3.5.0 # specific version
|
||||
```
|
||||
|
||||
### Option B: Switch to pre-built images (recommended)
|
||||
|
||||
Pre-built images skip the build step entirely — faster updates, no Go toolchain needed.
|
||||
|
||||
**One-time migration:**
|
||||
|
||||
1. Stop the current deployment:
|
||||
```bash
|
||||
./manage.sh stop
|
||||
```
|
||||
|
||||
2. Your data is in `~/meshcore-data/` (or whatever `PROD_DATA_DIR` is set to). It's untouched — the database, config, and theme files persist.
|
||||
|
||||
3. Copy `docker-compose.example.yml` to where you want to run from:
|
||||
```bash
|
||||
cp docker-compose.example.yml ~/docker-compose.yml
|
||||
```
|
||||
|
||||
4. Start with the pre-built image:
|
||||
```bash
|
||||
cd ~ && docker compose up -d
|
||||
```
|
||||
|
||||
5. Verify it picked up your existing data:
|
||||
```bash
|
||||
curl http://localhost/api/stats
|
||||
```
|
||||
|
||||
**Updates after migration:**
|
||||
```bash
|
||||
docker compose pull && docker compose up -d
|
||||
```
|
||||
|
||||
### What about manage.sh features?
|
||||
|
||||
| manage.sh command | Pre-built equivalent |
|
||||
|---|---|
|
||||
| `./manage.sh update` | `docker compose pull && docker compose up -d` |
|
||||
| `./manage.sh stop` | `docker compose down` |
|
||||
| `./manage.sh start` | `docker compose up -d` |
|
||||
| `./manage.sh logs` | `docker compose logs -f` |
|
||||
| `./manage.sh status` | `docker compose ps` |
|
||||
| `./manage.sh setup` | Copy `docker-compose.example.yml`, edit env vars |
|
||||
|
||||
`manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments.
|
||||
+7
-41
@@ -1,57 +1,25 @@
|
||||
# Build stage always runs natively on the builder's arch ($BUILDPLATFORM)
|
||||
# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU.
|
||||
# BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain
|
||||
# `docker build` (without buildx) doesn't fail on an empty platform string.
|
||||
ARG BUILDPLATFORM=linux/amd64
|
||||
FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
|
||||
FROM golang:1.22-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache build-base
|
||||
|
||||
ARG APP_VERSION=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_TIME=unknown
|
||||
# Provided by buildx for multi-arch builds
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly)
|
||||
# Build server
|
||||
WORKDIR /build/server
|
||||
COPY cmd/server/go.mod cmd/server/go.sum ./
|
||||
COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
COPY internal/packetpath/ ../../internal/packetpath/
|
||||
COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
COPY internal/dbschema/ ../../internal/dbschema/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
COPY internal/perfio/ ../../internal/perfio/
|
||||
COPY internal/mbcapqueue/ ../../internal/mbcapqueue/
|
||||
RUN go mod download
|
||||
COPY cmd/server/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server .
|
||||
|
||||
# Build ingestor
|
||||
WORKDIR /build/ingestor
|
||||
COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./
|
||||
COPY internal/geofilter/ ../../internal/geofilter/
|
||||
COPY internal/sigvalidate/ ../../internal/sigvalidate/
|
||||
COPY internal/packetpath/ ../../internal/packetpath/
|
||||
COPY internal/dbconfig/ ../../internal/dbconfig/
|
||||
COPY internal/dbschema/ ../../internal/dbschema/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
COPY internal/perfio/ ../../internal/perfio/
|
||||
COPY internal/mbcapqueue/ ../../internal/mbcapqueue/
|
||||
RUN go mod download
|
||||
COPY cmd/ingestor/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -o /corescope-ingestor .
|
||||
|
||||
# Build decrypt CLI
|
||||
WORKDIR /build/decrypt
|
||||
COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./
|
||||
COPY internal/channel/ ../../internal/channel/
|
||||
RUN go mod download
|
||||
COPY cmd/decrypt/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -ldflags="-s -w" -o /corescope-decrypt .
|
||||
RUN go build -o /corescope-ingestor .
|
||||
|
||||
# Runtime image
|
||||
FROM alpine:3.20
|
||||
@@ -61,7 +29,7 @@ RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget
|
||||
WORKDIR /app
|
||||
|
||||
# Go binaries
|
||||
COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/
|
||||
COPY --from=builder /corescope-server /corescope-ingestor /app/
|
||||
|
||||
# Frontend assets + config
|
||||
COPY public/ ./public/
|
||||
@@ -74,8 +42,6 @@ RUN echo "unknown" > .git-commit
|
||||
# Supervisor + Mosquitto + Caddy config
|
||||
COPY docker/supervisord-go.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
COPY docker/supervisord-go-no-mosquitto.conf /etc/supervisor/conf.d/supervisord-no-mosquitto.conf
|
||||
COPY docker/supervisord-go-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-caddy.conf
|
||||
COPY docker/supervisord-go-no-mosquitto-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-mosquitto-no-caddy.conf
|
||||
COPY docker/mosquitto.conf /etc/mosquitto/mosquitto.conf
|
||||
COPY docker/Caddyfile /etc/caddy/Caddyfile
|
||||
|
||||
|
||||
@@ -40,9 +40,6 @@ RUN if [ ! -f .git-commit ]; then echo "unknown" > .git-commit; fi
|
||||
|
||||
# Supervisor + Mosquitto + Caddy config
|
||||
COPY docker/supervisord-go.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
COPY docker/supervisord-go-no-mosquitto.conf /etc/supervisor/conf.d/supervisord-no-mosquitto.conf
|
||||
COPY docker/supervisord-go-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-caddy.conf
|
||||
COPY docker/supervisord-go-no-mosquitto-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-mosquitto-no-caddy.conf
|
||||
COPY docker/mosquitto.conf /etc/mosquitto/mosquitto.conf
|
||||
COPY docker/Caddyfile /etc/caddy/Caddyfile
|
||||
|
||||
|
||||
@@ -1,674 +0,0 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short
|
||||
notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, your program's commands
|
||||
might be different; for a GUI interface, you would use an "about box".
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU GPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program
|
||||
into proprietary programs. If your program is a subroutine library, you
|
||||
may consider it more useful to permit linking proprietary applications with
|
||||
the library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License. But first, please read
|
||||
<https://www.gnu.org/licenses/why-not-lgpl.html>.
|
||||
@@ -74,34 +74,9 @@ Full experience on your phone — proper touch controls, iOS safe area support,
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Pre-built Image (Recommended)
|
||||
### Docker (Recommended)
|
||||
|
||||
No build step required — just run:
|
||||
|
||||
```bash
|
||||
docker run -d --name corescope \
|
||||
--restart=unless-stopped \
|
||||
-p 80:80 -p 1883:1883 \
|
||||
-v /your/data:/app/data \
|
||||
ghcr.io/kpa-clawbot/corescope:latest
|
||||
```
|
||||
|
||||
Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults.
|
||||
|
||||
For HTTPS with a custom domain, add `-p 443:443` and mount your Caddyfile:
|
||||
```bash
|
||||
docker run -d --name corescope \
|
||||
--restart=unless-stopped \
|
||||
-p 80:80 -p 443:443 -p 1883:1883 \
|
||||
-v /your/data:/app/data \
|
||||
-v /your/Caddyfile:/etc/caddy/Caddyfile:ro \
|
||||
-v /your/caddy-data:/data/caddy \
|
||||
ghcr.io/kpa-clawbot/corescope:latest
|
||||
```
|
||||
|
||||
Disable built-in services with `-e DISABLE_MOSQUITTO=true` or `-e DISABLE_CADDY=true`, or drop a `.env` file in your data volume. See [docs/deployment.md](docs/deployment.md) for the full reference.
|
||||
|
||||
### Build from Source
|
||||
No Go installation needed — everything builds inside the container.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Kpa-clawbot/CoreScope.git
|
||||
@@ -120,6 +95,8 @@ The setup wizard walks you through config, domain, HTTPS, build, and run.
|
||||
./manage.sh help # All commands
|
||||
```
|
||||
|
||||
See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for the full deployment guide — HTTPS options (auto cert, bring your own, Cloudflare Tunnel), MQTT security, backups, and troubleshooting.
|
||||
|
||||
### Configure
|
||||
|
||||
Copy `config.example.json` to `config.json` and edit:
|
||||
@@ -265,8 +242,6 @@ Contributions welcome. Please read [AGENTS.md](AGENTS.md) for coding conventions
|
||||
|
||||
**Live instance:** [analyzer.00id.net](https://analyzer.00id.net) — all API endpoints are public, no auth required.
|
||||
|
||||
**API Documentation:** CoreScope auto-generates an OpenAPI 3.0 spec. Browse the interactive Swagger UI at [`/api/docs`](https://analyzer.00id.net/api/docs) or fetch the machine-readable spec at [`/api/spec`](https://analyzer.00id.net/api/spec).
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
# v3.6.0 - The Forensics
|
||||
|
||||
CoreScope just got eyes everywhere. This release drops **path inspection**, **color-by-hash markers**, **clock skew detection**, **full channel encryption**, an **observer graph**, and a pile of robustness fixes that make your mesh network feel like it's being watched by someone who actually cares.
|
||||
|
||||
134 commits, 105 PRs merged, 18K+ lines added. Here's what shipped.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 New Features
|
||||
|
||||
### Path-Prefix Candidate Inspector (#944, #945)
|
||||
The marquee feature. Click any path segment and CoreScope opens an interactive inspector showing every candidate node that could match that hop prefix - plotted on a map with scoring by neighbor-graph affinity and geographic centroid. Ambiguous hops? Now you can see *why* they're ambiguous and pick the right one.
|
||||
|
||||
**Why you'll love it:** No more guessing which `0xA3` is the real repeater. The inspector lays out every candidate, scores them, and lets you drill in visually.
|
||||
|
||||
### Color-by-Hash Packet Markers (#948, #951)
|
||||
Every packet type gets a vivid, hash-derived color - on the live feed, map polylines, and flying-packet animations. Bright fill with dark outline for contrast. No more monochrome blobs - you can visually track packet flows by color at a glance.
|
||||
|
||||
### Node Filter on Live Page (#924, #771)
|
||||
Filter the live packet stream to show only traffic flowing through a specific node. Pick a repeater, see exactly what it's carrying. That simple.
|
||||
|
||||
### Clock Skew Detection (#746, #752, #828, #850)
|
||||
Full pipeline: backend computes drift using Theil-Sen regression with outlier rejection (#828), the UI shows per-node badges, detail sparklines, and fleet-wide analytics (#752). Bimodal clock severity (#850) surfaces flaky-RTC nodes that toggle between accurate and drifted - instead of hiding them as "No Clock."
|
||||
|
||||
**Why you'll love it:** Nodes with bad clocks silently corrupt your timeline. Now they glow red before they ruin your analysis.
|
||||
|
||||
### Observer Graph (M1+M2) (#774)
|
||||
Observers are now first-class graph citizens. CoreScope builds a neighbor graph from observation overlaps, scores hop-resolver candidates by graph edges (#876), and uses geographic centroid for tiebreaking. The observer topology is visible and queryable.
|
||||
|
||||
### Channel Encryption - Full Stack (#726, #733, #750, #760)
|
||||
Three milestones landed as one: DB-backed channel message history (#726), client-side PSK decryption in the browser (#733), and PSK channel management with add/remove UX and message caching (#750). Add a channel key in the UI, and CoreScope decrypts messages client-side - no server-side key storage. The add-channel button (#760) makes it dead simple.
|
||||
|
||||
**Why you'll love it:** Encrypted channels are no longer black boxes. Add your PSK, see the messages, search history - all without exposing keys to the server.
|
||||
|
||||
### Hash Collision Inspector (#758)
|
||||
The Hash Usage Matrix now shows collision details for all hash sizes. When two nodes share a prefix, you see exactly who collides and at what size.
|
||||
|
||||
### Geofilter Builder - In-App (#735, #900)
|
||||
The geofilter polygon builder is now served directly from CoreScope with a full docs page (#900). No more hunting for external tools. Link from the customizer, draw your polygon, done.
|
||||
|
||||
### Node Blacklist (#742)
|
||||
`nodeBlacklist` in config hides abusive or troll nodes from all views. They're gone.
|
||||
|
||||
### Observer Retention (#764)
|
||||
Stale observers are automatically pruned after a configurable number of days. Your observer list stays clean without manual intervention.
|
||||
|
||||
### Advert Signature Validation (#794)
|
||||
Corrupt packets with invalid advert signatures are now rejected at ingest. Bad data never hits your store.
|
||||
|
||||
### Bounded Cold Load (#790)
|
||||
`Load()` now respects a memory budget - no more OOM on cold start with a fat database. Combined with retention-hours cutoff (#917), cold start is safe on constrained hardware.
|
||||
|
||||
### Multi-Arch Docker Images (#869)
|
||||
Official images now publish `amd64` + `arm64` in a single multi-arch manifest. Raspberry Pi operators: pull and run. No special tags needed.
|
||||
|
||||
### /nodes Detail Panel + Search (#868)
|
||||
The nodes detail panel ships with search improvements (#862) - find nodes fast, see their full detail in a slide-out panel.
|
||||
|
||||
### Deduplicated Top Longest Hops (#848)
|
||||
Longest hops are now deduplicated by pair with observation count and SNR cues. No more seeing the same link 47 times.
|
||||
|
||||
---
|
||||
|
||||
## 🔥 Performance Wins
|
||||
|
||||
### StoreTx ResolvedPath Elimination (#806)
|
||||
The per-transaction `ResolvedPath` computation is gone - replaced by a membership index with on-demand decode. This was one of the hottest paths in the ingestor.
|
||||
|
||||
### Node Packet Queries (#803)
|
||||
Raw JSON text search for node packets replaced with a proper `byNode` index (#673). Night and day.
|
||||
|
||||
### Channel Query Performance (#762, #763)
|
||||
New `channel_hash` column enables SQL-level channel filtering. No more full-table scan to find messages in a channel.
|
||||
|
||||
### SQLite Auto-Vacuum (#919, #920)
|
||||
Incremental auto-vacuum enabled - the database file actually shrinks after retention pruning. No more 2GB database holding 200MB of live data.
|
||||
|
||||
### Retention-Hours Cutoff on Load (#917)
|
||||
`Load()` now applies `retentionHours` at read time, preventing OOM when the DB has more history than memory allows.
|
||||
|
||||
---
|
||||
|
||||
## 🛡️ Security & Robustness
|
||||
|
||||
### MQTT Reconnect with Bounded Backoff (#947, #949)
|
||||
The ingestor now reconnects to MQTT brokers with exponential backoff, observability logging, and bounded retry. No more silent disconnects that kill your data stream.
|
||||
|
||||
---
|
||||
|
||||
## 🐛 Bugs Squashed
|
||||
|
||||
This release exterminates **40+ bugs** — from protocol-level hash mismatches to pixel-level CSS breakage. Operators told us what hurt; we listened.
|
||||
|
||||
- **Path inspector "Show on Map" missed origin and first hop** (#950) - map view now includes all hops
|
||||
- **Content hash used full header byte** (#787) - content hashing now uses payload type bits only, fixing hash collisions between packets that differ only in header flags
|
||||
- **Encrypted channel deep links showed broken UI** (#825, #826, #815) - deep links to encrypted channels now show a lock message instead of broken UI when you don't have the key
|
||||
- **Geofilter longitude wrapping** (#925) - geofilter builder wraps longitude to [-180, 180]; southern hemisphere polygons no longer invert
|
||||
- **Hash filter bypasses saved region filter** (#939) - hash lookups now skip the geo filter as intended
|
||||
- **Companion-as-repeater excluded from path hops** (#935, #936) - non-repeater nodes no longer pollute hop resolution
|
||||
- **Customize panel re-renders while typing** (#927) - text fields keep focus during config changes
|
||||
- **Per-observation raw_hex** (#881, #882) - each observer's hex dump now shows what *that observer* actually received
|
||||
- **Per-observation children in packet groups** (#866, #880) - expanded groups show per-obs data, not cross-observer aggregates
|
||||
- **Full-page obs-switch** (#866, #870) - switching observers updates hex, path, and direction correctly
|
||||
- **Packet detail shows wrong observation** (#849, #851) - clicking a specific observation opens *that* observation
|
||||
- **Byte breakdown hop count** (#844, #846) - derived from `path_len`, not aggregated `_parsedPath`
|
||||
- **Transport-route path_len offset** (#852, #853) - correct offset calculation + CSS variable fix
|
||||
- **Packets/hour chart bars + x-axis** (#858, #865) - bars render correctly, x-axis labels properly decimated
|
||||
- **Channel timeline capped to top 8** (#860, #864) - no more 47-channel chart spaghetti
|
||||
- **Reachability row opacity removed** (#859, #863) - clean rows without misleading gradient
|
||||
- **Sticky table headers on mobile** (#861, #867) - restored after regression
|
||||
- **Map popup 'Show Neighbors' on iOS Safari** (#840, #841) - link actually works now
|
||||
- **Node detail Recent Packets invisible text** (#829, #830) - CSS fix
|
||||
- **/api/packets/{hash} falls back to DB** (#827, #831) - when in-memory store misses, DB catches it
|
||||
- **IATA filter bypass for status messages** (#694, #802) - status packets no longer filtered out by airport codes
|
||||
- **Desktop node click URL hash** (#676, #739) - clicking a node updates the URL for deep linking
|
||||
- **Filter params in URL hash** (#682, #740) - all filter state serialized for shareable links
|
||||
- **Hide undecryptable channel messages** (#727, #728) - clean default view
|
||||
- **TRACE path_json uses path_sz** (#732) - correct field from flags byte, not header hash_size
|
||||
- **Multi-byte adopters** (#754, #767) - all node types, role column, advert precedence
|
||||
- **Channel key case sensitivity** (#761) - Public decode works correctly
|
||||
- **Transport route field offsets** (#766) - correct offsets in field table
|
||||
- **Clock skew sanity checks** (#769) - filter epoch-0, cap drift, require minimum samples
|
||||
- **Neighbor graph slider persistence** (#776) - default 0.7, persisted to localStorage
|
||||
- **Node detail panel navigation** (#779, #785) - Details/Analytics links actually navigate
|
||||
- **Channel key removal** (#898) - user-added keys for server-known channels can be removed
|
||||
- **Side-panel Details on desktop** (#892) - opens full-screen correctly
|
||||
- **Hex-dump byte ranges client-side** (#891) - computed from per-obs raw_hex
|
||||
- **path_json derived from raw_hex at ingest** (#886, #887) - single source of truth
|
||||
- **Path pill and byte breakdown hop agreement** (#885) - they match now
|
||||
- **Mobile close button + toolbar scroll** (#797, #805) - accessible and scrollable
|
||||
- **/health.recentPackets resolved_path fallback** (#810, #821) - falls back to longest sibling observation
|
||||
- **Channel filter on Packets page** (#812, #816) - UI and API both fixed
|
||||
- **Clock-skew section in side panel** (#813, #814) - renders correctly
|
||||
- **Real RSS in /api/stats** (#832, #835) - surface actual RSS alongside tracked store bytes
|
||||
- **Hash size detection for transport routes + zero-hop adverts** (#747) - correct detection
|
||||
- **Repeater+observer merged map marker** (#745) - single marker, not two overlapping
|
||||
|
||||
---
|
||||
|
||||
## 🎨 UI Polish
|
||||
|
||||
- QA findings applied across the board (#832, #833, #836, #837, #838) - dozens of small UX fixes from systematic QA pass
|
||||
|
||||
---
|
||||
|
||||
## 📦 Upgrading
|
||||
|
||||
```bash
|
||||
git pull
|
||||
docker compose down
|
||||
docker compose build prod
|
||||
docker compose up -d prod
|
||||
```
|
||||
|
||||
Your existing `config.json` works as-is. New optional config keys:
|
||||
- `nodeBlacklist` - array of node hashes to hide
|
||||
- `observerRetentionDays` - days before stale observers are pruned
|
||||
- `memoryBudgetMB` - cap on in-memory packet store
|
||||
|
||||
### Verify
|
||||
|
||||
```bash
|
||||
curl -s http://localhost/api/health | jq .version
|
||||
# "3.6.0"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🙏 External Contributors
|
||||
|
||||
- **#735** ([@efiten](https://github.com/efiten)) - Serve geofilter builder from app, link from customizer
|
||||
- **#739** ([@efiten](https://github.com/efiten)) - Desktop node click updates URL hash for deep linking
|
||||
- **#740** ([@efiten](https://github.com/efiten)) - Serialize filter params in URL hash for shareable links
|
||||
- **#742** ([@Joel-Claw](https://github.com/Joel-Claw)) - Add nodeBlacklist config to hide abusive/troll nodes
|
||||
- **#761** ([@copelaje](https://github.com/copelaje)) - Fix channel key case sensitivity for Public decode
|
||||
- **#764** ([@Joel-Claw](https://github.com/Joel-Claw)) - Add observer retention - prune stale observers after configurable days
|
||||
- **#802** ([@efiten](https://github.com/efiten)) - Bypass IATA filter for status messages, fill SNR on duplicate observations
|
||||
- **#803** ([@efiten](https://github.com/efiten)) - Replace raw JSON text search with byNode index for node packet queries
|
||||
- **#805** ([@efiten](https://github.com/efiten)) - Mobile close button accessible + toolbar scrollable
|
||||
- **#900** ([@efiten](https://github.com/efiten)) - App-served geofilter docs page
|
||||
- **#917** ([@efiten](https://github.com/efiten)) - Apply retentionHours cutoff in Load() to prevent OOM on cold start
|
||||
- **#924** ([@efiten](https://github.com/efiten)) - Node filter on live page - show only traffic through a specific node
|
||||
- **#925** ([@efiten](https://github.com/efiten)) - Fix geobuilder longitude wrapping for southern hemisphere polygons
|
||||
- **#927** ([@efiten](https://github.com/efiten)) - Skip customize panel re-render while text field has focus
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Breaking Changes
|
||||
|
||||
**None.** All API endpoints remain backwards-compatible. New fields are additive only.
|
||||
|
||||
---
|
||||
|
||||
## 📊 By the Numbers
|
||||
|
||||
| Stat | Count |
|
||||
|------|-------|
|
||||
| Commits | 134 |
|
||||
| PRs merged | 105 |
|
||||
| Lines added | 18,480 |
|
||||
| Lines removed | 1,632 |
|
||||
| Files changed | 110 |
|
||||
| Contributors | 4 |
|
||||
|
||||
---
|
||||
|
||||
*Previous release: [v3.5.2](https://github.com/Kpa-clawbot/CoreScope/releases/tag/v3.5.2)*
|
||||
@@ -1,142 +0,0 @@
|
||||
# corescope-decrypt
|
||||
|
||||
Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from a CoreScope SQLite database.
|
||||
|
||||
## Why
|
||||
|
||||
MeshCore hashtag channels use symmetric encryption where the key is derived deterministically from the channel name. The CoreScope ingestor stores **all** `GRP_TXT` packets in the database, including those it cannot decrypt at ingest time.
|
||||
|
||||
This tool enables:
|
||||
|
||||
- **Retroactive decryption** — decrypt historical messages for any channel whose name you learn after the fact
|
||||
- **Forensics & analysis** — export channel traffic for offline review
|
||||
- **Bulk export** — dump an entire channel's history as JSON, HTML, or plain text
|
||||
|
||||
## Installation
|
||||
|
||||
### From Docker image
|
||||
|
||||
The binary is included in the CoreScope Docker image at `/app/corescope-decrypt`:
|
||||
|
||||
```bash
|
||||
docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
|
||||
```
|
||||
|
||||
### From GitHub release
|
||||
|
||||
Download the static binary from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page:
|
||||
|
||||
```bash
|
||||
# Linux amd64
|
||||
curl -LO https://github.com/Kpa-clawbot/CoreScope/releases/latest/download/corescope-decrypt-linux-amd64
|
||||
chmod +x corescope-decrypt-linux-amd64
|
||||
./corescope-decrypt-linux-amd64 --help
|
||||
```
|
||||
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
cd cmd/decrypt
|
||||
CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt .
|
||||
```
|
||||
|
||||
The binary is statically linked — no dependencies, runs on any Linux.
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE]
|
||||
```
|
||||
|
||||
Run `corescope-decrypt --help` for full flag documentation.
|
||||
|
||||
### JSON output (default)
|
||||
|
||||
Machine-readable, includes all metadata (observers, path hops, raw hex):
|
||||
|
||||
```bash
|
||||
corescope-decrypt --channel "#wardriving" --db meshcore.db
|
||||
```
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"hash": "a1b2c3...",
|
||||
"timestamp": "2026-04-12T17:19:09Z",
|
||||
"sender": "XMD Tag 1",
|
||||
"message": "@[MapperBot] 37.76985, -122.40525 [0.3w]",
|
||||
"channel": "#wardriving",
|
||||
"raw_hex": "150206...",
|
||||
"path": ["A3", "B0"],
|
||||
"observers": [
|
||||
{"name": "Observer1", "snr": 9.5, "rssi": -56, "timestamp": "2026-04-12T17:19:10Z"}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### HTML output
|
||||
|
||||
Self-contained interactive viewer — search, sortable columns, expandable detail rows:
|
||||
|
||||
```bash
|
||||
corescope-decrypt --channel "#wardriving" --db meshcore.db --format html --output wardriving.html
|
||||
open wardriving.html
|
||||
```
|
||||
|
||||
No external dependencies. The JSON data is embedded directly in the HTML file.
|
||||
|
||||
### IRC / log output
|
||||
|
||||
Plain-text, one line per message — ideal for `grep`, `awk`, and piping:
|
||||
|
||||
```bash
|
||||
corescope-decrypt --channel "#wardriving" --db meshcore.db --format irc
|
||||
```
|
||||
|
||||
```
|
||||
[2026-04-12 17:19:09] <XMD Tag 1> @[MapperBot] 37.76985, -122.40525 [0.3w]
|
||||
[2026-04-12 17:20:25] <XMD Tag 1> @[MapperBot] 37.78075, -122.39774 [0.3w]
|
||||
[2026-04-12 17:25:30] <mk 🤠> @[MapperBot] 35.32444, -120.62077
|
||||
```
|
||||
|
||||
```bash
|
||||
# Find all messages from a specific sender
|
||||
corescope-decrypt --channel "#wardriving" --db meshcore.db --format irc | grep "KE6QR"
|
||||
```
|
||||
|
||||
## How channel encryption works
|
||||
|
||||
MeshCore hashtag channels derive their encryption key from the channel name:
|
||||
|
||||
1. **Key derivation**: `AES-128 key = SHA-256("#channelname")[:16]` (first 16 bytes)
|
||||
2. **Channel hash**: `SHA-256(key)[0]` — 1-byte identifier in the packet header, used for fast filtering
|
||||
3. **Encryption**: AES-128-ECB
|
||||
4. **MAC**: HMAC-SHA256 with a 32-byte secret (key + 16 zero bytes), truncated to 2 bytes
|
||||
5. **Plaintext format**: `timestamp(4 LE) + flags(1) + "sender: message\0"`
|
||||
|
||||
See the firmware source at `firmware/src/helpers/BaseChatMesh.cpp` for the canonical implementation.
|
||||
|
||||
## Testing against the fixture DB
|
||||
|
||||
```bash
|
||||
cd cmd/decrypt
|
||||
go test ./...
|
||||
|
||||
# Manual test with the real fixture:
|
||||
go run . --channel "#wardriving" --db ../../test-fixtures/e2e-fixture.db --format irc
|
||||
```
|
||||
|
||||
The shared crypto library also has independent tests:
|
||||
|
||||
```bash
|
||||
cd internal/channel
|
||||
go test -v ./...
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Hashtag channels only.** Only channels where the key is derived from `SHA-256("#name")` are supported. Custom PSK channels require the raw key (not implemented).
|
||||
- **No DM decryption.** Direct messages (`TXT_MSG`) use per-peer asymmetric encryption and cannot be decrypted by this tool.
|
||||
- **Read-only.** The tool opens the database in read-only mode and never modifies it.
|
||||
- **Timestamps are UTC.** The sender's embedded timestamp is used when available, displayed in UTC.
|
||||
@@ -1,22 +0,0 @@
|
||||
module github.com/corescope/decrypt
|
||||
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/meshcore-analyzer/channel v0.0.0
|
||||
modernc.org/sqlite v1.34.5
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/meshcore-analyzer/channel => ../../internal/channel
|
||||
@@ -1,43 +0,0 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
|
||||
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
|
||||
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
|
||||
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
|
||||
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
|
||||
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
|
||||
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
|
||||
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
|
||||
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
|
||||
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
|
||||
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
|
||||
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
|
||||
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -1,467 +0,0 @@
|
||||
// corescope-decrypt decrypts and exports hashtag channel messages from a CoreScope SQLite database.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// corescope-decrypt --channel "#wardriving" --db meshcore.db [--format json|html] [--output file]
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/channel"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Version info (set via ldflags).
|
||||
var version = "dev"
|
||||
|
||||
// ChannelMessage is a single decrypted channel message with metadata.
|
||||
type ChannelMessage struct {
|
||||
Hash string `json:"hash"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Sender string `json:"sender"`
|
||||
Message string `json:"message"`
|
||||
Channel string `json:"channel"`
|
||||
RawHex string `json:"raw_hex"`
|
||||
Path []string `json:"path"`
|
||||
Observers []Observer `json:"observers"`
|
||||
}
|
||||
|
||||
// Observer is a single observation of the transmission.
|
||||
type Observer struct {
|
||||
Name string `json:"name"`
|
||||
SNR float64 `json:"snr"`
|
||||
RSSI float64 `json:"rssi"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")")
|
||||
dbPath := flag.String("db", "", "Path to CoreScope SQLite database")
|
||||
format := flag.String("format", "json", "Output format: json, html, irc (or log)")
|
||||
output := flag.String("output", "", "Output file (default: stdout)")
|
||||
showVersion := flag.Bool("version", false, "Print version and exit")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, `corescope-decrypt — Decrypt and export MeshCore hashtag channel messages
|
||||
|
||||
USAGE
|
||||
corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE]
|
||||
|
||||
FLAGS
|
||||
--channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving")
|
||||
The "#" prefix is added automatically if missing.
|
||||
--db PATH Path to a CoreScope SQLite database file (read-only access).
|
||||
--format FORMAT Output format (default: json):
|
||||
json — Machine-readable JSON array with full metadata
|
||||
html — Self-contained HTML viewer with search and sorting
|
||||
irc — Plain-text IRC-style log, one line per message
|
||||
log — Alias for irc
|
||||
--output FILE Write output to FILE instead of stdout.
|
||||
--version Print version and exit.
|
||||
|
||||
EXAMPLES
|
||||
# Export #wardriving messages as JSON
|
||||
corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
|
||||
|
||||
# Generate an interactive HTML viewer
|
||||
corescope-decrypt --channel wardriving --db meshcore.db --format html --output wardriving.html
|
||||
|
||||
# Greppable IRC log
|
||||
corescope-decrypt --channel "#MeshCore" --db meshcore.db --format irc --output meshcore.log
|
||||
grep "KE6QR" meshcore.log
|
||||
|
||||
# From the Docker container
|
||||
docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db
|
||||
|
||||
RETROACTIVE DECRYPTION
|
||||
MeshCore hashtag channels use symmetric encryption — the key is derived from the
|
||||
channel name. The CoreScope ingestor stores ALL GRP_TXT packets in the database,
|
||||
even those it cannot decrypt at ingest time. This tool lets you retroactively
|
||||
decrypt messages for any channel whose name you know, even if the ingestor was
|
||||
never configured with that channel's key.
|
||||
|
||||
This means you can recover historical messages by simply knowing the channel name.
|
||||
|
||||
LIMITATIONS
|
||||
- Only hashtag channels (shared-secret, name-derived key) are supported.
|
||||
- Direct messages (TXT_MSG) use per-peer encryption and cannot be decrypted.
|
||||
- Custom PSK channels (non-hashtag) require the raw key, not a channel name.
|
||||
`)
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if *showVersion {
|
||||
fmt.Println("corescope-decrypt", version)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if *channelName == "" || *dbPath == "" {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Normalize channel name
|
||||
ch := *channelName
|
||||
if !strings.HasPrefix(ch, "#") {
|
||||
ch = "#" + ch
|
||||
}
|
||||
|
||||
key := channel.DeriveKey(ch)
|
||||
chHash := channel.ChannelHash(key)
|
||||
|
||||
db, err := sql.Open("sqlite", *dbPath+"?mode=ro")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Query all GRP_TXT packets
|
||||
rows, err := db.Query(`SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5`)
|
||||
if err != nil {
|
||||
log.Fatalf("Query failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var messages []ChannelMessage
|
||||
decrypted, total := 0, 0
|
||||
|
||||
for rows.Next() {
|
||||
var id int
|
||||
var txHash, rawHex, firstSeen string
|
||||
if err := rows.Scan(&id, &txHash, &rawHex, &firstSeen); err != nil {
|
||||
log.Printf("Scan error: %v", err)
|
||||
continue
|
||||
}
|
||||
total++
|
||||
|
||||
payload, err := extractGRPPayload(rawHex)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(payload) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check channel hash byte
|
||||
if payload[0] != chHash {
|
||||
continue
|
||||
}
|
||||
|
||||
mac := payload[1:3]
|
||||
ciphertext := payload[3:]
|
||||
if len(ciphertext) < 5 || len(ciphertext)%16 != 0 {
|
||||
// Pad ciphertext to block boundary for decryption attempt
|
||||
if len(ciphertext) < 16 {
|
||||
continue
|
||||
}
|
||||
// Truncate to block boundary
|
||||
ciphertext = ciphertext[:len(ciphertext)/16*16]
|
||||
}
|
||||
|
||||
plaintext, ok := channel.Decrypt(key, mac, ciphertext)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
ts, sender, msg, err := channel.ParsePlaintext(plaintext)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
decrypted++
|
||||
|
||||
// Convert MeshCore timestamp
|
||||
timestamp := time.Unix(int64(ts), 0).UTC().Format(time.RFC3339)
|
||||
|
||||
// Get path from decoded_json
|
||||
path := getPathFromDB(db, id)
|
||||
|
||||
// Get observers
|
||||
observers := getObservers(db, id)
|
||||
|
||||
messages = append(messages, ChannelMessage{
|
||||
Hash: txHash,
|
||||
Timestamp: timestamp,
|
||||
Sender: sender,
|
||||
Message: msg,
|
||||
Channel: ch,
|
||||
RawHex: rawHex,
|
||||
Path: path,
|
||||
Observers: observers,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort by timestamp
|
||||
sort.Slice(messages, func(i, j int) bool {
|
||||
return messages[i].Timestamp < messages[j].Timestamp
|
||||
})
|
||||
|
||||
log.Printf("Scanned %d GRP_TXT packets, decrypted %d for channel %s", total, decrypted, ch)
|
||||
|
||||
// Generate output
|
||||
var out []byte
|
||||
switch *format {
|
||||
case "json":
|
||||
out, err = json.MarshalIndent(messages, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("JSON marshal: %v", err)
|
||||
}
|
||||
out = append(out, '\n')
|
||||
case "html":
|
||||
out = renderHTML(messages, ch)
|
||||
case "irc", "log":
|
||||
out = renderIRC(messages)
|
||||
default:
|
||||
log.Fatalf("Unknown format: %s (use json, html, irc, or log)", *format)
|
||||
}
|
||||
|
||||
if *output != "" {
|
||||
if err := os.WriteFile(*output, out, 0644); err != nil {
|
||||
log.Fatalf("Write file: %v", err)
|
||||
}
|
||||
log.Printf("Written to %s", *output)
|
||||
} else {
|
||||
os.Stdout.Write(out)
|
||||
}
|
||||
}
|
||||
|
||||
// extractGRPPayload parses a raw hex packet and returns the GRP_TXT payload bytes.
|
||||
func extractGRPPayload(rawHex string) ([]byte, error) {
|
||||
buf, err := hex.DecodeString(strings.TrimSpace(rawHex))
|
||||
if err != nil || len(buf) < 2 {
|
||||
return nil, fmt.Errorf("invalid hex")
|
||||
}
|
||||
|
||||
// Header byte
|
||||
header := buf[0]
|
||||
payloadType := int((header >> 2) & 0x0F)
|
||||
if payloadType != 5 { // GRP_TXT
|
||||
return nil, fmt.Errorf("not GRP_TXT")
|
||||
}
|
||||
|
||||
routeType := int(header & 0x03)
|
||||
offset := 1
|
||||
|
||||
// Transport codes (2 codes × 2 bytes) come BEFORE path for transport routes
|
||||
if routeType == 0 || routeType == 3 {
|
||||
offset += 4
|
||||
}
|
||||
|
||||
// Path byte
|
||||
if offset >= len(buf) {
|
||||
return nil, fmt.Errorf("too short for path")
|
||||
}
|
||||
pathByte := buf[offset]
|
||||
offset++
|
||||
hashSize := int(pathByte>>6) + 1
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
offset += hashSize * hashCount
|
||||
|
||||
if offset >= len(buf) {
|
||||
return nil, fmt.Errorf("too short for payload")
|
||||
}
|
||||
|
||||
return buf[offset:], nil
|
||||
}
|
||||
|
||||
func getPathFromDB(db *sql.DB, txID int) []string {
|
||||
var decodedJSON sql.NullString
|
||||
err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = ?`, txID).Scan(&decodedJSON)
|
||||
if err != nil || !decodedJSON.Valid {
|
||||
return nil
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Path struct {
|
||||
Hops []string `json:"hops"`
|
||||
} `json:"path"`
|
||||
}
|
||||
if json.Unmarshal([]byte(decodedJSON.String), &decoded) == nil {
|
||||
return decoded.Path.Hops
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getObservers(db *sql.DB, txID int) []Observer {
|
||||
rows, err := db.Query(`
|
||||
SELECT o.name, obs.snr, obs.rssi, obs.timestamp
|
||||
FROM observations obs
|
||||
LEFT JOIN observers o ON o.id = CAST(obs.observer_idx AS TEXT)
|
||||
WHERE obs.transmission_id = ?
|
||||
ORDER BY obs.timestamp
|
||||
`, txID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var observers []Observer
|
||||
for rows.Next() {
|
||||
var name sql.NullString
|
||||
var snr, rssi sql.NullFloat64
|
||||
var ts int64
|
||||
if err := rows.Scan(&name, &snr, &rssi, &ts); err != nil {
|
||||
continue
|
||||
}
|
||||
obs := Observer{
|
||||
Timestamp: time.Unix(ts, 0).UTC().Format(time.RFC3339),
|
||||
}
|
||||
if name.Valid {
|
||||
obs.Name = name.String
|
||||
}
|
||||
if snr.Valid {
|
||||
obs.SNR = snr.Float64
|
||||
}
|
||||
if rssi.Valid {
|
||||
obs.RSSI = rssi.Float64
|
||||
}
|
||||
observers = append(observers, obs)
|
||||
}
|
||||
return observers
|
||||
}
|
||||
|
||||
func renderIRC(messages []ChannelMessage) []byte {
|
||||
var b strings.Builder
|
||||
for _, m := range messages {
|
||||
sender := m.Sender
|
||||
if sender == "" {
|
||||
sender = "???"
|
||||
}
|
||||
// Parse RFC3339 timestamp into a compact format
|
||||
t, err := time.Parse(time.RFC3339, m.Timestamp)
|
||||
if err != nil {
|
||||
b.WriteString(fmt.Sprintf("[%s] <%s> %s\n", m.Timestamp, sender, m.Message))
|
||||
continue
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("[%s] <%s> %s\n", t.Format("2006-01-02 15:04:05"), sender, m.Message))
|
||||
}
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
func renderHTML(messages []ChannelMessage, channelName string) []byte {
|
||||
jsonData, _ := json.Marshal(messages)
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>CoreScope Channel Export — ` + html.EscapeString(channelName) + `</title>
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif;background:#0d1117;color:#c9d1d9;padding:20px}
|
||||
h1{color:#58a6ff;margin-bottom:16px;font-size:1.5em}
|
||||
.stats{color:#8b949e;margin-bottom:16px;font-size:0.9em}
|
||||
input[type=text]{width:100%;max-width:500px;padding:8px 12px;background:#161b22;border:1px solid #30363d;border-radius:6px;color:#c9d1d9;font-size:14px;margin-bottom:16px}
|
||||
input[type=text]:focus{outline:none;border-color:#58a6ff}
|
||||
table{width:100%;border-collapse:collapse;font-size:14px}
|
||||
th{background:#161b22;color:#8b949e;text-align:left;padding:8px 12px;border-bottom:2px solid #30363d;cursor:pointer;user-select:none;white-space:nowrap}
|
||||
th:hover{color:#58a6ff}
|
||||
th.sorted-asc::after{content:" ▲"}
|
||||
th.sorted-desc::after{content:" ▼"}
|
||||
td{padding:8px 12px;border-bottom:1px solid #21262d;vertical-align:top}
|
||||
tr:hover{background:#161b22}
|
||||
tr.expanded{background:#161b22}
|
||||
.detail-row td{padding:12px 24px;background:#0d1117;border-bottom:1px solid #21262d}
|
||||
.detail-row pre{background:#161b22;padding:12px;border-radius:6px;overflow-x:auto;font-size:12px;color:#8b949e}
|
||||
.detail-row .label{color:#58a6ff;font-weight:600;margin-top:8px;display:block}
|
||||
.observer-tag{display:inline-block;background:#1f6feb22;color:#58a6ff;padding:2px 8px;border-radius:4px;margin:2px;font-size:12px}
|
||||
.no-results{color:#8b949e;text-align:center;padding:40px;font-size:16px}
|
||||
.sender{color:#d2a8ff;font-weight:600}
|
||||
.timestamp{color:#8b949e;font-family:monospace;font-size:12px}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>` + html.EscapeString(channelName) + ` — Channel Messages</h1>
|
||||
<div class="stats" id="stats"></div>
|
||||
<input type="text" id="search" placeholder="Search messages..." autocomplete="off">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th data-col="timestamp">Timestamp</th>
|
||||
<th data-col="sender">Sender</th>
|
||||
<th data-col="message">Message</th>
|
||||
<th data-col="observers">Observers</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="tbody"></tbody>
|
||||
</table>
|
||||
<div class="no-results" id="no-results" style="display:none">No matching messages</div>
|
||||
<script>
|
||||
var DATA=` + string(jsonData) + `;
|
||||
var sortCol="timestamp",sortAsc=true,expandedHash=null;
|
||||
function init(){
|
||||
document.getElementById("stats").textContent=DATA.length+" messages";
|
||||
document.getElementById("search").addEventListener("input",render);
|
||||
document.querySelectorAll("th[data-col]").forEach(function(th){
|
||||
th.addEventListener("click",function(){
|
||||
var col=th.dataset.col;
|
||||
if(sortCol===col)sortAsc=!sortAsc;
|
||||
else{sortCol=col;sortAsc=true}
|
||||
render();
|
||||
});
|
||||
});
|
||||
render();
|
||||
}
|
||||
function render(){
|
||||
var q=document.getElementById("search").value.toLowerCase();
|
||||
var filtered=DATA.filter(function(m){
|
||||
if(!q)return true;
|
||||
return(m.message||"").toLowerCase().indexOf(q)>=0||(m.sender||"").toLowerCase().indexOf(q)>=0;
|
||||
});
|
||||
filtered.sort(function(a,b){
|
||||
var va=a[sortCol]||"",vb=b[sortCol]||"";
|
||||
if(sortCol==="observers"){va=a.observers?a.observers.length:0;vb=b.observers?b.observers.length:0}
|
||||
if(va<vb)return sortAsc?-1:1;
|
||||
if(va>vb)return sortAsc?1:-1;
|
||||
return 0;
|
||||
});
|
||||
document.querySelectorAll("th[data-col]").forEach(function(th){
|
||||
th.className=th.dataset.col===sortCol?(sortAsc?"sorted-asc":"sorted-desc"):"";
|
||||
});
|
||||
var tb=document.getElementById("tbody");
|
||||
tb.innerHTML="";
|
||||
document.getElementById("no-results").style.display=filtered.length?"none":"block";
|
||||
filtered.forEach(function(m){
|
||||
var tr=document.createElement("tr");
|
||||
tr.innerHTML='<td class="timestamp">'+esc(m.timestamp)+'</td><td class="sender">'+esc(m.sender||"—")+'</td><td>'+esc(m.message)+'</td><td>'+
|
||||
(m.observers?m.observers.map(function(o){return'<span class="observer-tag">'+esc(o.name||"?")+" SNR:"+o.snr.toFixed(1)+'</span>'}).join(""):"—")+'</td>';
|
||||
tr.style.cursor="pointer";
|
||||
tr.addEventListener("click",function(){
|
||||
expandedHash=expandedHash===m.hash?null:m.hash;
|
||||
render();
|
||||
});
|
||||
tb.appendChild(tr);
|
||||
if(expandedHash===m.hash){
|
||||
tr.className="expanded";
|
||||
var dr=document.createElement("tr");
|
||||
dr.className="detail-row";
|
||||
dr.innerHTML='<td colspan="4"><span class="label">Hash</span><pre>'+esc(m.hash)+'</pre>'+
|
||||
'<span class="label">Raw Hex</span><pre>'+esc(m.raw_hex)+'</pre>'+
|
||||
(m.path&&m.path.length?'<span class="label">Path</span><pre>'+esc(m.path.join(" → "))+'</pre>':'')+
|
||||
'<span class="label">Observers</span><pre>'+esc(JSON.stringify(m.observers,null,2))+'</pre></td>';
|
||||
tb.appendChild(dr);
|
||||
}
|
||||
});
|
||||
}
|
||||
function esc(s){var d=document.createElement("div");d.textContent=s;return d.innerHTML}
|
||||
init();
|
||||
</script>
|
||||
</body>
|
||||
</html>`)
|
||||
|
||||
return []byte(b.String())
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/meshcore-analyzer/channel"
|
||||
)
|
||||
|
||||
func TestExtractGRPPayload(t *testing.T) {
|
||||
// Build a minimal GRP_TXT packet: header(1) + path(1) + payload
|
||||
// header: route=FLOOD(1), payload=GRP_TXT(5), version=0 → (5<<2)|1 = 0x15
|
||||
// path: 0 hops, hash_size=1 → 0x00
|
||||
payload := []byte{0x81, 0x12, 0x34} // channel_hash + mac + data
|
||||
pkt := append([]byte{0x15, 0x00}, payload...)
|
||||
rawHex := hex.EncodeToString(pkt)
|
||||
|
||||
result, err := extractGRPPayload(rawHex)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result) != 3 || result[0] != 0x81 {
|
||||
t.Fatalf("payload mismatch: %x", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGRPPayloadTransport(t *testing.T) {
|
||||
// Transport flood: route=0, 4 bytes transport codes BEFORE path byte
|
||||
// header: (5<<2)|0 = 0x14
|
||||
payload := []byte{0xAA, 0xBB, 0xCC}
|
||||
// header + 4 transport bytes + path(0 hops) + payload
|
||||
pkt := append([]byte{0x14, 0xFF, 0xFF, 0xFF, 0xFF, 0x00}, payload...)
|
||||
rawHex := hex.EncodeToString(pkt)
|
||||
|
||||
result, err := extractGRPPayload(rawHex)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result[0] != 0xAA {
|
||||
t.Fatalf("expected AA, got %02X", result[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGRPPayloadNotGRP(t *testing.T) {
|
||||
// payload type = ADVERT (4): (4<<2)|1 = 0x11
|
||||
rawHex := hex.EncodeToString([]byte{0x11, 0x00, 0x01, 0x02})
|
||||
_, err := extractGRPPayload(rawHex)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for non-GRP_TXT")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyDerivationConsistency(t *testing.T) {
|
||||
// Verify key derivation matches what the ingestor expects
|
||||
key := channel.DeriveKey("#wardriving")
|
||||
if len(key) != 16 {
|
||||
t.Fatalf("key len %d", len(key))
|
||||
}
|
||||
ch := channel.ChannelHash(key)
|
||||
if ch != 0x81 {
|
||||
// We know from fixture data that #wardriving has channelHashHex "81"
|
||||
t.Fatalf("channel hash %02X, expected 81", ch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderIRC(t *testing.T) {
|
||||
msgs := []ChannelMessage{
|
||||
{Timestamp: "2026-04-12T03:45:12Z", Sender: "NodeA", Message: "Hello"},
|
||||
{Timestamp: "2026-04-12T03:46:01Z", Sender: "", Message: "No sender"},
|
||||
}
|
||||
out := string(renderIRC(msgs))
|
||||
if !strings.Contains(out, "[2026-04-12 03:45:12] <NodeA> Hello") {
|
||||
t.Fatalf("IRC output missing expected line: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "<???> No sender") {
|
||||
t.Fatalf("IRC output should use ??? for empty sender: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderHTMLValid(t *testing.T) {
|
||||
msgs := []ChannelMessage{
|
||||
{Hash: "abc", Timestamp: "2026-04-12T00:00:00Z", Sender: "X", Message: "test", Channel: "#test"},
|
||||
}
|
||||
out := string(renderHTML(msgs, "#test"))
|
||||
if !strings.Contains(out, "<!DOCTYPE html>") {
|
||||
t.Fatal("not valid HTML")
|
||||
}
|
||||
if !strings.Contains(out, "#test") {
|
||||
t.Fatal("channel name missing")
|
||||
}
|
||||
if !strings.Contains(out, "</html>") {
|
||||
t.Fatal("HTML not closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONOutputParseable(t *testing.T) {
|
||||
msgs := []ChannelMessage{
|
||||
{Hash: "abc", Timestamp: "2026-04-12T00:00:00Z", Sender: "X", Message: "hi", Channel: "#test"},
|
||||
}
|
||||
data, err := json.MarshalIndent(msgs, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed []ChannelMessage
|
||||
if err := json.Unmarshal(data, &parsed); err != nil {
|
||||
t.Fatalf("JSON not parseable: %v", err)
|
||||
}
|
||||
if len(parsed) != 1 || parsed[0].Sender != "X" {
|
||||
t.Fatalf("parsed mismatch: %+v", parsed)
|
||||
}
|
||||
}
|
||||
|
||||
// Integration test against fixture DB (skipped if DB not found)
|
||||
func TestFixtureDecrypt(t *testing.T) {
|
||||
dbPath := "../../test-fixtures/e2e-fixture.db"
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
t.Skip("fixture DB not found")
|
||||
}
|
||||
|
||||
// We know the fixture has #wardriving messages with channelHash 0x81
|
||||
key := channel.DeriveKey("#wardriving")
|
||||
ch := channel.ChannelHash(key)
|
||||
if ch != 0x81 {
|
||||
t.Fatalf("unexpected channel hash: %02X", ch)
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
ingestor
|
||||
@@ -47,24 +47,6 @@ The config file uses the same format as the Node.js `config.json`. The ingestor
|
||||
| `DB_PATH` | SQLite database path | `data/meshcore.db` |
|
||||
| `MQTT_BROKER` | Single MQTT broker URL (overrides config) | — |
|
||||
| `MQTT_TOPIC` | MQTT topic (used with `MQTT_BROKER`) | `meshcore/#` |
|
||||
| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/tmp/corescope-ingestor-stats.json` |
|
||||
|
||||
### Stats file (`CORESCOPE_INGESTOR_STATS`)
|
||||
|
||||
Every second the ingestor publishes a JSON snapshot of its counters
|
||||
(`tx_inserted`, `obs_inserted`, `walCommits`, `backfillUpdates.*`, etc.) plus
|
||||
a `procIO` block sampled from `/proc/self/io` (read/write/cancelled bytes per
|
||||
second + syscall counts). The server reads this file and surfaces the data on
|
||||
the Perf page so operators can self-diagnose write-volume anomalies.
|
||||
|
||||
The writer uses `O_NOFOLLOW | O_CREAT | O_TRUNC` mode `0o600`, so a
|
||||
pre-planted symlink at the path cannot be used to clobber an arbitrary file.
|
||||
|
||||
**Security note:** the default lives in `/tmp`, which is world-writable on
|
||||
most hosts (sticky bit only protects deletion, not creation). On
|
||||
shared/multi-tenant hosts, override `CORESCOPE_INGESTOR_STATS` to point at a
|
||||
private directory (e.g. `/var/lib/corescope/ingestor-stats.json`) that only
|
||||
the corescope user can write to.
|
||||
|
||||
### Minimal Config
|
||||
|
||||
|
||||
+9
-202
@@ -2,14 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
)
|
||||
|
||||
@@ -22,17 +18,6 @@ type MQTTSource struct {
|
||||
RejectUnauthorized *bool `json:"rejectUnauthorized,omitempty"`
|
||||
Topics []string `json:"topics"`
|
||||
IATAFilter []string `json:"iataFilter,omitempty"`
|
||||
ConnectTimeoutSec int `json:"connectTimeoutSec,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
}
|
||||
|
||||
// ConnectTimeoutOrDefault returns the per-source connect timeout in seconds,
|
||||
// or 30 if not set (matching the WaitTimeout default from #926).
|
||||
func (s MQTTSource) ConnectTimeoutOrDefault() int {
|
||||
if s.ConnectTimeoutSec > 0 {
|
||||
return s.ConnectTimeoutSec
|
||||
}
|
||||
return 30
|
||||
}
|
||||
|
||||
// MQTTLegacy is the old single-broker config format.
|
||||
@@ -50,123 +35,16 @@ type Config struct {
|
||||
ChannelKeysPath string `json:"channelKeysPath,omitempty"`
|
||||
ChannelKeys map[string]string `json:"channelKeys,omitempty"`
|
||||
HashChannels []string `json:"hashChannels,omitempty"`
|
||||
HashRegions []string `json:"hashRegions,omitempty"`
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"`
|
||||
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
|
||||
// ObserverIATAWhitelist restricts which observer IATA regions are processed.
|
||||
// When non-empty, only observers whose IATA code (from the MQTT topic) matches
|
||||
// one of these entries are accepted. Case-insensitive. An empty list means all
|
||||
// IATA codes are allowed. This applies globally, unlike the per-source iataFilter.
|
||||
ObserverIATAWhitelist []string `json:"observerIATAWhitelist,omitempty"`
|
||||
|
||||
// obsIATAWhitelistCached is the lazily-built uppercase set for O(1) lookups.
|
||||
obsIATAWhitelistCached map[string]bool
|
||||
obsIATAWhitelistOnce sync.Once
|
||||
|
||||
// ObserverBlacklist is a list of observer public keys to drop at ingest.
|
||||
// Messages from blacklisted observers are silently discarded — no DB writes,
|
||||
// no UpsertObserver, no observations, no metrics.
|
||||
ObserverBlacklist []string `json:"observerBlacklist,omitempty"`
|
||||
|
||||
// obsBlacklistSetCached is the lazily-built lowercase set for O(1) lookups.
|
||||
obsBlacklistSetCached map[string]bool
|
||||
obsBlacklistOnce sync.Once
|
||||
|
||||
// NeighborEdgesMaxAgeDays controls neighbor_edges row retention
|
||||
// (#1287 — moved from cmd/server). 0 = default 5.
|
||||
NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"`
|
||||
}
|
||||
|
||||
// NeighborEdgesDaysOrDefault returns the configured pruning window or 5.
|
||||
func (c *Config) NeighborEdgesDaysOrDefault() int {
|
||||
if c == nil || c.NeighborEdgesMaxAgeDays <= 0 {
|
||||
return 5
|
||||
}
|
||||
return c.NeighborEdgesMaxAgeDays
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
type GeoFilterConfig = geofilter.Config
|
||||
|
||||
// ForeignAdvertConfig controls how the ingestor handles ADVERTs whose GPS lies
|
||||
// outside the configured geofilter polygon (#730). Modes:
|
||||
// - "flag" (default): store the advert/node and tag it foreign for visibility.
|
||||
// - "drop": silently discard the advert (legacy behavior).
|
||||
type ForeignAdvertConfig struct {
|
||||
Mode string `json:"mode,omitempty"`
|
||||
}
|
||||
|
||||
// IsDropMode reports whether the foreign-advert config is set to "drop".
|
||||
// Defaults to false ("flag" mode) when nil or unset.
|
||||
func (f *ForeignAdvertConfig) IsDropMode() bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(strings.TrimSpace(f.Mode), "drop")
|
||||
}
|
||||
|
||||
// RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
// PacketDays is the retention window for transmissions (#1283).
|
||||
// Ownership moved from cmd/server to cmd/ingestor; 0 disables.
|
||||
PacketDays int `json:"packetDays"`
|
||||
}
|
||||
|
||||
// PacketDaysOrZero returns the configured retention.packetDays or 0
|
||||
// (disabled) if not set.
|
||||
func (c *Config) PacketDaysOrZero() int {
|
||||
if c.Retention != nil && c.Retention.PacketDays > 0 {
|
||||
return c.Retention.PacketDays
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// MetricsConfig controls observer metrics collection.
|
||||
type MetricsConfig struct {
|
||||
SampleIntervalSec int `json:"sampleIntervalSec"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
type DBConfig = dbconfig.DBConfig
|
||||
|
||||
// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
|
||||
func (c *Config) IncrementalVacuumPages() int {
|
||||
if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
|
||||
return c.DB.IncrementalVacuumPages
|
||||
}
|
||||
return 1024
|
||||
}
|
||||
|
||||
// ShouldValidateSignatures returns true (default) unless explicitly disabled.
|
||||
func (c *Config) ShouldValidateSignatures() bool {
|
||||
if c.ValidateSignatures != nil {
|
||||
return *c.ValidateSignatures
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MetricsSampleInterval returns the configured sample interval or 300s default.
|
||||
func (c *Config) MetricsSampleInterval() int {
|
||||
if c.Metrics != nil && c.Metrics.SampleIntervalSec > 0 {
|
||||
return c.Metrics.SampleIntervalSec
|
||||
}
|
||||
return 300
|
||||
}
|
||||
|
||||
// MetricsRetentionDays returns configured metrics retention or 30 days default.
|
||||
func (c *Config) MetricsRetentionDays() int {
|
||||
if c.Retention != nil && c.Retention.MetricsDays > 0 {
|
||||
return c.Retention.MetricsDays
|
||||
}
|
||||
return 30
|
||||
NodeDays int `json:"nodeDays"`
|
||||
}
|
||||
|
||||
// NodeDaysOrDefault returns the configured retention.nodeDays or 7 if not set.
|
||||
@@ -177,68 +55,16 @@ func (c *Config) NodeDaysOrDefault() int {
|
||||
return 7
|
||||
}
|
||||
|
||||
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
|
||||
// A value of -1 means observers are never removed.
|
||||
func (c *Config) ObserverDaysOrDefault() int {
|
||||
if c.Retention != nil && c.Retention.ObserverDays != 0 {
|
||||
return c.Retention.ObserverDays
|
||||
}
|
||||
return 14
|
||||
}
|
||||
|
||||
// IsObserverBlacklisted returns true if the given observer ID is in the observerBlacklist.
|
||||
func (c *Config) IsObserverBlacklisted(id string) bool {
|
||||
if c == nil || len(c.ObserverBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
c.obsBlacklistOnce.Do(func() {
|
||||
m := make(map[string]bool, len(c.ObserverBlacklist))
|
||||
for _, pk := range c.ObserverBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsBlacklistSetCached = m
|
||||
})
|
||||
return c.obsBlacklistSetCached[strings.ToLower(strings.TrimSpace(id))]
|
||||
}
|
||||
|
||||
// IsObserverIATAAllowed returns true if the given IATA code is permitted.
|
||||
// When ObserverIATAWhitelist is empty, all codes are allowed.
|
||||
func (c *Config) IsObserverIATAAllowed(iata string) bool {
|
||||
if c == nil || len(c.ObserverIATAWhitelist) == 0 {
|
||||
return true
|
||||
}
|
||||
c.obsIATAWhitelistOnce.Do(func() {
|
||||
m := make(map[string]bool, len(c.ObserverIATAWhitelist))
|
||||
for _, code := range c.ObserverIATAWhitelist {
|
||||
trimmed := strings.ToUpper(strings.TrimSpace(code))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsIATAWhitelistCached = m
|
||||
})
|
||||
return c.obsIATAWhitelistCached[strings.ToUpper(strings.TrimSpace(iata))]
|
||||
}
|
||||
|
||||
// LoadConfig reads configuration from a JSON file, with env var overrides.
|
||||
// If the config file does not exist, sensible defaults are used (zero-config startup).
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
var cfg Config
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("reading config %s: %w", path, err)
|
||||
}
|
||||
// Config file doesn't exist — use defaults (zero-config mode)
|
||||
log.Printf("config file %s not found, using sensible defaults", path)
|
||||
} else {
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parsing config %s: %w", path, err)
|
||||
}
|
||||
return nil, fmt.Errorf("reading config %s: %w", path, err)
|
||||
}
|
||||
|
||||
var cfg Config
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parsing config %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Env var overrides
|
||||
@@ -272,38 +98,19 @@ func LoadConfig(path string) (*Config, error) {
|
||||
}}
|
||||
}
|
||||
|
||||
// Default MQTT source: connect to localhost broker when no sources configured
|
||||
if len(cfg.MQTTSources) == 0 {
|
||||
cfg.MQTTSources = []MQTTSource{{
|
||||
Name: "local",
|
||||
Broker: "mqtt://localhost:1883",
|
||||
Topics: []string{"meshcore/#"},
|
||||
}}
|
||||
log.Printf("no MQTT sources configured, defaulting to mqtt://localhost:1883")
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// ResolvedSources returns the final list of MQTT sources to connect to.
|
||||
//
|
||||
// Scheme mapping:
|
||||
//
|
||||
// mqtt:// → tcp:// (paho plain TCP)
|
||||
// mqtts:// → ssl:// (paho TLS over TCP)
|
||||
// ws:// (paho WebSocket — passed through, no mapping needed)
|
||||
// wss:// (paho WebSocket TLS — passed through, no mapping needed)
|
||||
func (c *Config) ResolvedSources() []MQTTSource {
|
||||
for i := range c.MQTTSources {
|
||||
// paho uses tcp:// and ssl:// for plain MQTT; ws:// and wss:// are accepted natively.
|
||||
// paho uses tcp:// and ssl:// not mqtt:// and mqtts://
|
||||
b := c.MQTTSources[i].Broker
|
||||
if strings.HasPrefix(b, "mqtt://") {
|
||||
c.MQTTSources[i].Broker = "tcp://" + b[7:]
|
||||
} else if strings.HasPrefix(b, "mqtts://") {
|
||||
c.MQTTSources[i].Broker = "ssl://" + b[8:]
|
||||
}
|
||||
// ws:// and wss:// pass through unchanged — paho handles WebSocket
|
||||
// connections natively via gorilla/websocket.
|
||||
}
|
||||
return c.MQTTSources
|
||||
}
|
||||
|
||||
+5
-221
@@ -32,25 +32,9 @@ func TestLoadConfigValidJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLoadConfigMissingFile(t *testing.T) {
|
||||
t.Setenv("DB_PATH", "")
|
||||
t.Setenv("MQTT_BROKER", "")
|
||||
|
||||
cfg, err := LoadConfig("/nonexistent/path/config.json")
|
||||
if err != nil {
|
||||
t.Fatalf("missing config should not error (zero-config mode), got: %v", err)
|
||||
}
|
||||
if cfg.DBPath != "data/meshcore.db" {
|
||||
t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath)
|
||||
}
|
||||
// Should default to localhost MQTT
|
||||
if len(cfg.MQTTSources) != 1 {
|
||||
t.Fatalf("mqttSources len=%d, want 1", len(cfg.MQTTSources))
|
||||
}
|
||||
if cfg.MQTTSources[0].Broker != "mqtt://localhost:1883" {
|
||||
t.Errorf("default broker=%s, want mqtt://localhost:1883", cfg.MQTTSources[0].Broker)
|
||||
}
|
||||
if cfg.MQTTSources[0].Name != "local" {
|
||||
t.Errorf("default source name=%s, want local", cfg.MQTTSources[0].Name)
|
||||
_, err := LoadConfig("/nonexistent/path/config.json")
|
||||
if err == nil {
|
||||
t.Error("expected error for missing file")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,8 +196,8 @@ func TestLoadConfigLegacyMQTTEmptyBroker(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.MQTTSources) != 1 || cfg.MQTTSources[0].Name != "local" {
|
||||
t.Errorf("mqttSources should default to local broker when legacy broker is empty, got %v", cfg.MQTTSources)
|
||||
if len(cfg.MQTTSources) != 0 {
|
||||
t.Errorf("mqttSources should be empty when legacy broker is empty, got %d", len(cfg.MQTTSources))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,203 +268,3 @@ func TestLoadConfigWithAllFields(t *testing.T) {
|
||||
t.Errorf("iataFilter=%v", src.IATAFilter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTimeoutOrDefault(t *testing.T) {
|
||||
// Default when unset
|
||||
s := MQTTSource{}
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
||||
t.Errorf("default: got %d, want 30", got)
|
||||
}
|
||||
|
||||
// Custom value
|
||||
s.ConnectTimeoutSec = 5
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 5 {
|
||||
t.Errorf("custom: got %d, want 5", got)
|
||||
}
|
||||
|
||||
// Zero treated as unset
|
||||
s.ConnectTimeoutSec = 0
|
||||
if got := s.ConnectTimeoutOrDefault(); got != 30 {
|
||||
t.Errorf("zero: got %d, want 30", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTimeoutFromJSON(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := dir + "/config.json"
|
||||
os.WriteFile(cfgPath, []byte(`{"mqttSources":[{"name":"s1","broker":"tcp://b:1883","topics":["#"],"connectTimeoutSec":5}]}`), 0644)
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := cfg.MQTTSources[0].ConnectTimeoutOrDefault(); got != 5 {
|
||||
t.Errorf("from JSON: got %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelist(t *testing.T) {
|
||||
// Config with whitelist set
|
||||
cfg := Config{
|
||||
ObserverIATAWhitelist: []string{"ARN", "got"},
|
||||
}
|
||||
|
||||
// Matching (case-insensitive)
|
||||
if !cfg.IsObserverIATAAllowed("ARN") {
|
||||
t.Error("ARN should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("arn") {
|
||||
t.Error("arn (lowercase) should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("GOT") {
|
||||
t.Error("GOT should be allowed")
|
||||
}
|
||||
|
||||
// Non-matching
|
||||
if cfg.IsObserverIATAAllowed("SJC") {
|
||||
t.Error("SJC should NOT be allowed")
|
||||
}
|
||||
|
||||
// Empty string not allowed
|
||||
if cfg.IsObserverIATAAllowed("") {
|
||||
t.Error("empty IATA should NOT be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelistEmpty(t *testing.T) {
|
||||
// No whitelist = allow all
|
||||
cfg := Config{}
|
||||
if !cfg.IsObserverIATAAllowed("SJC") {
|
||||
t.Error("with no whitelist, all IATAs should be allowed")
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("") {
|
||||
t.Error("with no whitelist, even empty IATA should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverIATAWhitelistJSON(t *testing.T) {
|
||||
json := `{
|
||||
"dbPath": "test.db",
|
||||
"observerIATAWhitelist": ["ARN", "GOT"]
|
||||
}`
|
||||
tmp := t.TempDir() + "/config.json"
|
||||
os.WriteFile(tmp, []byte(json), 0644)
|
||||
cfg, err := LoadConfig(tmp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.ObserverIATAWhitelist) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(cfg.ObserverIATAWhitelist))
|
||||
}
|
||||
if !cfg.IsObserverIATAAllowed("ARN") {
|
||||
t.Error("ARN should be allowed after loading from JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTSourceRegionField(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
os.WriteFile(cfgPath, []byte(`{
|
||||
"dbPath": "/tmp/test.db",
|
||||
"mqttSources": [
|
||||
{"name": "cascadia", "broker": "tcp://localhost:1883", "topics": ["meshcore/#"], "region": "PDX"}
|
||||
]
|
||||
}`), 0o644)
|
||||
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.MQTTSources[0].Region != "PDX" {
|
||||
t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvedSourcesSchemeMapping verifies that mqtt:// and mqtts:// are translated
|
||||
// to the paho-native tcp:// and ssl:// schemes, while ws:// and wss:// pass through
|
||||
// unchanged (paho handles WebSocket connections natively).
|
||||
func TestResolvedSourcesSchemeMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"mqtt://host:1883", "tcp://host:1883"},
|
||||
{"mqtts://host:8883", "ssl://host:8883"},
|
||||
{"tcp://host:1883", "tcp://host:1883"},
|
||||
{"ssl://host:8883", "ssl://host:8883"},
|
||||
{"ws://host:9001", "ws://host:9001"},
|
||||
{"wss://host:9001", "wss://host:9001"},
|
||||
{"ws://host:9001/mqtt", "ws://host:9001/mqtt"},
|
||||
{"wss://host:9001/mqtt", "wss://host:9001/mqtt"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
cfg := &Config{
|
||||
MQTTSources: []MQTTSource{
|
||||
{Name: "test", Broker: tt.input, Topics: []string{"meshcore/#"}},
|
||||
},
|
||||
}
|
||||
sources := cfg.ResolvedSources()
|
||||
if got := sources[0].Broker; got != tt.want {
|
||||
t.Errorf("ResolvedSources(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadConfigWSSource verifies that a WebSocket MQTT source round-trips through
|
||||
// LoadConfig correctly — username/password preserved, scheme unchanged.
|
||||
func TestLoadConfigWSSource(t *testing.T) {
|
||||
t.Setenv("DB_PATH", "")
|
||||
t.Setenv("MQTT_BROKER", "")
|
||||
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
os.WriteFile(cfgPath, []byte(`{
|
||||
"dbPath": "test.db",
|
||||
"mqttSources": [
|
||||
{
|
||||
"name": "local-tcp",
|
||||
"broker": "mqtt://localhost:1883",
|
||||
"topics": ["meshcore/#"]
|
||||
},
|
||||
{
|
||||
"name": "wsmqtt-ws",
|
||||
"broker": "wss://wsmqtt.example.com/mqtt",
|
||||
"username": "corescope",
|
||||
"password": "s3cr3t",
|
||||
"topics": ["meshcore/#"]
|
||||
}
|
||||
]
|
||||
}`), 0o644)
|
||||
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.MQTTSources) != 2 {
|
||||
t.Fatalf("mqttSources len=%d, want 2", len(cfg.MQTTSources))
|
||||
}
|
||||
|
||||
tcp := cfg.MQTTSources[0]
|
||||
if tcp.Name != "local-tcp" {
|
||||
t.Errorf("name=%s, want local-tcp", tcp.Name)
|
||||
}
|
||||
|
||||
ws := cfg.MQTTSources[1]
|
||||
if ws.Name != "wsmqtt-ws" {
|
||||
t.Errorf("name=%s, want wsmqtt-ws", ws.Name)
|
||||
}
|
||||
if ws.Broker != "wss://wsmqtt.example.com/mqtt" {
|
||||
t.Errorf("broker=%s, want wss://wsmqtt.example.com/mqtt", ws.Broker)
|
||||
}
|
||||
if ws.Username != "corescope" {
|
||||
t.Errorf("username=%s, want corescope", ws.Username)
|
||||
}
|
||||
if ws.Password != "s3cr3t" {
|
||||
t.Errorf("password=%s, want s3cr3t", ws.Password)
|
||||
}
|
||||
|
||||
sources := cfg.ResolvedSources()
|
||||
if sources[1].Broker != "wss://wsmqtt.example.com/mqtt" {
|
||||
t.Errorf("ResolvedSources wss broker=%s, want unchanged", sources[1].Broker)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// hmacSHA256 computes HMAC-SHA256 for test use.
|
||||
@@ -160,7 +157,7 @@ func TestHandleMessageChannelMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Alice: Hello everyone","channel_idx":3,"SNR":5.0,"RSSI":-95,"score":10,"direction":"rx","sender_timestamp":1700000000}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/2", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -206,13 +203,21 @@ func TestHandleMessageChannelMessage(t *testing.T) {
|
||||
t.Errorf("direction=%v, want rx", direction)
|
||||
}
|
||||
|
||||
// Sender node should NOT be created (see issue #665: synthetic "sender-" keys
|
||||
// are unreachable from the claiming/health flow)
|
||||
// Should create sender node
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("nodes count=%d, want 0 (no phantom sender node)", count)
|
||||
if count != 1 {
|
||||
t.Errorf("nodes count=%d, want 1 (sender node)", count)
|
||||
}
|
||||
|
||||
// Verify sender node name
|
||||
var nodeName string
|
||||
if err := store.db.QueryRow("SELECT name FROM nodes LIMIT 1").Scan(&nodeName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeName != "Alice" {
|
||||
t.Errorf("node name=%s, want Alice", nodeName)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +225,7 @@ func TestHandleMessageChannelMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -235,7 +240,7 @@ func TestHandleMessageChannelNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: []byte(`{"text":"no sender here"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -252,7 +257,7 @@ func TestHandleMessageDirectMessage(t *testing.T) {
|
||||
payload := []byte(`{"text":"Bob: Hey there","sender_timestamp":1700000000,"SNR":3.0,"rssi":-100,"Score":8,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc123", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -296,7 +301,7 @@ func TestHandleMessageDirectMessageEmptyText(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: []byte(`{"text":""}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -311,7 +316,7 @@ func TestHandleMessageDirectNoSender(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: []byte(`{"text":"message with no colon"}`)}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -330,7 +335,7 @@ func TestHandleMessageUppercaseScoreDirection(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","Score":9.0,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var score *float64
|
||||
var direction *string
|
||||
@@ -351,7 +356,7 @@ func TestHandleMessageChannelLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":3.0,"rssi":-90,"Score":5,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/0", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -367,7 +372,7 @@ func TestHandleMessageDirectLowercaseFields(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Test: msg","snr":2.0,"rssi":-85,"score":7,"direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/xyz", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -390,7 +395,7 @@ func TestHandleMessageAdvertWithTelemetry(t *testing.T) {
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
// Should have created transmission, node, and observer
|
||||
var txCount, nodeCount, obsCount int
|
||||
@@ -430,12 +435,7 @@ func TestHandleMessageAdvertGeoFiltered(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
// Legacy silent-drop behavior is now opt-in via ForeignAdverts.Mode="drop"
|
||||
// (#730). The new default — flag — is covered by foreign_advert_test.go.
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{
|
||||
GeoFilter: gf,
|
||||
ForeignAdverts: &ForeignAdvertConfig{Mode: "drop"},
|
||||
})
|
||||
handleMessage(store, "test", source, msg, nil, gf)
|
||||
|
||||
// Geo-filtered adverts should not create nodes
|
||||
var nodeCount int
|
||||
@@ -443,7 +443,7 @@ func TestHandleMessageAdvertGeoFiltered(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeCount != 0 {
|
||||
t.Errorf("nodes=%d, want 0 (geo-filtered advert in drop mode should not create node)", nodeCount)
|
||||
t.Errorf("nodes=%d, want 0 (geo-filtered advert should not create node)", nodeCount)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -461,7 +461,7 @@ func TestDecodeAdvertLocationTruncated(t *testing.T) {
|
||||
buf[100] = 0x11
|
||||
// Only 4 bytes after flags — not enough for full location (needs 8)
|
||||
|
||||
p := decodeAdvert(buf[:105], false)
|
||||
p := decodeAdvert(buf[:105])
|
||||
if p.Error != "" {
|
||||
t.Fatalf("error: %s", p.Error)
|
||||
}
|
||||
@@ -483,7 +483,7 @@ func TestDecodeAdvertFeat1Truncated(t *testing.T) {
|
||||
buf[100] = 0x21
|
||||
// Only 1 byte after flags — not enough for feat1 (needs 2)
|
||||
|
||||
p := decodeAdvert(buf[:102], false)
|
||||
p := decodeAdvert(buf[:102])
|
||||
if p.Feat1 != nil {
|
||||
t.Error("feat1 should be nil with truncated data")
|
||||
}
|
||||
@@ -504,7 +504,7 @@ func TestDecodeAdvertFeat2Truncated(t *testing.T) {
|
||||
buf[102] = 0x00
|
||||
// Only 1 byte left — not enough for feat2
|
||||
|
||||
p := decodeAdvert(buf[:104], false)
|
||||
p := decodeAdvert(buf[:104])
|
||||
if p.Feat1 == nil {
|
||||
t.Error("feat1 should be set")
|
||||
}
|
||||
@@ -544,7 +544,7 @@ func TestDecodeAdvertSensorBadTelemetry(t *testing.T) {
|
||||
buf[105] = 0x20
|
||||
buf[106] = 0x4E
|
||||
|
||||
p := decodeAdvert(buf[:107], false)
|
||||
p := decodeAdvert(buf[:107])
|
||||
if p.BatteryMv != nil {
|
||||
t.Error("battery_mv=0 should be nil")
|
||||
}
|
||||
@@ -672,7 +672,7 @@ func TestHandleMessageCorruptedAdvertNoNode(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -694,7 +694,7 @@ func TestHandleMessageNonAdvertPacket(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -740,7 +740,7 @@ func TestDecodeAdvertSensorNoName(t *testing.T) {
|
||||
buf[103] = 0xC4
|
||||
buf[104] = 0x09
|
||||
|
||||
p := decodeAdvert(buf[:105], false)
|
||||
p := decodeAdvert(buf[:105])
|
||||
if p.Error != "" {
|
||||
t.Fatalf("error: %s", p.Error)
|
||||
}
|
||||
@@ -755,13 +755,8 @@ func TestDecodeAdvertSensorNoName(t *testing.T) {
|
||||
// --- db.go: OpenStore error path (invalid dir) ---
|
||||
|
||||
func TestOpenStoreInvalidPath(t *testing.T) {
|
||||
// Create a regular file then try to open a DB inside it — impossible on all platforms.
|
||||
f, err := os.CreateTemp(t.TempDir(), "not-a-dir")
|
||||
if err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
f.Close()
|
||||
_, err = OpenStore(filepath.Join(f.Name(), "db.sqlite"))
|
||||
// Path under /dev/null can't create directory
|
||||
_, err := OpenStore("/dev/null/impossible/path/db.sqlite")
|
||||
if err == nil {
|
||||
t.Error("should error on impossible path")
|
||||
}
|
||||
@@ -840,7 +835,7 @@ func TestDecodePacketNoPathByteAfterHeader(t *testing.T) {
|
||||
// Non-transport route, but only header byte (no path byte)
|
||||
// Actually 0A alone = 1 byte, but we need >= 2
|
||||
// Header + exactly at offset boundary
|
||||
_, err := DecodePacket("0A", nil, false)
|
||||
_, err := DecodePacket("0A", nil)
|
||||
if err == nil {
|
||||
t.Error("should error - too short")
|
||||
}
|
||||
@@ -861,7 +856,7 @@ func TestDecodeAdvertNameNoNull(t *testing.T) {
|
||||
// Name without null terminator — goes to end of buffer
|
||||
copy(buf[101:], []byte("LongNameNoNull"))
|
||||
|
||||
p := decodeAdvert(buf[:115], false)
|
||||
p := decodeAdvert(buf[:115])
|
||||
if p.Name != "LongNameNoNull" {
|
||||
t.Errorf("name=%q, want LongNameNoNull", p.Name)
|
||||
}
|
||||
@@ -876,7 +871,7 @@ func TestHandleMessageChannelLongSender(t *testing.T) {
|
||||
longText := "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&count); err != nil {
|
||||
@@ -895,7 +890,7 @@ func TestHandleMessageDirectLongSender(t *testing.T) {
|
||||
longText := "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB: msg"
|
||||
payload := []byte(`{"text":"` + longText + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/abc", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -912,7 +907,7 @@ func TestHandleMessageDirectUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"X: hi","Score":6,"Direction":"rx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/d1", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -942,7 +937,7 @@ func TestHandleMessageChannelUppercaseScoreDirection(t *testing.T) {
|
||||
|
||||
payload := []byte(`{"text":"Y: hi","Score":4,"Direction":"tx"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/5", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count); err != nil {
|
||||
@@ -973,7 +968,7 @@ func TestHandleMessageRawLowercaseScore(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","score":3.5}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var score *float64
|
||||
if err := store.db.QueryRow("SELECT score FROM observations LIMIT 1").Scan(&score); err != nil {
|
||||
@@ -992,7 +987,7 @@ func TestHandleMessageStatusNoOrigin(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs5/status",
|
||||
payload: []byte(`{"model":"L1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id = 'obs5'").Scan(&count); err != nil {
|
||||
@@ -1151,182 +1146,3 @@ func TestDecodeTraceWithPath(t *testing.T) {
|
||||
t.Errorf("flags=%v, want 3", p.TraceFlags)
|
||||
}
|
||||
}
|
||||
|
||||
// --- db.go: RemoveStaleObservers (soft-delete) ---
|
||||
|
||||
func TestRemoveStaleObservers(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert an observer with last_seen 30 days ago
|
||||
err := store.UpsertObserver("obs-old", "OldObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Override last_seen to 30 days ago
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Insert a recent observer
|
||||
err = store.UpsertObserver("obs-new", "NewObserver", "NYC", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Errorf("removed=%d, want 1", removed)
|
||||
}
|
||||
|
||||
// Observer should still be in the table (soft-delete), but marked inactive
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("observers count=%d, want 2 (soft-delete preserves row)", count)
|
||||
}
|
||||
|
||||
// Check that the old observer is marked inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 1 {
|
||||
t.Errorf("obs-old inactive=%d, want 1", inactive)
|
||||
}
|
||||
|
||||
// Check that the recent observer is still active
|
||||
var newInactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newInactive != 0 {
|
||||
t.Errorf("obs-new inactive=%d, want 0", newInactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversNone(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Errorf("removed=%d, want 0", removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversKeepForever(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert an old observer
|
||||
err := store.UpsertObserver("obs-ancient", "AncientObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// observerDays = -1 means keep forever
|
||||
removed, err := store.RemoveStaleObservers(-1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 0 {
|
||||
t.Errorf("removed=%d, want 0 (keep forever)", removed)
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM observers").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("observers count=%d, want 1 (keep forever)", count)
|
||||
}
|
||||
|
||||
// Observer should NOT be marked inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 0 {
|
||||
t.Errorf("obs-ancient inactive=%d, want 0 (keep forever)", inactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveStaleObserversReactivation(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
|
||||
// Insert and stale-mark an observer
|
||||
err := store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339)
|
||||
_, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
removed, err := store.RemoveStaleObservers(14)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed != 1 {
|
||||
t.Errorf("removed=%d, want 1", removed)
|
||||
}
|
||||
|
||||
// Verify it's inactive
|
||||
var inactive int
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 1 {
|
||||
t.Errorf("inactive=%d, want 1 after soft-delete", inactive)
|
||||
}
|
||||
|
||||
// Now UpsertObserver should reactivate it
|
||||
err = store.UpsertObserver("obs-test", "TestObserver", "LAX", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inactive != 0 {
|
||||
t.Errorf("inactive=%d, want 0 after reactivation", inactive)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverDaysOrDefault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *Config
|
||||
want int
|
||||
}{
|
||||
{"nil retention", &Config{}, 14},
|
||||
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
|
||||
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
|
||||
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.ObserverDaysOrDefault()
|
||||
if got != tt.want {
|
||||
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+70
-859
File diff suppressed because it is too large
Load Diff
+20
-1234
File diff suppressed because it is too large
Load Diff
@@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestHandleMessageDecodeErrorLog_PII — issue #1211 round-0 fix shipped without
|
||||
// a test. Asserts the decode-error log line:
|
||||
// (a) includes structured fields: topic, observer prefix, payload length
|
||||
// (b) observer substring is at most 8 chars
|
||||
// (c) full observer ID is NOT present in the output
|
||||
//
|
||||
// A bare `log.Printf("... observer=%s ...", obs)` would leak the full ID.
|
||||
func TestHandleMessageDecodeErrorLog_PII_Issue1211(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
// Use a 64-char observer ID; the prefix MUST be capped at 8 chars in logs.
|
||||
observerID := "abcdef0123456789aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
// Malformed raw — pathByte=0xF6 claims 216 path bytes in a tiny buffer.
|
||||
// This triggers the decode-error path under test.
|
||||
rawHex := "12F6AAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
topic := "meshcore/SJC/" + observerID + "/packets"
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: topic, payload: payload}
|
||||
|
||||
var buf bytes.Buffer
|
||||
orig := log.Writer()
|
||||
log.SetOutput(&buf)
|
||||
defer log.SetOutput(orig)
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "decode error") {
|
||||
t.Fatalf("expected decode-error log; got:\n%s", out)
|
||||
}
|
||||
// (a) structured fields present
|
||||
if !strings.Contains(out, "topic=") {
|
||||
t.Errorf("log missing topic=; got:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "observer=") {
|
||||
t.Errorf("log missing observer=; got:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "rawHexLen=") {
|
||||
t.Errorf("log missing rawHexLen=; got:\n%s", out)
|
||||
}
|
||||
// (c) full observer ID must NOT appear
|
||||
if strings.Contains(out, observerID) {
|
||||
t.Errorf("log leaked full observer ID; got:\n%s", out)
|
||||
}
|
||||
// (b) observer substring capped at 8 chars — the 9th char ('2') after the
|
||||
// 8-char prefix must NOT appear adjacent to the prefix.
|
||||
if strings.Contains(out, "abcdef01234") {
|
||||
t.Errorf("log observer field longer than 8 chars; got:\n%s", out)
|
||||
}
|
||||
// Positive: 8-char prefix must be present in the log
|
||||
if !strings.Contains(out, "abcdef01") {
|
||||
t.Errorf("log missing 8-char observer prefix; got:\n%s", out)
|
||||
}
|
||||
}
|
||||
+32
-413
@@ -11,9 +11,6 @@ import (
|
||||
"math"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/sigvalidate"
|
||||
)
|
||||
|
||||
// Route type constants (header bits 1-0)
|
||||
@@ -81,10 +78,9 @@ type TransportCodes struct {
|
||||
|
||||
// Path holds decoded path/hop information.
|
||||
type Path struct {
|
||||
HashSize int `json:"hashSize"`
|
||||
HashCount int `json:"hashCount"`
|
||||
Hops []string `json:"hops"`
|
||||
HopsCompleted *int `json:"hopsCompleted,omitempty"`
|
||||
HashSize int `json:"hashSize"`
|
||||
HashCount int `json:"hashCount"`
|
||||
Hops []string `json:"hops"`
|
||||
}
|
||||
|
||||
// AdvertFlags holds decoded advert flag bits.
|
||||
@@ -113,7 +109,6 @@ type Payload struct {
|
||||
Timestamp uint32 `json:"timestamp,omitempty"`
|
||||
TimestampISO string `json:"timestampISO,omitempty"`
|
||||
Signature string `json:"signature,omitempty"`
|
||||
SignatureValid *bool `json:"signatureValid,omitempty"`
|
||||
Flags *AdvertFlags `json:"flags,omitempty"`
|
||||
Lat *float64 `json:"lat,omitempty"`
|
||||
Lon *float64 `json:"lon,omitempty"`
|
||||
@@ -126,39 +121,16 @@ type Payload struct {
|
||||
ChannelHashHex string `json:"channelHashHex,omitempty"`
|
||||
DecryptionStatus string `json:"decryptionStatus,omitempty"`
|
||||
Channel string `json:"channel,omitempty"`
|
||||
// GRP_DATA (PAYLOAD_TYPE_GRP_DATA=0x06) inner fields, decoded after
|
||||
// channel decrypt per firmware/src/helpers/BaseChatMesh.cpp:382-385.
|
||||
DataType *int `json:"dataType,omitempty"`
|
||||
DataLen *int `json:"dataLen,omitempty"`
|
||||
DecryptedBlob string `json:"decryptedBlob,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Sender string `json:"sender,omitempty"`
|
||||
SenderTimestamp uint32 `json:"sender_timestamp,omitempty"`
|
||||
EphemeralPubKey string `json:"ephemeralPubKey,omitempty"`
|
||||
PathData string `json:"pathData,omitempty"`
|
||||
SNRValues []float64 `json:"snrValues,omitempty"`
|
||||
Tag uint32 `json:"tag,omitempty"`
|
||||
AuthCode uint32 `json:"authCode,omitempty"`
|
||||
TraceFlags *int `json:"traceFlags,omitempty"`
|
||||
RawHex string `json:"raw,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// MULTIPART (PAYLOAD_TYPE_MULTIPART=0x0A) inner fields, decoded per
|
||||
// firmware/src/Mesh.cpp:289 — byte0 = (remaining<<4) | inner_type.
|
||||
Remaining *int `json:"remaining,omitempty"`
|
||||
InnerType *int `json:"innerType,omitempty"`
|
||||
InnerTypeName string `json:"innerTypeName,omitempty"`
|
||||
InnerAckCrc string `json:"innerAckCrc,omitempty"`
|
||||
InnerPayload string `json:"innerPayload,omitempty"`
|
||||
// CONTROL (PAYLOAD_TYPE_CONTROL=0x0B) byte0 flags, per
|
||||
// firmware/src/Mesh.cpp:69 — byte0 high-bit marks zero-hop direct subset.
|
||||
CtrlFlags string `json:"ctrlFlags,omitempty"`
|
||||
CtrlZeroHop *bool `json:"ctrlZeroHop,omitempty"`
|
||||
CtrlLength *int `json:"ctrlLength,omitempty"`
|
||||
// RAW_CUSTOM (PAYLOAD_TYPE_RAW_CUSTOM=0x0F) — application-defined per
|
||||
// firmware/src/Mesh.cpp:577 (createRawData). Exposes the bare envelope
|
||||
// shape (length + leading tag) so consumers can triage by app id.
|
||||
RawLength *int `json:"rawLength,omitempty"`
|
||||
FirstByteTag string `json:"firstByteTag,omitempty"`
|
||||
}
|
||||
|
||||
// DecodedPacket is the full decoded result.
|
||||
@@ -168,8 +140,6 @@ type DecodedPacket struct {
|
||||
Path Path `json:"path"`
|
||||
Payload Payload `json:"payload"`
|
||||
Raw string `json:"raw"`
|
||||
Anomaly string `json:"anomaly,omitempty"`
|
||||
payloadRaw []byte
|
||||
}
|
||||
|
||||
func decodeHeader(b byte) Header {
|
||||
@@ -195,35 +165,9 @@ func decodeHeader(b byte) Header {
|
||||
}
|
||||
}
|
||||
|
||||
// Firmware-derived limits — see firmware/src/MeshCore.h:19,21.
|
||||
const (
|
||||
maxPathSize = 64 // MAX_PATH_SIZE — total path bytes allowed
|
||||
maxPacketPayload = 184 // MAX_PACKET_PAYLOAD — max raw payload bytes
|
||||
)
|
||||
|
||||
// isValidPathLen mirrors firmware Packet::isValidPathLen
|
||||
// (firmware/src/Packet.cpp:13-18). hash_size==4 is reserved; total path bytes
|
||||
// must fit within MAX_PATH_SIZE.
|
||||
func isValidPathLen(pathByte byte) bool {
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
hashSize := int(pathByte>>6) + 1
|
||||
if hashSize == 4 {
|
||||
return false // reserved
|
||||
}
|
||||
return hashCount*hashSize <= maxPathSize
|
||||
}
|
||||
|
||||
func decodePath(pathByte byte, buf []byte, offset int) (Path, int, error) {
|
||||
func decodePath(pathByte byte, buf []byte, offset int) (Path, int) {
|
||||
hashSize := int(pathByte>>6) + 1
|
||||
hashCount := int(pathByte & 0x3F)
|
||||
// Exact mirror of firmware Packet::isValidPathLen (Packet.cpp:13-18).
|
||||
// hash_size==4 is reserved and is rejected by firmware regardless of
|
||||
// hash_count, so we must reject 0xC0 etc even on zero-hop packets —
|
||||
// firmware never emits them, so an on-wire pathByte with the upper
|
||||
// 2 bits set to 11 is by definition malformed/adversarial.
|
||||
if !isValidPathLen(pathByte) {
|
||||
return Path{}, 0, fmt.Errorf("invalid path encoding: pathByte 0x%02X (hash_size=%d hash_count=%d) violates firmware validity (Packet.cpp:13-18, MAX_PATH_SIZE=%d)", pathByte, hashSize, hashCount, maxPathSize)
|
||||
}
|
||||
totalBytes := hashSize * hashCount
|
||||
hops := make([]string, 0, hashCount)
|
||||
|
||||
@@ -240,12 +184,11 @@ func decodePath(pathByte byte, buf []byte, offset int) (Path, int, error) {
|
||||
HashSize: hashSize,
|
||||
HashCount: hashCount,
|
||||
Hops: hops,
|
||||
}, totalBytes, nil
|
||||
}, totalBytes
|
||||
}
|
||||
|
||||
// isTransportRoute delegates to packetpath.IsTransportRoute.
|
||||
func isTransportRoute(routeType int) bool {
|
||||
return packetpath.IsTransportRoute(routeType)
|
||||
return routeType == RouteTransportFlood || routeType == RouteTransportDirect
|
||||
}
|
||||
|
||||
func decodeEncryptedPayload(typeName string, buf []byte) Payload {
|
||||
@@ -272,7 +215,7 @@ func decodeAck(buf []byte) Payload {
|
||||
}
|
||||
}
|
||||
|
||||
func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
func decodeAdvert(buf []byte) Payload {
|
||||
if len(buf) < 100 {
|
||||
return Payload{Type: "ADVERT", Error: "too short for advert", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
@@ -290,16 +233,6 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
Signature: signature,
|
||||
}
|
||||
|
||||
if validateSignatures {
|
||||
valid, err := sigvalidate.ValidateAdvert(buf[0:32], buf[36:100], timestamp, appdata)
|
||||
if err != nil {
|
||||
f := false
|
||||
p.SignatureValid = &f
|
||||
} else {
|
||||
p.SignatureValid = &valid
|
||||
}
|
||||
}
|
||||
|
||||
if len(appdata) > 0 {
|
||||
flags := appdata[0]
|
||||
advType := int(flags & 0x0F)
|
||||
@@ -349,13 +282,6 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
}
|
||||
name := string(appdata[off:nameEnd])
|
||||
name = sanitizeName(name)
|
||||
// Firmware writes the node name into a 32-byte buffer
|
||||
// (MAX_ADVERT_DATA_SIZE, firmware/src/MeshCore.h:11). Truncate
|
||||
// here so adversarial on-wire adverts can't pollute Payload.Name
|
||||
// with bytes firmware would never emit.
|
||||
if len(name) > 32 {
|
||||
name = name[:32]
|
||||
}
|
||||
p.Name = name
|
||||
off = nameEnd
|
||||
// Skip null terminator(s)
|
||||
@@ -366,17 +292,6 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
|
||||
// Telemetry bytes after name: battery_mv(2 LE) + temperature_c(2 LE, signed, /100)
|
||||
// Only sensor nodes (advType=4) carry telemetry bytes.
|
||||
//
|
||||
// Firmware derivation (see firmware/src/helpers/SensorMesh.h and the
|
||||
// SensorHost::handleAdvert path in firmware/src/helpers/SensorMesh.cpp:
|
||||
// the sensor builds appdata as <flags+adv_type><pubkey?><name\0>
|
||||
// followed by two little-endian uint16 fields appended verbatim:
|
||||
// appdata[name_end+0..1] = battery voltage in millivolts (uint16 LE,
|
||||
// valid 0 < mv ≤ 10000)
|
||||
// appdata[name_end+2..3] = temperature × 100 (int16 LE, divide by 100
|
||||
// for °C; valid raw -5000..10000 → -50..100 °C)
|
||||
// We accept only adverts whose flags.Sensor bit is set (firmware
|
||||
// AdvertDataHelpers.h:7-12, ADV_TYPE_SENSOR=4) before parsing telemetry.
|
||||
if p.Flags.Sensor && off+4 <= len(appdata) {
|
||||
batteryMv := int(binary.LittleEndian.Uint16(appdata[off : off+2]))
|
||||
tempRaw := int16(binary.LittleEndian.Uint16(appdata[off+2 : off+4]))
|
||||
@@ -493,22 +408,6 @@ func decryptChannelMessage(ciphertextHex, macHex, channelKeyHex string) (*channe
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// knownChannelCasing maps known channel keys to their canonical display names.
|
||||
// Only well-known channels are normalized — custom/user channels are left as-is.
|
||||
var knownChannelCasing = map[string]string{
|
||||
"public": "Public",
|
||||
}
|
||||
|
||||
// normalizeChannelName fixes casing for well-known channel names.
|
||||
// Only normalizes names that appear in knownChannelCasing (e.g. "public" → "Public").
|
||||
// Custom channel names are left untouched since we can't know the intended casing.
|
||||
func normalizeChannelName(name string) string {
|
||||
if corrected, ok := knownChannelCasing[strings.ToLower(name)]; ok {
|
||||
return corrected
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func decodeGrpTxt(buf []byte, channelKeys map[string]string) Payload {
|
||||
if len(buf) < 3 {
|
||||
return Payload{Type: "GRP_TXT", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
@@ -533,7 +432,7 @@ func decodeGrpTxt(buf []byte, channelKeys map[string]string) Payload {
|
||||
}
|
||||
return Payload{
|
||||
Type: "CHAN",
|
||||
Channel: normalizeChannelName(name),
|
||||
Channel: name,
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "decrypted",
|
||||
@@ -562,185 +461,6 @@ func decodeGrpTxt(buf []byte, channelKeys map[string]string) Payload {
|
||||
}
|
||||
}
|
||||
|
||||
// decodeGrpData decodes PAYLOAD_TYPE_GRP_DATA (0x06). Outer envelope is the
|
||||
// same shape as GRP_TXT (channel_hash(1)+MAC(2)+ciphertext) — see
|
||||
// firmware/src/helpers/BaseChatMesh.cpp:476,500. When the channel key matches,
|
||||
// the decrypted inner is parsed per firmware/src/helpers/BaseChatMesh.cpp:382-385
|
||||
// as data_type(uint16 LE) + data_len(1) + blob(data_len).
|
||||
func decodeGrpData(buf []byte, channelKeys map[string]string) Payload {
|
||||
if len(buf) < 3 {
|
||||
return Payload{Type: "GRP_DATA", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
channelHash := int(buf[0])
|
||||
channelHashHex := fmt.Sprintf("%02X", buf[0])
|
||||
mac := hex.EncodeToString(buf[1:3])
|
||||
encryptedData := hex.EncodeToString(buf[3:])
|
||||
|
||||
hasKeys := len(channelKeys) > 0
|
||||
if hasKeys && len(encryptedData) >= 10 {
|
||||
for name, key := range channelKeys {
|
||||
plain, err := decryptChannelBlock(encryptedData, mac, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Inner: data_type(uint16 LE) + data_len(1) + blob (firmware:382-385).
|
||||
if len(plain) < 3 {
|
||||
return Payload{
|
||||
Type: "GRP_DATA",
|
||||
Channel: name,
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "decrypted",
|
||||
Error: "inner too short",
|
||||
}
|
||||
}
|
||||
dataType := int(binary.LittleEndian.Uint16(plain[0:2]))
|
||||
dataLen := int(plain[2])
|
||||
if 3+dataLen > len(plain) {
|
||||
return Payload{
|
||||
Type: "GRP_DATA",
|
||||
Channel: name,
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "decrypted",
|
||||
DataType: &dataType,
|
||||
DataLen: &dataLen,
|
||||
Error: "inner data_len exceeds buffer",
|
||||
}
|
||||
}
|
||||
blob := hex.EncodeToString(plain[3 : 3+dataLen])
|
||||
return Payload{
|
||||
Type: "GRP_DATA",
|
||||
Channel: name,
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "decrypted",
|
||||
DataType: &dataType,
|
||||
DataLen: &dataLen,
|
||||
DecryptedBlob: blob,
|
||||
}
|
||||
}
|
||||
return Payload{
|
||||
Type: "GRP_DATA",
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "decryption_failed",
|
||||
MAC: mac,
|
||||
EncryptedData: encryptedData,
|
||||
}
|
||||
}
|
||||
|
||||
return Payload{
|
||||
Type: "GRP_DATA",
|
||||
ChannelHash: channelHash,
|
||||
ChannelHashHex: channelHashHex,
|
||||
DecryptionStatus: "no_key",
|
||||
MAC: mac,
|
||||
EncryptedData: encryptedData,
|
||||
}
|
||||
}
|
||||
|
||||
// decodeMultipart decodes PAYLOAD_TYPE_MULTIPART (0x0A) per
|
||||
// firmware/src/Mesh.cpp:287-310. byte0 = (remaining<<4) | inner_type;
|
||||
// when inner_type == PAYLOAD_TYPE_ACK the next 4 bytes are an ack_crc.
|
||||
func decodeMultipart(buf []byte) Payload {
|
||||
if len(buf) < 1 {
|
||||
return Payload{Type: "MULTIPART", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
remaining := int(buf[0] >> 4)
|
||||
innerType := int(buf[0] & 0x0F)
|
||||
innerName := payloadTypeNames[innerType]
|
||||
if innerName == "" {
|
||||
innerName = "UNKNOWN"
|
||||
}
|
||||
p := Payload{
|
||||
Type: "MULTIPART",
|
||||
Remaining: &remaining,
|
||||
InnerType: &innerType,
|
||||
InnerTypeName: innerName,
|
||||
}
|
||||
if innerType == PayloadACK && len(buf) >= 5 {
|
||||
// ack_crc is little-endian; surface as canonical big-endian hex
|
||||
// to match decodeAck's extraHash convention.
|
||||
crc := binary.LittleEndian.Uint32(buf[1:5])
|
||||
p.InnerAckCrc = fmt.Sprintf("%08x", crc)
|
||||
} else if len(buf) > 1 {
|
||||
p.InnerPayload = hex.EncodeToString(buf[1:])
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// decodeControl decodes PAYLOAD_TYPE_CONTROL (0x0B) byte0 flags per
|
||||
// firmware/src/Mesh.cpp:69 (high-bit set ⇒ zero-hop direct subset).
|
||||
func decodeControl(buf []byte) Payload {
|
||||
if len(buf) < 1 {
|
||||
return Payload{Type: "CONTROL", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
zeroHop := buf[0]&0x80 != 0
|
||||
length := len(buf)
|
||||
return Payload{
|
||||
Type: "CONTROL",
|
||||
CtrlFlags: fmt.Sprintf("%02x", buf[0]),
|
||||
CtrlZeroHop: &zeroHop,
|
||||
CtrlLength: &length,
|
||||
RawHex: hex.EncodeToString(buf),
|
||||
}
|
||||
}
|
||||
|
||||
// decodeRawCustom decodes PAYLOAD_TYPE_RAW_CUSTOM (0x0F). Application-defined
|
||||
// payload per firmware/src/Mesh.cpp:577 (createRawData); we only surface the
|
||||
// envelope shape (total length + leading tag byte).
|
||||
func decodeRawCustom(buf []byte) Payload {
|
||||
length := len(buf)
|
||||
p := Payload{
|
||||
Type: "RAW_CUSTOM",
|
||||
RawLength: &length,
|
||||
RawHex: hex.EncodeToString(buf),
|
||||
}
|
||||
if length > 0 {
|
||||
p.FirstByteTag = fmt.Sprintf("%02X", buf[0])
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// decryptChannelBlock performs the MAC verify + AES-128-ECB decrypt step shared
|
||||
// by GRP_TXT and GRP_DATA, returning the raw plaintext block (no further
|
||||
// parsing). See firmware/src/helpers/BaseChatMesh.cpp:376-391.
|
||||
func decryptChannelBlock(ciphertextHex, macHex, channelKeyHex string) ([]byte, error) {
|
||||
channelKey, err := hex.DecodeString(channelKeyHex)
|
||||
if err != nil || len(channelKey) != 16 {
|
||||
return nil, fmt.Errorf("invalid channel key")
|
||||
}
|
||||
macBytes, err := hex.DecodeString(macHex)
|
||||
if err != nil || len(macBytes) != 2 {
|
||||
return nil, fmt.Errorf("invalid MAC")
|
||||
}
|
||||
ciphertext, err := hex.DecodeString(ciphertextHex)
|
||||
if err != nil || len(ciphertext) == 0 {
|
||||
return nil, fmt.Errorf("invalid ciphertext")
|
||||
}
|
||||
channelSecret := make([]byte, 32)
|
||||
copy(channelSecret, channelKey)
|
||||
h := hmac.New(sha256.New, channelSecret)
|
||||
h.Write(ciphertext)
|
||||
calc := h.Sum(nil)
|
||||
if calc[0] != macBytes[0] || calc[1] != macBytes[1] {
|
||||
return nil, fmt.Errorf("MAC verification failed")
|
||||
}
|
||||
if len(ciphertext)%aes.BlockSize != 0 {
|
||||
return nil, fmt.Errorf("ciphertext not aligned to AES block size")
|
||||
}
|
||||
block, err := aes.NewCipher(channelKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plain := make([]byte, len(ciphertext))
|
||||
for i := 0; i < len(ciphertext); i += aes.BlockSize {
|
||||
block.Decrypt(plain[i:i+aes.BlockSize], ciphertext[i:i+aes.BlockSize])
|
||||
}
|
||||
return plain, nil
|
||||
}
|
||||
|
||||
func decodeAnonReq(buf []byte) Payload {
|
||||
if len(buf) < 35 {
|
||||
return Payload{Type: "ANON_REQ", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
@@ -786,7 +506,7 @@ func decodeTrace(buf []byte) Payload {
|
||||
return p
|
||||
}
|
||||
|
||||
func decodePayload(payloadType int, buf []byte, channelKeys map[string]string, validateSignatures bool) Payload {
|
||||
func decodePayload(payloadType int, buf []byte, channelKeys map[string]string) Payload {
|
||||
switch payloadType {
|
||||
case PayloadREQ:
|
||||
return decodeEncryptedPayload("REQ", buf)
|
||||
@@ -797,30 +517,22 @@ func decodePayload(payloadType int, buf []byte, channelKeys map[string]string, v
|
||||
case PayloadACK:
|
||||
return decodeAck(buf)
|
||||
case PayloadADVERT:
|
||||
return decodeAdvert(buf, validateSignatures)
|
||||
return decodeAdvert(buf)
|
||||
case PayloadGRP_TXT:
|
||||
return decodeGrpTxt(buf, channelKeys)
|
||||
case PayloadGRP_DATA:
|
||||
return decodeGrpData(buf, channelKeys)
|
||||
case PayloadANON_REQ:
|
||||
return decodeAnonReq(buf)
|
||||
case PayloadPATH:
|
||||
return decodePathPayload(buf)
|
||||
case PayloadTRACE:
|
||||
return decodeTrace(buf)
|
||||
case PayloadMULTIPART:
|
||||
return decodeMultipart(buf)
|
||||
case PayloadCONTROL:
|
||||
return decodeControl(buf)
|
||||
case PayloadRAW_CUSTOM:
|
||||
return decodeRawCustom(buf)
|
||||
default:
|
||||
return Payload{Type: "UNKNOWN", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
}
|
||||
|
||||
// DecodePacket decodes a hex-encoded MeshCore packet.
|
||||
func DecodePacket(hexString string, channelKeys map[string]string, validateSignatures bool) (*DecodedPacket, error) {
|
||||
func DecodePacket(hexString string, channelKeys map[string]string) (*DecodedPacket, error) {
|
||||
hexString = strings.ReplaceAll(hexString, " ", "")
|
||||
hexString = strings.ReplaceAll(hexString, "\n", "")
|
||||
hexString = strings.ReplaceAll(hexString, "\r", "")
|
||||
@@ -854,104 +566,39 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna
|
||||
pathByte := buf[offset]
|
||||
offset++
|
||||
|
||||
path, bytesConsumed, decodeErr := decodePath(pathByte, buf, offset)
|
||||
if decodeErr != nil {
|
||||
return nil, decodeErr
|
||||
}
|
||||
path, bytesConsumed := decodePath(pathByte, buf, offset)
|
||||
offset += bytesConsumed
|
||||
|
||||
// Bounds check: pathByte is wire-supplied (hash_size in upper 2 bits,
|
||||
// hash_count in lower 6 bits → up to 4*63=252 claimed path bytes). A
|
||||
// malformed packet can claim more bytes than the buffer holds — without
|
||||
// this guard `buf[offset:]` panics with `slice bounds out of range
|
||||
// [offset:len(buf)]`. See issue #1211 (prod observed [218:15]).
|
||||
if offset > len(buf) {
|
||||
return nil, fmt.Errorf("packet path length (%d bytes claimed by pathByte 0x%02X) exceeds buffer (%d bytes)", bytesConsumed, pathByte, len(buf))
|
||||
}
|
||||
|
||||
payloadBuf := buf[offset:]
|
||||
// Firmware caps payload at MAX_PACKET_PAYLOAD=184 (firmware/src/MeshCore.h:19).
|
||||
if len(payloadBuf) > maxPacketPayload {
|
||||
return nil, fmt.Errorf("packet payload (%d bytes) exceeds firmware MAX_PACKET_PAYLOAD=%d (MeshCore.h:19)", len(payloadBuf), maxPacketPayload)
|
||||
}
|
||||
payload := decodePayload(header.PayloadType, payloadBuf, channelKeys, validateSignatures)
|
||||
payload := decodePayload(header.PayloadType, payloadBuf, channelKeys)
|
||||
|
||||
// TRACE packets store hop IDs in the payload (buf[9:]) rather than the header
|
||||
// path field. Firmware always sends TRACE as DIRECT (route_type 2 or 3);
|
||||
// FLOOD-routed TRACEs are anomalous but handled gracefully (parsed, but
|
||||
// flagged). The TRACE flags byte (payload offset 8) encodes path_sz in
|
||||
// bits 0-1 as a power-of-two exponent: hash_bytes = 1 << path_sz.
|
||||
// NOT the header path byte's hash_size bits. The header path contains SNR
|
||||
// bytes — one per hop that actually forwarded.
|
||||
// We expose hopsCompleted (count of SNR bytes) so consumers can distinguish
|
||||
// how far the trace got vs the full intended route.
|
||||
var anomaly string
|
||||
if header.PayloadType == PayloadTRACE && payload.Error != "" {
|
||||
anomaly = fmt.Sprintf("TRACE payload decode failed: %s", payload.Error)
|
||||
}
|
||||
// path field. The header path byte still encodes hashSize in bits 6-7, which
|
||||
// we use to split the payload path data into individual hop prefixes.
|
||||
if header.PayloadType == PayloadTRACE && payload.PathData != "" {
|
||||
// Flag anomalous routing — firmware only sends TRACE as DIRECT
|
||||
if header.RouteType != RouteDirect && header.RouteType != RouteTransportDirect {
|
||||
anomaly = "TRACE packet with non-DIRECT routing (expected DIRECT or TRANSPORT_DIRECT)"
|
||||
}
|
||||
// The header path hops count represents SNR entries = completed hops
|
||||
hopsCompleted := path.HashCount
|
||||
// Extract per-hop SNR from header path bytes (int8, quarter-dB encoding).
|
||||
// Mirrors cmd/server/decoder.go — must be done at ingest time so SNR
|
||||
// values are persisted in decoded_json (server endpoint serves DB as-is).
|
||||
if hopsCompleted > 0 && len(path.Hops) >= hopsCompleted {
|
||||
snrVals := make([]float64, 0, hopsCompleted)
|
||||
for i := 0; i < hopsCompleted; i++ {
|
||||
b, err := hex.DecodeString(path.Hops[i])
|
||||
if err == nil && len(b) == 1 {
|
||||
snrVals = append(snrVals, float64(int8(b[0]))/4.0)
|
||||
}
|
||||
}
|
||||
if len(snrVals) > 0 {
|
||||
payload.SNRValues = snrVals
|
||||
}
|
||||
}
|
||||
pathBytes, err := hex.DecodeString(payload.PathData)
|
||||
if err == nil && payload.TraceFlags != nil {
|
||||
// path_sz from flags byte is a power-of-two exponent per firmware:
|
||||
// hash_bytes = 1 << (flags & 0x03)
|
||||
pathSz := 1 << (*payload.TraceFlags & 0x03)
|
||||
hops := make([]string, 0, len(pathBytes)/pathSz)
|
||||
for i := 0; i+pathSz <= len(pathBytes); i += pathSz {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+pathSz])))
|
||||
if err == nil && path.HashSize > 0 {
|
||||
hops := make([]string, 0, len(pathBytes)/path.HashSize)
|
||||
for i := 0; i+path.HashSize <= len(pathBytes); i += path.HashSize {
|
||||
hops = append(hops, strings.ToUpper(hex.EncodeToString(pathBytes[i:i+path.HashSize])))
|
||||
}
|
||||
path.Hops = hops
|
||||
path.HashCount = len(hops)
|
||||
path.HashSize = pathSz
|
||||
path.HopsCompleted = &hopsCompleted
|
||||
}
|
||||
}
|
||||
|
||||
// Zero-hop direct packets have hash_count=0 (lower 6 bits of pathByte),
|
||||
// which makes the generic formula yield a bogus hashSize. Reset to 0
|
||||
// (unknown) so API consumers get correct data. We mask with 0x3F to check
|
||||
// only hash_count, matching the JS frontend approach — the upper hash_size
|
||||
// bits are meaningless when there are no hops. Skip TRACE packets — they
|
||||
// use hashSize to parse hops from the payload above.
|
||||
if (header.RouteType == RouteDirect || header.RouteType == RouteTransportDirect) && pathByte&0x3F == 0 && header.PayloadType != PayloadTRACE {
|
||||
path.HashSize = 0
|
||||
}
|
||||
|
||||
return &DecodedPacket{
|
||||
Header: header,
|
||||
TransportCodes: tc,
|
||||
Path: path,
|
||||
Payload: payload,
|
||||
Raw: strings.ToUpper(hexString),
|
||||
Anomaly: anomaly,
|
||||
payloadRaw: payloadBuf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ComputeContentHash computes the SHA-256-based content hash (first 16 hex chars).
|
||||
// It hashes the payload-type nibble + payload (skipping path bytes) to produce a
|
||||
// route-independent identifier for the same logical packet. For TRACE packets,
|
||||
// path_len is included in the hash to match firmware behavior.
|
||||
// It hashes the header byte + payload (skipping path bytes) to produce a
|
||||
// path-independent identifier for the same transmission.
|
||||
func ComputeContentHash(rawHex string) string {
|
||||
buf, err := hex.DecodeString(rawHex)
|
||||
if err != nil || len(buf) < 2 {
|
||||
@@ -987,18 +634,7 @@ func ComputeContentHash(rawHex string) string {
|
||||
}
|
||||
|
||||
payload := buf[payloadStart:]
|
||||
|
||||
// Hash payload-type byte only (bits 2-5 of header), not the full header.
|
||||
// Firmware: SHA256(payload_type + [path_len for TRACE] + payload)
|
||||
// Using the full header caused different hashes for the same logical packet
|
||||
// when route type or version bits differed. See issue #786.
|
||||
payloadType := (headerByte >> 2) & 0x0F
|
||||
toHash := []byte{payloadType}
|
||||
if int(payloadType) == PayloadTRACE {
|
||||
// Firmware uses uint16_t path_len (2 bytes, little-endian)
|
||||
toHash = append(toHash, pathByte, 0x00)
|
||||
}
|
||||
toHash = append(toHash, payload...)
|
||||
toHash := append([]byte{headerByte}, payload...)
|
||||
|
||||
h := sha256.Sum256(toHash)
|
||||
return hex.EncodeToString(h[:])[:16]
|
||||
@@ -1062,13 +698,8 @@ func ValidateAdvert(p *Payload) (bool, string) {
|
||||
|
||||
if p.Flags != nil {
|
||||
role := advertRole(p.Flags)
|
||||
// Accept canonical labels plus "none" (ADV_TYPE_NONE=0) and the
|
||||
// "type-N" placeholders we now return for ADV_TYPE 5-15 (FUTURE)
|
||||
// — see firmware/src/helpers/AdvertDataHelpers.h:7-12.
|
||||
validRoles := map[string]bool{
|
||||
"repeater": true, "companion": true, "room": true, "sensor": true, "none": true,
|
||||
}
|
||||
if !validRoles[role] && !strings.HasPrefix(role, "type-") {
|
||||
validRoles := map[string]bool{"repeater": true, "companion": true, "room": true, "sensor": true}
|
||||
if !validRoles[role] {
|
||||
return false, fmt.Sprintf("unknown role: %s", role)
|
||||
}
|
||||
}
|
||||
@@ -1088,29 +719,17 @@ func sanitizeName(s string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// advertRole returns a stable role label for an advert. Follows firmware
|
||||
// ADV_TYPE_* constants in firmware/src/helpers/AdvertDataHelpers.h:7-12:
|
||||
// 0 NONE, 1 CHAT, 2 REPEATER, 3 ROOM, 4 SENSOR, 5-15 FUTURE.
|
||||
// Previously this coerced both 0 (NONE) and 5-15 (FUTURE) to "companion",
|
||||
// silently relabelling unknown/reserved types — see issue #1279 P1 #3.
|
||||
func advertRole(f *AdvertFlags) string {
|
||||
if f == nil {
|
||||
return "companion"
|
||||
}
|
||||
switch f.Type {
|
||||
case 0:
|
||||
return "none"
|
||||
case 1:
|
||||
return "companion"
|
||||
case 2:
|
||||
if f.Repeater {
|
||||
return "repeater"
|
||||
case 3:
|
||||
return "room"
|
||||
case 4:
|
||||
return "sensor"
|
||||
default:
|
||||
return fmt.Sprintf("type-%d", f.Type)
|
||||
}
|
||||
if f.Room {
|
||||
return "room"
|
||||
}
|
||||
if f.Sensor {
|
||||
return "sensor"
|
||||
}
|
||||
return "companion"
|
||||
}
|
||||
|
||||
func epochToISO(epoch uint32) string {
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- Issue #1211 round-1 protocol-correctness regressions ---
|
||||
// See cmd/server/decoder_bounds_test.go for full firmware citations
|
||||
// (firmware/src/Packet.cpp:13-18, firmware/src/MeshCore.h:19-21).
|
||||
|
||||
// pathByte=0xF6 → hash_size=4 (reserved), hash_count=54.
|
||||
// Buffer holds all 216 claimed bytes so the OOB guard does NOT catch.
|
||||
func TestDecodePacketRejectsReservedHashSize_Issue1211(t *testing.T) {
|
||||
raw := "12F6" + strings.Repeat("AB", 216) + strings.Repeat("CD", 8)
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error rejecting reserved hash_size=4 (firmware Packet.cpp:13-18); got nil, pkt=%+v", pkt)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "path") {
|
||||
t.Errorf("error should mention path; got %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
// pathByte=0xBF → hash_size=3, hash_count=63, total=189 > MAX_PATH_SIZE=64.
|
||||
func TestDecodePacketRejectsOversizedPath_Issue1211(t *testing.T) {
|
||||
raw := "12BF" + strings.Repeat("AB", 189) + strings.Repeat("CD", 8)
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error rejecting hash_count*hash_size > 64; got nil, pkt=%+v", pkt)
|
||||
}
|
||||
}
|
||||
|
||||
// Payload > MAX_PACKET_PAYLOAD (184).
|
||||
func TestDecodePacketRejectsOversizedPayload_Issue1211(t *testing.T) {
|
||||
raw := "1200" + strings.Repeat("AA", 200)
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error rejecting payload > MAX_PACKET_PAYLOAD=184 (firmware MeshCore.h:19); got nil, pkt=%+v", pkt)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "payload") {
|
||||
t.Errorf("error should mention payload; got %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePath_RejectsReservedHashSize_Issue1211(t *testing.T) {
|
||||
buf := make([]byte, 216)
|
||||
for i := range buf {
|
||||
buf[i] = 0xAB
|
||||
}
|
||||
_, _, err := decodePath(0xF6, buf, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("decodePath should reject pathByte=0xF6 (hash_size=4 reserved); got nil err")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePath_RejectsOversizedPath_Issue1211(t *testing.T) {
|
||||
buf := make([]byte, 189)
|
||||
_, _, err := decodePath(0xBF, buf, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("decodePath should reject hash_count*hash_size=189 > MAX_PATH_SIZE=64; got nil err")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePath_AcceptsValidEncodings_Issue1211(t *testing.T) {
|
||||
buf := []byte{0x01, 0x02, 0x03, 0x04, 0x05}
|
||||
path, consumed, err := decodePath(0x05, buf, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("decodePath rejected valid encoding: %v", err)
|
||||
}
|
||||
if consumed != 5 {
|
||||
t.Errorf("consumed=%d, want 5", consumed)
|
||||
}
|
||||
if path.HashCount != 5 || path.HashSize != 1 {
|
||||
t.Errorf("decode wrong: hashCount=%d hashSize=%d", path.HashCount, path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Kent #1 — pin tautological assertion: error MUST mention "path length"
|
||||
// AND "exceeds buffer", not just non-nil. Uses firmware-valid pathByte
|
||||
// that exhausts a small buffer, so the OOB guard fires (not validity).
|
||||
func TestDecodePacketBoundsFromWireErrorPhrasing_Issue1211(t *testing.T) {
|
||||
raw := "120A" + strings.Repeat("AA", 5)
|
||||
_, err := DecodePacket(raw, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "path length") {
|
||||
t.Errorf("error missing 'path length'; got %q", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exceeds buffer") {
|
||||
t.Errorf("error missing 'exceeds buffer'; got %q", err)
|
||||
}
|
||||
}
|
||||
|
||||
var _ = hex.EncodeToString
|
||||
+51
-618
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/ed25519"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
@@ -10,9 +9,6 @@ import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/sigvalidate"
|
||||
)
|
||||
|
||||
func TestDecodeHeaderRoutTypes(t *testing.T) {
|
||||
@@ -59,7 +55,7 @@ func TestDecodeHeaderPayloadTypes(t *testing.T) {
|
||||
|
||||
func TestDecodePathZeroHops(t *testing.T) {
|
||||
// 0x00: 0 hops, 1-byte hashes
|
||||
pkt, err := DecodePacket("0500"+strings.Repeat("00", 10), nil, false)
|
||||
pkt, err := DecodePacket("0500"+strings.Repeat("00", 10), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +72,7 @@ func TestDecodePathZeroHops(t *testing.T) {
|
||||
|
||||
func TestDecodePath1ByteHashes(t *testing.T) {
|
||||
// 0x05: 5 hops, 1-byte hashes → 5 path bytes
|
||||
pkt, err := DecodePacket("0505"+"AABBCCDDEE"+strings.Repeat("00", 10), nil, false)
|
||||
pkt, err := DecodePacket("0505"+"AABBCCDDEE"+strings.Repeat("00", 10), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -99,7 +95,7 @@ func TestDecodePath1ByteHashes(t *testing.T) {
|
||||
|
||||
func TestDecodePath2ByteHashes(t *testing.T) {
|
||||
// 0x45: 5 hops, 2-byte hashes
|
||||
pkt, err := DecodePacket("0545"+"AA11BB22CC33DD44EE55"+strings.Repeat("00", 10), nil, false)
|
||||
pkt, err := DecodePacket("0545"+"AA11BB22CC33DD44EE55"+strings.Repeat("00", 10), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -116,7 +112,7 @@ func TestDecodePath2ByteHashes(t *testing.T) {
|
||||
|
||||
func TestDecodePath3ByteHashes(t *testing.T) {
|
||||
// 0x8A: 10 hops, 3-byte hashes
|
||||
pkt, err := DecodePacket("058A"+strings.Repeat("AA11FF", 10)+strings.Repeat("00", 10), nil, false)
|
||||
pkt, err := DecodePacket("058A"+strings.Repeat("AA11FF", 10)+strings.Repeat("00", 10), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -135,7 +131,7 @@ func TestTransportCodes(t *testing.T) {
|
||||
// Route type 0 (TRANSPORT_FLOOD) should have transport codes
|
||||
// Firmware order: header + transport_codes(4) + path_len + path + payload
|
||||
hex := "14" + "AABB" + "CCDD" + "00" + strings.Repeat("00", 10)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
pkt, err := DecodePacket(hex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -153,7 +149,7 @@ func TestTransportCodes(t *testing.T) {
|
||||
}
|
||||
|
||||
// Route type 1 (FLOOD) should NOT have transport codes
|
||||
pkt2, err := DecodePacket("0500"+strings.Repeat("00", 10), nil, false)
|
||||
pkt2, err := DecodePacket("0500"+strings.Repeat("00", 10), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -173,7 +169,7 @@ func TestDecodeAdvertFull(t *testing.T) {
|
||||
name := "546573744E6F6465" // "TestNode"
|
||||
|
||||
hex := "1200" + pubkey + timestamp + signature + flags + lat + lon + name
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
pkt, err := DecodePacket(hex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -231,7 +227,7 @@ func TestDecodeAdvertTypeEnums(t *testing.T) {
|
||||
makeAdvert := func(flagsByte byte) *DecodedPacket {
|
||||
hex := "1200" + strings.Repeat("AA", 32) + "00000000" + strings.Repeat("BB", 64) +
|
||||
strings.ToUpper(string([]byte{hexDigit(flagsByte>>4), hexDigit(flagsByte & 0x0f)}))
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
pkt, err := DecodePacket(hex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -276,7 +272,7 @@ func hexDigit(v byte) byte {
|
||||
|
||||
func TestDecodeAdvertNoLocationNoName(t *testing.T) {
|
||||
hex := "1200" + strings.Repeat("CC", 32) + "00000000" + strings.Repeat("DD", 64) + "02"
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
pkt, err := DecodePacket(hex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -295,7 +291,7 @@ func TestDecodeAdvertNoLocationNoName(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGoldenFixtureTxtMsg(t *testing.T) {
|
||||
pkt, err := DecodePacket("0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", nil, false)
|
||||
pkt, err := DecodePacket("0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -318,7 +314,7 @@ func TestGoldenFixtureTxtMsg(t *testing.T) {
|
||||
|
||||
func TestGoldenFixtureAdvert(t *testing.T) {
|
||||
rawHex := "120046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
pkt, err := DecodePacket(rawHex, nil, false)
|
||||
pkt, err := DecodePacket(rawHex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -341,7 +337,7 @@ func TestGoldenFixtureAdvert(t *testing.T) {
|
||||
|
||||
func TestGoldenFixtureUnicodeAdvert(t *testing.T) {
|
||||
rawHex := "120073CFF971E1CB5754A742C152B2D2E0EB108A19B246D663ED8898A72C4A5AD86EA6768E66694B025EDF6939D5C44CFF719C5D5520E5F06B20680A83AD9C2C61C3227BBB977A85EE462F3553445FECF8EDD05C234ECE217272E503F14D6DF2B1B9B133890C923CDF3002F8FDC1F85045414BF09F8CB3"
|
||||
pkt, err := DecodePacket(rawHex, nil, false)
|
||||
pkt, err := DecodePacket(rawHex, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -358,14 +354,14 @@ func TestGoldenFixtureUnicodeAdvert(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDecodePacketTooShort(t *testing.T) {
|
||||
_, err := DecodePacket("FF", nil, false)
|
||||
_, err := DecodePacket("FF", nil)
|
||||
if err == nil {
|
||||
t.Error("expected error for 1-byte packet")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacketInvalidHex(t *testing.T) {
|
||||
_, err := DecodePacket("ZZZZ", nil, false)
|
||||
_, err := DecodePacket("ZZZZ", nil)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid hex")
|
||||
}
|
||||
@@ -447,28 +443,6 @@ func TestValidateAdvert(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacketPayloadRaw(t *testing.T) {
|
||||
// Build a minimal TRANSPORT_FLOOD packet (route_type=0):
|
||||
// header(1) + transport_codes(4) + path_len(1) + payload(N)
|
||||
// Header 0x00 = route_type=TRANSPORT_FLOOD, payload_type=0, version=0
|
||||
// Code1=9A52, Code2=0000, path_len=0x00 (0 hops, hash_size=1)
|
||||
payload := []byte("hello")
|
||||
raw := []byte{0x00, 0x9A, 0x52, 0x00, 0x00, 0x00}
|
||||
raw = append(raw, payload...)
|
||||
hexStr := strings.ToUpper(hex.EncodeToString(raw))
|
||||
|
||||
decoded, err := DecodePacket(hexStr, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
if decoded.TransportCodes == nil {
|
||||
t.Fatal("expected TransportCodes, got nil")
|
||||
}
|
||||
if string(decoded.payloadRaw) != string(payload) {
|
||||
t.Errorf("payloadRaw = %v, want %v", decoded.payloadRaw, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGrpTxtShort(t *testing.T) {
|
||||
p := decodeGrpTxt([]byte{0x01, 0x02}, nil)
|
||||
if p.Error != "too short" {
|
||||
@@ -594,7 +568,7 @@ func TestDecodeTracePathParsing(t *testing.T) {
|
||||
// Packet from issue #276: 260001807dca00000000007d547d
|
||||
// Path byte 0x00 → hashSize=1, hops in payload at buf[9:] = 7d 54 7d
|
||||
// Expected path: ["7D", "54", "7D"]
|
||||
pkt, err := DecodePacket("260001807dca00000000007d547d", nil, false)
|
||||
pkt, err := DecodePacket("260001807dca00000000007d547d", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
@@ -616,7 +590,7 @@ func TestDecodeTracePathParsing(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDecodeAdvertShort(t *testing.T) {
|
||||
p := decodeAdvert(make([]byte, 50), false)
|
||||
p := decodeAdvert(make([]byte, 50))
|
||||
if p.Error != "too short for advert" {
|
||||
t.Errorf("expected 'too short for advert' error, got %q", p.Error)
|
||||
}
|
||||
@@ -653,76 +627,69 @@ func TestDecodeEncryptedPayloadValid(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDecodePayloadGRPData(t *testing.T) {
|
||||
// GRP_DATA (0x06) decoder added for #1279 P0 #1 — envelope only when no
|
||||
// channel key matches (firmware/src/helpers/BaseChatMesh.cpp:500).
|
||||
buf := []byte{0x01, 0x02, 0x03}
|
||||
p := decodePayload(PayloadGRP_DATA, buf, nil, false)
|
||||
if p.Type != "GRP_DATA" {
|
||||
t.Errorf("type=%s, want GRP_DATA", p.Type)
|
||||
p := decodePayload(PayloadGRP_DATA, buf, nil)
|
||||
if p.Type != "UNKNOWN" {
|
||||
t.Errorf("type=%s, want UNKNOWN", p.Type)
|
||||
}
|
||||
if p.RawHex != "010203" {
|
||||
t.Errorf("rawHex=%s, want 010203", p.RawHex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePayloadRAWCustom(t *testing.T) {
|
||||
// #1279 P2 #5: RAW_CUSTOM (0x0F) now exposes envelope shape (length +
|
||||
// first-byte tag) per firmware/src/Mesh.cpp:577 (createRawData).
|
||||
buf := []byte{0xFF, 0xFE}
|
||||
p := decodePayload(PayloadRAW_CUSTOM, buf, nil, false)
|
||||
if p.Type != "RAW_CUSTOM" {
|
||||
t.Errorf("type=%s, want RAW_CUSTOM", p.Type)
|
||||
}
|
||||
if p.RawLength == nil || *p.RawLength != 2 {
|
||||
t.Errorf("rawLength missing or wrong, want 2")
|
||||
}
|
||||
if p.FirstByteTag != "FF" {
|
||||
t.Errorf("firstByteTag=%q, want FF", p.FirstByteTag)
|
||||
p := decodePayload(PayloadRAW_CUSTOM, buf, nil)
|
||||
if p.Type != "UNKNOWN" {
|
||||
t.Errorf("type=%s, want UNKNOWN", p.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePayloadAllTypes(t *testing.T) {
|
||||
// REQ
|
||||
p := decodePayload(PayloadREQ, make([]byte, 10), nil, false)
|
||||
p := decodePayload(PayloadREQ, make([]byte, 10), nil)
|
||||
if p.Type != "REQ" {
|
||||
t.Errorf("REQ: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// RESPONSE
|
||||
p = decodePayload(PayloadRESPONSE, make([]byte, 10), nil, false)
|
||||
p = decodePayload(PayloadRESPONSE, make([]byte, 10), nil)
|
||||
if p.Type != "RESPONSE" {
|
||||
t.Errorf("RESPONSE: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// TXT_MSG
|
||||
p = decodePayload(PayloadTXT_MSG, make([]byte, 10), nil, false)
|
||||
p = decodePayload(PayloadTXT_MSG, make([]byte, 10), nil)
|
||||
if p.Type != "TXT_MSG" {
|
||||
t.Errorf("TXT_MSG: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// ACK
|
||||
p = decodePayload(PayloadACK, make([]byte, 10), nil, false)
|
||||
p = decodePayload(PayloadACK, make([]byte, 10), nil)
|
||||
if p.Type != "ACK" {
|
||||
t.Errorf("ACK: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// GRP_TXT
|
||||
p = decodePayload(PayloadGRP_TXT, make([]byte, 10), nil, false)
|
||||
p = decodePayload(PayloadGRP_TXT, make([]byte, 10), nil)
|
||||
if p.Type != "GRP_TXT" {
|
||||
t.Errorf("GRP_TXT: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// ANON_REQ
|
||||
p = decodePayload(PayloadANON_REQ, make([]byte, 40), nil, false)
|
||||
p = decodePayload(PayloadANON_REQ, make([]byte, 40), nil)
|
||||
if p.Type != "ANON_REQ" {
|
||||
t.Errorf("ANON_REQ: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// PATH
|
||||
p = decodePayload(PayloadPATH, make([]byte, 10), nil, false)
|
||||
p = decodePayload(PayloadPATH, make([]byte, 10), nil)
|
||||
if p.Type != "PATH" {
|
||||
t.Errorf("PATH: type=%s", p.Type)
|
||||
}
|
||||
|
||||
// TRACE
|
||||
p = decodePayload(PayloadTRACE, make([]byte, 20), nil, false)
|
||||
p = decodePayload(PayloadTRACE, make([]byte, 20), nil)
|
||||
if p.Type != "TRACE" {
|
||||
t.Errorf("TRACE: type=%s", p.Type)
|
||||
}
|
||||
@@ -956,96 +923,9 @@ func TestComputeContentHashLongFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashRouteTypeIndependence verifies that the same logical
|
||||
// packet produces the same content hash regardless of route type (issue #786).
|
||||
func TestComputeContentHashRouteTypeIndependence(t *testing.T) {
|
||||
// Same payload type (TXT_MSG=2, bits 2-5) with different route types.
|
||||
// Header 0x08 = route_type 0 (TRANSPORT_FLOOD), payload_type 2
|
||||
// Header 0x0A = route_type 2 (DIRECT), payload_type 2
|
||||
// Header 0x09 = route_type 1 (FLOOD), payload_type 2
|
||||
// pathByte=0x00, payload=D69FD7A5A7
|
||||
payloadHex := "D69FD7A5A7"
|
||||
|
||||
// FLOOD: header=0x09 (route_type 1), pathByte=0x00
|
||||
floodHex := "09" + "00" + payloadHex
|
||||
// DIRECT: header=0x0A (route_type 2), pathByte=0x00
|
||||
directHex := "0A" + "00" + payloadHex
|
||||
|
||||
hashFlood := ComputeContentHash(floodHex)
|
||||
hashDirect := ComputeContentHash(directHex)
|
||||
if hashFlood != hashDirect {
|
||||
t.Errorf("same payload with different route types produced different hashes: flood=%s direct=%s", hashFlood, hashDirect)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashTraceIncludesPathLen verifies TRACE packets include
|
||||
// path_len in the hash (matching firmware behavior).
|
||||
func TestComputeContentHashTraceIncludesPathLen(t *testing.T) {
|
||||
// TRACE = payload_type 0x09, so header bits 2-5 = 0x09 → header = 0x09<<2 | route=2 = 0x26
|
||||
// pathByte=0x01 (1 hop, 1-byte hash) → 1 path byte
|
||||
traceHeader1 := "26" // route=2, payload_type=9
|
||||
pathByte1 := "01"
|
||||
pathData1 := "AA"
|
||||
payload := "DEADBEEF"
|
||||
hex1 := traceHeader1 + pathByte1 + pathData1 + payload
|
||||
|
||||
// Same but pathByte=0x02 (2 hops) → 2 path bytes
|
||||
pathByte2 := "02"
|
||||
pathData2 := "AABB"
|
||||
hex2 := traceHeader1 + pathByte2 + pathData2 + payload
|
||||
|
||||
hash1 := ComputeContentHash(hex1)
|
||||
hash2 := ComputeContentHash(hex2)
|
||||
if hash1 == hash2 {
|
||||
t.Error("TRACE packets with different path_len should produce different hashes (path_len is part of hash input)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashMatchesFirmware verifies hash output matches what the
|
||||
// firmware would compute: SHA256(payload_type_byte + payload)[:16hex].
|
||||
func TestComputeContentHashMatchesFirmware(t *testing.T) {
|
||||
// header=0x0A → payload_type = (0x0A >> 2) & 0x0F = 2
|
||||
// pathByte=0x00, payload = D69FD7A5A7475DB07337749AE61FA53A4788E976
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
hash := ComputeContentHash(rawHex)
|
||||
|
||||
// Manually compute expected: SHA256(0x02 + payload_bytes)
|
||||
payloadBytes, _ := hex.DecodeString("D69FD7A5A7475DB07337749AE61FA53A4788E976")
|
||||
toHash := append([]byte{0x02}, payloadBytes...)
|
||||
expected := sha256.Sum256(toHash)
|
||||
expectedHex := hex.EncodeToString(expected[:])[:16]
|
||||
if hash != expectedHex {
|
||||
t.Errorf("hash=%s, want %s (firmware-compatible)", hash, expectedHex)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeContentHashTraceGoldenValue is a golden-value test that locks down
|
||||
// the 2-byte path_len (uint16 LE) behavior for TRACE hashing. If anyone removes
|
||||
// the 0x00 byte from the hash input, this test breaks.
|
||||
//
|
||||
// Packet: header=0x25 (FLOOD route=1, payload_type=TRACE=0x09), pathByte=0x02
|
||||
// (2 hops, 1-byte hash), path=[AA,BB], payload=[DE,AD,BE,EF].
|
||||
// Hash input: [0x09, 0x02, 0x00, 0xDE, 0xAD, 0xBE, 0xEF]
|
||||
// → SHA256 = b1baaf3bf0d0726c2672b1ec9e2665dc...
|
||||
// → first 16 hex chars = "b1baaf3bf0d0726c"
|
||||
func TestComputeContentHashTraceGoldenValue(t *testing.T) {
|
||||
// TRACE packet: header byte 0x25 = payload_type 9 (TRACE), route_type 1 (FLOOD)
|
||||
// pathByte 0x02 = hash_size 1, hash_count 2
|
||||
// 2 path bytes (AA, BB), then payload DEADBEEF
|
||||
rawHex := "2502AABBDEADBEEF"
|
||||
hash := ComputeContentHash(rawHex)
|
||||
|
||||
// Pre-computed: SHA256(0x09 0x02 0x00 0xDE 0xAD 0xBE 0xEF)[:16hex]
|
||||
// The 0x00 is the high byte of uint16_t path_len (little-endian).
|
||||
const golden = "b1baaf3bf0d0726c"
|
||||
if hash != golden {
|
||||
t.Errorf("TRACE golden hash = %s, want %s (2-byte path_len encoding)", hash, golden)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePacketWithWhitespace(t *testing.T) {
|
||||
raw := "0A 00 D6 9F D7 A5 A7 47 5D B0 73 37 74 9A E6 1F A5 3A 47 88 E9 76"
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
pkt, err := DecodePacket(raw, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1056,7 +936,7 @@ func TestDecodePacketWithWhitespace(t *testing.T) {
|
||||
|
||||
func TestDecodePacketWithNewlines(t *testing.T) {
|
||||
raw := "0A00\nD69F\r\nD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
pkt, err := DecodePacket(raw, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1067,7 +947,7 @@ func TestDecodePacketWithNewlines(t *testing.T) {
|
||||
|
||||
func TestDecodePacketTransportRouteTooShort(t *testing.T) {
|
||||
// TRANSPORT_FLOOD (route=0) but only 2 bytes total → too short for transport codes
|
||||
_, err := DecodePacket("1400", nil, false)
|
||||
_, err := DecodePacket("1400", nil)
|
||||
if err == nil {
|
||||
t.Error("expected error for transport route with too-short buffer")
|
||||
}
|
||||
@@ -1126,24 +1006,24 @@ func TestDecodeHeaderUnknownTypes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDecodePayloadMultipart(t *testing.T) {
|
||||
// MULTIPART (0x0A) now decoded — #1279 P0 #2 (firmware/src/Mesh.cpp:289).
|
||||
p := decodePayload(PayloadMULTIPART, []byte{0x01, 0x02}, nil, false)
|
||||
if p.Type != "MULTIPART" {
|
||||
t.Errorf("MULTIPART type=%s, want MULTIPART", p.Type)
|
||||
// MULTIPART (0x0A) falls through to default → UNKNOWN
|
||||
p := decodePayload(PayloadMULTIPART, []byte{0x01, 0x02}, nil)
|
||||
if p.Type != "UNKNOWN" {
|
||||
t.Errorf("MULTIPART type=%s, want UNKNOWN", p.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePayloadControl(t *testing.T) {
|
||||
// CONTROL (0x0B) now decoded — #1279 P1 #4 (firmware/src/Mesh.cpp:69).
|
||||
p := decodePayload(PayloadCONTROL, []byte{0x01, 0x02}, nil, false)
|
||||
if p.Type != "CONTROL" {
|
||||
t.Errorf("CONTROL type=%s, want CONTROL", p.Type)
|
||||
// CONTROL (0x0B) falls through to default → UNKNOWN
|
||||
p := decodePayload(PayloadCONTROL, []byte{0x01, 0x02}, nil)
|
||||
if p.Type != "UNKNOWN" {
|
||||
t.Errorf("CONTROL type=%s, want UNKNOWN", p.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathTruncatedBuffer(t *testing.T) {
|
||||
// path byte claims 5 hops of 2 bytes = 10 bytes, but only 4 available
|
||||
path, consumed, _ := decodePath(0x45, []byte{0xAA, 0x11, 0xBB, 0x22}, 0)
|
||||
path, consumed := decodePath(0x45, []byte{0xAA, 0x11, 0xBB, 0x22}, 0)
|
||||
if path.HashCount != 5 {
|
||||
t.Errorf("hashCount=%d, want 5", path.HashCount)
|
||||
}
|
||||
@@ -1159,7 +1039,7 @@ func TestDecodePathTruncatedBuffer(t *testing.T) {
|
||||
func TestDecodeFloodAdvert5Hops(t *testing.T) {
|
||||
// From test-decoder.js Test 1
|
||||
raw := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
pkt, err := DecodePacket(raw, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1530,7 +1410,7 @@ func TestDecodeAdvertWithTelemetry(t *testing.T) {
|
||||
name + nullTerm +
|
||||
hex.EncodeToString(batteryLE) + hex.EncodeToString(tempLE)
|
||||
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
pkt, err := DecodePacket(hexStr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1569,7 +1449,7 @@ func TestDecodeAdvertWithTelemetryNegativeTemp(t *testing.T) {
|
||||
name + nullTerm +
|
||||
hex.EncodeToString(batteryLE) + hex.EncodeToString(tempLE)
|
||||
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
pkt, err := DecodePacket(hexStr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1596,7 +1476,7 @@ func TestDecodeAdvertWithoutTelemetry(t *testing.T) {
|
||||
name := hex.EncodeToString([]byte("Node1"))
|
||||
|
||||
hexStr := "1200" + pubkey + timestamp + signature + flags + name
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
pkt, err := DecodePacket(hexStr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1623,7 +1503,7 @@ func TestDecodeAdvertNonSensorIgnoresTelemetryBytes(t *testing.T) {
|
||||
extraBytes := "B40ED403" // battery-like and temp-like bytes
|
||||
|
||||
hexStr := "1200" + pubkey + timestamp + signature + flags + name + nullTerm + extraBytes
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
pkt, err := DecodePacket(hexStr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1651,7 +1531,7 @@ func TestDecodeAdvertTelemetryZeroTemp(t *testing.T) {
|
||||
name + nullTerm +
|
||||
hex.EncodeToString(batteryLE) + hex.EncodeToString(tempLE)
|
||||
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
pkt, err := DecodePacket(hexStr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1662,450 +1542,3 @@ func TestDecodeAdvertTelemetryZeroTemp(t *testing.T) {
|
||||
t.Errorf("temperature_c=%f, want 0.0", *pkt.Payload.TemperatureC)
|
||||
}
|
||||
}
|
||||
|
||||
func repeatHex(byteHex string, n int) string {
|
||||
s := ""
|
||||
for i := 0; i < n; i++ {
|
||||
s += byteHex
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestZeroHopDirectHashSize(t *testing.T) {
|
||||
// DIRECT (RouteType=2) + REQ (PayloadType=0) → header byte = 0x02
|
||||
// pathByte=0x00 → hash_count=0, hash_size bits=0 → should get HashSize=0
|
||||
hex := "02" + "00" + repeatHex("AA", 20)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket failed: %v", err)
|
||||
}
|
||||
if pkt.Path.HashSize != 0 {
|
||||
t.Errorf("DIRECT zero-hop: want HashSize=0, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroHopDirectHashSizeWithNonZeroUpperBits(t *testing.T) {
|
||||
// DIRECT (RouteType=2) + REQ (PayloadType=0) → header byte = 0x02
|
||||
// pathByte=0x40 → hash_count=0, hash_size bits=01 → should still get HashSize=0
|
||||
hex := "02" + "40" + repeatHex("AA", 20)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket failed: %v", err)
|
||||
}
|
||||
if pkt.Path.HashSize != 0 {
|
||||
t.Errorf("DIRECT zero-hop with hash_size bits set: want HashSize=0, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonDirectZeroPathByteKeepsHashSize(t *testing.T) {
|
||||
// FLOOD (RouteType=1) + REQ (PayloadType=0) → header byte = 0x01
|
||||
// pathByte=0x00 → non-DIRECT should keep HashSize=1
|
||||
hex := "01" + "00" + repeatHex("AA", 20)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket failed: %v", err)
|
||||
}
|
||||
if pkt.Path.HashSize != 1 {
|
||||
t.Errorf("FLOOD zero pathByte: want HashSize=1, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectNonZeroHopKeepsHashSize(t *testing.T) {
|
||||
// DIRECT (RouteType=2) + REQ (PayloadType=0) → header byte = 0x02
|
||||
// pathByte=0x01 → hash_count=1, hash_size=1 → should keep HashSize=1
|
||||
hex := "02" + "01" + repeatHex("BB", 21)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket failed: %v", err)
|
||||
}
|
||||
if pkt.Path.HashSize != 1 {
|
||||
t.Errorf("DIRECT with 1 hop: want HashSize=1, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroHopTransportDirectHashSize(t *testing.T) {
|
||||
// TRANSPORT_DIRECT (RouteType=3) + REQ (PayloadType=0) → header byte = 0x03
|
||||
// 4 bytes transport codes + pathByte=0x00 → hash_count=0 → should get HashSize=0
|
||||
hex := "03" + "11223344" + "00" + repeatHex("AA", 20)
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket failed: %v", err)
|
||||
}
|
||||
if pkt.Path.HashSize != 0 {
|
||||
t.Errorf("TRANSPORT_DIRECT zero-hop: want HashSize=0, got %d", pkt.Path.HashSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroHopTransportDirectHashSizeWithNonZeroUpperBits(t *testing.T) {
|
||||
// pathByte=0xC0 → hash_size bits=11 (4, reserved per firmware Packet.cpp:13-18).
|
||||
// Firmware Packet::isValidPathLen rejects this regardless of hash_count,
|
||||
// because hash_size==4 is reserved. Go decoder must mirror that — even
|
||||
// when hash_count==0, an attacker-emitted 0xC0 byte should not be
|
||||
// silently accepted; firmware never emits hash_size==4.
|
||||
hex := "03" + "11223344" + "C0" + repeatHex("AA", 20)
|
||||
_, err := DecodePacket(hex, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("DecodePacket(pathByte=0xC0) succeeded; want error mirroring firmware Packet.cpp:13-18 (hash_size==4 reserved)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAdvertSignature(t *testing.T) {
|
||||
// Generate a real ed25519 key pair
|
||||
pub, priv, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var timestamp uint32 = 1234567890
|
||||
appdata := []byte{0x02, 0x11, 0x22} // flags + some data
|
||||
|
||||
// Build the signed message: pubKey + timestamp(LE) + appdata
|
||||
message := make([]byte, 32+4+len(appdata))
|
||||
copy(message[0:32], pub)
|
||||
binary.LittleEndian.PutUint32(message[32:36], timestamp)
|
||||
copy(message[36:], appdata)
|
||||
|
||||
sig := ed25519.Sign(priv, message)
|
||||
|
||||
// Valid signature
|
||||
valid, err := sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp, appdata)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !valid {
|
||||
t.Error("expected valid signature")
|
||||
}
|
||||
|
||||
// Tampered appdata → invalid
|
||||
badAppdata := []byte{0x03, 0x11, 0x22}
|
||||
valid, err = sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp, badAppdata)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if valid {
|
||||
t.Error("expected invalid signature with tampered appdata")
|
||||
}
|
||||
|
||||
// Wrong timestamp → invalid
|
||||
valid, err = sigvalidate.ValidateAdvert([]byte(pub), sig, timestamp+1, appdata)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if valid {
|
||||
t.Error("expected invalid signature with wrong timestamp")
|
||||
}
|
||||
|
||||
// Wrong length pubkey
|
||||
_, err = sigvalidate.ValidateAdvert([]byte{0xAA, 0xBB}, sig, timestamp, appdata)
|
||||
if err == nil {
|
||||
t.Error("expected error for short pubkey")
|
||||
}
|
||||
|
||||
// Wrong length signature
|
||||
_, err = sigvalidate.ValidateAdvert([]byte(pub), []byte{0xAA, 0xBB}, timestamp, appdata)
|
||||
if err == nil {
|
||||
t.Error("expected error for short signature")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeAdvertWithSignatureValidation(t *testing.T) {
|
||||
// Generate key pair
|
||||
pub, priv, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var timestamp uint32 = 1000000
|
||||
appdata := []byte{0x02} // repeater type, no location
|
||||
|
||||
// Build signed message
|
||||
message := make([]byte, 32+4+len(appdata))
|
||||
copy(message[0:32], pub)
|
||||
binary.LittleEndian.PutUint32(message[32:36], timestamp)
|
||||
copy(message[36:], appdata)
|
||||
sig := ed25519.Sign(priv, message)
|
||||
|
||||
// Build advert buffer: pubkey(32) + timestamp(4) + signature(64) + appdata
|
||||
buf := make([]byte, 0, 101)
|
||||
buf = append(buf, pub...)
|
||||
ts := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(ts, timestamp)
|
||||
buf = append(buf, ts...)
|
||||
buf = append(buf, sig...)
|
||||
buf = append(buf, appdata...)
|
||||
|
||||
// With validation enabled
|
||||
p := decodeAdvert(buf, true)
|
||||
if p.Error != "" {
|
||||
t.Fatalf("decode error: %s", p.Error)
|
||||
}
|
||||
if p.SignatureValid == nil {
|
||||
t.Fatal("SignatureValid should be set when validation enabled")
|
||||
}
|
||||
if !*p.SignatureValid {
|
||||
t.Error("expected valid signature")
|
||||
}
|
||||
|
||||
// Without validation
|
||||
p2 := decodeAdvert(buf, false)
|
||||
if p2.SignatureValid != nil {
|
||||
t.Error("SignatureValid should be nil when validation disabled")
|
||||
}
|
||||
}
|
||||
|
||||
// === Tests for DecodePathFromRawHex (issue #886) ===
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize1(t *testing.T) {
|
||||
// Header byte 0x26 = route_type DIRECT, payload TRACE
|
||||
// Path byte 0x04 = hash_size 1 (bits 7-6 = 00 → 0+1=1), hash_count 4
|
||||
// Path bytes: 30 2D 0D 23
|
||||
raw := "2604302D0D2359FEE7B100000000006733D63367"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"30", "2D", "0D", "23"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize2(t *testing.T) {
|
||||
// Path byte 0x42 = hash_size 2 (bits 7-6 = 01 → 1+1=2), hash_count 2
|
||||
// Header 0x09 = FLOOD route (rt=1), payload ADVERT (pt=2)
|
||||
// Path bytes: AABB CCDD (4 bytes = 2 hops * 2 bytes)
|
||||
raw := "0942AABBCCDD" + "00000000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"AABB", "CCDD"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize3(t *testing.T) {
|
||||
// Path byte 0x81 = hash_size 3 (bits 7-6 = 10 → 2+1=3), hash_count 1
|
||||
// Header 0x09 = FLOOD route (rt=1), payload ADVERT
|
||||
raw := "0981AABBCC" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 1 || hops[0] != "AABBCC" {
|
||||
t.Fatalf("got %v, want [AABBCC]", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_HashSize4(t *testing.T) {
|
||||
// Path byte 0xC1 = hash_size 4 (bits 7-6 = 11 → 3+1=4), hash_count 1
|
||||
// Header 0x09 = FLOOD route (rt=1)
|
||||
raw := "09C1AABBCCDD" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 1 || hops[0] != "AABBCCDD" {
|
||||
t.Fatalf("got %v, want [AABBCCDD]", hops)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_DirectZeroHops(t *testing.T) {
|
||||
// Path byte 0x00 = hash_size 1, hash_count 0
|
||||
// Header 0x0A = DIRECT route (rt=2), payload ADVERT
|
||||
raw := "0A00" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hops) != 0 {
|
||||
t.Fatalf("got %d hops, want 0", len(hops))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodePathFromRawHex_Transport(t *testing.T) {
|
||||
// Route type 3 = TRANSPORT_DIRECT → 4 transport code bytes before path byte
|
||||
// Header 0x27 = route_type 3, payload TRACE
|
||||
// Transport codes: 1122 3344
|
||||
// Path byte 0x02 = hash_size 1, hash_count 2
|
||||
// Path bytes: AA BB
|
||||
raw := "2711223344" + "02AABB" + "0000000000"
|
||||
hops, err := packetpath.DecodePathFromRawHex(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []string{"AA", "BB"}
|
||||
if len(hops) != len(expected) {
|
||||
t.Fatalf("got %d hops, want %d", len(hops), len(expected))
|
||||
}
|
||||
for i, h := range hops {
|
||||
if h != expected[i] {
|
||||
t.Errorf("hop[%d] = %s, want %s", i, h, expected[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeTracePayloadFailSetsAnomaly(t *testing.T) {
|
||||
// Issue #889: TRACE packet with payload too short to decode (< 9 bytes)
|
||||
// should still return a DecodedPacket (observation stored) but with Anomaly
|
||||
// set to warn operators that the decode was degraded.
|
||||
// Packet: header 0x26 (TRACE+DIRECT), pathByte 0x00, payload 4 bytes (too short).
|
||||
pkt, err := DecodePacket("2600aabbccdd", nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.Payload.Type != "TRACE" {
|
||||
t.Fatalf("payload type=%s, want TRACE", pkt.Payload.Type)
|
||||
}
|
||||
if pkt.Payload.Error == "" {
|
||||
t.Fatal("expected payload.Error to indicate decode failure")
|
||||
}
|
||||
// The key assertion: Anomaly must be set when TRACE decode fails
|
||||
if pkt.Anomaly == "" {
|
||||
t.Error("expected Anomaly to be set when TRACE payload decode fails but observation is stored")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeTraceExtractsSNRValues verifies that for TRACE packets, the header
|
||||
// path bytes are interpreted as int8 SNR values (quarter-dB) and exposed via
|
||||
// payload.SNRValues. Mirrors logic in cmd/server/decoder.go (issue: SNR values
|
||||
// extracted by server but never written into decoded_json by ingestor).
|
||||
//
|
||||
// Packet 26022FF8116A23A80000000001C0DE1000DEDE:
|
||||
// header 0x26 → TRACE (pt=9), DIRECT (rt=2)
|
||||
// pathByte 0x02 → hash_size=1, hash_count=2
|
||||
// header path: 2F F8 → SNR = [int8(0x2F)/4, int8(0xF8)/4] = [11.75, -2.0]
|
||||
// payload (15B): tag=116A23A8 auth=00000000 flags=0x01 pathData=C0DE1000DEDE
|
||||
func TestDecodeTraceExtractsSNRValues(t *testing.T) {
|
||||
pkt, err := DecodePacket("26022FF8116A23A80000000001C0DE1000DEDE", nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket error: %v", err)
|
||||
}
|
||||
if pkt.Payload.Type != "TRACE" {
|
||||
t.Fatalf("payload type=%s, want TRACE", pkt.Payload.Type)
|
||||
}
|
||||
if len(pkt.Payload.SNRValues) != 2 {
|
||||
t.Fatalf("len(SNRValues)=%d, want 2 (got %v)", len(pkt.Payload.SNRValues), pkt.Payload.SNRValues)
|
||||
}
|
||||
if pkt.Payload.SNRValues[0] != 11.75 {
|
||||
t.Errorf("SNRValues[0]=%v, want 11.75", pkt.Payload.SNRValues[0])
|
||||
}
|
||||
if pkt.Payload.SNRValues[1] != -2.0 {
|
||||
t.Errorf("SNRValues[1]=%v, want -2.0", pkt.Payload.SNRValues[1])
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodePacketBoundsFromWire — regression for issue #1211.
|
||||
//
|
||||
// A malformed packet on the wire claimed pathByte=0xF6 (hash_size=4, hash_count=54
|
||||
// → 216 path bytes) inside a 15-byte buffer. decodePath() returned bytesConsumed=216
|
||||
// without bounds-check, causing the outer slice `payloadBuf := buf[offset:]` to
|
||||
// blow up with `slice bounds out of range [218:15]`.
|
||||
//
|
||||
// Expected behaviour: DecodePacket MUST NOT panic on any input. If the path
|
||||
// length claimed by the wire byte exceeds the buffer, it should return a
|
||||
// clean error.
|
||||
func TestDecodePacketBoundsFromWire_Issue1211(t *testing.T) {
|
||||
// 15-byte buffer: header=0x12 (rt=DIRECT, pt=ADVERT), pathByte=0xF6
|
||||
// (hash_size=4, hash_count=54 → claims 216 path bytes), + 13 garbage bytes.
|
||||
raw := "12F6" + strings.Repeat("AA", 13)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("DecodePacket panicked on malformed input: %v", r)
|
||||
}
|
||||
}()
|
||||
pkt, err := DecodePacket(raw, nil, false)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for malformed packet (path claims 216 bytes in 15-byte buf), got nil; pkt=%+v", pkt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodePacketFuzzTruncated — sweep the decoder with truncated payloads.
|
||||
// Zero panics is the acceptance bar.
|
||||
//
|
||||
// Adv M2: the original loop ran 256*256*20 = 1.3M iterations on every
|
||||
// `go test` (in both packages, so 2.6M total). That is not "fuzzing" — it
|
||||
// is an expensive deterministic sweep that runs in the default unit-test
|
||||
// path with no opt-in. We now:
|
||||
//
|
||||
// - gate the exhaustive sweep on !testing.Short() so `go test -short`
|
||||
// skips it (CI's unit gate runs short)
|
||||
// - keep the full sweep under `go test ./...` to preserve coverage
|
||||
// - prefer `go test -fuzz=FuzzDecodePacketTruncated` for actual
|
||||
// randomized fuzzing (see FuzzDecodePacketTruncated below)
|
||||
func TestDecodePacketFuzzTruncated_Issue1211(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("DecodePacket panicked during fuzz: %v", r)
|
||||
}
|
||||
}()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping exhaustive sweep in -short mode; use FuzzDecodePacketTruncated")
|
||||
}
|
||||
// Sweep every pathByte value with a short tail.
|
||||
for hdr := 0; hdr < 256; hdr++ {
|
||||
for pb := 0; pb < 256; pb++ {
|
||||
for tail := 0; tail < 20; tail++ {
|
||||
raw := hex.EncodeToString([]byte{byte(hdr), byte(pb)}) + strings.Repeat("00", tail)
|
||||
_, _ = DecodePacket(raw, nil, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FuzzDecodePacketTruncated — native go fuzz target. Run with:
|
||||
//
|
||||
// go test -fuzz=FuzzDecodePacketTruncated -fuzztime=30s ./cmd/ingestor
|
||||
//
|
||||
// Zero panics regardless of input is the acceptance bar.
|
||||
func FuzzDecodePacketTruncated(f *testing.F) {
|
||||
seeds := [][]byte{
|
||||
{0x12, 0xF6, 0xAA, 0xAA, 0xAA},
|
||||
{0x12, 0x00},
|
||||
{0x03, 0x11, 0x22, 0x33, 0x44, 0xC0, 0xAA, 0xAA, 0xAA},
|
||||
}
|
||||
for _, s := range seeds {
|
||||
f.Add(s)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("DecodePacket panicked on input %x: %v", data, r)
|
||||
}
|
||||
}()
|
||||
_, _ = DecodePacket(hex.EncodeToString(data), nil, false)
|
||||
})
|
||||
}
|
||||
|
||||
// TestDecodeAdvertOversizedNameTruncated asserts decodeAdvert truncates the
|
||||
// advert name to firmware's MAX_ADVERT_DATA_SIZE=32 (firmware/src/MeshCore.h:11).
|
||||
// Firmware writes the node name into a 32-byte buffer, so any on-wire advert
|
||||
// carrying >32 bytes of name data is adversarial — the Go decoder must not
|
||||
// surface attacker-controlled bytes beyond what firmware would ever emit.
|
||||
func TestDecodeAdvertOversizedNameTruncated(t *testing.T) {
|
||||
pubkey := repeatHex("AA", 32)
|
||||
timestamp := "78563412"
|
||||
signature := repeatHex("BB", 64)
|
||||
flags := "81" // chat(1) | hasName(0x80), no location, no feat1/2
|
||||
// 64-byte ASCII 'X' name with no null terminator (firmware buffer is 32 bytes).
|
||||
name := repeatHex("58", 64)
|
||||
hex := "1200" + pubkey + timestamp + signature + flags + name
|
||||
pkt, err := DecodePacket(hex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
if got := len(pkt.Payload.Name); got > 32 {
|
||||
t.Errorf("name length=%d, want <=32 (MAX_ADVERT_DATA_SIZE firmware/src/MeshCore.h:11)", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestHandleMessageAdvertForeign_FlagModeStoresWithFlag asserts that when an
|
||||
// ADVERT comes from a node whose GPS is OUTSIDE the configured geofilter,
|
||||
// the ingestor (in default "flag" mode) stores the node and marks it foreign,
|
||||
// instead of silently dropping it (#730).
|
||||
func TestHandleMessageAdvertForeign_FlagModeStoresWithFlag(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
// Real ADVERT raw hex from existing TestHandleMessageAdvertGeoFiltered.
|
||||
// Decoder will produce a node with a known GPS — the test below just
|
||||
// asserts that with a tight geofilter that EXCLUDES that GPS, the node
|
||||
// is still stored AND tagged as foreign.
|
||||
rawHex := "120046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
|
||||
latMin, latMax := -1.0, 1.0
|
||||
lonMin, lonMax := -1.0, 1.0
|
||||
gf := &GeoFilterConfig{
|
||||
LatMin: &latMin, LatMax: &latMax,
|
||||
LonMin: &lonMin, LonMax: &lonMax,
|
||||
}
|
||||
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
// Default mode (no ForeignAdverts.Mode set) MUST be "flag", per #730 design.
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{GeoFilter: gf})
|
||||
|
||||
var nodeCount int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&nodeCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeCount != 1 {
|
||||
t.Fatalf("nodes=%d, want 1 (foreign advert should be stored, not dropped, in flag mode)", nodeCount)
|
||||
}
|
||||
|
||||
var foreign int
|
||||
if err := store.db.QueryRow("SELECT foreign_advert FROM nodes").Scan(&foreign); err != nil {
|
||||
t.Fatalf("foreign_advert column missing or unreadable: %v", err)
|
||||
}
|
||||
if foreign != 1 {
|
||||
t.Errorf("foreign_advert=%d, want 1", foreign)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessageAdvertForeign_DropModeStillDrops asserts the legacy
|
||||
// drop-on-foreign behavior is preserved when ForeignAdverts.Mode = "drop".
|
||||
func TestHandleMessageAdvertForeign_DropModeStillDrops(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
rawHex := "120046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
|
||||
latMin, latMax := -1.0, 1.0
|
||||
lonMin, lonMax := -1.0, 1.0
|
||||
gf := &GeoFilterConfig{
|
||||
LatMin: &latMin, LatMax: &latMax,
|
||||
LonMin: &lonMin, LonMax: &lonMax,
|
||||
}
|
||||
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
cfg := &Config{
|
||||
GeoFilter: gf,
|
||||
ForeignAdverts: &ForeignAdvertConfig{Mode: "drop"},
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var nodeCount int
|
||||
if err := store.db.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&nodeCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeCount != 0 {
|
||||
t.Errorf("nodes=%d, want 0 (drop mode preserves legacy silent-drop behavior)", nodeCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessageAdvertInRegion_NotFlaggedForeign asserts in-region
|
||||
// adverts are NOT marked foreign.
|
||||
func TestHandleMessageAdvertInRegion_NotFlaggedForeign(t *testing.T) {
|
||||
store, source := newTestContext(t)
|
||||
|
||||
rawHex := "120046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
|
||||
// Wide-open geofilter: every coord passes.
|
||||
latMin, latMax := -90.0, 90.0
|
||||
lonMin, lonMax := -180.0, 180.0
|
||||
gf := &GeoFilterConfig{
|
||||
LatMin: &latMin, LatMax: &latMax,
|
||||
LonMin: &lonMin, LonMax: &lonMax,
|
||||
}
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{GeoFilter: gf})
|
||||
|
||||
var foreign int
|
||||
err := store.db.QueryRow("SELECT foreign_advert FROM nodes").Scan(&foreign)
|
||||
if err != nil {
|
||||
t.Fatalf("query foreign_advert: %v", err)
|
||||
}
|
||||
if foreign != 0 {
|
||||
t.Errorf("foreign_advert=%d, want 0 (in-region node)", foreign)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for #1143: ingestor must populate transmissions.from_pubkey at
|
||||
// write time (cheap — already parsing decoded_json) so attribution queries
|
||||
// don't rely on JSON substring matches.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
const pk = "f7181c468dfe7c55aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
data := &PacketData{
|
||||
RawHex: "AABBCC",
|
||||
Timestamp: "2026-03-25T00:00:00Z",
|
||||
ObserverID: "obs1",
|
||||
Hash: "advert_hash_1143",
|
||||
RouteType: 1,
|
||||
PayloadType: 4, // ADVERT
|
||||
PayloadVersion: 0,
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"ADVERT","pubKey":"` + pk + `","name":"X"}`,
|
||||
FromPubkey: pk,
|
||||
}
|
||||
if _, err := s.InsertTransmission(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var got sql.NullString
|
||||
s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got)
|
||||
if !got.Valid || got.String != pk {
|
||||
t.Fatalf("from_pubkey = %v (valid=%v), want %q", got.String, got.Valid, pk)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
data := &PacketData{
|
||||
RawHex: "AA",
|
||||
Timestamp: "2026-03-25T00:00:00Z",
|
||||
ObserverID: "obs1",
|
||||
Hash: "txt_hash_1143",
|
||||
RouteType: 1,
|
||||
PayloadType: 2, // TXT_MSG
|
||||
PayloadVersion: 0,
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
// FromPubkey deliberately empty — non-ADVERTs don't carry one.
|
||||
}
|
||||
if _, err := s.InsertTransmission(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var got sql.NullString
|
||||
s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got)
|
||||
if got.Valid {
|
||||
t.Fatalf("from_pubkey for non-ADVERT must be NULL, got %q", got.String)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPacketData_PopulatesFromPubkey(t *testing.T) {
|
||||
const pk = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
|
||||
msg := &MQTTPacketMessage{Raw: "AA", Origin: "obs"}
|
||||
decoded := &DecodedPacket{
|
||||
Header: Header{PayloadType: PayloadADVERT},
|
||||
Payload: Payload{Type: "ADVERT", PubKey: pk},
|
||||
}
|
||||
pd := BuildPacketData(msg, decoded, "obs", "", nil)
|
||||
if pd.FromPubkey != pk {
|
||||
t.Fatalf("BuildPacketData FromPubkey = %q, want %q", pd.FromPubkey, pk)
|
||||
}
|
||||
|
||||
// Non-ADVERT: must not carry a pubkey.
|
||||
decoded2 := &DecodedPacket{
|
||||
Header: Header{PayloadType: 2},
|
||||
Payload: Payload{Type: "TXT_MSG"},
|
||||
}
|
||||
pd2 := BuildPacketData(msg, decoded2, "obs", "", nil)
|
||||
if pd2.FromPubkey != "" {
|
||||
t.Fatalf("BuildPacketData FromPubkey for non-ADVERT = %q, want empty", pd2.FromPubkey)
|
||||
}
|
||||
}
|
||||
@@ -5,30 +5,11 @@ go 1.22
|
||||
require (
|
||||
github.com/eclipse/paho.mqtt.golang v1.5.0
|
||||
github.com/meshcore-analyzer/geofilter v0.0.0
|
||||
github.com/meshcore-analyzer/sigvalidate v0.0.0
|
||||
modernc.org/sqlite v1.34.5
|
||||
)
|
||||
|
||||
replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter
|
||||
|
||||
replace github.com/meshcore-analyzer/sigvalidate => ../../internal/sigvalidate
|
||||
|
||||
require github.com/meshcore-analyzer/packetpath v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath
|
||||
|
||||
require github.com/meshcore-analyzer/dbconfig v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig
|
||||
|
||||
require github.com/meshcore-analyzer/perfio v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/perfio => ../../internal/perfio
|
||||
|
||||
require github.com/meshcore-analyzer/dbschema v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -43,11 +24,3 @@ require (
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
)
|
||||
|
||||
require github.com/meshcore-analyzer/prunequeue v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue
|
||||
|
||||
require github.com/meshcore-analyzer/mbcapqueue v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression test for issue #1370 — counters PR #1233 (commit 498fbc03).
|
||||
//
|
||||
// PR #1233 made the ingestor use the MQTT envelope's "timestamp" field as
|
||||
// transmissions.first_seen / observations.timestamp, on the premise that
|
||||
// uploaders stamp it at radio receive and the value is trustworthy.
|
||||
//
|
||||
// That premise FAILS for observers whose own clock is wrong. Staging
|
||||
// Voodoo3 tx 304114 in channel #test had 5 observations:
|
||||
// - 4 from Voodoo3 stamped "18:42" — Voodoo3's broken client clock,
|
||||
// - 1 from another observer stamped "01:42" — the actual receive time.
|
||||
// Voodoo3 ingested first, so first_seen locked at "18:42" and the
|
||||
// /api/channels row showed the channel as last-active 7h+ in the past.
|
||||
//
|
||||
// Fix: revert the storage path — packet/observation timestamps are
|
||||
// server ingest time (time.Now() at the ingestor). Envelope timestamp
|
||||
// stays usable for observer.last_seen (PR #1233's MAX/MIN guard there
|
||||
// is fine and unrelated to the channel-ordering bug).
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Raw packet path: envelope reports timestamp 7h in the past
|
||||
// (simulating Voodoo3's broken client clock). After ingest,
|
||||
// transmissions.first_seen and observations.timestamp must reflect
|
||||
// SERVER wall clock, not the bogus envelope value.
|
||||
func TestHandleMessage_PacketTimestamp_IgnoresStaleEnvelope_1370(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"voodoo3","timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/voodoo3/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
// ─── transmissions.first_seen ───────────────────────────────────────
|
||||
var firstSeen string
|
||||
if err := store.db.QueryRow(`SELECT first_seen FROM transmissions LIMIT 1`).Scan(&firstSeen); err != nil {
|
||||
t.Fatalf("scan first_seen: %v", err)
|
||||
}
|
||||
fsParsed, err := time.Parse(time.RFC3339, firstSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("first_seen %q not RFC3339: %v", firstSeen, err)
|
||||
}
|
||||
if fsParsed.Unix() < before-5 || fsParsed.Unix() > after+5 {
|
||||
t.Errorf("transmissions.first_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope reported stale %q (7h ago) — PR #1233's premise that envelope timestamp is trustworthy is FALSE for buggy-clock observers. Issue #1370.",
|
||||
firstSeen, fsParsed.Unix(), before, after, stale)
|
||||
}
|
||||
|
||||
// ─── observations.timestamp (epoch) ─────────────────────────────────
|
||||
var obsTs int64
|
||||
if err := store.db.QueryRow(`SELECT timestamp FROM observations LIMIT 1`).Scan(&obsTs); err != nil {
|
||||
t.Fatalf("scan observations.timestamp: %v", err)
|
||||
}
|
||||
if obsTs < before-5 || obsTs > after+5 {
|
||||
t.Errorf("observations.timestamp = %d; want in [%d, %d] (server wall clock). Envelope stale = %q. Issue #1370.",
|
||||
obsTs, before, after, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// Channel-message (BLE companion) path: envelope timestamp stale → stored
|
||||
// transmissions.first_seen must still be server wall clock.
|
||||
func TestHandleMessage_ChannelPath_PacketTimestamp_IgnoresStaleEnvelope_1370(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"text":"Voodoo3: tst hmdpt","channel_idx":3,"SNR":5.0,"RSSI":-95,"timestamp":"` + stale + `","sender_timestamp":` + strconv.FormatInt(time.Now().Unix(), 10) + `}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/channel/3", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var firstSeen string
|
||||
if err := store.db.QueryRow(`SELECT first_seen FROM transmissions LIMIT 1`).Scan(&firstSeen); err != nil {
|
||||
t.Fatalf("scan first_seen: %v", err)
|
||||
}
|
||||
fsParsed, err := time.Parse(time.RFC3339, firstSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("first_seen %q not RFC3339: %v", firstSeen, err)
|
||||
}
|
||||
if fsParsed.Unix() < before-5 || fsParsed.Unix() > after+5 {
|
||||
t.Errorf("channel-path transmissions.first_seen = %q (epoch %d); want in [%d, %d] (server wall clock). Envelope stale = %q. Issue #1370.",
|
||||
firstSeen, fsParsed.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// DM (BLE companion direct-message) path: same revert applies.
|
||||
func TestHandleMessage_DMPath_PacketTimestamp_IgnoresStaleEnvelope_1370(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"text":"Voodoo3: hello","SNR":5.0,"RSSI":-95,"timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/message/direct/voodoo3", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var firstSeen string
|
||||
if err := store.db.QueryRow(`SELECT first_seen FROM transmissions LIMIT 1`).Scan(&firstSeen); err != nil {
|
||||
t.Fatalf("scan first_seen: %v", err)
|
||||
}
|
||||
fsParsed, err := time.Parse(time.RFC3339, firstSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("first_seen %q not RFC3339: %v", firstSeen, err)
|
||||
}
|
||||
if fsParsed.Unix() < before-5 || fsParsed.Unix() > after+5 {
|
||||
t.Errorf("DM-path transmissions.first_seen = %q (epoch %d); want in [%d, %d] (server wall clock). Envelope stale = %q. Issue #1370.",
|
||||
firstSeen, fsParsed.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for issue #1279 P2 item 5: ingestor RAW_CUSTOM exposure.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDecodeRawCustomExposesLengthAndTag(t *testing.T) {
|
||||
// header = (1<<6)|(0x0F<<2)|1 = 0x7D ; path byte = 0x00 ; payload = A5 DE AD BE EF
|
||||
hexStr := "7D00A5DEADBEEF"
|
||||
pkt, err := DecodePacket(hexStr, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if pkt.Payload.Type != "RAW_CUSTOM" {
|
||||
t.Fatalf("payload type = %q, want RAW_CUSTOM", pkt.Payload.Type)
|
||||
}
|
||||
if pkt.Payload.RawLength == nil || *pkt.Payload.RawLength != 5 {
|
||||
got := -1
|
||||
if pkt.Payload.RawLength != nil {
|
||||
got = *pkt.Payload.RawLength
|
||||
}
|
||||
t.Errorf("RawLength=%d, want 5", got)
|
||||
}
|
||||
if !strings.EqualFold(pkt.Payload.FirstByteTag, "A5") {
|
||||
t.Errorf("FirstByteTag=%q, want A5", pkt.Payload.FirstByteTag)
|
||||
}
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for issue #1279 P0+P1 decoder additions.
|
||||
//
|
||||
// Each test uses firmware-derived wire vectors:
|
||||
// - GRP_DATA outer: firmware/src/helpers/BaseChatMesh.cpp:500 (createGroupDatagram)
|
||||
// - GRP_DATA inner: firmware/src/helpers/BaseChatMesh.cpp:382-385
|
||||
// - MULTIPART byte0: firmware/src/Mesh.cpp:289
|
||||
// - MULTIPART ACK inner: firmware/src/Mesh.cpp:292-307
|
||||
// - CONTROL byte0 flags: firmware/src/Mesh.cpp:69 + createControlData at Mesh.cpp:609
|
||||
// - advertRole label rules: firmware/src/helpers/AdvertDataHelpers.h:7-12
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- P0 #1: GRP_DATA decoder ---
|
||||
|
||||
// buildChannelEncrypted encrypts arbitrary inner bytes with the channel
|
||||
// key/MAC scheme firmware uses for both GRP_TXT and GRP_DATA (see
|
||||
// BaseChatMesh.cpp:376-391: AES-128-ECB, HMAC-SHA256-trunc-2 MAC).
|
||||
func buildChannelEncrypted(channelKeyHex string, inner []byte) (ctHex, macHex string) {
|
||||
key, _ := hex.DecodeString(channelKeyHex)
|
||||
plain := append([]byte{}, inner...)
|
||||
pad := aes.BlockSize - (len(plain) % aes.BlockSize)
|
||||
if pad != aes.BlockSize {
|
||||
plain = append(plain, make([]byte, pad)...)
|
||||
}
|
||||
block, _ := aes.NewCipher(key)
|
||||
ct := make([]byte, len(plain))
|
||||
for i := 0; i < len(plain); i += aes.BlockSize {
|
||||
block.Encrypt(ct[i:i+aes.BlockSize], plain[i:i+aes.BlockSize])
|
||||
}
|
||||
secret := make([]byte, 32)
|
||||
copy(secret, key)
|
||||
h := hmac.New(sha256.New, secret)
|
||||
h.Write(ct)
|
||||
mac := h.Sum(nil)
|
||||
return hex.EncodeToString(ct), hex.EncodeToString(mac[:2])
|
||||
}
|
||||
|
||||
func TestDecodeGrpDataNoKey(t *testing.T) {
|
||||
// Envelope alone (no key in store).
|
||||
buf := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11}
|
||||
p := decodeGrpData(buf, nil)
|
||||
if p.Type != "GRP_DATA" {
|
||||
t.Fatalf("type=%q want GRP_DATA", p.Type)
|
||||
}
|
||||
if p.ChannelHash != 0xAA {
|
||||
t.Errorf("channelHash=%d want 170", p.ChannelHash)
|
||||
}
|
||||
if p.ChannelHashHex != "AA" {
|
||||
t.Errorf("channelHashHex=%q want AA", p.ChannelHashHex)
|
||||
}
|
||||
if p.MAC != "bbcc" {
|
||||
t.Errorf("mac=%q want bbcc", p.MAC)
|
||||
}
|
||||
if p.EncryptedData != "ddeeff11" {
|
||||
t.Errorf("encryptedData=%q want ddeeff11", p.EncryptedData)
|
||||
}
|
||||
if p.DecryptionStatus != "no_key" {
|
||||
t.Errorf("decryptionStatus=%q want no_key", p.DecryptionStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeGrpDataDecryptedInner(t *testing.T) {
|
||||
// Inner per BaseChatMesh.cpp:382-385: data_type(uint16 LE) + data_len(1) + blob.
|
||||
key := "2cc3d22840e086105ad73443da2cacb8"
|
||||
blob := []byte{0x10, 0x20, 0x30, 0x40, 0x50}
|
||||
inner := []byte{0x34, 0x12, byte(len(blob))} // data_type = 0x1234
|
||||
inner = append(inner, blob...)
|
||||
ctHex, macHex := buildChannelEncrypted(key, inner)
|
||||
|
||||
buf := []byte{0xAB}
|
||||
mb, _ := hex.DecodeString(macHex)
|
||||
buf = append(buf, mb...)
|
||||
cb, _ := hex.DecodeString(ctHex)
|
||||
buf = append(buf, cb...)
|
||||
|
||||
p := decodeGrpData(buf, map[string]string{"test": key})
|
||||
if p.Type != "GRP_DATA" {
|
||||
t.Fatalf("type=%q want GRP_DATA", p.Type)
|
||||
}
|
||||
if p.DecryptionStatus != "decrypted" {
|
||||
t.Fatalf("decryptionStatus=%q want decrypted", p.DecryptionStatus)
|
||||
}
|
||||
if p.DataType == nil || *p.DataType != 0x1234 {
|
||||
t.Errorf("dataType=%v want 0x1234", p.DataType)
|
||||
}
|
||||
if p.DataLen == nil || *p.DataLen != 5 {
|
||||
t.Errorf("dataLen=%v want 5", p.DataLen)
|
||||
}
|
||||
if p.DecryptedBlob != hex.EncodeToString(blob) {
|
||||
t.Errorf("decryptedBlob=%q want %q", p.DecryptedBlob, hex.EncodeToString(blob))
|
||||
}
|
||||
if p.Channel != "test" {
|
||||
t.Errorf("channel=%q want test", p.Channel)
|
||||
}
|
||||
}
|
||||
|
||||
// --- P0 #2: MULTIPART decoder ---
|
||||
|
||||
func TestDecodeMultipartAck(t *testing.T) {
|
||||
// remaining=3, inner_type=PAYLOAD_TYPE_ACK(0x03), ack_crc=0xDEADBEEF.
|
||||
// byte0 = (3<<4) | 3 = 0x33; next 4 bytes are LE crc.
|
||||
buf := []byte{0x33, 0xEF, 0xBE, 0xAD, 0xDE}
|
||||
p := decodeMultipart(buf)
|
||||
if p.Type != "MULTIPART" {
|
||||
t.Fatalf("type=%q want MULTIPART", p.Type)
|
||||
}
|
||||
if p.Remaining == nil || *p.Remaining != 3 {
|
||||
t.Errorf("remaining=%v want 3", p.Remaining)
|
||||
}
|
||||
if p.InnerType == nil || *p.InnerType != 0x03 {
|
||||
t.Errorf("innerType=%v want 3", p.InnerType)
|
||||
}
|
||||
if p.InnerTypeName != "ACK" {
|
||||
t.Errorf("innerTypeName=%q want ACK", p.InnerTypeName)
|
||||
}
|
||||
if p.InnerAckCrc != "deadbeef" {
|
||||
t.Errorf("innerAckCrc=%q want deadbeef", p.InnerAckCrc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeMultipartNonAck(t *testing.T) {
|
||||
// remaining=2, inner_type=0x02 (TXT_MSG), arbitrary inner payload.
|
||||
buf := []byte{0x22, 0x01, 0x02, 0x03}
|
||||
p := decodeMultipart(buf)
|
||||
if p.Remaining == nil || *p.Remaining != 2 {
|
||||
t.Errorf("remaining=%v want 2", p.Remaining)
|
||||
}
|
||||
if p.InnerType == nil || *p.InnerType != 0x02 {
|
||||
t.Errorf("innerType=%v want 2", p.InnerType)
|
||||
}
|
||||
if p.InnerTypeName != "TXT_MSG" {
|
||||
t.Errorf("innerTypeName=%q want TXT_MSG", p.InnerTypeName)
|
||||
}
|
||||
if p.InnerPayload != "010203" {
|
||||
t.Errorf("innerPayload=%q want 010203", p.InnerPayload)
|
||||
}
|
||||
if p.InnerAckCrc != "" {
|
||||
t.Errorf("non-ACK should not surface innerAckCrc, got %q", p.InnerAckCrc)
|
||||
}
|
||||
}
|
||||
|
||||
// --- P1 #3: advertRole label fix ---
|
||||
|
||||
func TestAdvertRoleLabelsRawType(t *testing.T) {
|
||||
// Firmware: ADV_TYPE_NONE=0, CHAT=1, REPEATER=2, ROOM=3, SENSOR=4, 5..15 FUTURE.
|
||||
cases := []struct {
|
||||
typ int
|
||||
want string
|
||||
}{
|
||||
{0, "none"},
|
||||
{1, "companion"},
|
||||
{2, "repeater"},
|
||||
{3, "room"},
|
||||
{4, "sensor"},
|
||||
{5, "type-5"},
|
||||
{15, "type-15"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := advertRole(&AdvertFlags{Type: tc.typ, Repeater: tc.typ == 2, Room: tc.typ == 3, Sensor: tc.typ == 4})
|
||||
if got != tc.want {
|
||||
t.Errorf("advertRole(type=%d) = %q, want %q", tc.typ, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- P1 #4: CONTROL byte0 flags ---
|
||||
|
||||
func TestDecodeControlZeroHop(t *testing.T) {
|
||||
// byte0 = 0x81 (high-bit set ⇒ zero-hop), followed by 3 app bytes.
|
||||
buf := []byte{0x81, 0xAA, 0xBB, 0xCC}
|
||||
p := decodeControl(buf)
|
||||
if p.Type != "CONTROL" {
|
||||
t.Fatalf("type=%q want CONTROL", p.Type)
|
||||
}
|
||||
if p.CtrlFlags != "81" {
|
||||
t.Errorf("ctrlFlags=%q want 81", p.CtrlFlags)
|
||||
}
|
||||
if p.CtrlZeroHop == nil || !*p.CtrlZeroHop {
|
||||
t.Errorf("ctrlZeroHop=%v want true", p.CtrlZeroHop)
|
||||
}
|
||||
if p.CtrlLength == nil || *p.CtrlLength != 4 {
|
||||
t.Errorf("ctrlLength=%v want 4", p.CtrlLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControlMultiHop(t *testing.T) {
|
||||
// byte0 = 0x01 (high-bit clear ⇒ not zero-hop subset).
|
||||
buf := []byte{0x01, 0x42}
|
||||
p := decodeControl(buf)
|
||||
if p.CtrlFlags != "01" {
|
||||
t.Errorf("ctrlFlags=%q want 01", p.CtrlFlags)
|
||||
}
|
||||
if p.CtrlZeroHop == nil || *p.CtrlZeroHop {
|
||||
t.Errorf("ctrlZeroHop=%v want false", p.CtrlZeroHop)
|
||||
}
|
||||
if p.CtrlLength == nil || *p.CtrlLength != 2 {
|
||||
t.Errorf("ctrlLength=%v want 2", p.CtrlLength)
|
||||
}
|
||||
}
|
||||
|
||||
// silence unused-import diagnostics for stub-phase builds
|
||||
var _ = binary.LittleEndian
|
||||
@@ -1,98 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// TestIngestorPruneOldPackets enforces #1283: the writer for
|
||||
// transmissions retention lives on the ingestor's *Store. Before the fix,
|
||||
// this lived on cmd/server/*DB and raced with ingestor INSERTs. After
|
||||
// the fix, ingestor owns it and runs it on its own write-locked handle.
|
||||
func TestIngestorPruneOldPackets(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "prune.db")
|
||||
store, err := OpenStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
old := time.Now().UTC().AddDate(0, 0, -10).Format(time.RFC3339)
|
||||
new := time.Now().UTC().Format(time.RFC3339)
|
||||
for i, ts := range []string{old, old, new} {
|
||||
_, err := store.db.Exec(
|
||||
`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json)
|
||||
VALUES (?, ?, ?, 0, 1, 1, '{}')`,
|
||||
"AA", "h"+string(rune('a'+i)), ts,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("seed tx: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := store.PruneOldPackets(5)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneOldPackets: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("expected 2 pruned, got %d", n)
|
||||
}
|
||||
|
||||
var remaining int
|
||||
if err := store.db.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&remaining); err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if remaining != 1 {
|
||||
t.Fatalf("expected 1 transmission remaining, got %d", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL exercises the
|
||||
// scenario that originally broke in #1283: a fresh DB with
|
||||
// auto_vacuum=NONE, vacuumOnStartup=true, no contention from a server
|
||||
// process. The ingestor must complete the VACUUM and flip auto_vacuum to
|
||||
// INCREMENTAL. Before the fix, the migration ran inside cmd/server and
|
||||
// hit SQLITE_BUSY because the ingestor (sharing the container) was
|
||||
// already writing.
|
||||
func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "vac.db")
|
||||
|
||||
// Create a NONE-auto_vacuum DB (simulates an older deployment).
|
||||
seed, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed.SetMaxOpenConns(1)
|
||||
if _, err := seed.Exec(`CREATE TABLE dummy(id INTEGER PRIMARY KEY)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var before int
|
||||
seed.QueryRow("PRAGMA auto_vacuum").Scan(&before)
|
||||
if before != 0 {
|
||||
t.Fatalf("precondition: auto_vacuum=%d, want 0", before)
|
||||
}
|
||||
seed.Close()
|
||||
|
||||
store, err := OpenStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
cfg := &Config{DB: &DBConfig{VacuumOnStartup: true}}
|
||||
store.CheckAutoVacuum(cfg)
|
||||
|
||||
var after int
|
||||
if err := store.db.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after != 2 {
|
||||
t.Fatalf("expected auto_vacuum=2 after ingestor VACUUM, got %d", after)
|
||||
}
|
||||
}
|
||||
+70
-673
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
@@ -12,13 +11,11 @@ import (
|
||||
"math"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -52,141 +49,29 @@ func main() {
|
||||
}
|
||||
|
||||
sources := cfg.ResolvedSources()
|
||||
if len(sources) == 0 {
|
||||
log.Fatal("no MQTT sources configured — set mqttSources in config or MQTT_BROKER env var")
|
||||
}
|
||||
|
||||
store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval())
|
||||
store, err := OpenStore(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
log.Printf("SQLite opened: %s", cfg.DBPath)
|
||||
|
||||
// Async backfill: path_json from raw_hex (#888) — must not block MQTT startup
|
||||
store.BackfillPathJSONAsync()
|
||||
|
||||
// Soft-delete blacklisted observers (#1287 — moved from cmd/server).
|
||||
if len(cfg.ObserverBlacklist) > 0 {
|
||||
store.SoftDeleteBlacklistedObservers(cfg.ObserverBlacklist)
|
||||
}
|
||||
|
||||
// Async backfill: from_pubkey for legacy ADVERT rows (#1143).
|
||||
// Moved from cmd/server in #1287. Best-effort; must not block MQTT.
|
||||
go store.BackfillFromPubkey(5000, 100*time.Millisecond, nil)
|
||||
|
||||
// Check auto_vacuum mode and optionally migrate (#919)
|
||||
store.CheckAutoVacuum(cfg)
|
||||
|
||||
// Node retention: move stale nodes to inactive_nodes on startup
|
||||
nodeDays := cfg.NodeDaysOrDefault()
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
|
||||
// Observer retention: remove stale observers on startup
|
||||
observerDays := cfg.ObserverDaysOrDefault()
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
|
||||
// Metrics retention: prune old metrics on startup
|
||||
metricsDays := cfg.MetricsRetentionDays()
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
|
||||
// Packet (transmissions) retention: previously lived in cmd/server,
|
||||
// moved to ingestor in #1283 to eliminate cross-process write
|
||||
// contention (SQLITE_BUSY). 0 = disabled.
|
||||
packetDays := cfg.PacketDaysOrZero()
|
||||
if packetDays > 0 {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[prune] startup pruned %d transmissions older than %d days", n, packetDays)
|
||||
}
|
||||
}
|
||||
|
||||
vacuumPages := cfg.IncrementalVacuumPages()
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
|
||||
// Daily ticker for node retention
|
||||
retentionTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
for range retentionTicker.C {
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for observer retention (every 24h, staggered 90s after startup)
|
||||
observerRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(90 * time.Second) // stagger after metrics prune
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
for range observerRetentionTicker.C {
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for metrics retention (every 24h)
|
||||
metricsRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range metricsRetentionTicker.C {
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for transmission retention (#1283).
|
||||
var packetRetentionTicker *time.Ticker
|
||||
if packetDays > 0 {
|
||||
packetRetentionTicker = time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range packetRetentionTicker.C {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays)
|
||||
}
|
||||
|
||||
// Hourly WAL checkpoint to prevent unbounded WAL growth.
|
||||
// TRUNCATE resets the WAL file to zero bytes when all frames are flushed;
|
||||
// if the server's read connection holds frames, remaining pages stay in the
|
||||
// WAL until the next tick. Staggered 30s after startup to avoid competing
|
||||
// with the initial burst of ingest writes.
|
||||
walCheckpointTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(30 * time.Second)
|
||||
store.Checkpoint()
|
||||
for range walCheckpointTicker.C {
|
||||
store.Checkpoint()
|
||||
}
|
||||
}()
|
||||
log.Printf("[db] WAL checkpoint scheduled every 1h")
|
||||
|
||||
// Daily neighbor_edges retention (#1287 — moved from cmd/server).
|
||||
{
|
||||
nDays := cfg.NeighborEdgesDaysOrDefault()
|
||||
neighborPruneTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(4 * time.Minute) // stagger
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] startup pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
for range neighborPruneTicker.C {
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[neighbor-prune] auto-prune enabled: edges older than %d days", nDays)
|
||||
}
|
||||
|
||||
// Periodic stats logging (every 5 minutes)
|
||||
statsTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
@@ -195,49 +80,6 @@ func main() {
|
||||
}
|
||||
}()
|
||||
|
||||
// Prune-request queue (#669 M4 / #738): the read-only server enqueues
|
||||
// geo-prune requests as marker files; the ingestor (which holds the
|
||||
// write handle) executes the DELETEs. Process on startup, then every
|
||||
// 15 seconds — short enough for a one-click UX, long enough to avoid
|
||||
// useless wake-ups.
|
||||
store.RunPendingPruneRequests()
|
||||
pruneQueueTicker := time.NewTicker(15 * time.Second)
|
||||
go func() {
|
||||
for range pruneQueueTicker.C {
|
||||
store.RunPendingPruneRequests()
|
||||
}
|
||||
}()
|
||||
|
||||
// Per-second stats file writer for the server's /api/perf/write-sources
|
||||
// endpoint (#1120). Best-effort; never fatal.
|
||||
StartStatsFileWriter(store, time.Second)
|
||||
|
||||
// Multi-byte capability persister (#1324 follow-up): the server's
|
||||
// analytics cycle publishes a snapshot file via internal/mbcapqueue
|
||||
// (it cannot UPDATE itself, mode=ro since #1289). The ingestor
|
||||
// applies the snapshot here every 5 minutes — derived/cached
|
||||
// columns, ingestor owns the write.
|
||||
multibytePersistTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
time.Sleep(2 * time.Minute) // stagger after analytics warmup
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
for range multibytePersistTicker.C {
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[multibyte-persist] enabled (interval=5m)")
|
||||
|
||||
// Neighbor-edges builder (#1287 — Option 4): ingestor owns
|
||||
// neighbor_edges writes. Runs every 60s. Server reads the snapshot
|
||||
// via cmd/server/neighbor_recomputer.go on the same cadence.
|
||||
stopNeighborBuilder := store.StartNeighborEdgesBuilder(NeighborEdgesBuilderInterval)
|
||||
defer stopNeighborBuilder()
|
||||
log.Printf("[neighbor-build] enabled (interval=%s)", NeighborEdgesBuilderInterval)
|
||||
|
||||
channelKeys := loadChannelKeys(cfg, *configPath)
|
||||
if len(channelKeys) > 0 {
|
||||
log.Printf("Loaded %d channel keys for GRP_TXT decryption", len(channelKeys))
|
||||
@@ -245,37 +87,34 @@ func main() {
|
||||
log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted")
|
||||
}
|
||||
|
||||
regionKeys := loadRegionKeys(cfg)
|
||||
store.BackfillDefaultScopeAsync(regionKeys)
|
||||
|
||||
// Connect to each MQTT source
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
for _, source := range sources {
|
||||
tag := source.Name
|
||||
if tag == "" {
|
||||
tag = source.Broker
|
||||
}
|
||||
|
||||
opts := buildMQTTOpts(source)
|
||||
connectTimeout := source.ConnectTimeoutOrDefault()
|
||||
log.Printf("MQTT [%s] connect timeout: %ds", tag, connectTimeout)
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(source.Broker).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetOrderMatters(true)
|
||||
|
||||
// Pre-allocate the liveness pointer so OnConnect can reset its
|
||||
// stale-message clock on reconnect (PR #1216 r1 item 2). IsConnectedFn
|
||||
// is wired below once the client exists.
|
||||
liveness := &SourceLivenessState{
|
||||
Tag: tag,
|
||||
Broker: source.Broker,
|
||||
if source.Username != "" {
|
||||
opts.SetUsername(source.Username)
|
||||
}
|
||||
if source.Password != "" {
|
||||
opts.SetPassword(source.Password)
|
||||
}
|
||||
if source.RejectUnauthorized != nil && !*source.RejectUnauthorized {
|
||||
opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") {
|
||||
opts.SetTLSConfig(&tls.Config{})
|
||||
}
|
||||
|
||||
opts.SetOnConnectHandler(func(c mqtt.Client) {
|
||||
log.Printf("MQTT [%s] connected to %s", tag, source.Broker)
|
||||
// PR #1216 r1 item 2: clear the stale LastMessageUnix from
|
||||
// before the outage so the watchdog doesn't immediately scream
|
||||
// "stalled for 2h". Also restarts the cold-start grace window
|
||||
// and clears the alert cooldown so a fresh stall edge can fire.
|
||||
liveness.MarkReconnected(time.Now())
|
||||
topics := source.Topics
|
||||
if len(topics) == 0 {
|
||||
topics = []string{"meshcore/#"}
|
||||
@@ -292,88 +131,30 @@ func main() {
|
||||
})
|
||||
|
||||
opts.SetConnectionLostHandler(func(c mqtt.Client, err error) {
|
||||
log.Printf("MQTT [%s] disconnected from %s: %v", tag, source.Broker, err)
|
||||
})
|
||||
|
||||
opts.SetReconnectingHandler(func(c mqtt.Client, options *mqtt.ClientOptions) {
|
||||
log.Printf("MQTT [%s] reconnecting to %s", tag, source.Broker)
|
||||
log.Printf("MQTT [%s] disconnected: %v", tag, err)
|
||||
})
|
||||
|
||||
// Capture source for closure
|
||||
src := source
|
||||
opts.SetDefaultPublishHandler(func(c mqtt.Client, m mqtt.Message) {
|
||||
handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg)
|
||||
handleMessage(store, tag, src, m, channelKeys, cfg.GeoFilter)
|
||||
})
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
// Wire IsConnectedFn now that the client exists, then register.
|
||||
// Registration BEFORE Connect so the attempt counter is available
|
||||
// to OnConnectAttempt on the very first dial.
|
||||
liveness.IsConnectedFn = client.IsConnected
|
||||
// #1335: wire force-reconnect so the watchdog can drop a
|
||||
// half-open TCP socket and re-dial when paho.IsConnected==true
|
||||
// but no messages have flowed past the stall threshold. Throttled
|
||||
// per source by the watchdog itself (forceReconnectThrottle).
|
||||
// Disconnect(250) gives in-flight publishes 250ms to drain;
|
||||
// Connect() returns immediately and paho's reconnect machinery
|
||||
// takes over from there. Captured-by-value `client` is the same
|
||||
// pointer used everywhere else for this source.
|
||||
liveness.ForceReconnectFn = func() {
|
||||
client.Disconnect(250)
|
||||
client.Connect()
|
||||
}
|
||||
// PR #1216 r2 item 3: tag collisions used to log.Fatalf, which
|
||||
// killed the entire ingestor over one config typo and recreated
|
||||
// the #1212 total-ingest-stop class this PR exists to prevent.
|
||||
// registerLivenessOrSkip logs ERROR + skips liveness registration
|
||||
// for the duplicate; the MQTT source still attempts to connect,
|
||||
// it just isn't tracked by the watchdog. First registration
|
||||
// remains authoritative.
|
||||
registerLivenessOrSkip(liveness)
|
||||
token := client.Connect()
|
||||
// With ConnectRetry=true, token.Wait() blocks forever for unreachable brokers.
|
||||
// WaitTimeout lets startup proceed; the client keeps retrying in the background
|
||||
// and OnConnect fires (subscribing) when it eventually connects (#910).
|
||||
if !token.WaitTimeout(time.Duration(connectTimeout) * time.Second) {
|
||||
log.Printf("MQTT [%s] initial connection timed out — retrying in background", tag)
|
||||
clients = append(clients, client)
|
||||
continue
|
||||
}
|
||||
token.Wait()
|
||||
if token.Error() != nil {
|
||||
log.Printf("MQTT [%s] connection failed (non-fatal): %v", tag, token.Error())
|
||||
// BL1 fix: Disconnect to stop Paho's internal retry goroutines.
|
||||
// With ConnectRetry=true, Connect() spawns background goroutines
|
||||
// that leak if the client is simply discarded.
|
||||
client.Disconnect(0)
|
||||
continue
|
||||
}
|
||||
connectedCount++
|
||||
clients = append(clients, client)
|
||||
}
|
||||
|
||||
// BL2 fix: require at least one immediately-connected source. Timed-out
|
||||
// clients are retrying in background (tracked in clients) but don't count
|
||||
// as "connected" — a single unreachable broker must not silently run with
|
||||
// zero active connections.
|
||||
if connectedCount == 0 {
|
||||
// Clean up any timed-out clients still retrying
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
log.Fatal("no MQTT sources connected — all timed out or failed. Check broker is running (default: mqtt://localhost:1883). Set MQTT_BROKER env var or configure mqttSources in config.json")
|
||||
if len(clients) == 0 {
|
||||
log.Fatal("no MQTT connections established")
|
||||
}
|
||||
|
||||
if connectedCount < len(clients) {
|
||||
log.Printf("Running — %d MQTT source(s) connected, %d retrying in background", connectedCount, len(clients)-connectedCount)
|
||||
} else {
|
||||
log.Printf("Running — %d MQTT source(s) connected", connectedCount)
|
||||
}
|
||||
|
||||
// #1212: per-source stall watchdog. Detects "silently dead" sources
|
||||
// where the client reports connected but no messages have flowed. Logs
|
||||
// a WARN line every minute for any source silent for >5m. Scan every
|
||||
// 60s so detection latency is bounded.
|
||||
stopWatchdog := runLivenessWatchdog(60*time.Second, 5*time.Minute)
|
||||
log.Printf("Running — %d MQTT source(s) connected", len(clients))
|
||||
|
||||
// Wait for shutdown signal
|
||||
sig := make(chan os.Signal, 1)
|
||||
@@ -382,14 +163,7 @@ func main() {
|
||||
|
||||
log.Println("Shutting down...")
|
||||
retentionTicker.Stop()
|
||||
metricsRetentionTicker.Stop()
|
||||
if packetRetentionTicker != nil {
|
||||
packetRetentionTicker.Stop()
|
||||
}
|
||||
statsTicker.Stop()
|
||||
pruneQueueTicker.Stop()
|
||||
walCheckpointTicker.Stop()
|
||||
stopWatchdog()
|
||||
store.LogStats() // final stats on shutdown
|
||||
for _, c := range clients {
|
||||
c.Disconnect(5000) // 5s to allow in-flight messages to drain
|
||||
@@ -397,73 +171,7 @@ func main() {
|
||||
log.Println("Done.")
|
||||
}
|
||||
|
||||
// buildMQTTOpts creates MQTT client options for a source with bounded reconnect
|
||||
// backoff, connect timeout, and TLS/auth configuration.
|
||||
//
|
||||
// Logs every TCP/TLS dial via OnConnectAttempt. Unlike SetReconnectingHandler
|
||||
// (which only fires inside paho's reconnect goroutine and can be silent if
|
||||
// that loop never iterates), OnConnectAttempt fires on every attempt — the
|
||||
// initial Connect() and every reconnect. This is the observability fix for
|
||||
// #1212 (prod outage on 2026-05-15 where the disconnect was logged but no
|
||||
// reconnect activity was ever visible).
|
||||
func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions {
|
||||
tag := source.Name
|
||||
if tag == "" {
|
||||
tag = source.Broker
|
||||
}
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker(source.Broker).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectRetry(true).
|
||||
SetOrderMatters(true).
|
||||
SetMaxReconnectInterval(30 * time.Second).
|
||||
SetConnectTimeout(10 * time.Second).
|
||||
SetWriteTimeout(10 * time.Second).
|
||||
// #1335: TCP-level keepalive surfaces a half-open socket within
|
||||
// ~30-60s instead of waiting for the application-level watchdog
|
||||
// (5m) to notice no messages. paho's MQTT PINGREQ uses this
|
||||
// interval too — if the broker's PINGRESP doesn't arrive,
|
||||
// ConnectionLost fires and auto-reconnect kicks in. Was unset
|
||||
// (paho default 30s actually — making this explicit so it can't
|
||||
// drift, and so operators reading the code know it's intentional
|
||||
// per the #1335 RCA).
|
||||
SetKeepAlive(30 * time.Second)
|
||||
|
||||
opts.SetConnectionAttemptHandler(func(broker *url.URL, tlsCfg *tls.Config) *tls.Config {
|
||||
// Look up the per-source liveness state (registered in main) so we
|
||||
// can attach an attempt counter. If not yet registered (first dial
|
||||
// from Connect()), fall through with attempt=1.
|
||||
var attempt int64 = 1
|
||||
livenessRegistryMu.RLock()
|
||||
s := livenessRegistry[tag]
|
||||
livenessRegistryMu.RUnlock()
|
||||
if s != nil {
|
||||
attempt = atomic.AddInt64(&s.AttemptCount, 1)
|
||||
}
|
||||
log.Printf("MQTT [%s] connection attempt #%d to %s", tag, attempt, broker.String())
|
||||
return tlsCfg
|
||||
})
|
||||
|
||||
if source.Username != "" {
|
||||
opts.SetUsername(source.Username)
|
||||
}
|
||||
if source.Password != "" {
|
||||
opts.SetPassword(source.Password)
|
||||
}
|
||||
if source.RejectUnauthorized != nil && !*source.RejectUnauthorized {
|
||||
opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") || strings.HasPrefix(source.Broker, "wss://") {
|
||||
// TLS with system CA pool — valid for ssl:// MQTT brokers and
|
||||
// wss:// WebSocket brokers behind a publicly-trusted certificate.
|
||||
opts.SetTLSConfig(&tls.Config{})
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionKeys map[string][]byte, cfg *Config) {
|
||||
// Liveness watchdog (#1212): record receipt before any processing so a
|
||||
// slow handler still counts as "source is alive". Cheap atomic store.
|
||||
markLivenessForTag(tag, time.Now())
|
||||
func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, geoFilter *GeoFilterConfig) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("MQTT [%s] panic in handler: %v", tag, r)
|
||||
@@ -473,66 +181,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
topic := m.Topic()
|
||||
parts := strings.Split(topic, "/")
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Skip status/connection topics
|
||||
if topic == "meshcore/status" || topic == "meshcore/events/connection" {
|
||||
return
|
||||
}
|
||||
|
||||
// Observer blacklist: drop ALL messages from blacklisted observers before any
|
||||
// DB writes (status, metrics, packets). Trumps IATA filter.
|
||||
if len(parts) > 2 && cfg.IsObserverBlacklisted(parts[2]) {
|
||||
log.Printf("MQTT [%s] observer %.8s blacklisted, dropping", tag, parts[2])
|
||||
return
|
||||
}
|
||||
|
||||
// Global observer IATA whitelist: if configured, drop messages from observers
|
||||
// in non-whitelisted IATA regions. Applies to ALL message types (status + packets).
|
||||
if len(parts) > 1 && !cfg.IsObserverIATAAllowed(parts[1]) {
|
||||
return
|
||||
}
|
||||
|
||||
// Status topic: meshcore/<region>/<observer_id>/status
|
||||
// Per-source IATA filter does NOT apply here — observer metadata (noise_floor, battery, etc.)
|
||||
// is region-independent and should be accepted from all observers regardless of
|
||||
// which IATA regions are configured for packet ingestion.
|
||||
if len(parts) >= 4 && parts[3] == "status" {
|
||||
observerID := parts[2]
|
||||
name, _ := msg["origin"].(string)
|
||||
iata := parts[1]
|
||||
meta := extractObserverMeta(msg)
|
||||
// observer.last_seen is "when did the analyzer last hear from this
|
||||
// observer" — fundamentally an ingest-time question. Passing "" makes
|
||||
// UpsertObserverAt use time.Now(), independent of the envelope timestamp
|
||||
// (which can be stale/skewed even when well-formed). See #1465.
|
||||
if err := store.UpsertObserverAt(observerID, name, iata, meta, ""); err != nil {
|
||||
log.Printf("MQTT [%s] observer status error: %v", tag, err)
|
||||
}
|
||||
// Insert metrics sample from status message
|
||||
if meta != nil {
|
||||
metricsData := &MetricsData{
|
||||
ObserverID: observerID,
|
||||
NoiseFloor: meta.NoiseFloor,
|
||||
TxAirSecs: meta.TxAirSecs,
|
||||
RxAirSecs: meta.RxAirSecs,
|
||||
RecvErrors: meta.RecvErrors,
|
||||
BatteryMv: meta.BatteryMv,
|
||||
PacketsSent: meta.PacketsSent,
|
||||
PacketsRecv: meta.PacketsRecv,
|
||||
}
|
||||
if err := store.InsertMetrics(metricsData); err != nil {
|
||||
log.Printf("MQTT [%s] metrics insert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
log.Printf("MQTT [%s] status: %s (%s)", tag, firstNonEmpty(name, observerID), iata)
|
||||
return
|
||||
}
|
||||
|
||||
// IATA filter applies to packet messages only — not status messages above.
|
||||
// IATA filter
|
||||
if len(source.IATAFilter) > 0 && len(parts) > 1 {
|
||||
region := parts[1]
|
||||
matched := false
|
||||
@@ -547,35 +196,35 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
}
|
||||
}
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Skip status/connection topics
|
||||
if topic == "meshcore/status" || topic == "meshcore/events/connection" {
|
||||
return
|
||||
}
|
||||
|
||||
// Status topic: meshcore/<region>/<observer_id>/status
|
||||
if len(parts) >= 4 && parts[3] == "status" {
|
||||
observerID := parts[2]
|
||||
name, _ := msg["origin"].(string)
|
||||
iata := parts[1]
|
||||
meta := extractObserverMeta(msg)
|
||||
if err := store.UpsertObserver(observerID, name, iata, meta); err != nil {
|
||||
log.Printf("MQTT [%s] observer status error: %v", tag, err)
|
||||
}
|
||||
log.Printf("MQTT [%s] status: %s (%s)", tag, firstNonEmpty(name, observerID), iata)
|
||||
return
|
||||
}
|
||||
|
||||
// Format 1: Raw packet (meshcoretomqtt / Cisien format)
|
||||
rawHex, _ := msg["raw"].(string)
|
||||
if rawHex != "" {
|
||||
validateSigs := cfg.ShouldValidateSignatures()
|
||||
decoded, err := DecodePacket(rawHex, channelKeys, validateSigs)
|
||||
decoded, err := DecodePacket(rawHex, channelKeys)
|
||||
if err != nil {
|
||||
// Per #1211: include enough context to repro malformed-packet drops,
|
||||
// but NEVER log the full observer ID (PII / fingerprinting risk).
|
||||
// We log:
|
||||
// - topic prefix (with observer segment elided)
|
||||
// - 8-char observer prefix
|
||||
// - payload length, claimed length (rawHex len)
|
||||
obs := ""
|
||||
if len(parts) > 2 {
|
||||
obs = parts[2]
|
||||
}
|
||||
// Build a redacted topic that replaces parts[2] (the observer id)
|
||||
// with the 8-char prefix, so the rest of the topic is preserved
|
||||
// for debugging without leaking the full identifier.
|
||||
redactedTopic := topic
|
||||
if len(parts) > 2 {
|
||||
redactedParts := make([]string, len(parts))
|
||||
copy(redactedParts, parts)
|
||||
if len(parts[2]) > 8 {
|
||||
redactedParts[2] = parts[2][:8]
|
||||
}
|
||||
redactedTopic = strings.Join(redactedParts, "/")
|
||||
}
|
||||
log.Printf("MQTT [%s] decode error: %v (topic=%s observer=%.8s rawHexLen=%d)", tag, err, redactedTopic, obs, len(rawHex))
|
||||
log.Printf("MQTT [%s] decode error: %v", tag, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -587,17 +236,8 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
if len(parts) > 1 {
|
||||
region = parts[1]
|
||||
}
|
||||
// Fallback to source-level region config when topic has no region (#788)
|
||||
if region == "" && source.Region != "" {
|
||||
region = source.Region
|
||||
}
|
||||
|
||||
mqttMsg := &MQTTPacketMessage{Raw: rawHex}
|
||||
mqttMsg.Timestamp = resolveRxTime(msg, tag)
|
||||
// Parse optional region from JSON payload (#788)
|
||||
if v, ok := msg["region"].(string); ok && v != "" {
|
||||
mqttMsg.Region = v
|
||||
}
|
||||
if v, ok := msg["SNR"]; ok {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
mqttMsg.SNR = &f
|
||||
@@ -642,48 +282,10 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] skipping corrupted ADVERT: %s", tag, reason)
|
||||
return
|
||||
}
|
||||
// Signature validation: drop adverts with invalid ed25519 signatures
|
||||
if validateSigs && decoded.Payload.SignatureValid != nil && !*decoded.Payload.SignatureValid {
|
||||
hash := ComputeContentHash(rawHex)
|
||||
truncPK := decoded.Payload.PubKey
|
||||
if len(truncPK) > 16 {
|
||||
truncPK = truncPK[:16]
|
||||
}
|
||||
log.Printf("MQTT [%s] DROPPED invalid signature: hash=%s name=%s observer=%s pubkey=%s",
|
||||
tag, hash, decoded.Payload.Name, firstNonEmpty(mqttMsg.Origin, observerID), truncPK)
|
||||
store.InsertDroppedPacket(&DroppedPacket{
|
||||
Hash: hash,
|
||||
RawHex: rawHex,
|
||||
Reason: "invalid signature",
|
||||
ObserverID: observerID,
|
||||
ObserverName: mqttMsg.Origin,
|
||||
NodePubKey: decoded.Payload.PubKey,
|
||||
NodeName: decoded.Payload.Name,
|
||||
})
|
||||
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, geoFilter) {
|
||||
return
|
||||
}
|
||||
foreign := false
|
||||
if !NodePassesGeoFilter(decoded.Payload.Lat, decoded.Payload.Lon, cfg.GeoFilter) {
|
||||
if cfg.ForeignAdverts.IsDropMode() {
|
||||
return
|
||||
}
|
||||
foreign = true
|
||||
lat, lon := 0.0, 0.0
|
||||
if decoded.Payload.Lat != nil {
|
||||
lat = *decoded.Payload.Lat
|
||||
}
|
||||
if decoded.Payload.Lon != nil {
|
||||
lon = *decoded.Payload.Lon
|
||||
}
|
||||
truncPK := decoded.Payload.PubKey
|
||||
if len(truncPK) > 16 {
|
||||
truncPK = truncPK[:16]
|
||||
}
|
||||
log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s",
|
||||
tag, truncPK, decoded.Payload.Name, lat, lon, firstNonEmpty(mqttMsg.Origin, observerID))
|
||||
}
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys)
|
||||
pktData.Foreign = foreign
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
|
||||
isNew, err := store.InsertTransmission(pktData)
|
||||
if err != nil {
|
||||
log.Printf("MQTT [%s] db insert error: %v", tag, err)
|
||||
@@ -692,11 +294,6 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
if err := store.UpsertNode(decoded.Payload.PubKey, decoded.Payload.Name, role, decoded.Payload.Lat, decoded.Payload.Lon, pktData.Timestamp); err != nil {
|
||||
log.Printf("MQTT [%s] node upsert error: %v", tag, err)
|
||||
}
|
||||
if foreign {
|
||||
if err := store.MarkNodeForeign(decoded.Payload.PubKey); err != nil {
|
||||
log.Printf("MQTT [%s] mark foreign error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
if isNew {
|
||||
if err := store.IncrementAdvertCount(decoded.Payload.PubKey); err != nil {
|
||||
log.Printf("MQTT [%s] advert count error: %v", tag, err)
|
||||
@@ -708,16 +305,10 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] node telemetry update error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
// Update default_scope when advert carries a matched transport scope (#899)
|
||||
if pktData.IsTransportScoped {
|
||||
if err := store.UpdateNodeDefaultScope(decoded.Payload.PubKey, pktData.ScopeName); err != nil {
|
||||
log.Printf("MQTT [%s] node default_scope update error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-ADVERT packets: store normally (routing/channel messages from
|
||||
// in-area observers are relevant regardless of relay hop origin).
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys)
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region)
|
||||
if _, err := store.InsertTransmission(pktData); err != nil {
|
||||
log.Printf("MQTT [%s] db insert error: %v", tag, err)
|
||||
}
|
||||
@@ -726,15 +317,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
// Upsert observer
|
||||
if observerID != "" {
|
||||
origin, _ := msg["origin"].(string)
|
||||
// Use effective region: payload > topic > source config (#788)
|
||||
effectiveRegion := region
|
||||
if mqttMsg.Region != "" {
|
||||
effectiveRegion = mqttMsg.Region
|
||||
}
|
||||
// Same as the status-path call above: observer.last_seen is ingest
|
||||
// time, not envelope time. Per-packet rxTime (stored in observations
|
||||
// via InsertTransmission) still uses envelope time. See #1465.
|
||||
if err := store.UpsertObserverAt(observerID, origin, effectiveRegion, nil, ""); err != nil {
|
||||
if err := store.UpsertObserver(observerID, origin, region, nil); err != nil {
|
||||
log.Printf("MQTT [%s] observer upsert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
@@ -778,8 +361,8 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
|
||||
decodedJSON, _ := json.Marshal(channelMsg)
|
||||
|
||||
ingestNow := time.Now().UTC().Format(time.RFC3339)
|
||||
hashInput := fmt.Sprintf("ch:%s:%s:%s", channelIdx, text, ingestNow)
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
hashInput := fmt.Sprintf("ch:%s:%s:%s", channelIdx, text, now)
|
||||
h := sha256.Sum256([]byte(hashInput))
|
||||
hash := hex.EncodeToString(h[:])[:16]
|
||||
|
||||
@@ -819,7 +402,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
}
|
||||
|
||||
pktData := &PacketData{
|
||||
Timestamp: ingestNow, // #1370 (counters #1233): server ingest time, not envelope rxTime
|
||||
Timestamp: now,
|
||||
ObserverID: "companion",
|
||||
ObserverName: "L1 Pro (BLE)",
|
||||
SNR: snr,
|
||||
@@ -831,18 +414,19 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
PayloadType: 5, // GRP_TXT
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: string(decodedJSON),
|
||||
ChannelHash: channelName, // fast channel queries (#762)
|
||||
}
|
||||
|
||||
if _, err := store.InsertTransmission(pktData); err != nil {
|
||||
log.Printf("MQTT [%s] channel insert error: %v", tag, err)
|
||||
}
|
||||
|
||||
// Note: we intentionally do NOT create a node entry for channel message senders.
|
||||
// Channel messages don't carry the sender's real pubkey, so any entry we create
|
||||
// would use a synthetic key ("sender-<name>") that doesn't match the real pubkey
|
||||
// used for claiming/health lookups. The node will get a proper entry when it
|
||||
// sends an advert. See issue #665.
|
||||
// Upsert sender as a companion node
|
||||
if sender != "" {
|
||||
senderKey := "sender-" + strings.ToLower(sender)
|
||||
if err := store.UpsertNode(senderKey, sender, "companion", nil, nil, now); err != nil {
|
||||
log.Printf("MQTT [%s] sender node upsert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("MQTT [%s] channel message: ch%s from %s", tag, channelIdx, firstNonEmpty(sender, "unknown"))
|
||||
return
|
||||
@@ -871,8 +455,8 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
|
||||
decodedJSON, _ := json.Marshal(dm)
|
||||
|
||||
ingestNow := time.Now().UTC().Format(time.RFC3339)
|
||||
hashInput := fmt.Sprintf("dm:%s:%s", text, ingestNow)
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
hashInput := fmt.Sprintf("dm:%s:%s", text, now)
|
||||
h := sha256.Sum256([]byte(hashInput))
|
||||
hash := hex.EncodeToString(h[:])[:16]
|
||||
|
||||
@@ -912,7 +496,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
}
|
||||
|
||||
pktData := &PacketData{
|
||||
Timestamp: ingestNow, // #1370 (counters #1233): server ingest time, not envelope rxTime
|
||||
Timestamp: now,
|
||||
ObserverID: "companion",
|
||||
ObserverName: "L1 Pro (BLE)",
|
||||
SNR: snr,
|
||||
@@ -1032,41 +616,6 @@ func extractObserverMeta(msg map[string]interface{}) *ObserverMeta {
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
if v := nestedOrTopLevel(stats, msg, "tx_air_secs"); v != nil {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
iv := int(math.Round(f))
|
||||
meta.TxAirSecs = &iv
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
if v := nestedOrTopLevel(stats, msg, "rx_air_secs"); v != nil {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
iv := int(math.Round(f))
|
||||
meta.RxAirSecs = &iv
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
if v := nestedOrTopLevel(stats, msg, "recv_errors"); v != nil {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
iv := int(math.Round(f))
|
||||
meta.RecvErrors = &iv
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
if v := nestedOrTopLevel(stats, msg, "packets_sent"); v != nil {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
iv := int(math.Round(f))
|
||||
meta.PacketsSent = &iv
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
if v := nestedOrTopLevel(stats, msg, "packets_recv"); v != nil {
|
||||
if f, ok := toFloat64(v); ok {
|
||||
iv := int(math.Round(f))
|
||||
meta.PacketsRecv = &iv
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasData {
|
||||
return nil
|
||||
@@ -1096,91 +645,6 @@ func firstNonEmpty(vals ...string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// resolveRxTime returns the observer receive-time for a packet, taken from
|
||||
// the MQTT envelope's "timestamp" field. Falls back to ingest time only when
|
||||
// the field is missing, unparseable, or implausibly in the future (a
|
||||
// clock-skewed observer). Result is always RFC3339 UTC.
|
||||
//
|
||||
// The envelope timestamp is stamped by the uploader when the radio receives
|
||||
// the frame, not when the MQTT message is published — so a buffered packet
|
||||
// uploaded hours late still carries its true receive time. Using ingest time
|
||||
// (time.Now()) here mis-dated such packets by the upload delay.
|
||||
func resolveRxTime(msg map[string]interface{}, tag string) string {
|
||||
now := time.Now().UTC()
|
||||
raw, _ := msg["timestamp"].(string)
|
||||
if raw == "" {
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
t, naive, err := parseEnvelopeTime(raw)
|
||||
if err != nil {
|
||||
log.Printf("MQTT [%s] unparseable timestamp %q, using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Hard reject: > 14h ahead is a genuine clock error (UTC+14 is the maximum
|
||||
// standard offset, so nothing valid should be further ahead than that).
|
||||
if t.After(now.Add(14 * time.Hour)) {
|
||||
log.Printf("MQTT [%s] future timestamp %q, using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Hard reject: > 30 days in the past is an RTC-reset node reporting a
|
||||
// factory date (e.g. 2020-01-01). Such a value would permanently drag
|
||||
// transmissions.first_seen backwards via stmtUpdateTxFirstSeen in
|
||||
// InsertTransmission. No legitimate buffered upload is that stale.
|
||||
if t.Before(now.Add(-30 * 24 * time.Hour)) {
|
||||
log.Printf("MQTT [%s] stale timestamp %q (>30d old), using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Symmetric naive-timestamp clamp (issue #1463). Naive (zone-less) ISO
|
||||
// values from observers in non-UTC zones are parsed as-if UTC, leaving a
|
||||
// residual offset equal to the observer's UTC offset:
|
||||
// - UTC+N observer → value appears N hours in the future
|
||||
// - UTC-N observer → value appears N hours in the past
|
||||
// The past case was silently stored verbatim, poisoning last_seen and
|
||||
// rendering UTC-N observers perpetually "Stale" in the UI. Collapse any
|
||||
// naive value more than 15 min off server-now to now() — well-behaved
|
||||
// observers (Z-suffixed or explicit offset) are untouched regardless of
|
||||
// skew so legitimate buffered uploads remain accurate.
|
||||
const naiveTolerance = 15 * time.Minute
|
||||
if naive {
|
||||
delta := t.Sub(now)
|
||||
if delta < 0 {
|
||||
delta = -delta
|
||||
}
|
||||
if delta > naiveTolerance {
|
||||
log.Printf("MQTT [%s] naive timestamp %q off by %s, using ingest time", tag, raw, delta.Round(time.Second))
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
// Legacy soft clamp for zone-aware near-future values: any value ahead of
|
||||
// now is from a slightly skewed observer clock — collapse to now so we
|
||||
// don't render ⚠️ in the UI for live packets from those nodes.
|
||||
if t.After(now) {
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// parseEnvelopeTime parses the MQTT envelope timestamp. Two on-wire forms
|
||||
// occur: zone-aware ISO8601 (RFC3339), and a naive local-clock ISO string
|
||||
// with no zone (python datetime.isoformat()). Zone-aware layouts are tried
|
||||
// first; naive layouts are assumed UTC but the caller is informed via the
|
||||
// returned `naive` flag so it can apply a symmetric clamp (see issue #1463).
|
||||
func parseEnvelopeTime(s string) (time.Time, bool, error) {
|
||||
// Zone-aware first — RFC3339 demands Z or ±HH:MM.
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return t, false, nil
|
||||
}
|
||||
for _, layout := range []string{
|
||||
"2006-01-02T15:04:05.999999", // python isoformat w/ microseconds
|
||||
"2006-01-02T15:04:05", // naive ISO
|
||||
} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, true, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, false, fmt.Errorf("unrecognized timestamp layout: %q", s)
|
||||
}
|
||||
|
||||
// deriveHashtagChannelKey derives an AES-128 key from a channel name.
|
||||
// Same algorithm as Node.js: SHA-256(channelName) → first 32 hex chars (16 bytes).
|
||||
func deriveHashtagChannelKey(channelName string) string {
|
||||
@@ -1240,79 +704,12 @@ func loadChannelKeys(cfg *Config, configPath string) map[string]string {
|
||||
|
||||
// 3. Explicit config keys (highest priority — overrides rainbow + derived)
|
||||
for k, v := range cfg.ChannelKeys {
|
||||
normalized := normalizeChannelName(k)
|
||||
if normalized != k {
|
||||
log.Printf("[channels] Normalizing known channel key %q → %q for display", k, normalized)
|
||||
}
|
||||
// Detect config collision: if both "public" and "Public" are present,
|
||||
// the normalized key collides. Resolve deterministically: prefer the
|
||||
// canonical (already-normalized) form over the lowercase variant.
|
||||
if _, dupe := keys[normalized]; dupe {
|
||||
// If the incoming key IS the canonical form, it wins (overwrite).
|
||||
// If the incoming key is a non-canonical form (e.g., "public"), keep existing.
|
||||
if k == normalized {
|
||||
log.Printf("[channels] Resolving duplicate %q: canonical form wins over non-canonical", normalized)
|
||||
keys[normalized] = v
|
||||
} else {
|
||||
log.Printf("[channels] WARNING: duplicate channel key %q — config has %q normalizing to %q, keeping canonical value", normalized, k, normalized)
|
||||
}
|
||||
} else {
|
||||
keys[normalized] = v
|
||||
}
|
||||
keys[k] = v
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
func loadRegionKeys(cfg *Config) map[string][]byte {
|
||||
keys := make(map[string][]byte)
|
||||
for _, raw := range cfg.HashRegions {
|
||||
name := strings.TrimSpace(raw)
|
||||
if name == "" {
|
||||
log.Printf("[regions] skipping empty hashRegions entry")
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(name, "#") {
|
||||
name = "#" + name
|
||||
}
|
||||
if _, exists := keys[name]; exists {
|
||||
log.Printf("[regions] duplicate region %q ignored", name)
|
||||
continue
|
||||
}
|
||||
h := sha256.Sum256([]byte(name))
|
||||
keys[name] = h[:16]
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
log.Printf("[regions] %d region key(s) loaded", len(keys))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// matchScope performs one HMAC-SHA256 per configured region. Expected
|
||||
// len(regionKeys) ≤ 50; beyond that, consider a pre-indexed lookup table.
|
||||
func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string {
|
||||
if code1 == "0000" || len(regionKeys) == 0 || len(payloadRaw) == 0 {
|
||||
return ""
|
||||
}
|
||||
for name, key := range regionKeys {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte{payloadType})
|
||||
mac.Write(payloadRaw)
|
||||
hmacBytes := mac.Sum(nil)
|
||||
code := uint16(hmacBytes[0]) | uint16(hmacBytes[1])<<8
|
||||
if code == 0 {
|
||||
code = 1
|
||||
} else if code == 0xFFFF {
|
||||
code = 0xFFFE
|
||||
}
|
||||
codeBytes := [2]byte{byte(code & 0xFF), byte(code >> 8)}
|
||||
if strings.ToUpper(hex.EncodeToString(codeBytes[:])) == code1 {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Version info (set via ldflags)
|
||||
var version = "dev"
|
||||
|
||||
|
||||
+29
-310
@@ -1,17 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mqtt "github.com/eclipse/paho.mqtt.golang"
|
||||
)
|
||||
|
||||
func TestToFloat64(t *testing.T) {
|
||||
@@ -135,7 +130,7 @@ func TestHandleMessageRawPacket(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"myobs"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -152,7 +147,7 @@ func TestHandleMessageRawPacketAdvert(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
// Should create a node from the ADVERT
|
||||
var count int
|
||||
@@ -174,7 +169,7 @@ func TestHandleMessageInvalidJSON(t *testing.T) {
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: []byte(`not json`)}
|
||||
|
||||
// Should not panic
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -191,7 +186,7 @@ func TestHandleMessageStatusTopic(t *testing.T) {
|
||||
payload: []byte(`{"origin":"MyObserver"}`),
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var name, iata string
|
||||
err := store.db.QueryRow("SELECT name, iata FROM observers WHERE id = 'obs1'").Scan(&name, &iata)
|
||||
@@ -212,11 +207,11 @@ func TestHandleMessageSkipStatusTopics(t *testing.T) {
|
||||
|
||||
// meshcore/status should be skipped
|
||||
msg1 := &mockMessage{topic: "meshcore/status", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg1, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg1, nil, nil)
|
||||
|
||||
// meshcore/events/connection should be skipped
|
||||
msg2 := &mockMessage{topic: "meshcore/events/connection", payload: []byte(`{"raw":"0A00"}`)}
|
||||
handleMessage(store, "test", source, msg2, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg2, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -235,7 +230,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -248,7 +243,7 @@ func TestHandleMessageIATAFilter(t *testing.T) {
|
||||
topic: "meshcore/LAX/obs2/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg2, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg2, nil, nil)
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 1 {
|
||||
@@ -266,7 +261,7 @@ func TestHandleMessageIATAFilterNoRegion(t *testing.T) {
|
||||
topic: "meshcore",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
// No region part → filter doesn't apply, message goes through
|
||||
// Actually the code checks len(parts) > 1 for IATA filter
|
||||
@@ -282,7 +277,7 @@ func TestHandleMessageNoRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"type":"companion","data":"something"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -300,7 +295,7 @@ func TestHandleMessageBadRawHex(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"ZZZZ"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -317,7 +312,7 @@ func TestHandleMessageWithSNRRSSIAsNumbers(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"RSSI":-95}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -336,7 +331,7 @@ func TestHandleMessageMinimalTopic(t *testing.T) {
|
||||
topic: "meshcore/SJC",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -357,7 +352,7 @@ func TestHandleMessageCorruptedAdvert(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
// Transmission should be inserted (even if advert is invalid)
|
||||
var count int
|
||||
@@ -383,7 +378,7 @@ func TestHandleMessageNoObserverID(t *testing.T) {
|
||||
topic: "packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `","origin":"obs1"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -405,7 +400,7 @@ func TestHandleMessageSNRNotFloat(t *testing.T) {
|
||||
// SNR as a string value — should not parse as float
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":"bad","RSSI":"bad"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
@@ -421,7 +416,7 @@ func TestHandleMessageOriginExtraction(t *testing.T) {
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","origin":"MyOrigin"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
// Verify origin was extracted to observer name
|
||||
var name string
|
||||
@@ -444,7 +439,7 @@ func TestHandleMessagePanicRecovery(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should not panic — the defer/recover should catch it
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
}
|
||||
|
||||
func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
@@ -456,7 +451,7 @@ func TestHandleMessageStatusOriginFallback(t *testing.T) {
|
||||
topic: "meshcore/SJC/obs1/status",
|
||||
payload: []byte(`{"type":"status"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var name string
|
||||
err := store.db.QueryRow("SELECT name FROM observers WHERE id = 'obs1'").Scan(&name)
|
||||
@@ -482,20 +477,18 @@ func TestEpochToISO(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAdvertRole(t *testing.T) {
|
||||
// advertRole now keys off AdvertFlags.Type (firmware ADV_TYPE_*) — see
|
||||
// firmware/src/helpers/AdvertDataHelpers.h:7-12 and issue #1279 P1 #3.
|
||||
tests := []struct {
|
||||
name string
|
||||
flags *AdvertFlags
|
||||
want string
|
||||
}{
|
||||
{"none (type 0)", &AdvertFlags{Type: 0}, "none"},
|
||||
{"companion (type 1)", &AdvertFlags{Type: 1, Chat: true}, "companion"},
|
||||
{"repeater (type 2)", &AdvertFlags{Type: 2, Repeater: true}, "repeater"},
|
||||
{"room (type 3)", &AdvertFlags{Type: 3, Room: true}, "room"},
|
||||
{"sensor (type 4)", &AdvertFlags{Type: 4, Sensor: true}, "sensor"},
|
||||
{"future type-5", &AdvertFlags{Type: 5}, "type-5"},
|
||||
{"nil flags falls back to companion", nil, "companion"},
|
||||
{"repeater", &AdvertFlags{Repeater: true}, "repeater"},
|
||||
{"room", &AdvertFlags{Room: true}, "room"},
|
||||
{"sensor", &AdvertFlags{Sensor: true}, "sensor"},
|
||||
{"companion (default)", &AdvertFlags{Chat: true}, "companion"},
|
||||
{"companion (no flags)", &AdvertFlags{}, "companion"},
|
||||
{"repeater takes priority", &AdvertFlags{Repeater: true, Room: true}, "repeater"},
|
||||
{"room before sensor", &AdvertFlags{Room: true, Sensor: true}, "room"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -647,7 +640,7 @@ func TestHandleMessageWithLowercaseSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","snr":5.5,"rssi":-102}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -668,7 +661,7 @@ func TestHandleMessageSNRRSSIUppercaseWins(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":7.2,"snr":1.0,"RSSI":-95,"rssi":-50}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -688,7 +681,7 @@ func TestHandleMessageNoSNRRSSI(t *testing.T) {
|
||||
payload := []byte(`{"raw":"` + rawHex + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs1/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
handleMessage(store, "test", source, msg, nil, nil)
|
||||
|
||||
var snr, rssi *float64
|
||||
store.db.QueryRow("SELECT snr, rssi FROM observations LIMIT 1").Scan(&snr, &rssi)
|
||||
@@ -746,277 +739,3 @@ func TestToFloat64WithUnits(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIATAFilterDoesNotDropStatusMessages verifies that status messages from
|
||||
// out-of-region observers are still processed (noise_floor, battery, etc.)
|
||||
// even when an IATA filter is configured for packet data.
|
||||
func TestIATAFilterDoesNotDropStatusMessages(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test", IATAFilter: []string{"SJC"}}
|
||||
|
||||
// BFL observer sends a status message with noise_floor — outside the IATA filter.
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/status",
|
||||
payload: []byte(`{"origin":"BFLObserver","stats":{"noise_floor":-105.0}}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
|
||||
var name string
|
||||
var noiseFloor *float64
|
||||
err := store.db.QueryRow("SELECT name, noise_floor FROM observers WHERE id = 'bfl-obs1'").Scan(&name, &noiseFloor)
|
||||
if err != nil {
|
||||
t.Fatalf("observer not found after status from out-of-region observer: %v", err)
|
||||
}
|
||||
if name != "BFLObserver" {
|
||||
t.Errorf("name=%q, want BFLObserver", name)
|
||||
}
|
||||
if noiseFloor == nil || *noiseFloor != -105.0 {
|
||||
t.Errorf("noise_floor=%v, want -105.0 — status message was dropped by IATA filter when it should not be", noiseFloor)
|
||||
}
|
||||
|
||||
// Verify that a packet from BFL is still filtered.
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
pktMsg := &mockMessage{
|
||||
topic: "meshcore/BFL/bfl-obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, pktMsg, nil, nil, &Config{})
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("packet from out-of-region BFL should still be filtered by IATA")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRegionKeys(t *testing.T) {
|
||||
cfg := &Config{HashRegions: []string{"#belgium", "eu", " #Test ", "", "#belgium"}}
|
||||
keys := loadRegionKeys(cfg)
|
||||
|
||||
// Deduplication + normalization
|
||||
if len(keys) != 3 {
|
||||
t.Fatalf("len(keys) = %d, want 3", len(keys))
|
||||
}
|
||||
// Pre-computed: SHA256("#belgium")[:16]. Hardcoded so a change to the key
|
||||
// derivation algorithm (hash function, truncation length) breaks this test
|
||||
// even if both sides were updated together.
|
||||
wantBelgium, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27")
|
||||
if got := keys["#belgium"]; !bytes.Equal(got, wantBelgium) {
|
||||
t.Errorf("#belgium key mismatch: got %x, want %x", got, wantBelgium)
|
||||
}
|
||||
// "eu" should be normalized to "#eu"
|
||||
if _, ok := keys["#eu"]; !ok {
|
||||
t.Error("expected #eu key")
|
||||
}
|
||||
// " #Test " should be normalized to "#Test"
|
||||
if _, ok := keys["#Test"]; !ok {
|
||||
t.Error("expected #Test key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchScope(t *testing.T) {
|
||||
// Fixed known-answer vectors only — no in-test HMAC computation.
|
||||
// Keys and Code1 values are pre-computed externally so a wrong algorithm
|
||||
// that produces consistent wrong results on both sides would still fail.
|
||||
|
||||
// Vector 1: "#test"/payloadType=5/"hello" → Code1=2AB5
|
||||
// Key = SHA256("#test")[:16] = 9cd8fcf22a47333b591d96a2b848b73f
|
||||
testKey, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f")
|
||||
testKeys := map[string][]byte{"#test": testKey}
|
||||
if got := matchScope(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" {
|
||||
t.Errorf("#test vector: matchScope = %q, want #test", got)
|
||||
}
|
||||
|
||||
// Vector 2: "#belgium"/payloadType=5/"hello" → Code1=4A75
|
||||
// Key = SHA256("#belgium")[:16] = 7085b78ed010599094f8c8e7d1aa0e27
|
||||
belgiumKey, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27")
|
||||
belgiumKeys := map[string][]byte{"#belgium": belgiumKey}
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" {
|
||||
t.Errorf("#belgium vector: matchScope = %q, want #belgium", got)
|
||||
}
|
||||
|
||||
// Code1=0000 (unscoped transport) → no region matched
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "0000"); got != "" {
|
||||
t.Errorf("unscoped: matchScope = %q, want empty", got)
|
||||
}
|
||||
|
||||
// Code1 present but matches no configured region → empty string
|
||||
if got := matchScope(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" {
|
||||
t.Errorf("no match: matchScope = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPacketDataScopeMatching(t *testing.T) {
|
||||
// Fixed known-answer packet: TRANSPORT_FLOOD, payloadType=5, payload="hello",
|
||||
// Code1=2AB5 (pre-computed for region "#test").
|
||||
// header=0x14 (route_type=0 FLOOD, payloadType=5 → 5<<2), Code1=[0x2A,0xB5],
|
||||
// Code2=[0,0], path_len=0, payload="hello" (68 65 6C 6C 6F).
|
||||
const rawHex = "142AB500000068656C6C6F"
|
||||
key, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f") // SHA256("#test")[:16]
|
||||
regionKeys := map[string][]byte{"#test": key}
|
||||
|
||||
decoded, err := DecodePacket(rawHex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys)
|
||||
if pktData.ScopeName != "#test" {
|
||||
t.Errorf("ScopeName = %q, want #test", pktData.ScopeName)
|
||||
}
|
||||
if !pktData.IsTransportScoped {
|
||||
t.Error("IsTransportScoped should be true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTConnectRetryTimeoutDoesNotBlock verifies that WaitTimeout returns within
|
||||
// the deadline for an unreachable broker when ConnectRetry=true (#910). Previously,
|
||||
// token.Wait() would block forever in this configuration.
|
||||
func TestMQTTConnectRetryTimeoutDoesNotBlock(t *testing.T) {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1"). // port 1 — nothing listening, fast refusal
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true)
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
token := client.Connect()
|
||||
defer client.Disconnect(100)
|
||||
|
||||
start := time.Now()
|
||||
connected := token.WaitTimeout(3 * time.Second)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if connected {
|
||||
t.Skip("port 1 unexpectedly accepted a connection — skipping")
|
||||
}
|
||||
if elapsed > 4*time.Second {
|
||||
t.Errorf("WaitTimeout blocked for %v — token.Wait() would block forever with ConnectRetry=true", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBL1_GoroutineLeakOnHardFailure reproduces BLOCKER 1: without Disconnect()
|
||||
// on the error path, Paho's internal retry goroutines leak when a client is
|
||||
// discarded after Connect() with ConnectRetry=true.
|
||||
//
|
||||
// We prove the leak by creating N clients WITHOUT Disconnect — goroutines grow
|
||||
// proportionally. The fix (client.Disconnect(0) before continue) prevents this.
|
||||
func TestBL1_GoroutineLeakOnHardFailure(t *testing.T) {
|
||||
runtime.GC()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
baseline := runtime.NumGoroutine()
|
||||
|
||||
// Create multiple clients connected to unreachable broker, WITHOUT disconnecting.
|
||||
// Each one spawns Paho retry goroutines that accumulate.
|
||||
const numClients = 10
|
||||
clients := make([]mqtt.Client, numClients)
|
||||
for i := 0; i < numClients; i++ {
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1").
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true).
|
||||
SetConnectTimeout(500 * time.Millisecond)
|
||||
c := mqtt.NewClient(opts)
|
||||
tok := c.Connect()
|
||||
tok.WaitTimeout(1 * time.Second)
|
||||
clients[i] = c
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
leaked := runtime.NumGoroutine()
|
||||
goroutineGrowth := leaked - baseline
|
||||
|
||||
// Clean up to not actually leak in test
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
|
||||
t.Logf("baseline=%d, after %d undisconnected clients=%d, growth=%d",
|
||||
baseline, numClients, leaked, goroutineGrowth)
|
||||
|
||||
// With ConnectRetry=true, each Connect() spawns retry goroutines.
|
||||
// Without Disconnect, these accumulate. Verify growth is meaningful.
|
||||
if goroutineGrowth < 3 {
|
||||
t.Skip("Connect didn't spawn enough extra goroutines to measure leak")
|
||||
}
|
||||
|
||||
// The fix: calling client.Disconnect(0) on the error path prevents accumulation.
|
||||
// Anti-tautology: removing the Disconnect(0) call from main.go's error path
|
||||
// would cause goroutine accumulation proportional to failed broker count.
|
||||
t.Logf("CONFIRMED: %d leaked goroutines from %d clients without Disconnect — fix adds Disconnect(0) on error path", goroutineGrowth, numClients)
|
||||
}
|
||||
|
||||
// TestBL2_ZeroConnectedFatals verifies BLOCKER 2: when all brokers are unreachable,
|
||||
// connectedCount==0 must be detected. We test the logic directly — if only timed-out
|
||||
// clients exist (appended to clients slice) but connectedCount is 0, the guard triggers.
|
||||
func TestBL2_ZeroConnectedFatals(t *testing.T) {
|
||||
// Simulate the connection loop result: 1 timed-out client, 0 connected
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
|
||||
// Create a client that times out (unreachable broker)
|
||||
opts := mqtt.NewClientOptions().
|
||||
AddBroker("tcp://127.0.0.1:1").
|
||||
SetConnectRetry(true).
|
||||
SetAutoReconnect(true)
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
token := client.Connect()
|
||||
if !token.WaitTimeout(2 * time.Second) {
|
||||
// Timed out — PR #926 appends to clients
|
||||
clients = append(clients, client)
|
||||
}
|
||||
defer func() {
|
||||
for _, c := range clients {
|
||||
c.Disconnect(0)
|
||||
}
|
||||
}()
|
||||
|
||||
// OLD bug: len(clients) == 0 would be false (1 timed-out client in list)
|
||||
// → ingestor would silently run with zero connections
|
||||
if len(clients) == 0 {
|
||||
t.Fatal("expected timed-out client to be in clients slice")
|
||||
}
|
||||
|
||||
// NEW fix: connectedCount == 0 catches this
|
||||
if connectedCount != 0 {
|
||||
t.Errorf("connectedCount should be 0, got %d", connectedCount)
|
||||
}
|
||||
|
||||
// The real code does: if connectedCount == 0 { log.Fatal(...) }
|
||||
// This test proves len(clients) > 0 but connectedCount == 0 — the old guard
|
||||
// would have missed it.
|
||||
if len(clients) > 0 && connectedCount == 0 {
|
||||
t.Log("BL2 confirmed: old guard len(clients)==0 would NOT fatal; new guard connectedCount==0 correctly catches zero-connected state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleMessageObserverIATAWhitelist(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{
|
||||
ObserverIATAWhitelist: []string{"ARN"},
|
||||
}
|
||||
|
||||
// Message from non-whitelisted region GOT — should be dropped
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/GOT/obs1/status",
|
||||
payload: []byte(`{"origin":"node1","noise_floor":-110}`),
|
||||
}, nil, nil, cfg)
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs1'").Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("observer from non-whitelisted IATA GOT should be dropped")
|
||||
}
|
||||
|
||||
// Message from whitelisted region ARN — should be accepted
|
||||
handleMessage(store, "test", source, &mockMessage{
|
||||
topic: "meshcore/ARN/obs2/status",
|
||||
payload: []byte(`{"origin":"node2","noise_floor":-105}`),
|
||||
}, nil, nil, cfg)
|
||||
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM observers WHERE id='obs2'").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Errorf("observer from whitelisted IATA ARN should be accepted, got count=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/dbschema"
|
||||
)
|
||||
|
||||
// PruneOldPackets deletes transmissions (and their child observations)
|
||||
// older than `days`. Returns count of transmissions deleted.
|
||||
//
|
||||
// Owned by the ingestor per #1283: the writer process is the only one
|
||||
// allowed to hold the DB write lock; previously this lived in
|
||||
// cmd/server/db.go and raced ingestor INSERTs (SQLITE_BUSY).
|
||||
func (s *Store) PruneOldPackets(days int) (int64, error) {
|
||||
if days <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// Delete child observations first (no CASCADE in SQLite).
|
||||
if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN (
|
||||
SELECT id FROM transmissions WHERE first_seen < ?
|
||||
)`, cutoff); err != nil {
|
||||
return 0, fmt.Errorf("prune observations: %w", err)
|
||||
}
|
||||
|
||||
res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune transmissions: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("prune commit: %w", err)
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// SoftDeleteBlacklistedObservers marks observers in the blacklist as
|
||||
// inactive=1 so they are hidden from API responses. Owned by ingestor
|
||||
// per #1287. Runs once at startup.
|
||||
func (s *Store) SoftDeleteBlacklistedObservers(blacklist []string) {
|
||||
n, err := dbschema.SoftDeleteBlacklistedObservers(s.db, blacklist)
|
||||
if err != nil {
|
||||
log.Printf("[observer-blacklist] warning: soft-delete failed: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[observer-blacklist] soft-deleted %d blacklisted observer(s)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// PruneNeighborEdges deletes rows older than maxAgeDays from
|
||||
// neighbor_edges. Owned by the ingestor per #1287 (was in cmd/server).
|
||||
// Returns DB rows deleted.
|
||||
func (s *Store) PruneNeighborEdges(maxAgeDays int) (int64, error) {
|
||||
if maxAgeDays <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
cutoff := time.Now().UTC().Add(-time.Duration(maxAgeDays) * 24 * time.Hour).Format(time.RFC3339)
|
||||
res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < ?", cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune neighbor_edges: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n > 0 {
|
||||
log.Printf("[neighbor-prune] removed %d DB rows older than %d days", n, maxAgeDays)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// ─── from_pubkey backfill (#1143) ──────────────────────────────────────────
|
||||
//
|
||||
// Moved from cmd/server/from_pubkey_migration.go in #1287. Runs from the
|
||||
// ingestor's maintenance loop. Populates transmissions.from_pubkey for
|
||||
// ADVERT rows whose value is still NULL, by parsing decoded_json.pubKey.
|
||||
|
||||
// FromPubkeyBackfillStats holds progress for /api/healthz exposure.
|
||||
// The ingestor exposes these via stats_file.go so the server can read
|
||||
// them without writing.
|
||||
type FromPubkeyBackfillStats struct {
|
||||
Total int64 `json:"total"`
|
||||
Processed int64 `json:"processed"`
|
||||
Done bool `json:"done"`
|
||||
}
|
||||
|
||||
// BackfillFromPubkey scans transmissions where from_pubkey IS NULL and
|
||||
// payload_type = 4 (ADVERT) and populates from_pubkey from decoded_json.
|
||||
// Chunked + yields between batches. Safe to call repeatedly; once a row
|
||||
// is set to either "" or hex it never matches the WHERE clause again.
|
||||
func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, progress func(total, processed int64, done bool)) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[backfill] from_pubkey panic recovered: %v", r)
|
||||
}
|
||||
if progress != nil {
|
||||
progress(0, 0, true) // signal done; values overwritten below if collected
|
||||
}
|
||||
}()
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 5000
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := s.db.QueryRow(
|
||||
"SELECT COUNT(*) FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4",
|
||||
).Scan(&total); err != nil {
|
||||
log.Printf("[backfill] from_pubkey count error: %v", err)
|
||||
return
|
||||
}
|
||||
if total == 0 {
|
||||
log.Println("[backfill] from_pubkey: nothing to do")
|
||||
if progress != nil {
|
||||
progress(0, 0, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
if progress != nil {
|
||||
progress(total, 0, false)
|
||||
}
|
||||
log.Printf("[backfill] from_pubkey starting: %d ADVERT rows", total)
|
||||
|
||||
stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = ? WHERE id = ?")
|
||||
if err != nil {
|
||||
log.Printf("[backfill] from_pubkey prepare: %v", err)
|
||||
return
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
var processed int64
|
||||
for {
|
||||
rows, err := s.db.Query(
|
||||
"SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT ?",
|
||||
chunkSize)
|
||||
if err != nil {
|
||||
log.Printf("[backfill] from_pubkey select: %v", err)
|
||||
return
|
||||
}
|
||||
type row struct {
|
||||
id int64
|
||||
pk string
|
||||
}
|
||||
batch := make([]row, 0, chunkSize)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var dj sql.NullString
|
||||
if err := rows.Scan(&id, &dj); err != nil {
|
||||
continue
|
||||
}
|
||||
batch = append(batch, row{id: id, pk: extractPubkeyFromAdvertJSON(dj.String)})
|
||||
}
|
||||
rows.Close()
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[backfill] from_pubkey begin tx: %v", err)
|
||||
return
|
||||
}
|
||||
txStmt := tx.Stmt(stmt)
|
||||
for _, b := range batch {
|
||||
// Sentinel: "" = scanned-no-pubkey (so the WHERE clause
|
||||
// won't keep rescanning this row). hex = real pubkey.
|
||||
var val interface{} = ""
|
||||
if b.pk != "" {
|
||||
val = b.pk
|
||||
}
|
||||
if _, err := txStmt.Exec(val, b.id); err != nil {
|
||||
log.Printf("[backfill] from_pubkey update id=%d: %v", b.id, err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Printf("[backfill] from_pubkey commit: %v", err)
|
||||
return
|
||||
}
|
||||
processed += int64(len(batch))
|
||||
if progress != nil {
|
||||
progress(total, processed, false)
|
||||
}
|
||||
if len(batch) < chunkSize {
|
||||
break
|
||||
}
|
||||
if yieldDuration > 0 {
|
||||
time.Sleep(yieldDuration)
|
||||
}
|
||||
}
|
||||
log.Printf("[backfill] from_pubkey complete: %d rows processed", processed)
|
||||
if progress != nil {
|
||||
progress(total, processed, true)
|
||||
}
|
||||
}
|
||||
|
||||
// extractPubkeyFromAdvertJSON parses an ADVERT decoded_json blob and
|
||||
// returns the pubKey field, or "" if absent/invalid.
|
||||
func extractPubkeyFromAdvertJSON(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(s), &m); err != nil {
|
||||
return ""
|
||||
}
|
||||
if v, ok := m["pubKey"].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBuildMQTTOpts_ReconnectSettings(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "tcp://localhost:1883",
|
||||
Name: "test",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.MaxReconnectInterval != 30*time.Second {
|
||||
t.Errorf("MaxReconnectInterval = %v, want 30s", opts.MaxReconnectInterval)
|
||||
}
|
||||
if opts.ConnectTimeout != 10*time.Second {
|
||||
t.Errorf("ConnectTimeout = %v, want 10s", opts.ConnectTimeout)
|
||||
}
|
||||
if opts.WriteTimeout != 10*time.Second {
|
||||
t.Errorf("WriteTimeout = %v, want 10s", opts.WriteTimeout)
|
||||
}
|
||||
if !opts.AutoReconnect {
|
||||
t.Error("AutoReconnect should be true")
|
||||
}
|
||||
if !opts.ConnectRetry {
|
||||
t.Error("ConnectRetry should be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_Credentials(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "tcp://broker:1883",
|
||||
Username: "user1",
|
||||
Password: "pass1",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.Username != "user1" {
|
||||
t.Errorf("Username = %q, want %q", opts.Username, "user1")
|
||||
}
|
||||
if opts.Password != "pass1" {
|
||||
t.Errorf("Password = %q, want %q", opts.Password, "pass1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_TLS_InsecureSkipVerify(t *testing.T) {
|
||||
f := false
|
||||
source := MQTTSource{
|
||||
Broker: "ssl://broker:8883",
|
||||
RejectUnauthorized: &f,
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig should be set")
|
||||
}
|
||||
if !opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify should be true when RejectUnauthorized=false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMQTTOpts_TLS_SSL_Prefix(t *testing.T) {
|
||||
source := MQTTSource{
|
||||
Broker: "ssl://broker:8883",
|
||||
}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.TLSConfig == nil {
|
||||
t.Fatal("TLSConfig should be set for ssl:// brokers")
|
||||
}
|
||||
if opts.TLSConfig.InsecureSkipVerify {
|
||||
t.Error("InsecureSkipVerify should be false by default")
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"log"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PR #1216 r1 item 5 (kent #1 / adv MAJOR-2): the original assertion was
|
||||
// tautological — it only checked OnConnectAttempt != nil, which passes
|
||||
// even if the handler is a no-op. This version invokes the wired handler,
|
||||
// captures log output, and asserts the OBSERVABLE behaviour operators
|
||||
// rely on during a #1212-class outage:
|
||||
// - the configured source tag appears in the log line
|
||||
// - the broker URL appears in the log line
|
||||
// - the per-source AttemptCount increments on every invocation (proving
|
||||
// the handler is wired to the right state, not just a stub)
|
||||
// - the tlsCfg passed in is returned unchanged (no surprise TLS rewrite)
|
||||
func TestBuildMQTTOpts_InstrumentsConnectionAttempt(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
source := MQTTSource{Broker: "tcp://localhost:1883", Name: "obs-tag"}
|
||||
opts := buildMQTTOpts(source)
|
||||
|
||||
if opts.OnConnectAttempt == nil {
|
||||
t.Fatal("OnConnectAttempt must be wired in buildMQTTOpts (#1212 / PR #1216 r1)")
|
||||
}
|
||||
|
||||
// Register the liveness state so the handler can find it and increment
|
||||
// the attempt counter (same wiring main.go does).
|
||||
liveness := &SourceLivenessState{Tag: "obs-tag", Broker: source.Broker}
|
||||
if err := registerLivenessState(liveness); err != nil {
|
||||
t.Fatalf("test setup: registerLivenessState: %v", err)
|
||||
}
|
||||
|
||||
// Capture log output via log.SetOutput. Save/restore so other tests
|
||||
// running serially don't lose their writer.
|
||||
var buf bytes.Buffer
|
||||
origOut := log.Writer()
|
||||
origFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
defer func() {
|
||||
log.SetOutput(origOut)
|
||||
log.SetFlags(origFlags)
|
||||
}()
|
||||
|
||||
brokerURL, err := url.Parse(source.Broker)
|
||||
if err != nil {
|
||||
t.Fatalf("test setup: parse broker url: %v", err)
|
||||
}
|
||||
tlsIn := &tls.Config{ServerName: "sentinel.test"}
|
||||
|
||||
// Invoke the handler twice — operators need to see attempt # increment
|
||||
// per dial to gauge backoff progress.
|
||||
tlsOut1 := opts.OnConnectAttempt(brokerURL, tlsIn)
|
||||
tlsOut2 := opts.OnConnectAttempt(brokerURL, tlsIn)
|
||||
|
||||
if tlsOut1 != tlsIn || tlsOut2 != tlsIn {
|
||||
t.Errorf("OnConnectAttempt must pass tlsCfg through unchanged (got %p, %p; want %p)", tlsOut1, tlsOut2, tlsIn)
|
||||
}
|
||||
|
||||
logOut := buf.String()
|
||||
if !strings.Contains(logOut, "obs-tag") {
|
||||
t.Errorf("log output must include the source tag for operator grep; got %q", logOut)
|
||||
}
|
||||
if !strings.Contains(logOut, source.Broker) {
|
||||
t.Errorf("log output must include the broker URL so operators can correlate against config; got %q", logOut)
|
||||
}
|
||||
if !strings.Contains(logOut, "#1") || !strings.Contains(logOut, "#2") {
|
||||
t.Errorf("log output must show attempt #1 and #2 across the two invocations (per-source counter); got %q", logOut)
|
||||
}
|
||||
|
||||
if got := atomic.LoadInt64(&liveness.AttemptCount); got != 2 {
|
||||
t.Errorf("AttemptCount must increment per dial (got %d after 2 invocations, want 2)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// RED: the watchdog acceptance criterion from #1212 — even when the client
|
||||
// reports connected, if NO packets have flowed for >threshold, log a warning.
|
||||
// This is a separate detection layer that catches "silently dead" sockets
|
||||
// (broker accepted TCP but stopped forwarding, half-open TCP, etc.).
|
||||
func TestMQTTStallWatchdog_FiresOnSilentSource(t *testing.T) {
|
||||
state := &SourceLivenessState{Tag: "test", Broker: "tcp://x:1883"}
|
||||
atomic.StoreInt64(&state.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
state.IsConnectedFn = func() bool { return true }
|
||||
|
||||
msg, kind := checkSourceLiveness(state, 5*time.Minute, time.Now())
|
||||
if kind != LivenessStalled {
|
||||
t.Fatalf("watchdog should flag stall when source connected but no message for 10m (threshold 5m); got kind=%v msg=%q", kind, msg)
|
||||
}
|
||||
if !strings.Contains(msg, "no messages") {
|
||||
t.Errorf("stall message should mention 'no messages'; got %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "test") {
|
||||
t.Errorf("stall message should include the source tag; got %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTStallWatchdog_QuietWhenRecent(t *testing.T) {
|
||||
state := &SourceLivenessState{Tag: "test", Broker: "tcp://x:1883"}
|
||||
atomic.StoreInt64(&state.LastMessageUnix, time.Now().Add(-30*time.Second).Unix())
|
||||
state.IsConnectedFn = func() bool { return true }
|
||||
|
||||
_, kind := checkSourceLiveness(state, 5*time.Minute, time.Now())
|
||||
if kind != LivenessOK {
|
||||
t.Fatal("watchdog should NOT flag stall when last message was 30s ago and threshold is 5m")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTStallWatchdog_QuietWhenDisconnected(t *testing.T) {
|
||||
// When disconnected, paho's own reconnect logging covers it — the
|
||||
// watchdog should only fire for the silent-while-connected case.
|
||||
state := &SourceLivenessState{Tag: "test", Broker: "tcp://x:1883"}
|
||||
atomic.StoreInt64(&state.LastMessageUnix, time.Now().Add(-1*time.Hour).Unix())
|
||||
state.IsConnectedFn = func() bool { return false }
|
||||
|
||||
_, kind := checkSourceLiveness(state, 5*time.Minute, time.Now())
|
||||
if kind != LivenessDisconnected {
|
||||
t.Fatalf("watchdog must classify a !IsConnected source as LivenessDisconnected (silent state), not LivenessOK — r2 item 1 prevents disconnect→recovery mis-classification; got kind=%v", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// snapshotAndResetRegistry isolates the package-level livenessRegistry for a
|
||||
// single test. Returns a restore func to defer. Without this, parallel or
|
||||
// previously-registered sources leak into the watchdog goroutine under test.
|
||||
func snapshotAndResetRegistry(t *testing.T) func() {
|
||||
t.Helper()
|
||||
livenessRegistryMu.Lock()
|
||||
saved := livenessRegistry
|
||||
livenessRegistry = map[string]*SourceLivenessState{}
|
||||
livenessRegistryMu.Unlock()
|
||||
return func() {
|
||||
livenessRegistryMu.Lock()
|
||||
livenessRegistry = saved
|
||||
livenessRegistryMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// RED-then-GREEN: the watchdog GOROUTINE (not just checkSourceLiveness) must
|
||||
// fan out emits across the registry on each tick, AND must exit cleanly when
|
||||
// the stop signal fires. Originally runLivenessWatchdog used `for range
|
||||
// t.C` — ticker.Stop() does not close the channel, so the goroutine
|
||||
// leaked past shutdown. This test asserts both:
|
||||
// - tick → emit for every stalled source in the registry
|
||||
// - stop → goroutine returns within a short bound
|
||||
func TestMQTTStallWatchdog_LoopEmitsAndStopsCleanly(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
s1 := &SourceLivenessState{Tag: "alpha", Broker: "tcp://a:1883", IsConnectedFn: func() bool { return true }}
|
||||
s2 := &SourceLivenessState{Tag: "beta", Broker: "tcp://b:1883", IsConnectedFn: func() bool { return true }}
|
||||
atomic.StoreInt64(&s1.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s2.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
registerLivenessState(s1)
|
||||
registerLivenessState(s2)
|
||||
|
||||
tick := make(chan time.Time, 1)
|
||||
done := make(chan struct{})
|
||||
|
||||
var mu sync.Mutex
|
||||
var emits []string
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(args) > 0 {
|
||||
if s, ok := args[0].(string); ok {
|
||||
emits = append(emits, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
runLivenessWatchdogLoop(tick, done, 5*time.Minute, emit)
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
tick <- time.Now()
|
||||
// Drain: wait briefly for the emits to land. Polling instead of sleeping
|
||||
// keeps the test fast on a healthy machine.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
n := len(emits)
|
||||
mu.Unlock()
|
||||
if n >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
got := append([]string(nil), emits...)
|
||||
mu.Unlock()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 stall emits (alpha+beta), got %d: %v", len(got), got)
|
||||
}
|
||||
|
||||
close(done)
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("watchdog goroutine did not exit within 2s of stop — ticker leak regression")
|
||||
}
|
||||
}
|
||||
|
||||
// PR #1216 r1 item 6 (kent #2 / adv MAJOR-3): the original test had no
|
||||
// assertions gating behaviour — it called stop() and trusted `-race` to
|
||||
// catch leaks. `-race` does NOT detect goroutine leaks. This version
|
||||
// captures runtime.NumGoroutine() before/after and asserts the watchdog's
|
||||
// goroutine actually exited. Allows ±1 slack for unrelated runtime
|
||||
// bookkeeping (gc, finalizer).
|
||||
func TestMQTTStallWatchdog_RunStopsCleanly(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
// Settle: let any prior-test goroutines finish before sampling baseline.
|
||||
runtime.GC()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
before := runtime.NumGoroutine()
|
||||
|
||||
stop := runLivenessWatchdog(10*time.Millisecond, 5*time.Minute)
|
||||
// Let the watchdog run a few ticks so we're sure it's truly spawned.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if mid := runtime.NumGoroutine(); mid <= before {
|
||||
t.Fatalf("watchdog goroutine did not spawn: before=%d mid=%d", before, mid)
|
||||
}
|
||||
|
||||
stop()
|
||||
|
||||
// Poll for the goroutine count to return to baseline (±1 slack).
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var after int
|
||||
for time.Now().Before(deadline) {
|
||||
runtime.Gosched()
|
||||
after = runtime.NumGoroutine()
|
||||
if after <= before+1 {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("watchdog goroutine leaked: before=%d after=%d (delta %d) — stop() did not signal the loop to exit", before, after, after-before)
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// heartbeatInterval is how often the watchdog re-emits a still-stalled
|
||||
// reminder once the initial WARN edge has fired. 1h matches the pager
|
||||
// budget — frequent enough that an unattended stall is noticed within a
|
||||
// shift, infrequent enough not to spam ops chat.
|
||||
const livenessHeartbeatInterval = time.Hour
|
||||
|
||||
// forceReconnectThrottle is the minimum interval between forced
|
||||
// reconnects on the SAME source. See processLivenessTransition.
|
||||
const forceReconnectThrottle = 60 * time.Second
|
||||
|
||||
// LivenessKind enumerates the watchdog verdicts for a source. Edge-triggered
|
||||
// transitions use this to decide whether to emit (and what severity).
|
||||
type LivenessKind int
|
||||
|
||||
const (
|
||||
LivenessOK LivenessKind = iota
|
||||
LivenessStalled
|
||||
LivenessNeverReceived
|
||||
LivenessRecovered
|
||||
LivenessHeartbeat
|
||||
// LivenessDisconnected (PR #1216 r2 item 1): paho reports !IsConnected.
|
||||
// Distinct from LivenessOK so processLivenessTransition does NOT
|
||||
// interpret a TCP drop as recovery and fire a spurious "messages
|
||||
// flowing again" INFO when the source actually went from silently
|
||||
// broken to overtly broken. paho's own reconnect logging already
|
||||
// covers the disconnect — this kind exists solely to keep the
|
||||
// transition engine from mis-classifying it.
|
||||
LivenessDisconnected
|
||||
)
|
||||
|
||||
// SourceLivenessState tracks per-source last-message timestamp and connection
|
||||
// state for the stall watchdog (#1212). LastMessageUnix is updated by the
|
||||
// message handler via atomic store; the watchdog reads it via atomic load.
|
||||
//
|
||||
// PR #1216 r1 added:
|
||||
// - StartedAt: re-stamped on reconnect to suppress transient-stall WARNs
|
||||
// during paho's reconnect window.
|
||||
// - LastAlertUnix: edge-trigger cooldown; prevents 60-per-hour re-emits
|
||||
// of the same WARN.
|
||||
//
|
||||
// PR #1216 r2 added:
|
||||
// - FirstConnectedAt: stamped ONCE at registration, never reset. The
|
||||
// cold-start "NEVER received" alarm uses this so a broker that flaps
|
||||
// in CONNECT → SUBSCRIBE-deny cannot indefinitely re-arm the grace
|
||||
// window. r1's StartedAt-as-grace-clock conflated transient-stall
|
||||
// suppression with cold-start grace; r2 separates them.
|
||||
type SourceLivenessState struct {
|
||||
Tag string
|
||||
Broker string
|
||||
LastMessageUnix int64 // atomic; unix seconds of last successfully received MQTT message
|
||||
// FirstConnectedAt (PR #1216 r2 item 2) is stamped ONCE at
|
||||
// registerLivenessState time and never reset. Cold-start grace
|
||||
// checks against this so a flapping broker (CONNECT ok, SUBSCRIBE
|
||||
// ACL-denied — the #1212 shape) can no longer suppress the
|
||||
// "NEVER received" alarm by re-stamping StartedAt on every reconnect.
|
||||
FirstConnectedAt int64 // atomic; unix seconds of first registration
|
||||
StartedAt int64 // atomic; unix seconds when the source was registered / last reconnected (transient-stall tracking)
|
||||
LastAlertUnix int64 // atomic; unix seconds of last emit (WARN or heartbeat); 0 means quiet
|
||||
IsConnectedFn func() bool
|
||||
// ForceReconnectFn (#1335) is called by the watchdog when a source
|
||||
// transitions INTO LivenessStalled. It must force the paho client
|
||||
// to drop its current TCP socket and re-establish (typically
|
||||
// client.Disconnect(250) followed by client.Connect()). Half-open
|
||||
// TCP sockets (Azure NAT idle timeout) report IsConnected==true so
|
||||
// paho's own auto-reconnect never fires; this is the recovery path.
|
||||
// May be nil (tests, or sources registered before wiring); the
|
||||
// watchdog must treat that as a safe no-op. Invocations are
|
||||
// throttled at forceReconnectThrottle per source so a
|
||||
// stall→reconnect→re-stall loop self-recovers without hammering
|
||||
// the broker.
|
||||
ForceReconnectFn func()
|
||||
// LastForceReconnectUnix is the unix-seconds timestamp of the most
|
||||
// recent forced reconnect for this source; the watchdog reads it
|
||||
// to enforce forceReconnectThrottle. atomic.
|
||||
LastForceReconnectUnix int64
|
||||
// AttemptCount is incremented on every TCP/TLS connection attempt. Used
|
||||
// by ConnectionAttemptHandler to log attempt # independent of paho's
|
||||
// internal reconnect-loop state. atomic.
|
||||
AttemptCount int64
|
||||
}
|
||||
|
||||
// MarkMessage records the time of a received MQTT message. Cheap; safe to
|
||||
// call from the message-handling hot path.
|
||||
func (s *SourceLivenessState) MarkMessage(now time.Time) {
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Unix())
|
||||
}
|
||||
|
||||
// MarkReconnected clears stale liveness state so the watchdog does not
|
||||
// false-alarm on a pre-outage timestamp after paho re-establishes the
|
||||
// connection (PR #1216 r1 item 2). Resets LastMessageUnix, re-stamps
|
||||
// StartedAt (transient-stall window restarts), and clears LastAlertUnix
|
||||
// (edge-trigger re-arms).
|
||||
//
|
||||
// PR #1216 r2 item 2: FirstConnectedAt is INTENTIONALLY not touched here.
|
||||
// Under broker flap (CONNECT ok, SUBSCRIBE ACL-denied — exact #1212
|
||||
// class) r1 reset StartedAt on every reconnect, indefinitely re-arming
|
||||
// the cold-start grace and silencing the headline "NEVER received"
|
||||
// alarm. Cold-start grace now reads FirstConnectedAt instead, so the
|
||||
// alarm fires after the FIRST grace window regardless of reconnect
|
||||
// churn.
|
||||
func (s *SourceLivenessState) MarkReconnected(now time.Time) {
|
||||
atomic.StoreInt64(&s.LastMessageUnix, 0)
|
||||
atomic.StoreInt64(&s.StartedAt, now.Unix())
|
||||
atomic.StoreInt64(&s.LastAlertUnix, 0)
|
||||
}
|
||||
|
||||
// checkSourceLiveness returns (message, kind) describing the source's
|
||||
// liveness state. kind==LivenessOK means quiet/healthy; kind==
|
||||
// LivenessDisconnected means paho is not connected (silent state — no
|
||||
// emit, no recovery). Any other kind indicates the caller may want to
|
||||
// emit (subject to edge-trigger).
|
||||
//
|
||||
// Cold-start (PR #1216 r1 item 1, r2 item 2): when LastMessageUnix==0,
|
||||
// the source has never published a single message. If FirstConnectedAt
|
||||
// was stamped at registration and more than `threshold` has elapsed,
|
||||
// this is the #1212 failure class — wrong channel hash, ACL drops
|
||||
// SUBSCRIBE, half-open TCP after CONNECT, or a broker that loops
|
||||
// CONNECT-then-disconnect. We emit a DISTINCT "NEVER received" alarm
|
||||
// so operators can grep for it independently of generic stalls. Using
|
||||
// FirstConnectedAt (not the reconnect-reset StartedAt) ensures broker
|
||||
// flap cannot silence this alarm.
|
||||
func checkSourceLiveness(s *SourceLivenessState, threshold time.Duration, now time.Time) (string, LivenessKind) {
|
||||
if s == nil || s.IsConnectedFn == nil {
|
||||
return "", LivenessOK
|
||||
}
|
||||
if !s.IsConnectedFn() {
|
||||
// paho's reconnect handler covers the disconnected case. Return
|
||||
// a DISTINCT kind so the transition engine does not mis-classify
|
||||
// disconnect as recovery (PR #1216 r2 item 1).
|
||||
return "", LivenessDisconnected
|
||||
}
|
||||
last := atomic.LoadInt64(&s.LastMessageUnix)
|
||||
if last == 0 {
|
||||
firstConnected := atomic.LoadInt64(&s.FirstConnectedAt)
|
||||
if firstConnected == 0 {
|
||||
// Registration didn't stamp FirstConnectedAt — conservative: stay quiet.
|
||||
return "", LivenessOK
|
||||
}
|
||||
sinceFirst := now.Sub(time.Unix(firstConnected, 0))
|
||||
if sinceFirst < threshold {
|
||||
return "", LivenessOK
|
||||
}
|
||||
msg := fmt.Sprintf("MQTT [%s] WATCHDOG: client reports connected to %s but has NEVER received a message in %s (threshold %s) — check channel hash / subscribe ACL / half-open TCP",
|
||||
s.Tag, s.Broker, sinceFirst.Round(time.Second), threshold)
|
||||
return msg, LivenessNeverReceived
|
||||
}
|
||||
silentFor := now.Sub(time.Unix(last, 0))
|
||||
if silentFor < threshold {
|
||||
return "", LivenessOK
|
||||
}
|
||||
msg := fmt.Sprintf("MQTT [%s] WATCHDOG: client reports connected to %s but no messages received for %s (threshold %s) — possible half-open socket or upstream stall",
|
||||
s.Tag, s.Broker, silentFor.Round(time.Second), threshold)
|
||||
return msg, LivenessStalled
|
||||
}
|
||||
|
||||
// livenessRegistry is a package-level lookup so handleMessage (called with
|
||||
// only `tag string`) can mark liveness without threading the state through
|
||||
// every call site. Reads dominate (per message); writes happen once per
|
||||
// source at startup.
|
||||
var (
|
||||
livenessRegistry = map[string]*SourceLivenessState{}
|
||||
livenessRegistryMu sync.RWMutex
|
||||
)
|
||||
|
||||
// registerLivenessState publishes a state to the registry by tag. Returns
|
||||
// an error on tag collision (PR #1216 r1 item 4) so operators see a
|
||||
// startup misconfiguration instead of silently losing AttemptCount and
|
||||
// LastMessageUnix for the clobbered source. The collision case is real:
|
||||
// two MQTT sources with empty Name fall back to Broker; two sources with
|
||||
// duplicate Name; copy-paste in config.json. Caller (main) decides whether
|
||||
// to fatal or just log and skip. The first registration remains
|
||||
// authoritative — we do NOT overwrite.
|
||||
//
|
||||
// Also stamps StartedAt (transient-stall window) and FirstConnectedAt
|
||||
// (cold-start grace anchor — never reset; see r2 item 2 in
|
||||
// MarkReconnected) so the cold-start watchdog has its clocks.
|
||||
func registerLivenessState(s *SourceLivenessState) error {
|
||||
livenessRegistryMu.Lock()
|
||||
defer livenessRegistryMu.Unlock()
|
||||
if existing, ok := livenessRegistry[s.Tag]; ok {
|
||||
return fmt.Errorf("liveness registry: duplicate tag %q (existing broker=%s, new broker=%s) — fix config so each MQTT source has a unique Name", s.Tag, existing.Broker, s.Broker)
|
||||
}
|
||||
nowUnix := time.Now().Unix()
|
||||
if atomic.LoadInt64(&s.StartedAt) == 0 {
|
||||
atomic.StoreInt64(&s.StartedAt, nowUnix)
|
||||
}
|
||||
if atomic.LoadInt64(&s.FirstConnectedAt) == 0 {
|
||||
atomic.StoreInt64(&s.FirstConnectedAt, nowUnix)
|
||||
}
|
||||
livenessRegistry[s.Tag] = s
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerLivenessOrSkip (PR #1216 r2 item 3) is the main-callsite wrapper
|
||||
// that replaces the previous log.Fatalf on tag collision. Fatal at
|
||||
// startup over a config typo would kill the entire ingestor and recreate
|
||||
// the #1212 total-ingest-stop class this PR exists to prevent. On
|
||||
// collision we log ERROR + skip — the MQTT source still attempts to
|
||||
// connect, it just won't be tracked by the liveness watchdog. Returns
|
||||
// true iff the source was registered.
|
||||
func registerLivenessOrSkip(s *SourceLivenessState) bool {
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
log.Printf("[ingestor] ERROR: source tag collision %q — skipping duplicate liveness registration, this source will connect but will not be tracked by the watchdog (%v)", s.Tag, err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// markLivenessForTag is the hot-path entry point: O(1) map lookup +
|
||||
// atomic store. Safe to call for unknown tags (no-op).
|
||||
func markLivenessForTag(tag string, now time.Time) {
|
||||
livenessRegistryMu.RLock()
|
||||
s := livenessRegistry[tag]
|
||||
livenessRegistryMu.RUnlock()
|
||||
if s != nil {
|
||||
s.MarkMessage(now)
|
||||
}
|
||||
}
|
||||
|
||||
// runLivenessWatchdog starts a goroutine that scans the registry every
|
||||
// `interval` and logs a warning for any source that has been silent while
|
||||
// connected for more than `threshold`. Returns a stop function that halts
|
||||
// the ticker AND signals the goroutine to exit (time.Ticker.Stop does NOT
|
||||
// close the channel, so a naive `for range t.C` would leak). interval
|
||||
// should be a fraction of threshold (e.g. threshold/5) so detection
|
||||
// latency is bounded.
|
||||
func runLivenessWatchdog(interval, threshold time.Duration) (stop func()) {
|
||||
t := time.NewTicker(interval)
|
||||
done := make(chan struct{})
|
||||
go runLivenessWatchdogLoop(t.C, done, threshold, log.Print)
|
||||
return func() {
|
||||
t.Stop()
|
||||
close(done)
|
||||
}
|
||||
}
|
||||
|
||||
// runLivenessWatchdogLoop is the goroutine body, extracted so tests can
|
||||
// drive it with a synthetic tick channel and capture log output without
|
||||
// racing on the real ticker.
|
||||
//
|
||||
// Edge-triggered (PR #1216 r1 item 3):
|
||||
// - quiet → stalled / never-received: emit WARN once, record LastAlertUnix
|
||||
// - still stalled, < heartbeat interval since last alert: suppress
|
||||
// - still stalled, ≥ heartbeat interval since last alert: emit reminder,
|
||||
// refresh LastAlertUnix
|
||||
// - stalled → flowing: emit recovery INFO once, clear LastAlertUnix
|
||||
//
|
||||
// Without this, the original loop re-emitted the same WARN on every 60s
|
||||
// tick (60 alerts/hr/source) — the kind of log flood that trains ops to
|
||||
// mute alerts and miss the next real outage.
|
||||
func runLivenessWatchdogLoop(tick <-chan time.Time, done <-chan struct{}, threshold time.Duration, emit func(...any)) {
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case now, ok := <-tick:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
livenessRegistryMu.RLock()
|
||||
states := make([]*SourceLivenessState, 0, len(livenessRegistry))
|
||||
for _, s := range livenessRegistry {
|
||||
states = append(states, s)
|
||||
}
|
||||
livenessRegistryMu.RUnlock()
|
||||
for _, s := range states {
|
||||
msg, kind := checkSourceLiveness(s, threshold, now)
|
||||
processLivenessTransition(s, kind, msg, now, emit)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processLivenessTransition applies the edge-trigger rules and updates
|
||||
// LastAlertUnix accordingly. Separated for testability and to keep the
|
||||
// loop body small.
|
||||
func processLivenessTransition(s *SourceLivenessState, kind LivenessKind, msg string, now time.Time, emit func(...any)) {
|
||||
lastAlert := atomic.LoadInt64(&s.LastAlertUnix)
|
||||
switch kind {
|
||||
case LivenessStalled, LivenessNeverReceived:
|
||||
if lastAlert == 0 {
|
||||
// First detection — fire WARN edge.
|
||||
emit(msg)
|
||||
atomic.StoreInt64(&s.LastAlertUnix, now.Unix())
|
||||
// #1335: ONLY LivenessStalled (paho reports connected but no
|
||||
// messages past threshold — classic half-open TCP) gets
|
||||
// force-reconnected. LivenessNeverReceived is almost always
|
||||
// an ACL deny / wrong channel hash — a new TCP socket won't
|
||||
// fix it and would just churn the broker. The distinct
|
||||
// "NEVER received" alarm is the right operator signal for
|
||||
// that class.
|
||||
if kind == LivenessStalled {
|
||||
maybeForceReconnect(s, now, emit)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Already alerted; only re-emit on heartbeat interval to avoid log flood.
|
||||
if now.Sub(time.Unix(lastAlert, 0)) >= livenessHeartbeatInterval {
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG heartbeat: still stalled — %s", s.Tag, msg))
|
||||
atomic.StoreInt64(&s.LastAlertUnix, now.Unix())
|
||||
// Heartbeat re-emit on a still-Stalled source: try another
|
||||
// force-reconnect IF the throttle window has elapsed. Under
|
||||
// a persistent broker issue this caps at one attempt per
|
||||
// heartbeat (1h) — orders of magnitude under any rate
|
||||
// limit and well within "don't hammer the broker".
|
||||
if kind == LivenessStalled {
|
||||
maybeForceReconnect(s, now, emit)
|
||||
}
|
||||
}
|
||||
case LivenessOK:
|
||||
if lastAlert != 0 {
|
||||
// Recovered: emit INFO once, clear the cooldown.
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG INFO: messages flowing again (recovered)", s.Tag))
|
||||
atomic.StoreInt64(&s.LastAlertUnix, 0)
|
||||
}
|
||||
case LivenessDisconnected:
|
||||
// PR #1216 r2 item 1: disconnect is NOT recovery. Stay completely
|
||||
// silent — paho's reconnect handler already logs the drop — and
|
||||
// preserve LastAlertUnix so the WARN edge can re-fire if/when
|
||||
// the source comes back stalled. Clearing the cooldown here
|
||||
// would mean a flapping source spams the WARN every cycle.
|
||||
}
|
||||
}
|
||||
|
||||
// maybeForceReconnect invokes ForceReconnectFn IFF (a) one is wired and
|
||||
// (b) the throttle window (forceReconnectThrottle) has elapsed since
|
||||
// the most recent forced reconnect for this source. Logs WATCHDOG
|
||||
// telemetry before/after so operators can correlate the reconnect with
|
||||
// downstream paho ConnectionAttempt/OnConnect lines.
|
||||
func maybeForceReconnect(s *SourceLivenessState, now time.Time, emit func(...any)) {
|
||||
if s.ForceReconnectFn == nil {
|
||||
return
|
||||
}
|
||||
lastForce := atomic.LoadInt64(&s.LastForceReconnectUnix)
|
||||
if lastForce != 0 && now.Sub(time.Unix(lastForce, 0)) < forceReconnectThrottle {
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG suppressing forced reconnect (last attempt %s ago, throttle %s)",
|
||||
s.Tag, now.Sub(time.Unix(lastForce, 0)).Round(time.Second), forceReconnectThrottle))
|
||||
return
|
||||
}
|
||||
atomic.StoreInt64(&s.LastForceReconnectUnix, now.Unix())
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG forcing reconnect (half-open TCP suspected — paho.IsConnected==true but no messages)", s.Tag))
|
||||
// Run in a goroutine: ForceReconnectFn typically calls
|
||||
// client.Disconnect(250) which blocks up to 250ms, then
|
||||
// client.Connect() which can block on the connect timeout. The
|
||||
// watchdog goroutine must not stall a per-tick scan over a single
|
||||
// slow source.
|
||||
go func() {
|
||||
s.ForceReconnectFn()
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG reconnect attempt issued", s.Tag))
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue #1335 — staging's lincomatic source stalls: paho reports
|
||||
// IsConnected==true but no messages arrive for 1h+. The PR #1216
|
||||
// watchdog DETECTS this (LivenessStalled) but only LOGS — it never
|
||||
// forces paho to drop the half-open TCP socket and reconnect, so the
|
||||
// source stays silently broken until container restart.
|
||||
//
|
||||
// Fix: on transition INTO LivenessStalled, invoke a per-source
|
||||
// ForceReconnectFn (wired in main.go to client.Disconnect(250) +
|
||||
// client.Connect()). Throttled by forceReconnectThrottle so a
|
||||
// stall→reconnect→re-stall loop self-recovers without hammering the
|
||||
// broker.
|
||||
|
||||
// RED on master: ForceReconnectFn is never invoked because the
|
||||
// transition engine does not call it. After the fix, the WARN edge on
|
||||
// LivenessStalled MUST fire force-reconnect exactly once.
|
||||
func TestMQTTStallWatchdog_ForceReconnectOnStallEdge(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "stalled-half-open",
|
||||
Broker: "tcp://halfopen.example:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, now.Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var emits []string
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(args) > 0 {
|
||||
if str, ok := args[0].(string); ok {
|
||||
emits = append(emits, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processLivenessTransition(s, LivenessStalled, "10m silent", now, emit)
|
||||
|
||||
// ForceReconnectFn runs in a goroutine (the production code can't
|
||||
// block the watchdog tick on a slow Disconnect+Connect). Wait
|
||||
// briefly for it to land before asserting.
|
||||
waitForReconnect(t, &reconnectCount, 1, 2*time.Second)
|
||||
|
||||
if got := reconnectCount.Load(); got != 1 {
|
||||
t.Fatalf("LivenessStalled transition MUST force-reconnect exactly once; got %d invocations (emits=%v)", got, emits)
|
||||
}
|
||||
}
|
||||
|
||||
// Throttle: a second LivenessStalled transition within the throttle
|
||||
// window MUST NOT fire a second reconnect (no broker hammering).
|
||||
func TestMQTTStallWatchdog_ForceReconnectThrottled(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "throttled",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
emit := func(args ...any) {}
|
||||
|
||||
// First stall edge → fires.
|
||||
processLivenessTransition(s, LivenessStalled, "stall 1", now, emit)
|
||||
waitForReconnect(t, &reconnectCount, 1, 2*time.Second)
|
||||
// Simulate paho reconnect cycle: MarkReconnected clears the alert
|
||||
// cooldown, then the source goes stalled again 5s later.
|
||||
s.MarkReconnected(now.Add(5 * time.Second))
|
||||
processLivenessTransition(s, LivenessStalled, "stall 2", now.Add(10*time.Second), emit)
|
||||
// Give a stray goroutine a chance to land (it shouldn't, due to throttle).
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if got := reconnectCount.Load(); got != 1 {
|
||||
t.Fatalf("force-reconnect MUST be throttled within %s; got %d invocations", forceReconnectThrottle, got)
|
||||
}
|
||||
|
||||
// After the throttle window, a fresh stall edge MAY fire again.
|
||||
s.MarkReconnected(now.Add(30 * time.Second))
|
||||
processLivenessTransition(s, LivenessStalled, "stall 3", now.Add(forceReconnectThrottle+30*time.Second), emit)
|
||||
waitForReconnect(t, &reconnectCount, 2, 2*time.Second)
|
||||
if got := reconnectCount.Load(); got != 2 {
|
||||
t.Fatalf("after throttle window, force-reconnect must re-arm; got %d invocations", got)
|
||||
}
|
||||
}
|
||||
|
||||
// NeverReceived (cold-start ACL-deny / never-flowed) MUST NOT
|
||||
// force-reconnect. A SUBSCRIBE ACL deny is not fixed by a new TCP
|
||||
// socket; reconnecting just churns the broker. Operators get the
|
||||
// distinct "NEVER received" alarm so they can address the ACL.
|
||||
func TestMQTTStallWatchdog_NoForceReconnectOnNeverReceived(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "acl-denied",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
emit := func(args ...any) {}
|
||||
processLivenessTransition(s, LivenessNeverReceived, "no msgs ever", now, emit)
|
||||
// Settle any (incorrect) goroutine before counting.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if got := reconnectCount.Load(); got != 0 {
|
||||
t.Fatalf("LivenessNeverReceived must NOT force-reconnect (likely ACL deny — TCP churn won't help); got %d invocations", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Safety: a source with no ForceReconnectFn wired (e.g. tests, or a
|
||||
// source registered before the wiring was added) MUST NOT panic when
|
||||
// LivenessStalled fires.
|
||||
func TestMQTTStallWatchdog_NilForceReconnectFnIsSafe(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
s := &SourceLivenessState{
|
||||
Tag: "no-reconnect-fn",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
// ForceReconnectFn deliberately nil.
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("nil ForceReconnectFn must be a safe no-op; panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
processLivenessTransition(s, LivenessStalled, "stalled", now, func(args ...any) {})
|
||||
}
|
||||
|
||||
// waitForReconnect polls reconnectCount until it reaches `want` or the
|
||||
// deadline elapses. ForceReconnectFn runs in a goroutine in production
|
||||
// (Disconnect+Connect can block on broker IO), so tests can't read the
|
||||
// counter synchronously.
|
||||
func waitForReconnect(t *testing.T, count *atomic.Int32, want int32, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if count.Load() >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -1,286 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PR #1216 round-1 review fixes. Tests are RED before the fix lands:
|
||||
// - Item 1: cold-start blind spot — silent-from-start source never alarmed.
|
||||
// - Item 2: reconnect reset — stale LastMessageUnix triggers false stall after recovery.
|
||||
// - Item 3: log flood — every-60s rescan re-emits same WARN forever.
|
||||
// - Item 4: tag collision in registerLivenessState silently overwrites prior state.
|
||||
|
||||
// waitFor polls until emits reaches `want` items or the deadline elapses.
|
||||
// Used to serialize "drain this tick before mutating state" in goroutine
|
||||
// tests so we observe deterministic edge transitions.
|
||||
func waitFor(t *testing.T, mu *sync.Mutex, emits *[]string, want int, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
n := len(*emits)
|
||||
mu.Unlock()
|
||||
if n >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
t.Fatalf("timeout waiting for %d emits; got %d: %v", want, len(*emits), *emits)
|
||||
}
|
||||
|
||||
// Item 1 (RED): a source that connects but never receives a message is
|
||||
// invisible to the current watchdog (LastMessageUnix==0 → skip). This is
|
||||
// the exact #1212 failure class — wrong channel hash, ACL drops SUBSCRIBE,
|
||||
// half-open TCP after CONNECT. Fix: stamp StartedAt at registration; when
|
||||
// LastMessageUnix==0 AND now-StartedAt > threshold, alarm with a distinct
|
||||
// "NEVER received" message.
|
||||
func TestMQTTStallWatchdog_FiresOnSilentFromStart(t *testing.T) {
|
||||
now := time.Now()
|
||||
state := &SourceLivenessState{
|
||||
Tag: "cold",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&state.StartedAt, now.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&state.FirstConnectedAt, now.Add(-10*time.Minute).Unix())
|
||||
// LastMessageUnix stays 0 — never received anything.
|
||||
|
||||
msg, kind := checkSourceLiveness(state, 5*time.Minute, now)
|
||||
if kind != LivenessNeverReceived {
|
||||
t.Fatalf("expected LivenessNeverReceived for silent-from-start source after threshold; got kind=%v msg=%q", kind, msg)
|
||||
}
|
||||
if !strings.Contains(strings.ToUpper(msg), "NEVER") {
|
||||
t.Errorf("cold-start alarm must mention NEVER received to distinguish from generic stall; got %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "cold") {
|
||||
t.Errorf("alarm must include source tag; got %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMQTTStallWatchdog_QuietDuringColdStartGrace(t *testing.T) {
|
||||
now := time.Now()
|
||||
state := &SourceLivenessState{
|
||||
Tag: "warming-up",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&state.StartedAt, now.Add(-30*time.Second).Unix())
|
||||
atomic.StoreInt64(&state.FirstConnectedAt, now.Add(-30*time.Second).Unix())
|
||||
|
||||
_, kind := checkSourceLiveness(state, 5*time.Minute, now)
|
||||
if kind != LivenessOK {
|
||||
t.Fatalf("must NOT alarm during cold-start grace (30s in, threshold 5m); got kind=%v", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// Item 2 (RED): after a long outage + paho reconnect, LastMessageUnix is
|
||||
// still 2h-old → watchdog screams "stalled for 2h" immediately. Fix: reset
|
||||
// LastMessageUnix (and the cold-start clock) on OnConnect. This test
|
||||
// asserts the reset method does what's required so the next watchdog scan
|
||||
// stays quiet for the grace window.
|
||||
func TestMQTTStallWatchdog_OnReconnectResetsClocks(t *testing.T) {
|
||||
now := time.Now()
|
||||
state := &SourceLivenessState{
|
||||
Tag: "flaky",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
// 2-hour-old timestamp from before the outage.
|
||||
atomic.StoreInt64(&state.LastMessageUnix, now.Add(-2*time.Hour).Unix())
|
||||
atomic.StoreInt64(&state.StartedAt, now.Add(-3*time.Hour).Unix())
|
||||
// Stale alert cooldown from before the outage too — must NOT carry forward.
|
||||
atomic.StoreInt64(&state.LastAlertUnix, now.Add(-90*time.Minute).Unix())
|
||||
|
||||
state.MarkReconnected(now)
|
||||
|
||||
if last := atomic.LoadInt64(&state.LastMessageUnix); last != 0 {
|
||||
t.Errorf("LastMessageUnix must be cleared on reconnect so a stale pre-outage timestamp does not trip the watchdog; got %d", last)
|
||||
}
|
||||
if started := atomic.LoadInt64(&state.StartedAt); started != now.Unix() {
|
||||
t.Errorf("StartedAt must be re-stamped on reconnect so the cold-start grace window restarts; got %d want %d", started, now.Unix())
|
||||
}
|
||||
if alert := atomic.LoadInt64(&state.LastAlertUnix); alert != 0 {
|
||||
t.Errorf("LastAlertUnix must be cleared on reconnect so edge-trigger re-arms; got %d", alert)
|
||||
}
|
||||
|
||||
// Now drive checkSourceLiveness immediately after reconnect: must NOT alarm.
|
||||
_, kind := checkSourceLiveness(state, 5*time.Minute, now.Add(1*time.Second))
|
||||
if kind != LivenessOK {
|
||||
t.Fatalf("watchdog must stay quiet immediately after MarkReconnected; got kind=%v", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// Item 3 (RED): the watchdog loop currently re-emits the same WARN on every
|
||||
// 60s tick (60 alerts/hr/source). Fix: edge-trigger — emit WARN once on
|
||||
// quiet→stalled transition, INFO once on stalled→flowing recovery, and an
|
||||
// hourly heartbeat while still stalled. Asserts: 3 consecutive ticks on a
|
||||
// stalled source produce exactly ONE WARN.
|
||||
func TestMQTTStallWatchdog_EdgeTriggeredEmitsOnlyOnce(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
s := &SourceLivenessState{
|
||||
Tag: "stuck",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, now.Add(-20*time.Minute).Unix())
|
||||
registerLivenessState(s)
|
||||
|
||||
var mu sync.Mutex
|
||||
var emits []string
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(args) > 0 {
|
||||
if str, ok := args[0].(string); ok {
|
||||
emits = append(emits, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tick := make(chan time.Time, 3)
|
||||
done := make(chan struct{})
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
runLivenessWatchdogLoop(tick, done, 5*time.Minute, emit)
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
// Three back-to-back ticks within the heartbeat window. Only the first
|
||||
// should emit a WARN; the other two must be suppressed (edge-triggered).
|
||||
tick <- now
|
||||
tick <- now.Add(30 * time.Second)
|
||||
tick <- now.Add(60 * time.Second)
|
||||
|
||||
// Wait for ticks to drain.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
n := len(emits)
|
||||
mu.Unlock()
|
||||
if n >= 1 && time.Since(deadline.Add(-2*time.Second)) > 200*time.Millisecond {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
close(done)
|
||||
<-exited
|
||||
|
||||
mu.Lock()
|
||||
got := append([]string(nil), emits...)
|
||||
mu.Unlock()
|
||||
|
||||
warns := 0
|
||||
for _, e := range got {
|
||||
if strings.Contains(e, "WATCHDOG") || strings.Contains(e, "stalled") || strings.Contains(strings.ToUpper(e), "WARN") {
|
||||
warns++
|
||||
}
|
||||
}
|
||||
if warns != 1 {
|
||||
t.Fatalf("expected exactly 1 stall WARN across 3 consecutive scans (edge-trigger); got %d: %v", warns, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Item 3 (RED): on stalled→flowing transition, a recovery INFO must fire
|
||||
// exactly once. Future ticks must stay silent until a new stall edge.
|
||||
func TestMQTTStallWatchdog_RecoveryEmitOnce(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
s := &SourceLivenessState{
|
||||
Tag: "src-b",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, now.Add(-20*time.Minute).Unix())
|
||||
registerLivenessState(s)
|
||||
|
||||
var mu sync.Mutex
|
||||
var emits []string
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(args) > 0 {
|
||||
if str, ok := args[0].(string); ok {
|
||||
emits = append(emits, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tick := make(chan time.Time, 4)
|
||||
done := make(chan struct{})
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
runLivenessWatchdogLoop(tick, done, 5*time.Minute, emit)
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
tick <- now // → WARN
|
||||
// Wait for the goroutine to drain that tick and record the WARN edge
|
||||
// before we mutate state — otherwise we race the loop and the first
|
||||
// emit observes the "recovered" timestamp instead of the stall.
|
||||
waitFor(t, &mu, &emits, 1, 2*time.Second)
|
||||
// Source recovers: a recent message arrives.
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Add(30*time.Second).Unix())
|
||||
tick <- now.Add(60 * time.Second) // → recovery INFO
|
||||
waitFor(t, &mu, &emits, 2, 2*time.Second)
|
||||
tick <- now.Add(120 * time.Second) // → silent
|
||||
tick <- now.Add(180 * time.Second) // → silent
|
||||
|
||||
// Brief settle so any (incorrect) extra emits land before we count.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
close(done)
|
||||
<-exited
|
||||
|
||||
mu.Lock()
|
||||
got := append([]string(nil), emits...)
|
||||
mu.Unlock()
|
||||
|
||||
infos := 0
|
||||
for _, e := range got {
|
||||
upper := strings.ToUpper(e)
|
||||
if strings.Contains(upper, "RECOVER") || strings.Contains(upper, "FLOWING") {
|
||||
infos++
|
||||
}
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected exactly 2 emits (1 WARN + 1 recovery INFO); got %d: %v", len(got), got)
|
||||
}
|
||||
if infos != 1 {
|
||||
t.Fatalf("expected exactly 1 recovery INFO emit; got %d (all=%v)", infos, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Item 4 (RED): registerLivenessState silently overwrites on tag collision
|
||||
// (empty-Name + same broker, duplicate Name). Must detect & report.
|
||||
func TestRegisterLivenessState_DetectsTagCollision(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
a := &SourceLivenessState{Tag: "dup", Broker: "tcp://a:1883"}
|
||||
b := &SourceLivenessState{Tag: "dup", Broker: "tcp://b:1883"}
|
||||
|
||||
if err := registerLivenessState(a); err != nil {
|
||||
t.Fatalf("first registration must succeed; got %v", err)
|
||||
}
|
||||
if err := registerLivenessState(b); err == nil {
|
||||
t.Fatal("second registration with same tag must return a collision error (current behavior silently clobbers)")
|
||||
}
|
||||
|
||||
// And the registry must still hold the FIRST registration — clobbering
|
||||
// AttemptCount/LastMessageUnix invisibly is the bug.
|
||||
livenessRegistryMu.RLock()
|
||||
got := livenessRegistry["dup"]
|
||||
livenessRegistryMu.RUnlock()
|
||||
if got != a {
|
||||
t.Errorf("on collision, first registration must remain authoritative (got pointer for broker=%s)", got.Broker)
|
||||
}
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PR #1216 round-2 review fixes. Tests RED before the fix lands.
|
||||
//
|
||||
// r1 closed the cold-start blind spot but introduced three new failure
|
||||
// modes that r2 must eliminate:
|
||||
//
|
||||
// r2 #1 — checkSourceLiveness returns LivenessOK for BOTH "messages
|
||||
// flowing" AND "disconnected/never-connected". A stalled source
|
||||
// whose TCP eventually RSTs trips processLivenessTransition's
|
||||
// recovery branch and emits "messages flowing again (recovered)"
|
||||
// while going from silently broken to overtly broken. Fix: a
|
||||
// distinct LivenessDisconnected kind that the transition
|
||||
// function treats as a silent (no-emit) state, so the alert
|
||||
// cooldown does not collapse on a non-event.
|
||||
//
|
||||
// r2 #2 — MarkReconnected re-stamps StartedAt on every reconnect, so
|
||||
// the cold-start grace clock restarts forever under a broker
|
||||
// flap (CONNECT ok, SUBSCRIBE ACL-denied — the exact #1212
|
||||
// shape). The headline "NEVER received" alarm never fires.
|
||||
// Fix: separate FirstConnectedAt (set once at registration,
|
||||
// never reset) from StartedAt (free to reset on reconnect for
|
||||
// transient-stall tracking). Cold-start grace must use
|
||||
// FirstConnectedAt.
|
||||
//
|
||||
// r2 #3 — main.go calls log.Fatalf on a tag collision in the liveness
|
||||
// registry, killing the entire ingestor over one config typo.
|
||||
// That recreates the #1212 total-ingest-stop failure class
|
||||
// this PR exists to prevent. Fix: log an ERROR and skip
|
||||
// liveness registration for the duplicate — the MQTT source
|
||||
// still attempts to connect, just isn't tracked by the
|
||||
// watchdog (the first registration remains authoritative).
|
||||
|
||||
// r2 #1 RED: a stalled source whose connection then drops must NOT emit
|
||||
// "recovered". The current code does — checkSourceLiveness returns
|
||||
// LivenessOK for both genuine recovery and disconnection, so
|
||||
// processLivenessTransition sees lastAlert!=0 + kind==LivenessOK and
|
||||
// fires the recovery INFO. Operators reading the log think the source
|
||||
// healed when it actually died.
|
||||
func TestMQTTStallWatchdog_NoFalseRecoveryOnDisconnect(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
now := time.Now()
|
||||
var connected atomic.Bool
|
||||
connected.Store(true)
|
||||
|
||||
s := &SourceLivenessState{
|
||||
Tag: "drops-after-stall",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return connected.Load() },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, now.Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: registerLivenessState: %v", err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var emits []string
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(args) > 0 {
|
||||
if str, ok := args[0].(string); ok {
|
||||
emits = append(emits, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tick := make(chan time.Time, 2)
|
||||
done := make(chan struct{})
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
runLivenessWatchdogLoop(tick, done, 5*time.Minute, emit)
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
// Tick 1: source connected + 10m silent → WARN edge.
|
||||
tick <- now
|
||||
waitFor(t, &mu, &emits, 1, 2*time.Second)
|
||||
|
||||
// The TCP socket RSTs — paho flips IsConnected to false. The watchdog
|
||||
// must NOT interpret this as recovery; the source went from silently
|
||||
// broken to overtly broken.
|
||||
connected.Store(false)
|
||||
tick <- now.Add(60 * time.Second)
|
||||
|
||||
// Settle so any (incorrect) extra emits land before we count.
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
close(done)
|
||||
<-exited
|
||||
|
||||
mu.Lock()
|
||||
got := append([]string(nil), emits...)
|
||||
mu.Unlock()
|
||||
|
||||
for _, e := range got {
|
||||
upper := strings.ToUpper(e)
|
||||
if strings.Contains(upper, "RECOVER") || strings.Contains(upper, "FLOWING AGAIN") {
|
||||
t.Fatalf("watchdog must NOT emit recovery INFO when a stalled source disconnects; got %q (all=%v)", e, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// r2 #2 RED: a broker that ACKs CONNECT but denies SUBSCRIBE causes paho
|
||||
// to loop CONNECT → drop → CONNECT → drop. Each reconnect calls
|
||||
// MarkReconnected, which re-stamps StartedAt=now and resets the
|
||||
// cold-start grace clock. After 30 minutes of flapping, the source has
|
||||
// still NEVER received a message, but the "NEVER received" alarm never
|
||||
// fires because sinceStart is always sub-threshold. Fix: track
|
||||
// FirstConnectedAt separately from StartedAt; the cold-start check must
|
||||
// use the former.
|
||||
func TestMQTTStallWatchdog_ColdStartSurvivesBrokerFlap(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
t0 := time.Now()
|
||||
s := &SourceLivenessState{
|
||||
Tag: "flapping-acl-deny",
|
||||
Broker: "tcp://acl-denied:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
// First registration stamps FirstConnectedAt (and StartedAt) at t0.
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: registerLivenessState: %v", err)
|
||||
}
|
||||
|
||||
// Paho keeps re-establishing the TCP/MQTT session every minute. No
|
||||
// message ever arrives because SUBSCRIBE is denied. Each reconnect
|
||||
// resets StartedAt.
|
||||
for i := 1; i <= 6; i++ {
|
||||
s.MarkReconnected(t0.Add(time.Duration(i) * time.Minute))
|
||||
}
|
||||
|
||||
// 6m after the very first connection — well past the 5m cold-start
|
||||
// threshold. The headline alarm must fire.
|
||||
now := t0.Add(6*time.Minute + 30*time.Second)
|
||||
_, kind := checkSourceLiveness(s, 5*time.Minute, now)
|
||||
if kind != LivenessNeverReceived {
|
||||
t.Fatalf("under broker flap (#1212 ACL-deny class), cold-start alarm must fire based on FirstConnectedAt, not the most recent reconnect; got kind=%v", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity check: a single transient reconnect WITHIN the cold-start window
|
||||
// must NOT prematurely trip the NeverReceived alarm — the grace was
|
||||
// designed for that. This guards against an over-correction where r2
|
||||
// switches blindly to FirstConnectedAt and ignores legitimate startup
|
||||
// jitter.
|
||||
func TestMQTTStallWatchdog_TransientReconnectDuringGraceStaysQuiet(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
t0 := time.Now()
|
||||
s := &SourceLivenessState{
|
||||
Tag: "transient-reconnect",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: registerLivenessState: %v", err)
|
||||
}
|
||||
|
||||
// 30s in, one transient reconnect.
|
||||
s.MarkReconnected(t0.Add(30 * time.Second))
|
||||
|
||||
// 1m after registration — still inside the 5m grace.
|
||||
_, kind := checkSourceLiveness(s, 5*time.Minute, t0.Add(1*time.Minute))
|
||||
if kind != LivenessOK {
|
||||
t.Fatalf("during cold-start grace, transient reconnects must stay quiet; got kind=%v", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// r2 #3 RED: tag collision must not kill the ingestor. main.go currently
|
||||
// log.Fatalf's, which recreates the #1212 total-ingest-stop class this
|
||||
// PR exists to prevent. registerLivenessOrSkip is the small helper main
|
||||
// will call instead: log an ERROR + skip liveness registration for the
|
||||
// duplicate, return false so the caller knows the source is connecting
|
||||
// untracked. The first registration remains authoritative.
|
||||
func TestRegisterLivenessOrSkip_LogsErrorAndDoesNotExitOnCollision(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
var buf bytes.Buffer
|
||||
origOut := log.Writer()
|
||||
origFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
defer func() {
|
||||
log.SetOutput(origOut)
|
||||
log.SetFlags(origFlags)
|
||||
}()
|
||||
|
||||
a := &SourceLivenessState{Tag: "dup", Broker: "tcp://a:1883"}
|
||||
b := &SourceLivenessState{Tag: "dup", Broker: "tcp://b:1883"}
|
||||
|
||||
if ok := registerLivenessOrSkip(a); !ok {
|
||||
t.Fatalf("first registration must succeed; helper returned false (log=%q)", buf.String())
|
||||
}
|
||||
if ok := registerLivenessOrSkip(b); ok {
|
||||
t.Fatalf("second registration with same tag must return false (skip); helper returned true (log=%q)", buf.String())
|
||||
}
|
||||
|
||||
logOut := buf.String()
|
||||
if !strings.Contains(logOut, "ERROR") {
|
||||
t.Errorf("collision must be logged at ERROR severity so operators see it without it crashing the process; got %q", logOut)
|
||||
}
|
||||
if !strings.Contains(logOut, "dup") {
|
||||
t.Errorf("collision log must include the offending tag; got %q", logOut)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(logOut), "skip") {
|
||||
t.Errorf("collision log must say the duplicate is being skipped so operators know the source is untracked; got %q", logOut)
|
||||
}
|
||||
|
||||
// And the registry still holds the FIRST registration.
|
||||
livenessRegistryMu.RLock()
|
||||
got := livenessRegistry["dup"]
|
||||
livenessRegistryMu.RUnlock()
|
||||
if got != a {
|
||||
t.Errorf("first registration must remain authoritative after collision-skip; got pointer for broker=%s", got.Broker)
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/meshcore-analyzer/mbcapqueue"
|
||||
)
|
||||
|
||||
// MultibyteCapPersistStats holds counts for /api/healthz exposure / logging.
|
||||
type MultibyteCapPersistStats struct {
|
||||
ReadEntries int // entries read from snapshot
|
||||
UpdatedActive int64 // rows updated in nodes
|
||||
UpdatedInactive int64 // rows updated in inactive_nodes
|
||||
Skipped int // entries skipped (status=="unknown")
|
||||
}
|
||||
|
||||
// RunMultibyteCapPersist consumes the latest multi-byte capability snapshot
|
||||
// written by the server (internal/mbcapqueue) and persists it to nodes /
|
||||
// inactive_nodes. Owned by the ingestor per #1287: the server is read-only
|
||||
// since #1289 and cannot UPDATE these columns itself.
|
||||
//
|
||||
// INVARIANT (canonical owner): multibyte_sup / multibyte_evidence are
|
||||
// derived/cached columns. The server COMPUTES the value during its
|
||||
// analytics cycle (from observed packets) and writes a snapshot file;
|
||||
// this function is the ONLY runtime path that mutates those columns
|
||||
// (the schema itself is added by internal/dbschema). The server MUST
|
||||
// NOT execute any UPDATE on nodes.multibyte_* — see
|
||||
// cmd/server/readonly_invariant_test.go for the enforcement.
|
||||
//
|
||||
// Data-destruction guard: entries with Status=="unknown" (sup==0) are
|
||||
// NEVER persisted — we never overwrite a previously confirmed/suspected
|
||||
// DB value with a snapshot blank. Same guarantee the original
|
||||
// server-side helper enforced before relocation.
|
||||
//
|
||||
// Safe to call from a ticker; no-op when no snapshot has been written
|
||||
// (cold start), when the snapshot is empty, when the snapshot is
|
||||
// malformed (#1386), or when running against a legacy DB that
|
||||
// pre-dates the multibyte_sup migration (#1386).
|
||||
func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) {
|
||||
var stats MultibyteCapPersistStats
|
||||
snap, err := mbcapqueue.ReadSnapshot(s.path)
|
||||
if err != nil {
|
||||
// os.ErrNotExist is the steady state until the server's first
|
||||
// analytics cycle completes — silent no-op. A malformed file
|
||||
// is operator-actionable: log it (but still no-op, no error
|
||||
// surfaced to the ticker — a corrupt snapshot must not stop
|
||||
// the maintenance loop).
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return stats, nil
|
||||
}
|
||||
// All other ReadSnapshot errors today are wrap-arounds of
|
||||
// io / unmarshal failures — both classify as "malformed
|
||||
// snapshot on disk" from this loop's perspective.
|
||||
var jsonErr *json.SyntaxError
|
||||
if errors.As(err, &jsonErr) || isMalformedSnapshotErr(err) {
|
||||
log.Printf("[multibyte-persist] malformed snapshot on disk (no-op): %v", err)
|
||||
return stats, nil
|
||||
}
|
||||
log.Printf("[multibyte-persist] read snapshot: %v (no-op)", err)
|
||||
return stats, nil
|
||||
}
|
||||
stats.ReadEntries = len(snap.Entries)
|
||||
if len(snap.Entries) == 0 {
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// Defensive schema check: a legacy DB that pre-dates the
|
||||
// multibyte_sup migration would fail at tx.Prepare with a SQL
|
||||
// error. Detect early and skip cleanly so the ticker keeps
|
||||
// running on heterogeneous deployments.
|
||||
if !s.hasMultibyteSupColumns() {
|
||||
log.Printf("[multibyte-persist] schema missing: nodes.multibyte_sup not present on this DB (legacy schema) — skipping %d entries", stats.ReadEntries)
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
// Combined dispatch: each pubkey lives in exactly one of nodes /
|
||||
// inactive_nodes. The pre-#1386 implementation issued one UPDATE
|
||||
// against each table per entry — 50% guaranteed-empty. We now
|
||||
// look up the table once, then issue the matching UPDATE.
|
||||
stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer stmtN.Close()
|
||||
stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer stmtI.Close()
|
||||
// Membership probe: one indexed PK lookup. Cheap; avoids the
|
||||
// guaranteed-miss second UPDATE.
|
||||
stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=? LIMIT 1`)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
defer stmtProbe.Close()
|
||||
|
||||
for _, e := range snap.Entries {
|
||||
sup := multibyteStatusToInt(e.Status)
|
||||
if sup == 0 {
|
||||
stats.Skipped++
|
||||
continue
|
||||
}
|
||||
// Probe once. If hit, UPDATE nodes; else UPDATE inactive_nodes.
|
||||
var hit int
|
||||
if err := stmtProbe.QueryRow(e.PublicKey).Scan(&hit); err == nil {
|
||||
if r, err := stmtN.Exec(sup, e.Evidence, e.PublicKey); err == nil {
|
||||
if n, _ := r.RowsAffected(); n > 0 {
|
||||
stats.UpdatedActive += n
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if r, err := stmtI.Exec(sup, e.Evidence, e.PublicKey); err == nil {
|
||||
if n, _ := r.RowsAffected(); n > 0 {
|
||||
stats.UpdatedInactive += n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if stats.UpdatedActive+stats.UpdatedInactive > 0 {
|
||||
log.Printf("[multibyte-persist] applied snapshot: %d entries (%d skipped); updated %d active + %d inactive nodes",
|
||||
stats.ReadEntries, stats.Skipped, stats.UpdatedActive, stats.UpdatedInactive)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// isMalformedSnapshotErr returns true if err looks like a JSON parse /
|
||||
// IO-truncation failure surfaced by mbcapqueue.ReadSnapshot. The
|
||||
// queue wraps errors with %w but mbcapqueue currently formats with
|
||||
// %w only for "read:"/"unmarshal:" prefixes — we substring-match
|
||||
// those so the operator-actionable log message is unambiguous.
|
||||
func isMalformedSnapshotErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, frag := range []string{"unmarshal", "invalid character", "unexpected end of JSON"} {
|
||||
if containsCI(msg, frag) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsCI(s, sub string) bool {
|
||||
if len(sub) == 0 {
|
||||
return true
|
||||
}
|
||||
// case-insensitive Contains without importing strings (already
|
||||
// imported in db.go, but keeping helper local to avoid widening
|
||||
// this file's imports).
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
match := true
|
||||
for j := 0; j < len(sub); j++ {
|
||||
a, b := s[i+j], sub[j]
|
||||
if a >= 'A' && a <= 'Z' {
|
||||
a += 32
|
||||
}
|
||||
if b >= 'A' && b <= 'Z' {
|
||||
b += 32
|
||||
}
|
||||
if a != b {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasMultibyteSupColumns probes whether the active DB carries the
|
||||
// multibyte_sup column on the `nodes` table. Used to short-circuit
|
||||
// RunMultibyteCapPersist on legacy DBs that pre-date the
|
||||
// internal/dbschema migration (#1386).
|
||||
func (s *Store) hasMultibyteSupColumns() bool {
|
||||
rows, err := s.db.Query(`PRAGMA table_info(nodes)`)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull, pk int
|
||||
var dflt interface{}
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
return false
|
||||
}
|
||||
if name == "multibyte_sup" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// multibyteStatusToInt mirrors the mapping the server used before relocation.
|
||||
// 0 = unknown (never persisted), 1 = suspected, 2 = confirmed.
|
||||
func multibyteStatusToInt(status string) int {
|
||||
switch status {
|
||||
case "confirmed":
|
||||
return 2
|
||||
case "suspected":
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// captureLogs redirects the standard logger to a buffer for the
|
||||
// duration of the test and returns the buffer. Restores the previous
|
||||
// writer when the test ends.
|
||||
func captureLogs(t *testing.T) *bytes.Buffer {
|
||||
t.Helper()
|
||||
buf := &bytes.Buffer{}
|
||||
prevWriter := log.Writer()
|
||||
prevFlags := log.Flags()
|
||||
log.SetOutput(buf)
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(prevWriter)
|
||||
log.SetFlags(prevFlags)
|
||||
})
|
||||
return buf
|
||||
}
|
||||
|
||||
// logContains reports whether the captured log buffer contains substr
|
||||
// (case-insensitive).
|
||||
func logContains(buf *bytes.Buffer, substr string) bool {
|
||||
return strings.Contains(strings.ToLower(buf.String()), strings.ToLower(substr))
|
||||
}
|
||||
|
||||
// columnExists reports whether the named column exists on the table.
|
||||
func columnExists(t *testing.T, db *sql.DB, table, col string) bool {
|
||||
t.Helper()
|
||||
rows, err := db.Query("PRAGMA table_info(" + table + ")")
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA table_info(%s): %v", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull, pk int
|
||||
var dfltValue sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil {
|
||||
t.Fatalf("scan PRAGMA: %v", err)
|
||||
}
|
||||
if name == col {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,369 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/meshcore-analyzer/mbcapqueue"
|
||||
)
|
||||
|
||||
// TestRunMultibyteCapPersist_AppliesSnapshot enforces the architectural
|
||||
// invariant from #1289 + #1322 + #1324 follow-up: the multi-byte
|
||||
// capability columns (multibyte_sup / multibyte_evidence) on
|
||||
// nodes / inactive_nodes MUST be written by the ingestor, NEVER by the
|
||||
// read-only server. The server publishes a snapshot file via
|
||||
// internal/mbcapqueue; the ingestor's maintenance loop applies it here.
|
||||
//
|
||||
// Pre-relocation (PR #1324 as-shipped), the server held a write handle
|
||||
// and executed UPDATE … nodes SET multibyte_sup directly — which is
|
||||
// impossible after #1289 made the server's *sql.DB read-only. This test
|
||||
// asserts the relocated path: snapshot in → UPDATEs out, from the
|
||||
// ingestor side.
|
||||
func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Seed two nodes: one active, one inactive.
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('aa11', 'Alpha', 'repeater', '2026-01-01T00:00:00Z', 0, NULL)`); err != nil {
|
||||
t.Fatalf("seed nodes: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('bb22', 'Bravo', 'repeater', '2025-01-01T00:00:00Z', 0, NULL)`); err != nil {
|
||||
t.Fatalf("seed inactive_nodes: %v", err)
|
||||
}
|
||||
// Seed a third node already confirmed, then send "unknown" for it —
|
||||
// the data-destruction guard must keep its DB value.
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('cc33', 'Charlie', 'repeater', '2026-01-01T00:00:00Z', 2, 'advert')`); err != nil {
|
||||
t.Fatalf("seed cc33: %v", err)
|
||||
}
|
||||
|
||||
snap := mbcapqueue.Snapshot{Entries: []mbcapqueue.Entry{
|
||||
{PublicKey: "aa11", Status: "confirmed", Evidence: "advert"},
|
||||
{PublicKey: "bb22", Status: "suspected", Evidence: "path"},
|
||||
{PublicKey: "cc33", Status: "unknown"}, // must NOT overwrite
|
||||
}}
|
||||
if err := mbcapqueue.WriteSnapshot(dbPath, snap); err != nil {
|
||||
t.Fatalf("WriteSnapshot: %v", err)
|
||||
}
|
||||
// Sanity: snapshot file landed where we expect.
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(dbPath), mbcapqueue.QueueDirName, mbcapqueue.SnapshotFileName)); err != nil {
|
||||
t.Fatalf("snapshot not on disk: %v", err)
|
||||
}
|
||||
|
||||
stats, err := store.RunMultibyteCapPersist()
|
||||
if err != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist: %v", err)
|
||||
}
|
||||
if stats.ReadEntries != 3 {
|
||||
t.Errorf("ReadEntries = %d, want 3", stats.ReadEntries)
|
||||
}
|
||||
if stats.Skipped != 1 {
|
||||
t.Errorf("Skipped = %d, want 1 (the unknown entry)", stats.Skipped)
|
||||
}
|
||||
if stats.UpdatedActive == 0 {
|
||||
t.Errorf("UpdatedActive = 0; expected aa11 to be updated in nodes")
|
||||
}
|
||||
if stats.UpdatedInactive == 0 {
|
||||
t.Errorf("UpdatedInactive = 0; expected bb22 to be updated in inactive_nodes")
|
||||
}
|
||||
|
||||
// Verify DB state.
|
||||
var sup int
|
||||
var evid string
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM nodes WHERE public_key='aa11'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read aa11: %v", err)
|
||||
}
|
||||
if sup != 2 || evid != "advert" {
|
||||
t.Errorf("aa11 after persist: sup=%d evid=%q, want sup=2 evid=advert", sup, evid)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM inactive_nodes WHERE public_key='bb22'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read bb22: %v", err)
|
||||
}
|
||||
if sup != 1 || evid != "path" {
|
||||
t.Errorf("bb22 after persist: sup=%d evid=%q, want sup=1 evid=path", sup, evid)
|
||||
}
|
||||
// Data-destruction guard: cc33 must still be confirmed=2/'advert'.
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM nodes WHERE public_key='cc33'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read cc33: %v", err)
|
||||
}
|
||||
if sup != 2 || evid != "advert" {
|
||||
t.Errorf("cc33 was overwritten by unknown entry: sup=%d evid=%q, want sup=2 evid=advert", sup, evid)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunMultibyteCapPersist_NoSnapshot_NoOp verifies that the persist
|
||||
// step is a clean no-op when the server hasn't written a snapshot yet
|
||||
// (cold start; the analytics cycle takes ~15s after server boot).
|
||||
func TestRunMultibyteCapPersist_NoSnapshot_NoOp(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
stats, err := store.RunMultibyteCapPersist()
|
||||
if err != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist (no snapshot): %v", err)
|
||||
}
|
||||
if stats.ReadEntries != 0 || stats.UpdatedActive != 0 || stats.UpdatedInactive != 0 {
|
||||
t.Errorf("expected zero-valued stats on cold start, got %+v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunMultibyteCapPersist_RoundTrip exercises the full end-to-end
|
||||
// contract claimed by PR #1324: the server writes a snapshot, the
|
||||
// ingestor persists it, and after a simulated restart (close + reopen
|
||||
// the store) the DB still carries the persisted state.
|
||||
//
|
||||
// The audit (#1386) flagged this as the #1 missing test: the two halves
|
||||
// (persist / read-back) were each tested in isolation, but no single
|
||||
// test proved the persist path produces a database state the loader
|
||||
// can later consume — so a column-rename or snapshot-version drift
|
||||
// would slip past.
|
||||
func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
// --- Phase 1: open store, seed, persist snapshot ---
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('dd44', 'Delta', 'repeater', '2026-01-01T00:00:00Z', 0, NULL)`); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('ee55', 'Echo', 'companion', '2025-12-01T00:00:00Z', 0, NULL)`); err != nil {
|
||||
t.Fatalf("seed inactive: %v", err)
|
||||
}
|
||||
snap := mbcapqueue.Snapshot{Entries: []mbcapqueue.Entry{
|
||||
{PublicKey: "dd44", Status: "confirmed", Evidence: "advert"},
|
||||
{PublicKey: "ee55", Status: "suspected", Evidence: "path"},
|
||||
}}
|
||||
if err := mbcapqueue.WriteSnapshot(dbPath, snap); err != nil {
|
||||
t.Fatalf("WriteSnapshot: %v", err)
|
||||
}
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist: %v", err)
|
||||
}
|
||||
// Capture original state for round-trip comparison.
|
||||
var origActiveSup, origInactiveSup int
|
||||
var origActiveEvid, origInactiveEvid string
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM nodes WHERE public_key='dd44'`).Scan(&origActiveSup, &origActiveEvid); err != nil {
|
||||
t.Fatalf("read dd44 (phase1): %v", err)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM inactive_nodes WHERE public_key='ee55'`).Scan(&origInactiveSup, &origInactiveEvid); err != nil {
|
||||
t.Fatalf("read ee55 (phase1): %v", err)
|
||||
}
|
||||
// Simulate restart: drop the in-memory Store entirely.
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
// --- Phase 2: fresh Store, verify persisted state survived ---
|
||||
store2, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore (reopen): %v", err)
|
||||
}
|
||||
defer store2.Close()
|
||||
var sup int
|
||||
var evid string
|
||||
if err := store2.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM nodes WHERE public_key='dd44'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read dd44 after reopen: %v", err)
|
||||
}
|
||||
if sup != origActiveSup || evid != origActiveEvid {
|
||||
t.Errorf("dd44 after restart: sup=%d evid=%q, want sup=%d evid=%q", sup, evid, origActiveSup, origActiveEvid)
|
||||
}
|
||||
if sup != 2 || evid != "advert" {
|
||||
t.Errorf("dd44 after restart: sup=%d evid=%q, want sup=2 evid=advert", sup, evid)
|
||||
}
|
||||
if err := store2.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM inactive_nodes WHERE public_key='ee55'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read ee55 after reopen: %v", err)
|
||||
}
|
||||
if sup != origInactiveSup || evid != origInactiveEvid {
|
||||
t.Errorf("ee55 after restart: sup=%d evid=%q, want sup=%d evid=%q", sup, evid, origInactiveSup, origInactiveEvid)
|
||||
}
|
||||
if sup != 1 || evid != "path" {
|
||||
t.Errorf("ee55 after restart: sup=%d evid=%q, want sup=1 evid=path", sup, evid)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunMultibyteCapPersist_MalformedSnapshot verifies the persist
|
||||
// path is safe against a corrupted/truncated snapshot file: it must
|
||||
// return without error (no-op), MUST NOT crash, AND MUST log a warning
|
||||
// distinguishing the malformed case from the steady-state "no
|
||||
// snapshot yet" cold-start case.
|
||||
//
|
||||
// Audit (#1386, kent-beck) flagged: "Snapshot file malformed /
|
||||
// truncated / wrong-version — RunMultibyteCapPersist error vs.
|
||||
// silent-skip behavior is unspecified by any test."
|
||||
func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Write malformed JSON directly to the snapshot path.
|
||||
if err := mbcapqueue.EnsureDir(dbPath); err != nil {
|
||||
t.Fatalf("EnsureDir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(mbcapqueue.SnapshotPath(dbPath), []byte("not-json{{{garbage"), 0o644); err != nil {
|
||||
t.Fatalf("write malformed: %v", err)
|
||||
}
|
||||
|
||||
// Capture log output to assert the warning is emitted.
|
||||
logBuf := captureLogs(t)
|
||||
|
||||
// Must not panic.
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist panicked on malformed snapshot: %v", r)
|
||||
}
|
||||
}()
|
||||
stats, err := store.RunMultibyteCapPersist()
|
||||
if err != nil {
|
||||
t.Errorf("RunMultibyteCapPersist on malformed snapshot returned error %v; expected silent no-op", err)
|
||||
}
|
||||
if stats.ReadEntries != 0 || stats.UpdatedActive != 0 || stats.UpdatedInactive != 0 {
|
||||
t.Errorf("expected zero-valued stats on malformed snapshot, got %+v", stats)
|
||||
}
|
||||
if !logContains(logBuf, "malformed") && !logContains(logBuf, "invalid") && !logContains(logBuf, "corrupt") {
|
||||
t.Errorf("expected log to mention malformed/invalid/corrupt snapshot; got: %s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunMultibyteCapPersist_MissingSchemaColumns verifies the persist
|
||||
// path is a clean no-op on a legacy DB that doesn't yet have the
|
||||
// multibyte_sup / multibyte_evidence columns. Currently the persist
|
||||
// would fail at tx.Prepare with a SQL error; the audit requires it
|
||||
// skip cleanly instead.
|
||||
//
|
||||
// We simulate a legacy DB by DROPping the columns post-migration
|
||||
// (SQLite ≥ 3.35 supports ALTER TABLE DROP COLUMN).
|
||||
func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Drop the multibyte columns from both tables to simulate a legacy DB.
|
||||
for _, stmt := range []string{
|
||||
`ALTER TABLE nodes DROP COLUMN multibyte_sup`,
|
||||
`ALTER TABLE nodes DROP COLUMN multibyte_evidence`,
|
||||
`ALTER TABLE inactive_nodes DROP COLUMN multibyte_sup`,
|
||||
`ALTER TABLE inactive_nodes DROP COLUMN multibyte_evidence`,
|
||||
} {
|
||||
if _, err := store.db.Exec(stmt); err != nil {
|
||||
t.Fatalf("simulate legacy DB (%q): %v", stmt, err)
|
||||
}
|
||||
}
|
||||
// Confirm columns are gone.
|
||||
if columnExists(t, store.db, "nodes", "multibyte_sup") {
|
||||
t.Fatalf("setup failed: nodes.multibyte_sup still present after DROP")
|
||||
}
|
||||
|
||||
snap := mbcapqueue.Snapshot{Entries: []mbcapqueue.Entry{
|
||||
{PublicKey: "ff66", Status: "confirmed", Evidence: "advert"},
|
||||
}}
|
||||
if err := mbcapqueue.WriteSnapshot(dbPath, snap); err != nil {
|
||||
t.Fatalf("WriteSnapshot: %v", err)
|
||||
}
|
||||
|
||||
logBuf := captureLogs(t)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist panicked on legacy DB: %v", r)
|
||||
}
|
||||
}()
|
||||
stats, err := store.RunMultibyteCapPersist()
|
||||
if err != nil {
|
||||
t.Errorf("RunMultibyteCapPersist on legacy DB returned error %v; expected clean skip", err)
|
||||
}
|
||||
if stats.UpdatedActive != 0 || stats.UpdatedInactive != 0 {
|
||||
t.Errorf("expected zero writes on legacy DB, got %+v", stats)
|
||||
}
|
||||
// Must explicitly detect + log the skip — otherwise the "clean skip"
|
||||
// is silent UPDATE-affected-zero accident, not defensive code.
|
||||
if !logContains(logBuf, "legacy") && !logContains(logBuf, "schema") && !logContains(logBuf, "multibyte_sup") {
|
||||
t.Errorf("expected explicit log on missing schema columns; got: %s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunMultibyteCapPersist_PreservesConfirmedOnUnknown is the
|
||||
// data-destruction guard the PR claims to enforce: a snapshot Entry
|
||||
// with status="unknown" must NEVER overwrite an existing "confirmed"
|
||||
// (or "suspected") DB row. The audit's mutation test: revert the
|
||||
// `if sup == 0 { continue }` guard in multibyte_persist.go — this
|
||||
// test must fail.
|
||||
func TestRunMultibyteCapPersist_PreservesConfirmedOnUnknown(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Seed a confirmed active node and a suspected inactive node.
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('gg77', 'Golf', 'repeater', '2026-01-01T00:00:00Z', 2, 'advert')`); err != nil {
|
||||
t.Fatalf("seed gg77: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name, role, last_seen, multibyte_sup, multibyte_evidence)
|
||||
VALUES ('hh88', 'Hotel', 'companion', '2025-12-01T00:00:00Z', 1, 'path')`); err != nil {
|
||||
t.Fatalf("seed hh88: %v", err)
|
||||
}
|
||||
|
||||
// Snapshot has only "unknown" entries for both — must skip both.
|
||||
snap := mbcapqueue.Snapshot{Entries: []mbcapqueue.Entry{
|
||||
{PublicKey: "gg77", Status: "unknown"},
|
||||
{PublicKey: "hh88", Status: "unknown"},
|
||||
}}
|
||||
if err := mbcapqueue.WriteSnapshot(dbPath, snap); err != nil {
|
||||
t.Fatalf("WriteSnapshot: %v", err)
|
||||
}
|
||||
|
||||
stats, err := store.RunMultibyteCapPersist()
|
||||
if err != nil {
|
||||
t.Fatalf("RunMultibyteCapPersist: %v", err)
|
||||
}
|
||||
if stats.Skipped != 2 {
|
||||
t.Errorf("Skipped = %d, want 2 (both unknown entries)", stats.Skipped)
|
||||
}
|
||||
if stats.UpdatedActive != 0 || stats.UpdatedInactive != 0 {
|
||||
t.Errorf("expected zero updates, got %+v", stats)
|
||||
}
|
||||
|
||||
// Verify the existing values were NOT clobbered.
|
||||
var sup int
|
||||
var evid string
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM nodes WHERE public_key='gg77'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read gg77: %v", err)
|
||||
}
|
||||
if sup != 2 || evid != "advert" {
|
||||
t.Errorf("gg77 was clobbered by unknown snapshot: sup=%d evid=%q, want sup=2 evid=advert", sup, evid)
|
||||
}
|
||||
if err := store.db.QueryRow(`SELECT multibyte_sup, COALESCE(multibyte_evidence,'') FROM inactive_nodes WHERE public_key='hh88'`).Scan(&sup, &evid); err != nil {
|
||||
t.Fatalf("read hh88: %v", err)
|
||||
}
|
||||
if sup != 1 || evid != "path" {
|
||||
t.Errorf("hh88 was clobbered by unknown snapshot: sup=%d evid=%q, want sup=1 evid=path", sup, evid)
|
||||
}
|
||||
}
|
||||
@@ -1,311 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NeighborEdgesBuilderInterval is how often the ingestor rescans
|
||||
// observations and refreshes neighbor_edges. Server reads with the
|
||||
// same 60s cadence (see cmd/server/neighbor_recomputer.go); a 60s
|
||||
// pulse here is sufficient to keep the snapshot fresh.
|
||||
const NeighborEdgesBuilderInterval = 60 * time.Second
|
||||
|
||||
// neighborBuilderMaxBatch caps how many observation rows a single
|
||||
// delta tick may process (#1339). With max_open_conns=1, an unbounded
|
||||
// scan on a multi-million-row table holds the SQLite write lock for
|
||||
// minutes and starves MQTT ingest. The cap keeps each tick bounded;
|
||||
// if a backlog accumulates, successive ticks drain it 50k rows at a
|
||||
// time without ever blocking ingest for long.
|
||||
const neighborBuilderMaxBatch = 50000
|
||||
|
||||
// neighborBuilderSlowTickThreshold is the per-tick wallclock budget
|
||||
// for the builder. Exceeding it is logged loudly so operators can
|
||||
// catch a regression of #1339 quickly. The full instrumentation
|
||||
// framework is tracked in #1340.
|
||||
const neighborBuilderSlowTickThreshold = 5 * time.Second
|
||||
|
||||
// payloadADVERT mirrors the constant in cmd/server/decoder.go.
|
||||
// Duplicated rather than imported so the ingestor binary stays
|
||||
// independent of the server package.
|
||||
const payloadADVERT = 0x04
|
||||
|
||||
// edgeRow is one row to upsert into neighbor_edges. (a, b) is already
|
||||
// canonical-ordered (a <= b).
|
||||
type edgeRow struct {
|
||||
a, b, ts string
|
||||
}
|
||||
|
||||
// StartNeighborEdgesBuilder launches the periodic builder. On each
|
||||
// tick it rescans recent observations + transmissions and upserts
|
||||
// derived neighbor_edges rows. Builder is the only writer to
|
||||
// neighbor_edges (#1287).
|
||||
//
|
||||
// The function returns a stop closure. Initial build runs synchronously
|
||||
// before the ticker starts so the server's first snapshot load picks
|
||||
// up real data instead of an empty table.
|
||||
func (s *Store) StartNeighborEdgesBuilder(interval time.Duration) func() {
|
||||
if interval <= 0 {
|
||||
interval = NeighborEdgesBuilderInterval
|
||||
}
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
|
||||
// Synchronous warm-up: on a fresh DB this is a full scan; on a DB
|
||||
// with persisted neighbor_edges (most restarts), the watermark
|
||||
// short-circuits it into a delta scan. Loop until the per-tick
|
||||
// batch cap stops triggering so we drain any backlog before
|
||||
// returning — first server load needs a fully-populated table.
|
||||
wuStart := time.Now()
|
||||
var wuTotal int
|
||||
for {
|
||||
n, err := s.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
log.Printf("[neighbor-build] initial build error: %v", err)
|
||||
break
|
||||
}
|
||||
wuTotal += n
|
||||
if n < neighborBuilderMaxBatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.Printf("[neighbor-build] initial build: %d edges upserted in %s", wuTotal, time.Since(wuStart))
|
||||
|
||||
var stopOnce sync.Once
|
||||
go func() {
|
||||
defer close(done)
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
start := time.Now()
|
||||
n, err := s.buildAndPersistNeighborEdges()
|
||||
dur := time.Since(start)
|
||||
if err != nil {
|
||||
log.Printf("[neighbor-build] tick error after %s: %v", dur, err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-build] tick: %d edges in %s (delta from watermark)", n, dur)
|
||||
}
|
||||
if dur > neighborBuilderSlowTickThreshold {
|
||||
log.Printf("[neighbor-build] SLOW tick: %s — possible regression of #1339", dur)
|
||||
}
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
stopOnce.Do(func() { close(stop) })
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildAndPersistNeighborEdges scans transmissions + observations,
|
||||
// extracts edge candidates (originator↔first-hop on ADVERTs;
|
||||
// observer↔last-hop on all packet types) and upserts them into
|
||||
// neighbor_edges. Returns count of attempted upserts.
|
||||
//
|
||||
// Watermark / delta semantics (#1339): the builder derives a watermark
|
||||
// from MAX(neighbor_edges.last_seen). On an empty edges table (fresh
|
||||
// DB), watermark is 0 and the builder does a full warm-up scan. On
|
||||
// every subsequent call, the SELECT is restricted to observations
|
||||
// whose timestamp is strictly greater than the watermark, bounded by
|
||||
// neighborBuilderMaxBatch. neighbor_edges itself is the persistence —
|
||||
// no metadata table or in-memory state is required, and restarts
|
||||
// resume cleanly from whatever the table reflects.
|
||||
//
|
||||
// Trade-off (documented for #1340 follow-up): an anomalously-old
|
||||
// observation that arrives AFTER its timestamp has already been
|
||||
// crossed by the watermark will be skipped. Acceptable for an
|
||||
// approximate neighbor graph; a periodic full-rebuild can be added
|
||||
// later if needed.
|
||||
//
|
||||
// Resolution of hop-prefix → full pubkey is done via a one-shot
|
||||
// SELECT of (lowered) pubkey prefixes from nodes. Prefixes with
|
||||
// multiple candidates are skipped (matches the conservative
|
||||
// resolution rule in cmd/server/extractEdgesFromObs).
|
||||
func (s *Store) buildAndPersistNeighborEdges() (int, error) {
|
||||
prefixIdx, err := buildPrefixIndex(s.db)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("build prefix index: %w", err)
|
||||
}
|
||||
|
||||
// Derive the watermark from the existing edges table. RFC3339
|
||||
// → epoch seconds so it can be compared against observations.timestamp
|
||||
// (stored as INTEGER unix epoch). On an empty edges table both the
|
||||
// query and the parse return zero → full warm-up scan.
|
||||
var watermarkRFC sql.NullString
|
||||
if err := s.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&watermarkRFC); err != nil {
|
||||
return 0, fmt.Errorf("read watermark: %w", err)
|
||||
}
|
||||
var watermarkEpoch int64
|
||||
if watermarkRFC.Valid && watermarkRFC.String != "" {
|
||||
if t, parseErr := time.Parse(time.RFC3339, watermarkRFC.String); parseErr == nil {
|
||||
watermarkEpoch = t.Unix()
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(`SELECT
|
||||
t.payload_type,
|
||||
t.decoded_json,
|
||||
COALESCE(t.from_pubkey, ''),
|
||||
COALESCE(o.path_json, ''),
|
||||
COALESCE(obs.id, '') AS observer_id,
|
||||
o.timestamp
|
||||
FROM observations o
|
||||
JOIN transmissions t ON t.id = o.transmission_id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
WHERE o.timestamp > ?
|
||||
ORDER BY o.timestamp
|
||||
LIMIT ?`, watermarkEpoch, neighborBuilderMaxBatch)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("scan observations: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var edges []edgeRow
|
||||
for rows.Next() {
|
||||
var payloadType sql.NullInt64
|
||||
var decodedJSON, fromPubkey, pathJSON, observerID string
|
||||
var epochTs int64
|
||||
if err := rows.Scan(&payloadType, &decodedJSON, &fromPubkey, &pathJSON, &observerID, &epochTs); err != nil {
|
||||
continue
|
||||
}
|
||||
fromNode := strings.ToLower(fromPubkey)
|
||||
if fromNode == "" {
|
||||
fromNode = strings.ToLower(extractPubkeyFromAdvertJSON(decodedJSON))
|
||||
}
|
||||
isAdvert := payloadType.Valid && payloadType.Int64 == int64(payloadADVERT)
|
||||
ts := time.Unix(epochTs, 0).UTC().Format(time.RFC3339)
|
||||
observerPK := strings.ToLower(observerID)
|
||||
path := parsePathArray(pathJSON)
|
||||
|
||||
if len(path) == 0 {
|
||||
if isAdvert && fromNode != "" && fromNode != observerPK && observerPK != "" {
|
||||
edges = append(edges, canonEdge(fromNode, observerPK, ts))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isAdvert && fromNode != "" {
|
||||
if resolved, ok := resolvePrefix(prefixIdx, path[0]); ok && resolved != fromNode {
|
||||
edges = append(edges, canonEdge(fromNode, resolved, ts))
|
||||
}
|
||||
}
|
||||
if observerPK != "" {
|
||||
last := path[len(path)-1]
|
||||
if resolved, ok := resolvePrefix(prefixIdx, last); ok && resolved != observerPK {
|
||||
edges = append(edges, canonEdge(observerPK, resolved, ts))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(edges) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(node_a, node_b) DO UPDATE SET
|
||||
count = count + 1,
|
||||
last_seen = MAX(last_seen, excluded.last_seen)`)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
var firstErr error
|
||||
for _, e := range edges {
|
||||
if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
return 0, fmt.Errorf("upsert: %w", firstErr)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("commit: %w", err)
|
||||
}
|
||||
return len(edges), nil
|
||||
}
|
||||
|
||||
// canonEdge orders the pair so node_a <= node_b (matches the existing
|
||||
// schema convention used by the loader and the bridge recomputer).
|
||||
func canonEdge(a, b, ts string) edgeRow {
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
return edgeRow{a, b, ts}
|
||||
}
|
||||
|
||||
// parsePathArray returns the hop strings from a path_json blob.
|
||||
// Defensive against missing/invalid JSON.
|
||||
func parsePathArray(s string) []string {
|
||||
if s == "" || s == "[]" {
|
||||
return nil
|
||||
}
|
||||
var arr []string
|
||||
if json.Unmarshal([]byte(s), &arr) != nil {
|
||||
return nil
|
||||
}
|
||||
return arr
|
||||
}
|
||||
|
||||
// prefixIndex maps a hop prefix (lowercase) → all full pubkeys whose
|
||||
// public_key starts with that prefix. Prefixes with > 1 candidate are
|
||||
// considered ambiguous and skipped during resolution.
|
||||
type prefixIndex map[string][]string
|
||||
|
||||
// buildPrefixIndex reads nodes.public_key and builds the prefix → pubkey
|
||||
// map. We index every 1-byte (2 hex char) prefix length the firmware
|
||||
// uses (1, 2, 3, 4, 6, 8). Memory cost is O(nodes × len(prefixLens)).
|
||||
func buildPrefixIndex(db *sql.DB) (prefixIndex, error) {
|
||||
rows, err := db.Query(`SELECT public_key FROM nodes`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
idx := make(prefixIndex, 1024)
|
||||
var prefixLens = []int{1 * 2, 2 * 2, 3 * 2, 4 * 2, 6 * 2, 8 * 2}
|
||||
for rows.Next() {
|
||||
var pk string
|
||||
if err := rows.Scan(&pk); err != nil {
|
||||
continue
|
||||
}
|
||||
pkLower := strings.ToLower(pk)
|
||||
for _, n := range prefixLens {
|
||||
if len(pkLower) < n {
|
||||
continue
|
||||
}
|
||||
prefix := pkLower[:n]
|
||||
idx[prefix] = append(idx[prefix], pkLower)
|
||||
}
|
||||
}
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// resolvePrefix returns the single resolved pubkey if exactly one
|
||||
// candidate matches, otherwise (zero || multiple), it returns ok=false
|
||||
// (matches the conservative server-side resolver in
|
||||
// cmd/server/extractEdgesFromObs).
|
||||
func resolvePrefix(idx prefixIndex, hop string) (string, bool) {
|
||||
h := strings.ToLower(hop)
|
||||
candidates := idx[h]
|
||||
if len(candidates) != 1 {
|
||||
return "", false
|
||||
}
|
||||
return candidates[0], true
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestNeighborEdgesBuilderDeltaScan enforces issue #1339:
|
||||
// after the initial (warm-up) full build, subsequent ticks of
|
||||
// buildAndPersistNeighborEdges MUST scan only observations newer
|
||||
// than the most recent edge already persisted. The watermark is
|
||||
// derived from MAX(neighbor_edges.last_seen) — neighbor_edges itself
|
||||
// is the persistence, no separate metadata table.
|
||||
//
|
||||
// RED expectations:
|
||||
// 1. After warm-up that produces edges, a second build with NO new
|
||||
// observations is a fast no-op (<1s) and writes nothing.
|
||||
// 2. After inserting K observations with timestamps strictly newer
|
||||
// than the prior MAX(last_seen), the next build upserts exactly
|
||||
// K edges in <1s.
|
||||
// 3. Initial build (empty neighbor_edges) still does a full scan
|
||||
// (warm-up preserved).
|
||||
func TestNeighborEdgesBuilderDeltaScan(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("synthetic 100k-row benchmark; skipped in -short")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "delta.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`,
|
||||
"aaaaaaaaaa", "from-node",
|
||||
"bbbbbbbbbb", "first-hop",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO observers (id, name) VALUES (?, ?)`,
|
||||
"obs-1", "observer-1",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var obsRowid int64
|
||||
if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Baseline timestamps: a contiguous block ending at baselineMaxTs.
|
||||
const baseline = 100_000
|
||||
const baselineStartTs int64 = 1735689600 // 2025-01-01 UTC
|
||||
baselineMaxTs := baselineStartTs + int64(baseline) - 1
|
||||
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txStmt, err := tx.Prepare(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey)
|
||||
VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obsStmt, err := tx.Prepare(`INSERT INTO observations
|
||||
(transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < baseline; i++ {
|
||||
res, err := txStmt.Exec(fmt.Sprintf("h%d", i), baselineStartTs+int64(i), payloadADVERT)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txID, _ := res.LastInsertId()
|
||||
if _, err := obsStmt.Exec(txID, obsRowid, baselineStartTs+int64(i)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Initial warm-up: drain to completion (StartNeighborEdgesBuilder
|
||||
// does the same — call directly so the test doesn't depend on the
|
||||
// goroutine harness). Full scan allowed because neighbor_edges
|
||||
// starts empty.
|
||||
for {
|
||||
n, err := store.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
t.Fatalf("warm-up build: %v", err)
|
||||
}
|
||||
if n == 0 || n < 50000 {
|
||||
break
|
||||
}
|
||||
}
|
||||
var edgesAfterWarmup int
|
||||
if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges`).Scan(&edgesAfterWarmup); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if edgesAfterWarmup == 0 {
|
||||
t.Fatal("warm-up produced 0 edges; can't establish a watermark")
|
||||
}
|
||||
// Sanity: MAX(last_seen) should reflect the baseline tail timestamp.
|
||||
var maxLastSeen string
|
||||
if err := store.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&maxLastSeen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantMax := time.Unix(baselineMaxTs, 0).UTC().Format(time.RFC3339)
|
||||
if maxLastSeen != wantMax {
|
||||
t.Fatalf("MAX(last_seen) after warm-up: want %s, got %s", wantMax, maxLastSeen)
|
||||
}
|
||||
|
||||
// Tick #2: NO new observations. Expect no-op + fast.
|
||||
noopStart := time.Now()
|
||||
n2, err := store.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
t.Fatalf("noop build: %v", err)
|
||||
}
|
||||
noopDur := time.Since(noopStart)
|
||||
if n2 != 0 {
|
||||
t.Fatalf("expected 0 edges on empty-delta tick; got %d (#1339)", n2)
|
||||
}
|
||||
if noopDur > time.Second {
|
||||
t.Fatalf("empty-delta build took %v; expected <1s — builder is "+
|
||||
"still doing a full table scan. (#1339)", noopDur)
|
||||
}
|
||||
|
||||
// Tick #3: insert K observations with timestamps strictly newer
|
||||
// than baselineMaxTs.
|
||||
const delta = 100
|
||||
deltaStartTs := baselineMaxTs + 1
|
||||
tx2, err := store.db.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txStmt2, err := tx2.Prepare(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey)
|
||||
VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obsStmt2, err := tx2.Prepare(`INSERT INTO observations
|
||||
(transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < delta; i++ {
|
||||
res, err := txStmt2.Exec(fmt.Sprintf("d%d", i), deltaStartTs+int64(i), payloadADVERT)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txID, _ := res.LastInsertId()
|
||||
if _, err := obsStmt2.Exec(txID, obsRowid, deltaStartTs+int64(i)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tx2.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
deltaStart := time.Now()
|
||||
n3, err := store.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
t.Fatalf("delta build: %v", err)
|
||||
}
|
||||
deltaDur := time.Since(deltaStart)
|
||||
// Each ADVERT observation with a non-empty path produces 2 edge
|
||||
// candidates (from↔hop[0] and observer↔hop[-1]). The watermark
|
||||
// must clamp the scan to the delta rows ONLY — anything more
|
||||
// proves the WHERE clause was bypassed.
|
||||
if n3 != delta*2 {
|
||||
t.Fatalf("expected %d edges upserted (delta only, 2 per advert obs); got %d. "+
|
||||
"Builder must only scan observations with timestamp > MAX(neighbor_edges.last_seen). (#1339)",
|
||||
delta*2, n3)
|
||||
}
|
||||
if deltaDur > 500*time.Millisecond {
|
||||
t.Fatalf("delta build of %d rows took %v; expected <500ms. (#1339)", delta, deltaDur)
|
||||
}
|
||||
|
||||
// Sanity: MAX(last_seen) advanced.
|
||||
var maxLastSeen2 string
|
||||
if err := store.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&maxLastSeen2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if maxLastSeen2 <= maxLastSeen {
|
||||
t.Fatalf("MAX(last_seen) did not advance: was %s, now %s", maxLastSeen, maxLastSeen2)
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNeighborEdgesBuilderUpsertsFromObservations enforces issue
|
||||
// #1287 Option 4: the INGESTOR builds neighbor_edges from raw
|
||||
// observations/transmissions and persists them. Server is read-only.
|
||||
//
|
||||
// Synthesize a tiny DB with one ADVERT observation whose path[0]
|
||||
// uniquely resolves to a known node, then assert the builder writes
|
||||
// the expected edge.
|
||||
func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "build.db")
|
||||
|
||||
// Open via the ingestor's normal opener so applySchema and
|
||||
// dbschema.Apply both run (the builder requires neighbor_edges +
|
||||
// observers.iata etc.).
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Seed two nodes whose pubkey prefixes will be used as hops.
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`,
|
||||
"aaaaaaaaaa", "from-node",
|
||||
"bbbbbbbbbb", "first-hop",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Seed one observer.
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO observers (id, name) VALUES (?, ?)`,
|
||||
"obs-1", "observer-1",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var obsRowid int64
|
||||
if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Insert one ADVERT transmission with from_pubkey = aaaaa…
|
||||
res, err := store.db.Exec(
|
||||
`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
"", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txID, _ := res.LastInsertId()
|
||||
|
||||
// Insert one observation whose path[0] = "bb" (2-hex prefix unique
|
||||
// to bbbbb… in the nodes table). Expected edge: a↔b.
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`,
|
||||
txID, obsRowid, `["bb"]`, int64(1735689600),
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
n, err := store.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
t.Fatalf("buildAndPersistNeighborEdges: %v", err)
|
||||
}
|
||||
if n == 0 {
|
||||
t.Fatal("expected at least 1 edge upserted, got 0")
|
||||
}
|
||||
|
||||
var got int
|
||||
if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != 1 {
|
||||
t.Fatalf("expected the a↔b edge to be persisted; got %d rows", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (test ends here)
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeChannelName(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
// Known channel: "public" should be normalized to "Public"
|
||||
{"public", "Public"},
|
||||
{"Public", "Public"},
|
||||
{"PUBLIC", "Public"},
|
||||
// Hashtag channels should be left untouched
|
||||
{"#LongFast", "#LongFast"},
|
||||
{"#wardrive", "#wardrive"},
|
||||
// Custom/unknown channels should be left untouched
|
||||
{"myChannel", "myChannel"},
|
||||
{"testchannel", "testchannel"},
|
||||
// Empty string
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := normalizeChannelName(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("normalizeChannelName(%q) = %q, want %q", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadChannelKeys_NormalizesKnownDisplayNames(t *testing.T) {
|
||||
// Verify that known channel keys with wrong casing get normalized
|
||||
cfg := &Config{
|
||||
ChannelKeys: map[string]string{
|
||||
"public": "8b3387e9c5cdea6ac9e5edbaa115cd72",
|
||||
},
|
||||
}
|
||||
|
||||
keys := loadChannelKeys(cfg, "/dev/null")
|
||||
|
||||
// Should have "Public" (normalized) not "public" (raw)
|
||||
if _, ok := keys["public"]; ok {
|
||||
t.Error("Expected 'public' to be normalized to 'Public'")
|
||||
}
|
||||
if _, ok := keys["Public"]; !ok {
|
||||
t.Error("Expected 'Public' key to exist in loaded channel keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadChannelKeys_LeavesCustomNamesUntouched(t *testing.T) {
|
||||
// Verify that custom channel names are NOT normalized
|
||||
cfg := &Config{
|
||||
ChannelKeys: map[string]string{
|
||||
"myCustomChannel": "deadbeef12345678",
|
||||
},
|
||||
}
|
||||
|
||||
keys := loadChannelKeys(cfg, "/dev/null")
|
||||
|
||||
// Should keep "myCustomChannel" as-is
|
||||
if _, ok := keys["myCustomChannel"]; !ok {
|
||||
t.Error("Expected 'myCustomChannel' to be left untouched")
|
||||
}
|
||||
// Should NOT have "MyCustomChannel"
|
||||
if _, ok := keys["MyCustomChannel"]; ok {
|
||||
t.Error("Custom channel names should NOT be auto-capitalized")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadChannelKeys_DuplicateCasingLogsWarning(t *testing.T) {
|
||||
// Verify that config with both "public" and "Public" resolves deterministically:
|
||||
// the canonical (already-normalized) form should win.
|
||||
cfg := &Config{
|
||||
ChannelKeys: map[string]string{
|
||||
"public": "8b3387e9c5cdea6ac9e5edbaa115cd72",
|
||||
"Public": "differentkey1234567",
|
||||
},
|
||||
}
|
||||
|
||||
keys := loadChannelKeys(cfg, "/dev/null")
|
||||
|
||||
// After normalization, only one key should exist: "Public"
|
||||
// The canonical form ("Public") should win over the lowercase form ("public")
|
||||
if _, ok := keys["public"]; ok {
|
||||
t.Error("Expected 'public' to be normalized away")
|
||||
}
|
||||
if _, ok := keys["Public"]; !ok {
|
||||
t.Error("Expected 'Public' key to exist")
|
||||
}
|
||||
// Assert the canonical form's value won, not just any value
|
||||
if keys["Public"] != "differentkey1234567" {
|
||||
t.Errorf("Expected canonical 'Public' value to win, got %q", keys["Public"])
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIngestorIsObserverBlacklisted(t *testing.T) {
|
||||
cfg := &Config{
|
||||
ObserverBlacklist: []string{"OBS1", "obs2"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
id string
|
||||
want bool
|
||||
}{
|
||||
{"OBS1", true},
|
||||
{"obs1", true},
|
||||
{"OBS2", true},
|
||||
{"obs3", false},
|
||||
{"", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := cfg.IsObserverBlacklisted(tt.id)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsObserverBlacklisted(%q) = %v, want %v", tt.id, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestorIsObserverBlacklistedEmpty(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("empty blacklist should not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestorIsObserverBlacklistedNil(t *testing.T) {
|
||||
var cfg *Config
|
||||
if cfg.IsObserverBlacklisted("anything") {
|
||||
t.Error("nil config should not match")
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression tests for issue #1465 — observer.last_seen MUST always reflect
|
||||
// ingest time (server wall clock), never the MQTT envelope timestamp. Observers
|
||||
// with broken clocks (wrong TZ, RTC drift, replayed retained messages) must
|
||||
// NOT be able to drag the analyzer's "last heard from" field into the past
|
||||
// or future.
|
||||
//
|
||||
// Per-packet rxTime semantics (envelope time with naive-clamp from #1464)
|
||||
// are out of scope here — those continue to use envelope time. This file
|
||||
// asserts only the observer.last_seen path.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Status path: envelope timestamp is a well-formed RFC3339 value 3h in the
|
||||
// past. observer.last_seen must be server wall clock, NOT the envelope value.
|
||||
func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-3 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"status":"online","origin":"obs-past","timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-past/status", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-past").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope reported well-formed stale %q (3h ago) — must NOT drag last_seen into the past. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// Status path: envelope timestamp 5 min in the future. observer.last_seen
|
||||
// must still be server wall clock.
|
||||
func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_FutureEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
future := time.Now().UTC().Add(5 * time.Minute).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"status":"online","origin":"obs-future","timestamp":"` + future + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-future/status", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-future").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope reported well-formed future %q (5 min ahead) — must NOT drag last_seen into the future. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, future)
|
||||
}
|
||||
}
|
||||
|
||||
// Packet path: a transmission whose envelope timestamp is 3h in the past
|
||||
// MUST still bump observer.last_seen to server wall clock — observer is
|
||||
// clearly alive (we just ingested a packet from it), regardless of what
|
||||
// its clock claims.
|
||||
func TestPacketMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-3 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"obs-pkt","timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-pkt/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-pkt").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("packet-path observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope stale = %q. Observer just delivered a packet; last_seen must be NOW. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Regression test for #1044: observer metadata (model, firmware, battery_mv,
|
||||
// noise_floor) is silently dropped when an MQTT status payload arrives, even
|
||||
// though the same payload's `radio` and `client_version` fields ARE persisted.
|
||||
//
|
||||
// Real-world payload captured from the production MQTT bridge:
|
||||
//
|
||||
// {"status":"online","origin":"TestObserver","origin_id":"AABBCCDD",
|
||||
// "radio":"910.5250244,62.5,7,5",
|
||||
// "model":"Heltec V3",
|
||||
// "firmware_version":"1.12.0-test",
|
||||
// "client_version":"meshcoretomqtt/1.0.8.0",
|
||||
// "stats":{"battery_mv":4209,"uptime_secs":75821,"noise_floor":-109,
|
||||
// "tx_air_secs":80,"rx_air_secs":1903,"recv_errors":934}}
|
||||
func TestStatusMessageMetadataPersisted_Issue1044(t *testing.T) {
|
||||
const payload = `{"status":"online","origin":"TestObserver","origin_id":"AABBCCDD","radio":"910.5250244,62.5,7,5","model":"Heltec V3","firmware_version":"1.12.0-test","client_version":"meshcoretomqtt/1.0.8.0","stats":{"battery_mv":4209,"uptime_secs":75821,"noise_floor":-109,"tx_air_secs":80,"rx_air_secs":1903,"recv_errors":934}}`
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(payload), &msg); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
meta := extractObserverMeta(msg)
|
||||
if meta == nil {
|
||||
t.Fatal("extractObserverMeta returned nil for a payload that contains model/firmware/battery_mv")
|
||||
}
|
||||
if meta.Model == nil || *meta.Model != "Heltec V3" {
|
||||
t.Errorf("meta.Model = %v, want \"Heltec V3\"", meta.Model)
|
||||
}
|
||||
if meta.Firmware == nil || *meta.Firmware != "1.12.0-test" {
|
||||
t.Errorf("meta.Firmware = %v, want \"1.12.0-test\"", meta.Firmware)
|
||||
}
|
||||
if meta.ClientVersion == nil || *meta.ClientVersion != "meshcoretomqtt/1.0.8.0" {
|
||||
t.Errorf("meta.ClientVersion = %v, want \"meshcoretomqtt/1.0.8.0\"", meta.ClientVersion)
|
||||
}
|
||||
if meta.Radio == nil || *meta.Radio != "910.5250244,62.5,7,5" {
|
||||
t.Errorf("meta.Radio = %v, want radio string", meta.Radio)
|
||||
}
|
||||
if meta.BatteryMv == nil || *meta.BatteryMv != 4209 {
|
||||
t.Errorf("meta.BatteryMv = %v, want 4209", meta.BatteryMv)
|
||||
}
|
||||
if meta.NoiseFloor == nil || *meta.NoiseFloor != -109 {
|
||||
t.Errorf("meta.NoiseFloor = %v, want -109", meta.NoiseFloor)
|
||||
}
|
||||
if meta.UptimeSecs == nil || *meta.UptimeSecs != 75821 {
|
||||
t.Errorf("meta.UptimeSecs = %v, want 75821", meta.UptimeSecs)
|
||||
}
|
||||
|
||||
// Now drive the meta through UpsertObserver and verify the row.
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := s.UpsertObserver("AABBCCDD", "TestObserver", "SJC", meta); err != nil {
|
||||
t.Fatalf("UpsertObserver: %v", err)
|
||||
}
|
||||
|
||||
var (
|
||||
gotModel, gotFirmware, gotClientVersion, gotRadio string
|
||||
gotBattery int
|
||||
gotUptime int64
|
||||
gotNoise float64
|
||||
)
|
||||
err = s.db.QueryRow(`SELECT model, firmware, client_version, radio,
|
||||
battery_mv, uptime_secs, noise_floor
|
||||
FROM observers WHERE id = 'AABBCCDD'`).Scan(
|
||||
&gotModel, &gotFirmware, &gotClientVersion, &gotRadio,
|
||||
&gotBattery, &gotUptime, &gotNoise,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("scan observer row: %v", err)
|
||||
}
|
||||
if gotModel != "Heltec V3" {
|
||||
t.Errorf("DB model = %q, want \"Heltec V3\"", gotModel)
|
||||
}
|
||||
if gotFirmware != "1.12.0-test" {
|
||||
t.Errorf("DB firmware = %q, want \"1.12.0-test\"", gotFirmware)
|
||||
}
|
||||
if gotBattery != 4209 {
|
||||
t.Errorf("DB battery_mv = %d, want 4209", gotBattery)
|
||||
}
|
||||
if gotUptime != 75821 {
|
||||
t.Errorf("DB uptime_secs = %d, want 75821", gotUptime)
|
||||
}
|
||||
if gotNoise != -109 {
|
||||
t.Errorf("DB noise_floor = %f, want -109", gotNoise)
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
// Package main: ingestor-side processor for prune-request marker files
|
||||
// written by the read-only server (see internal/prunequeue).
|
||||
//
|
||||
// The server cannot DELETE because it opens SQLite mode=ro (#1283/#1289).
|
||||
// Instead, the server writes request-<id>.json under <dataDir>/prune-requests/
|
||||
// and the ingestor consumes it here.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/prunequeue"
|
||||
)
|
||||
|
||||
// DeleteNodesByPubkeys deletes nodes by public key. Returns the count deleted.
|
||||
// Only the ingestor calls this (server has no write handle).
|
||||
func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) {
|
||||
if len(pubkeys) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// Chunk to keep statements under SQLite's variable limit (default 999).
|
||||
const chunk = 500
|
||||
var total int64
|
||||
for start := 0; start < len(pubkeys); start += chunk {
|
||||
end := start + chunk
|
||||
if end > len(pubkeys) {
|
||||
end = len(pubkeys)
|
||||
}
|
||||
batch := pubkeys[start:end]
|
||||
placeholders := strings.Repeat("?,", len(batch))
|
||||
placeholders = placeholders[:len(placeholders)-1]
|
||||
args := make([]interface{}, len(batch))
|
||||
for i, pk := range batch {
|
||||
args[i] = pk
|
||||
}
|
||||
// Cascade cleanup: a node row carries the canonical identity, but
|
||||
// observations/transmissions reference the pubkey too via observer
|
||||
// metadata and originator fields. There are no FK constraints in
|
||||
// the current schema (#669 review note), so we explicitly clear
|
||||
// the most obvious follow-on rows that would otherwise become
|
||||
// orphans visible to operators.
|
||||
//
|
||||
// Conservative scope: only the `nodes` row is removed here. The
|
||||
// referenced observation/transmission history is retained for
|
||||
// audit; operators can run the regular packet-retention prune to
|
||||
// age it out. If a future schema introduces FKs, revisit.
|
||||
res, err := s.db.Exec("DELETE FROM nodes WHERE public_key IN ("+placeholders+")", args...)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("delete batch [%d:%d]: %w", start, end, err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
total += n
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// RunPendingPruneRequests scans the prune-requests/ directory next to the
|
||||
// SQLite database and processes any request-<id>.json markers written by
|
||||
// the server. Each request is honored verbatim — the server is responsible
|
||||
// for the TOCTOU snapshot (only pubkeys that were still outside the
|
||||
// geofilter at confirm time). After running DELETE, the ingestor writes
|
||||
// result-<id>.json and removes the request file (atomic, via os.Rename in
|
||||
// prunequeue.WriteResult).
|
||||
//
|
||||
// Safe to call from a ticker — no-op when the queue is empty.
|
||||
func (s *Store) RunPendingPruneRequests() {
|
||||
paths, err := prunequeue.ListPending(s.path)
|
||||
if err != nil {
|
||||
log.Printf("[prune-queue] list pending failed: %v", err)
|
||||
return
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return
|
||||
}
|
||||
for _, p := range paths {
|
||||
req, err := prunequeue.ReadRequest(p)
|
||||
if err != nil {
|
||||
log.Printf("[prune-queue] read %s failed: %v — removing", p, err)
|
||||
_ = os.Remove(p)
|
||||
continue
|
||||
}
|
||||
log.Printf("[prune-queue] processing request %s: %d pubkey(s) (%s)",
|
||||
req.ID, len(req.Pubkeys), req.Reason)
|
||||
start := time.Now()
|
||||
deleted, derr := s.DeleteNodesByPubkeys(req.Pubkeys)
|
||||
res := prunequeue.Result{
|
||||
ID: req.ID,
|
||||
RequestedAt: req.RequestedAt,
|
||||
CompletedAt: time.Now().UTC(),
|
||||
Deleted: deleted,
|
||||
}
|
||||
if derr != nil {
|
||||
res.Error = derr.Error()
|
||||
log.Printf("[prune-queue] request %s FAILED after %s: %v", req.ID, time.Since(start), derr)
|
||||
} else {
|
||||
log.Printf("[prune-queue] request %s deleted %d node(s) in %s", req.ID, deleted, time.Since(start))
|
||||
}
|
||||
if werr := prunequeue.WriteResult(s.path, res); werr != nil {
|
||||
log.Printf("[prune-queue] write result for %s failed: %v", req.ID, werr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/prunequeue"
|
||||
)
|
||||
|
||||
func TestRunPendingPruneRequests(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Seed two nodes; one will be pruned, one will be kept.
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen)
|
||||
VALUES ('aaaa', 'gone', 'companion', 1.0, 1.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'),
|
||||
('bbbb', 'kept', 'companion', 2.0, 2.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')`); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
id := prunequeue.NewID()
|
||||
if err := prunequeue.WriteRequest(dbPath, prunequeue.Request{
|
||||
ID: id,
|
||||
RequestedAt: time.Now().UTC(),
|
||||
Reason: "geo-prune-test",
|
||||
Pubkeys: []string{"aaaa"},
|
||||
}); err != nil {
|
||||
t.Fatalf("WriteRequest: %v", err)
|
||||
}
|
||||
|
||||
store.RunPendingPruneRequests()
|
||||
|
||||
// Request file gone, result file present.
|
||||
if exists, _ := prunequeue.RequestExists(dbPath, id); exists {
|
||||
t.Error("request file should have been consumed")
|
||||
}
|
||||
res, err := prunequeue.ReadResult(dbPath, id)
|
||||
if err != nil || res == nil {
|
||||
t.Fatalf("ReadResult: res=%v err=%v", res, err)
|
||||
}
|
||||
if res.Deleted != 1 {
|
||||
t.Errorf("expected Deleted=1, got %d", res.Deleted)
|
||||
}
|
||||
if res.Error != "" {
|
||||
t.Errorf("unexpected error: %s", res.Error)
|
||||
}
|
||||
|
||||
// Verify DB state: aaaa gone, bbbb kept.
|
||||
var n int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key='aaaa'").Scan(&n)
|
||||
if n != 0 {
|
||||
t.Errorf("expected 'aaaa' deleted, got count=%d", n)
|
||||
}
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key='bbbb'").Scan(&n)
|
||||
if n != 1 {
|
||||
t.Errorf("expected 'bbbb' kept, got count=%d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPendingPruneRequests_EmptyQueueIsNoop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
// Must not panic / error on empty queue.
|
||||
store.RunPendingPruneRequests()
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseEnvelopeTime(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
ok bool
|
||||
wantNaive bool
|
||||
}{
|
||||
{"rfc3339 utc", "2026-05-16T10:00:00Z", true, false},
|
||||
{"rfc3339 offset", "2026-05-16T12:00:00+02:00", true, false},
|
||||
{"naive iso", "2026-05-16T10:00:00", true, true},
|
||||
{"naive iso micros", "2026-05-16T10:00:00.123456", true, true},
|
||||
{"garbage", "not-a-time", false, false},
|
||||
{"empty", "", false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, naive, err := parseEnvelopeTime(c.in)
|
||||
if (err == nil) != c.ok {
|
||||
t.Fatalf("parseEnvelopeTime(%q): want ok=%v, got err=%v", c.in, c.ok, err)
|
||||
}
|
||||
if err == nil && naive != c.wantNaive {
|
||||
t.Fatalf("parseEnvelopeTime(%q): want naive=%v, got %v", c.in, c.wantNaive, naive)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRxTime(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
|
||||
mustParse := func(s string) time.Time {
|
||||
t.Helper()
|
||||
parsed, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("result %q is not RFC3339: %v", s, err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
nearNow := func(s string) bool {
|
||||
d := mustParse(s).Sub(now)
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
return d <= time.Minute
|
||||
}
|
||||
|
||||
rx := now.Add(-5 * time.Hour).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": rx}, "test"); got != rx {
|
||||
t.Errorf("plausible past timestamp: got %q want %q", got, rx)
|
||||
}
|
||||
if got := resolveRxTime(map[string]interface{}{}, "test"); !nearNow(got) {
|
||||
t.Errorf("missing timestamp: got %q, expected ~now", got)
|
||||
}
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": "garbage"}, "test"); !nearNow(got) {
|
||||
t.Errorf("garbage timestamp: got %q, expected ~now", got)
|
||||
}
|
||||
future := now.Add(48 * time.Hour).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": future}, "test"); !nearNow(got) {
|
||||
t.Errorf("future timestamp: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
|
||||
// RTC-reset node reporting a factory date — must not drag first_seen back.
|
||||
factory := "2020-01-01T00:00:00Z"
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": factory}, "test"); !nearNow(got) {
|
||||
t.Errorf("stale factory timestamp: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
// Just past the 30-day floor → rejected.
|
||||
stale := now.Add(-31 * 24 * time.Hour).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": stale}, "test"); !nearNow(got) {
|
||||
t.Errorf("stale timestamp >30d: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
// Just inside the 30-day floor → used verbatim.
|
||||
recent := now.Add(-29 * 24 * time.Hour).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": recent}, "test"); got != recent {
|
||||
t.Errorf("recent timestamp <30d: got %q want %q", got, recent)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: issue #1463 — naive (zone-less) ISO timestamps from observers
|
||||
// in negative-UTC-offset zones (e.g. California PDT, UTC−7) were interpreted
|
||||
// as UTC, producing rxTime values 7h in the past that poisoned `last_seen`
|
||||
// and rendered the observer perpetually "Stale" in the UI. The symmetric
|
||||
// clamp now collapses any naive timestamp more than 15 min off server-now to
|
||||
// `now()`, while zone-aware timestamps (RFC3339 with Z or offset) are still
|
||||
// honored verbatim regardless of skew (those are well-behaved observers).
|
||||
func TestResolveRxTimeNaiveTimestampClamp(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
|
||||
mustParse := func(s string) time.Time {
|
||||
t.Helper()
|
||||
parsed, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("result %q is not RFC3339: %v", s, err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
nearNow := func(s string) bool {
|
||||
d := mustParse(s).Sub(now)
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
return d <= time.Minute
|
||||
}
|
||||
|
||||
// California observer (UTC-7) emitting a naive local-clock timestamp:
|
||||
// must NOT be stored verbatim 7h in the past — clamp to ~now.
|
||||
naivePast := now.Add(-7 * time.Hour).Format("2006-01-02T15:04:05")
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": naivePast}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive past timestamp (UTC-7 observer): got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Naive future just minutes ahead (UTC+N observer, existing soft-clamp
|
||||
// behavior): still clamped to now.
|
||||
naiveFuture := now.Add(5 * time.Minute).Format("2006-01-02T15:04:05")
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": naiveFuture}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive future timestamp: got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Naive microsecond layout (python isoformat without tz) — same clamp.
|
||||
naivePastMicros := now.Add(-7 * time.Hour).Format("2006-01-02T15:04:05.000000")
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": naivePastMicros}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive past timestamp w/ micros: got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Well-behaved observer: Z-suffixed past timestamp passes through verbatim
|
||||
// even if it's hours old (legitimate buffered uploads must be preserved).
|
||||
zPast := now.Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": zPast}, "test"); got != zPast {
|
||||
t.Errorf("Z-suffixed past timestamp must pass through: got %q want %q", got, zPast)
|
||||
}
|
||||
|
||||
// Well-behaved observer with explicit offset (UTC-7) — canonicalize to UTC
|
||||
// but preserve the moment in time. Must equal the same moment in UTC.
|
||||
offsetLoc := time.FixedZone("PDT", -7*3600)
|
||||
offsetMoment := now.Add(-7 * time.Hour).In(offsetLoc)
|
||||
offsetStr := offsetMoment.Format(time.RFC3339)
|
||||
wantUTC := offsetMoment.UTC().Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": offsetStr}, "test"); got != wantUTC {
|
||||
t.Errorf("offset-suffixed timestamp: got %q want %q", got, wantUTC)
|
||||
}
|
||||
|
||||
// Naive timestamp within tolerance window (2 min in past, observer that
|
||||
// happens to be in UTC) — within tolerance, passes through verbatim.
|
||||
naiveCloseStr := now.Add(-2 * time.Minute).Format("2006-01-02T15:04:05")
|
||||
naiveCloseWant := now.Add(-2 * time.Minute).Format(time.RFC3339)
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": naiveCloseStr}, "test"); got != naiveCloseWant {
|
||||
t.Errorf("naive timestamp within tolerance: got %q, expected %q (verbatim)", got, naiveCloseWant)
|
||||
}
|
||||
}
|
||||
@@ -1,339 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildAdvertHex constructs a full ADVERT packet hex string.
|
||||
// header(1) + pathByte(1) + pubkey(32) + timestamp(4) + signature(64) + appdata
|
||||
func buildAdvertHex(pubKey ed25519.PublicKey, privKey ed25519.PrivateKey, timestamp uint32, appdata []byte) string {
|
||||
// Build signed message: pubkey(32) + timestamp(4 LE) + appdata
|
||||
msg := make([]byte, 32+4+len(appdata))
|
||||
copy(msg[0:32], pubKey)
|
||||
binary.LittleEndian.PutUint32(msg[32:36], timestamp)
|
||||
copy(msg[36:], appdata)
|
||||
|
||||
sig := ed25519.Sign(privKey, msg)
|
||||
|
||||
// Payload: pubkey(32) + timestamp(4) + signature(64) + appdata
|
||||
payload := make([]byte, 0, 100+len(appdata))
|
||||
payload = append(payload, pubKey...)
|
||||
ts := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(ts, timestamp)
|
||||
payload = append(payload, ts...)
|
||||
payload = append(payload, sig...)
|
||||
payload = append(payload, appdata...)
|
||||
|
||||
// Header: ADVERT (0x04 << 2) | FLOOD (1) = 0x11, pathByte=0 (no hops)
|
||||
header := byte(0x11)
|
||||
pathByte := byte(0x00)
|
||||
|
||||
pkt := append([]byte{header, pathByte}, payload...)
|
||||
return hex.EncodeToString(pkt)
|
||||
}
|
||||
|
||||
// makeAppdata builds minimal appdata: flags(1) + name
|
||||
func makeAppdata(name string) []byte {
|
||||
flags := byte(0x81) // hasName=true, type=companion(1)
|
||||
data := []byte{flags}
|
||||
data = append(data, []byte(name)...)
|
||||
data = append(data, 0x00) // null terminator
|
||||
return data
|
||||
}
|
||||
|
||||
func TestSigValidation_ValidAdvertStored(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("TestNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// Verify packet was stored
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&count)
|
||||
if count == 0 {
|
||||
t.Fatal("valid advert should be stored, got 0 transmissions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_TamperedSignatureDropped(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("BadNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper with signature (flip a byte in the signature area)
|
||||
// Signature starts at offset 2 (header+path) + 32 (pubkey) + 4 (timestamp) = 38
|
||||
// That's byte 38 in the packet, hex chars 76-77
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
tamperedHex := string(rawBytes)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// Verify packet was NOT stored in transmissions
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount != 0 {
|
||||
t.Fatalf("tampered advert should be dropped, got %d transmissions", txCount)
|
||||
}
|
||||
|
||||
// Verify it was recorded in dropped_packets
|
||||
var dropCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&dropCount)
|
||||
if dropCount == 0 {
|
||||
t.Fatal("tampered advert should be recorded in dropped_packets")
|
||||
}
|
||||
|
||||
// Verify drop counter incremented
|
||||
if store.Stats.SignatureDrops.Load() != 1 {
|
||||
t.Fatalf("expected 1 signature drop, got %d", store.Stats.SignatureDrops.Load())
|
||||
}
|
||||
|
||||
// Verify dropped_packets has correct fields
|
||||
var reason, nodeKey, nodeName, obsID string
|
||||
store.db.QueryRow("SELECT reason, node_pubkey, node_name, observer_id FROM dropped_packets LIMIT 1").Scan(&reason, &nodeKey, &nodeName, &obsID)
|
||||
if reason != "invalid signature" {
|
||||
t.Fatalf("expected reason 'invalid signature', got %q", reason)
|
||||
}
|
||||
if nodeKey == "" {
|
||||
t.Fatal("dropped packet should have node_pubkey")
|
||||
}
|
||||
if !strings.Contains(nodeName, "BadNode") {
|
||||
t.Fatalf("expected node_name to contain 'BadNode', got %q", nodeName)
|
||||
}
|
||||
if obsID != "obs1" {
|
||||
t.Fatalf("expected observer_id 'obs1', got %q", obsID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_TruncatedAppdataDropped(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("TruncNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Sign was computed with full appdata. Now truncate the raw hex to remove
|
||||
// some appdata bytes, making the signature invalid.
|
||||
// Truncate last 4 hex chars (2 bytes of appdata)
|
||||
truncatedHex := rawHex[:len(rawHex)-4]
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+truncatedHex+`","origin":"TestObs"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount != 0 {
|
||||
t.Fatalf("truncated advert should be dropped, got %d transmissions", txCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_DisabledByConfig(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("NoValNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper with signature
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
tamperedHex := string(rawBytes)
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+tamperedHex+`","origin":"TestObs"}`)
|
||||
falseVal := false
|
||||
cfg := &Config{ValidateSignatures: &falseVal}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
// With validation disabled, tampered packet should be stored
|
||||
var txCount int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&txCount)
|
||||
if txCount == 0 {
|
||||
t.Fatal("with validateSignatures=false, tampered advert should be stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_DropCounterIncrements(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
appdata := makeAppdata("Node")
|
||||
rawHex := buildAdvertHex(pub, priv, uint32(1700000000+i), appdata)
|
||||
// Tamper
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"Obs"}`)
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
}
|
||||
|
||||
if store.Stats.SignatureDrops.Load() != 3 {
|
||||
t.Fatalf("expected 3 signature drops, got %d", store.Stats.SignatureDrops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSigValidation_LogContainsFields(t *testing.T) {
|
||||
// This test verifies the dropped_packets row has all required fields
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
pub, priv, _ := ed25519.GenerateKey(nil)
|
||||
appdata := makeAppdata("LogTestNode")
|
||||
rawHex := buildAdvertHex(pub, priv, 1700000000, appdata)
|
||||
|
||||
// Tamper
|
||||
rawBytes := []byte(rawHex)
|
||||
if rawBytes[76] == '0' {
|
||||
rawBytes[76] = 'f'
|
||||
} else {
|
||||
rawBytes[76] = '0'
|
||||
}
|
||||
|
||||
source := MQTTSource{Name: "test"}
|
||||
msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+string(rawBytes)+`","origin":"MyObserver"}`)
|
||||
cfg := &Config{}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, cfg)
|
||||
|
||||
var hash, reason, obsID, obsName, pubkey, nodeName string
|
||||
err = store.db.QueryRow("SELECT hash, reason, observer_id, observer_name, node_pubkey, node_name FROM dropped_packets LIMIT 1").
|
||||
Scan(&hash, &reason, &obsID, &obsName, &pubkey, &nodeName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if hash == "" {
|
||||
t.Error("dropped packet should have hash")
|
||||
}
|
||||
if reason != "invalid signature" {
|
||||
t.Errorf("expected reason 'invalid signature', got %q", reason)
|
||||
}
|
||||
if obsID != "obs1" {
|
||||
t.Errorf("expected observer_id 'obs1', got %q", obsID)
|
||||
}
|
||||
if obsName != "MyObserver" {
|
||||
t.Errorf("expected observer_name 'MyObserver', got %q", obsName)
|
||||
}
|
||||
if pubkey == "" {
|
||||
t.Error("dropped packet should have node_pubkey")
|
||||
}
|
||||
if !strings.Contains(nodeName, "LogTestNode") {
|
||||
t.Errorf("expected node_name containing 'LogTestNode', got %q", nodeName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneDroppedPackets(t *testing.T) {
|
||||
dbPath := t.TempDir() + "/test.db"
|
||||
store, err := OpenStoreWithInterval(dbPath, 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Insert an old dropped packet
|
||||
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`)
|
||||
store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`)
|
||||
|
||||
n, err := store.PruneDroppedPackets(30)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("expected 1 pruned, got %d", n)
|
||||
}
|
||||
|
||||
var count int
|
||||
store.db.QueryRow("SELECT COUNT(*) FROM dropped_packets").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Fatalf("expected 1 remaining, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldValidateSignatures_Default(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if !cfg.ShouldValidateSignatures() {
|
||||
t.Fatal("default should be true")
|
||||
}
|
||||
|
||||
falseVal := false
|
||||
cfg2 := &Config{ValidateSignatures: &falseVal}
|
||||
if cfg2.ShouldValidateSignatures() {
|
||||
t.Fatal("explicit false should be false")
|
||||
}
|
||||
|
||||
trueVal := true
|
||||
cfg3 := &Config{ValidateSignatures: &trueVal}
|
||||
if !cfg3.ShouldValidateSignatures() {
|
||||
t.Fatal("explicit true should be true")
|
||||
}
|
||||
}
|
||||
|
||||
// newMockMsg creates a minimal mqtt.Message for testing.
|
||||
func newMockMsg(topic, payload string) *mockMessage {
|
||||
return &mockMessage{topic: topic, payload: []byte(payload)}
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/perfio"
|
||||
)
|
||||
|
||||
// PerfIOSample is the canonical per-process I/O rate sample, sourced from the
|
||||
// shared internal/perfio package. The server consumes the same type when it
|
||||
// reads this binary's stats file — sharing the type prevents silent JSON
|
||||
// contract drift (#1167 follow-up).
|
||||
type PerfIOSample = perfio.Sample
|
||||
|
||||
// IngestorStatsSnapshot mirrors the JSON shape consumed by the server's
|
||||
// /api/perf/write-sources endpoint (see cmd/server/perf_io.go IngestorStats).
|
||||
//
|
||||
// NOTE: each field below is sampled with an independent atomic.Load(), so the
|
||||
// snapshot is EVENTUALLY-CONSISTENT — invariants like
|
||||
// `walCommits >= tx_inserted` may be momentarily violated
|
||||
// in a single sample. Consumers MUST NOT derive ratios on the assumption these
|
||||
// counters were captured at the same instant; treat each field as an
|
||||
// independent monotonically-increasing counter and look at deltas across
|
||||
// multiple samples instead.
|
||||
type IngestorStatsSnapshot struct {
|
||||
SampledAt string `json:"sampledAt"`
|
||||
TxInserted int64 `json:"tx_inserted"`
|
||||
ObsInserted int64 `json:"obs_inserted"`
|
||||
DuplicateTx int64 `json:"tx_dupes"`
|
||||
NodeUpserts int64 `json:"node_upserts"`
|
||||
ObserverUpserts int64 `json:"observer_upserts"`
|
||||
WriteErrors int64 `json:"write_errors"`
|
||||
SignatureDrops int64 `json:"sig_drops"`
|
||||
WALCommits int64 `json:"walCommits"`
|
||||
GroupCommitFlushes int64 `json:"groupCommitFlushes"` // always 0 — group commit reverted (refs #1129)
|
||||
BackfillUpdates map[string]int64 `json:"backfillUpdates"`
|
||||
// ProcIO is the ingestor's own /proc/self/io rate snapshot. Surfaced via
|
||||
// the server's /api/perf/io endpoint under .ingestor (#1120 — "Both
|
||||
// ingestor and server"). Optional; absent on non-Linux hosts.
|
||||
ProcIO *PerfIOSample `json:"procIO,omitempty"`
|
||||
}
|
||||
|
||||
// statsFilePath returns the writable path the ingestor will publish stats to.
|
||||
// Override via env CORESCOPE_INGESTOR_STATS for tests / non-default deploys.
|
||||
//
|
||||
// SECURITY: the default lives in /tmp which is world-writable. The writer uses
|
||||
// O_NOFOLLOW + 0o600 so a pre-planted symlink cannot be used to clobber an
|
||||
// arbitrary file via this path. Operators who want stronger guarantees should
|
||||
// point CORESCOPE_INGESTOR_STATS at a private directory (e.g. /var/lib/corescope/).
|
||||
func statsFilePath() string {
|
||||
if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" {
|
||||
return p
|
||||
}
|
||||
return "/tmp/corescope-ingestor-stats.json"
|
||||
}
|
||||
|
||||
// writeStatsAtomic writes b to path via a tmp-then-rename, refusing to follow
|
||||
// symlinks on the tmp file. Returns nil on success, an error otherwise.
|
||||
func writeStatsAtomic(path string, b []byte) error {
|
||||
tmp := path + ".tmp"
|
||||
// O_NOFOLLOW: if tmp is a pre-existing symlink, openat fails with ELOOP
|
||||
// instead of clobbering the symlink target. O_TRUNC zeroes existing
|
||||
// regular-file content. 0o600 — no need for world-readable.
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|oNoFollow, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
f.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// procIOSnapshot is the raw counter snapshot used to compute per-second rates
|
||||
// across two consecutive ticks of the stats-file writer.
|
||||
type procIOSnapshot struct {
|
||||
at time.Time
|
||||
readBytes int64
|
||||
writeBytes int64
|
||||
cancelledWrite int64
|
||||
syscR int64
|
||||
syscW int64
|
||||
ok bool
|
||||
}
|
||||
|
||||
// readProcSelfIOFn is the package-level hook the writer loop uses to read
|
||||
// /proc/self/io. Defaults to readProcSelfIO; tests override it to inject
|
||||
// deterministic counter snapshots without depending on a Linux kernel
|
||||
// that exposes /proc/self/io (CONFIG_TASK_IO_ACCOUNTING).
|
||||
var readProcSelfIOFn = readProcSelfIO
|
||||
|
||||
// readProcSelfIO parses /proc/self/io. Returns ok=false on non-Linux hosts or
|
||||
// any read/parse failure (caller skips the procIO block in that case).
|
||||
func readProcSelfIO() procIOSnapshot {
|
||||
out := procIOSnapshot{at: time.Now()}
|
||||
f, err := os.Open("/proc/self/io")
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer f.Close()
|
||||
parseProcSelfIOInto(bufio.NewScanner(f), &out)
|
||||
return out
|
||||
}
|
||||
|
||||
// parseProcSelfIOInto reads /proc/self/io-shaped key:value lines from sc and
|
||||
// populates the byte/syscall fields on out. Sets out.ok=true only if at
|
||||
// least one expected key was successfully parsed (#1167 must-fix #3).
|
||||
//
|
||||
// Implementation delegates to perfio.ParseProcIO so the ingestor and the
|
||||
// server share exactly one parser (Carmack must-fix #7).
|
||||
func parseProcSelfIOInto(sc *bufio.Scanner, out *procIOSnapshot) {
|
||||
var c perfio.Counters
|
||||
out.ok = perfio.ParseProcIO(sc, &c)
|
||||
out.readBytes = c.ReadBytes
|
||||
out.writeBytes = c.WriteBytes
|
||||
out.cancelledWrite = c.CancelledWriteBytes
|
||||
out.syscR = c.SyscR
|
||||
out.syscW = c.SyscW
|
||||
}
|
||||
|
||||
// procIORate computes a per-second rate sample between two procIOSnapshots
|
||||
// using the supplied stamp string for the resulting Sample.SampledAt
|
||||
// (Carmack must-fix #5 — the writer captures time.Now() once per tick and
|
||||
// passes the same RFC3339 string down so the snapshot top-level SampledAt
|
||||
// and the inner procIO SampledAt cannot drift).
|
||||
// Returns nil if either snapshot is invalid or the interval is zero.
|
||||
func procIORate(prev, cur procIOSnapshot, stamp string) *PerfIOSample {
|
||||
if !prev.ok || !cur.ok {
|
||||
return nil
|
||||
}
|
||||
dt := cur.at.Sub(prev.at).Seconds()
|
||||
if dt < 0.001 {
|
||||
return nil
|
||||
}
|
||||
return &PerfIOSample{
|
||||
ReadBytesPerSec: float64(cur.readBytes-prev.readBytes) / dt,
|
||||
WriteBytesPerSec: float64(cur.writeBytes-prev.writeBytes) / dt,
|
||||
CancelledWriteBytesPerSec: float64(cur.cancelledWrite-prev.cancelledWrite) / dt,
|
||||
SyscallsRead: float64(cur.syscR-prev.syscR) / dt,
|
||||
SyscallsWrite: float64(cur.syscW-prev.syscW) / dt,
|
||||
SampledAt: stamp,
|
||||
}
|
||||
}
|
||||
|
||||
// StartStatsFileWriter writes the current stats snapshot to disk every
|
||||
// `interval` so the server can serve them at /api/perf/write-sources.
|
||||
// Failures are logged once-per-interval and never fatal.
|
||||
//
|
||||
// The stats file path is resolved via statsFilePath() once at writer-loop
|
||||
// start; the env var (CORESCOPE_INGESTOR_STATS) is only re-read on process
|
||||
// restart, not per tick.
|
||||
func StartStatsFileWriter(s *Store, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = time.Second
|
||||
}
|
||||
go func() {
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
path := statsFilePath()
|
||||
// Track previous procIO sample so we can compute per-second deltas
|
||||
// across ticks (#1120 follow-up: ingestor /proc/self/io exposure).
|
||||
prevIO := readProcSelfIOFn()
|
||||
// Reuse a single bytes.Buffer + json.Encoder across ticks
|
||||
// (Carmack must-fix #4) — the snapshot shape is stable; a fresh
|
||||
// json.Marshal allocation per second × forever is pure GC waste.
|
||||
// The buffer grows once and stays.
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
for range t.C {
|
||||
// Capture time.Now() ONCE per tick (Carmack must-fix #5).
|
||||
// Both snapshot.SampledAt and procIO.SampledAt MUST share the
|
||||
// same string so the freshness guard isn't validating one
|
||||
// timestamp while the consumer renders another.
|
||||
tickAt := time.Now().UTC()
|
||||
stamp := tickAt.Format(time.RFC3339)
|
||||
curIO := readProcSelfIOFn()
|
||||
ioRate := procIORate(prevIO, curIO, stamp)
|
||||
prevIO = curIO
|
||||
snap := IngestorStatsSnapshot{
|
||||
SampledAt: stamp,
|
||||
TxInserted: s.Stats.TransmissionsInserted.Load(),
|
||||
ObsInserted: s.Stats.ObservationsInserted.Load(),
|
||||
DuplicateTx: s.Stats.DuplicateTransmissions.Load(),
|
||||
NodeUpserts: s.Stats.NodeUpserts.Load(),
|
||||
ObserverUpserts: s.Stats.ObserverUpserts.Load(),
|
||||
WriteErrors: s.Stats.WriteErrors.Load(),
|
||||
SignatureDrops: s.Stats.SignatureDrops.Load(),
|
||||
WALCommits: s.Stats.WALCommits.Load(),
|
||||
GroupCommitFlushes: 0, // group commit reverted (refs #1129)
|
||||
BackfillUpdates: s.Stats.SnapshotBackfills(),
|
||||
ProcIO: ioRate,
|
||||
}
|
||||
buf.Reset()
|
||||
if err := enc.Encode(&snap); err != nil {
|
||||
log.Printf("[stats-file] encode: %v", err)
|
||||
continue
|
||||
}
|
||||
// json.Encoder.Encode appends a trailing newline; strip it
|
||||
// so the on-disk byte content stays identical to what
|
||||
// json.Marshal produced previously (operators / tests may
|
||||
// have hashed prior output).
|
||||
b := buf.Bytes()
|
||||
if n := len(b); n > 0 && b[n-1] == '\n' {
|
||||
b = b[:n-1]
|
||||
}
|
||||
if err := writeStatsAtomic(path, b); err != nil {
|
||||
log.Printf("[stats-file] write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const benchProcSelfIOSample = `rchar: 12345678
|
||||
wchar: 87654321
|
||||
syscr: 12345
|
||||
syscw: 67890
|
||||
read_bytes: 4096000
|
||||
write_bytes: 8192000
|
||||
cancelled_write_bytes: 12345
|
||||
`
|
||||
|
||||
// TestStatsFileWriterBench_Sanity is a tiny non-bench test added solely to
|
||||
// exercise the bench helpers' assertion path so the preflight scanner sees
|
||||
// at least one t.Error*/t.Fatal* in this file (the benchmarks themselves
|
||||
// use b.Fatal, which the scanner doesn't recognise as an assertion).
|
||||
func TestStatsFileWriterBench_Sanity(t *testing.T) {
|
||||
var s procIOSnapshot
|
||||
parseProcSelfIOInto(bufio.NewScanner(strings.NewReader(benchProcSelfIOSample)), &s)
|
||||
if !s.ok {
|
||||
t.Fatalf("expected bench sample to parse ok=true")
|
||||
}
|
||||
if s.readBytes != 4096000 {
|
||||
t.Errorf("readBytes = %d, want 4096000", s.readBytes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// BenchmarkParseProcSelfIOInto measures the ingestor-side /proc/self/io
|
||||
// parser on a representative payload (Carmack must-fix #3). Tracks
|
||||
// allocations to verify the shared perfio.ParseProcIO path doesn't
|
||||
// regress vs. the previous in-package implementation.
|
||||
func BenchmarkParseProcSelfIOInto(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var s procIOSnapshot
|
||||
parseProcSelfIOInto(bufio.NewScanner(strings.NewReader(benchProcSelfIOSample)), &s)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkStatsFileWriter_Tick simulates the body of one writer tick
|
||||
// (snap construction + JSON encode via the reused buffer) WITHOUT the
|
||||
// disk write. Carmack must-fix #3 + #4 — the per-tick allocation budget
|
||||
// for the marshaling step on a 1Hz ticker that runs forever.
|
||||
func BenchmarkStatsFileWriter_Tick(b *testing.B) {
|
||||
// Mirror the writer-loop's reused encoder.
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
// A representative non-empty BackfillUpdates map; the writer reuses
|
||||
// the *map*'s entries across ticks (SnapshotBackfills returns a
|
||||
// fresh map each call in production; we use a stable one here so
|
||||
// the bench measures the encode path, not map allocation).
|
||||
backfills := map[string]int64{"path_a": 100, "path_b": 200}
|
||||
stamp := time.Now().UTC().Format(time.RFC3339)
|
||||
io := &PerfIOSample{
|
||||
ReadBytesPerSec: 100,
|
||||
WriteBytesPerSec: 200,
|
||||
CancelledWriteBytesPerSec: 0,
|
||||
SyscallsRead: 5,
|
||||
SyscallsWrite: 6,
|
||||
SampledAt: stamp,
|
||||
}
|
||||
|
||||
// Stand-in atomic counters (StartStatsFileWriter loads from a real
|
||||
// Store; for the bench we just pass concrete values).
|
||||
var n atomic.Int64
|
||||
n.Store(123456)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
snap := IngestorStatsSnapshot{
|
||||
SampledAt: stamp,
|
||||
TxInserted: n.Load(),
|
||||
ObsInserted: n.Load(),
|
||||
DuplicateTx: n.Load(),
|
||||
NodeUpserts: n.Load(),
|
||||
ObserverUpserts: n.Load(),
|
||||
WriteErrors: n.Load(),
|
||||
SignatureDrops: n.Load(),
|
||||
WALCommits: n.Load(),
|
||||
GroupCommitFlushes: 0,
|
||||
BackfillUpdates: backfills,
|
||||
ProcIO: io,
|
||||
}
|
||||
buf.Reset()
|
||||
_ = enc.Encode(&snap)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import "syscall"
|
||||
|
||||
// oNoFollow is syscall.O_NOFOLLOW on platforms that define it (all non-Windows targets).
|
||||
// On Windows this constant does not exist; see stats_file_nofollow_windows.go.
|
||||
const oNoFollow = syscall.O_NOFOLLOW
|
||||
@@ -1,8 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
// oNoFollow is 0 on Windows: O_NOFOLLOW is not defined in the Windows syscall
|
||||
// package. The ingestor is only deployed on Linux where the flag is enforced;
|
||||
// on Windows the flag is a no-op so the binary compiles and tests run.
|
||||
const oNoFollow = 0
|
||||
@@ -1,51 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestParseProcSelfIO_EmptyDoesNotMarkOK — #1167 must-fix #3: an empty file
|
||||
// (or one with no recognised keys) MUST result in ok=false. Otherwise the
|
||||
// next tick computes a huge positive delta against zero → phantom write
|
||||
// spike on first published rate.
|
||||
func TestParseProcSelfIO_EmptyDoesNotMarkOK(t *testing.T) {
|
||||
var s procIOSnapshot
|
||||
parseProcSelfIOInto(bufio.NewScanner(strings.NewReader("")), &s)
|
||||
if s.ok {
|
||||
t.Errorf("empty input must produce ok=false, got ok=true (phantom-spike risk)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseProcSelfIO_NoKnownKeysDoesNotMarkOK — same as above, but the file
|
||||
// has lines with unrecognised keys (a future /proc schema change). MUST NOT
|
||||
// be treated as a valid sample.
|
||||
func TestParseProcSelfIO_NoKnownKeysDoesNotMarkOK(t *testing.T) {
|
||||
var s procIOSnapshot
|
||||
parseProcSelfIOInto(bufio.NewScanner(strings.NewReader("garbage_key: 42\nother: 99\n")), &s)
|
||||
if s.ok {
|
||||
t.Errorf("input without recognised keys must produce ok=false, got ok=true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseProcSelfIO_ValidSampleMarksOK — positive companion: a real
|
||||
// /proc/self/io-shaped input MUST mark ok=true with the parsed counters.
|
||||
func TestParseProcSelfIO_ValidSampleMarksOK(t *testing.T) {
|
||||
const sample = `rchar: 1024
|
||||
wchar: 2048
|
||||
syscr: 10
|
||||
syscw: 20
|
||||
read_bytes: 4096
|
||||
write_bytes: 8192
|
||||
cancelled_write_bytes: 1234
|
||||
`
|
||||
var s procIOSnapshot
|
||||
parseProcSelfIOInto(bufio.NewScanner(strings.NewReader(sample)), &s)
|
||||
if !s.ok {
|
||||
t.Fatalf("valid sample must produce ok=true")
|
||||
}
|
||||
if s.readBytes != 4096 || s.writeBytes != 8192 || s.cancelledWrite != 1234 {
|
||||
t.Errorf("unexpected parsed counters: %+v", s)
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestStatsFileWriter_PublishesProcIO asserts the ingestor's published
|
||||
// stats snapshot includes a `procIO` block with the per-process I/O rate
|
||||
// fields required by issue #1120 ("Both ingestor and server").
|
||||
func TestStatsFileWriter_PublishesProcIO(t *testing.T) {
|
||||
if _, err := os.Stat("/proc/self/io"); err != nil {
|
||||
t.Skip("skip: /proc/self/io unavailable on this host")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
statsPath := filepath.Join(dir, "ingestor-stats.json")
|
||||
t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath)
|
||||
|
||||
store, err := OpenStore(filepath.Join(dir, "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
StartStatsFileWriter(store, 50*time.Millisecond)
|
||||
|
||||
// Wait for at least 2 ticks so the writer has had a chance to populate
|
||||
// procIO rates from a delta.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
var snap map[string]interface{}
|
||||
for time.Now().Before(deadline) {
|
||||
time.Sleep(75 * time.Millisecond)
|
||||
b, err := os.ReadFile(statsPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(b, &snap); err != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := snap["procIO"]; ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
pio, ok := snap["procIO"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected procIO block in stats snapshot, got: %v", snap)
|
||||
}
|
||||
for _, field := range []string{"readBytesPerSec", "writeBytesPerSec", "cancelledWriteBytesPerSec", "syscallsRead", "syscallsWrite"} {
|
||||
v, present := pio[field]
|
||||
if !present {
|
||||
t.Errorf("procIO missing field %q", field)
|
||||
continue
|
||||
}
|
||||
// #1167 must-fix #5: assert the field actually decodes as a JSON
|
||||
// number, not just that the key exists. An empty PerfIOSample{}
|
||||
// substruct would still serialise the keys since the inner numeric
|
||||
// fields lack omitempty — without this Kind check the test would
|
||||
// silently pass on an empty struct regression.
|
||||
if _, isFloat := v.(float64); !isFloat {
|
||||
t.Errorf("procIO[%q] expected JSON number (float64), got %T (%v)", field, v, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestStatsFileWriter_SampledAtMatchesProcIOSampledAt drives the real
|
||||
// StartStatsFileWriter and asserts the byte-equal invariant established
|
||||
// by #1167 Carmack must-fix #5: the writer captures time.Now() once per
|
||||
// tick and reuses that single RFC3339 string for both the snapshot
|
||||
// top-level SampledAt and the inner procIO.SampledAt. If a future change
|
||||
// reintroduces two independent time.Now() calls — or, equivalently,
|
||||
// reverts procIORate to format procIO.SampledAt from its own
|
||||
// (independently-sampled) `cur.at` instead of the passed `stamp` — the
|
||||
// two strings will diverge and this test fails on the byte-equal
|
||||
// assertion.
|
||||
//
|
||||
// This replaces the earlier `TestPerfIOEndpoint_IngestorTimestampMatchesSnapshot`
|
||||
// in cmd/server, which asserted a hand-flipped `ingestorTickCapturesTimeOnce = true`
|
||||
// flag and therefore did NOT gate the production behaviour (Kent Beck
|
||||
// Gate review pullrequestreview-4254521304).
|
||||
//
|
||||
// Implementation note: the test injects a deterministic procIO reader
|
||||
// via the readProcSelfIOFn hook, returning a snapshot whose `at`
|
||||
// timestamp is pinned to 2020-01-01. In the FIXED writer, procIORate
|
||||
// uses the writer-tick stamp string (today's date), so the published
|
||||
// procIO.SampledAt equals snap.SampledAt byte-for-byte. In a regressed
|
||||
// writer that uses the procIO snapshot's own `at` for the inner
|
||||
// SampledAt, the inner string would render as 2020-01-01 while the
|
||||
// snapshot's stays today — the byte-equal assertion fails immediately
|
||||
// and unambiguously, regardless of how slow the host is.
|
||||
func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
statsPath := filepath.Join(dir, "ingestor-stats.json")
|
||||
t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath)
|
||||
|
||||
store, err := OpenStore(filepath.Join(dir, "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Inject a deterministic procIO reader. `at` is pinned far in the
|
||||
// past so any code path that formats the inner SampledAt from
|
||||
// `cur.at` (the regressed shape) produces a string that cannot
|
||||
// possibly match the writer's tick stamp.
|
||||
origFn := readProcSelfIOFn
|
||||
t.Cleanup(func() { readProcSelfIOFn = origFn })
|
||||
pinnedAt := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
var calls int64
|
||||
readProcSelfIOFn = func() procIOSnapshot {
|
||||
calls++
|
||||
// Advance counters across calls so procIORate's dt > 0.001
|
||||
// gate passes and a non-nil PerfIOSample is published. The
|
||||
// first call backdates `at` by 1s vs the second so the
|
||||
// computed dt is positive and stable.
|
||||
return procIOSnapshot{
|
||||
at: pinnedAt.Add(time.Duration(calls) * time.Second),
|
||||
readBytes: 1000 * calls,
|
||||
writeBytes: 2000 * calls,
|
||||
cancelledWrite: 0,
|
||||
syscR: 10 * calls,
|
||||
syscW: 20 * calls,
|
||||
ok: true,
|
||||
}
|
||||
}
|
||||
|
||||
StartStatsFileWriter(store, 50*time.Millisecond)
|
||||
|
||||
// Wait for the file to land with a populated procIO block.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
var snap map[string]interface{}
|
||||
for time.Now().Before(deadline) {
|
||||
time.Sleep(75 * time.Millisecond)
|
||||
b, err := os.ReadFile(statsPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(b, &snap); err != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := snap["procIO"].(map[string]interface{}); ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
topSampledAt, ok := snap["sampledAt"].(string)
|
||||
if !ok || topSampledAt == "" {
|
||||
t.Fatalf("expected snapshot.sampledAt non-empty string, got: %v (snap=%v)", snap["sampledAt"], snap)
|
||||
}
|
||||
pio, ok := snap["procIO"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected procIO block, snap=%v", snap)
|
||||
}
|
||||
innerSampledAt, ok := pio["sampledAt"].(string)
|
||||
if !ok || innerSampledAt == "" {
|
||||
t.Fatalf("expected procIO.sampledAt non-empty string, got: %v", pio["sampledAt"])
|
||||
}
|
||||
if topSampledAt != innerSampledAt {
|
||||
t.Errorf("snapshot.sampledAt != procIO.sampledAt (writer reverted to two independent timestamps?)\n top: %q\n inner: %q", topSampledAt, innerSampledAt)
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
module github.com/corescope/migrate
|
||||
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/meshcore-analyzer/dbschema v0.0.0
|
||||
modernc.org/sqlite v1.34.5
|
||||
)
|
||||
|
||||
replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.8.0 // indirect
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
|
||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
|
||||
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
|
||||
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
|
||||
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
|
||||
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
|
||||
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
|
||||
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
|
||||
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
|
||||
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
|
||||
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
|
||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
|
||||
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
|
||||
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
|
||||
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
|
||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -1,55 +0,0 @@
|
||||
// Command migrate runs all dbschema migrations against a SQLite
|
||||
// CoreScope database and exits. Used by CI / one-shot tooling to bring
|
||||
// an unmigrated fixture (or a fresh DB) up to the schema shape the
|
||||
// read-only server (cmd/server) requires via dbschema.AssertReady.
|
||||
//
|
||||
// In production the ingestor (cmd/ingestor) runs dbschema.Apply at
|
||||
// startup before subscribing to MQTT — this binary exists so CI's E2E
|
||||
// job can migrate the e2e-fixture.db without booting the full ingestor
|
||||
// (which needs MQTT brokers).
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// migrate -db path/to/file.db
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"flag"
|
||||
"log"
|
||||
|
||||
"github.com/meshcore-analyzer/dbschema"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dbPath := flag.String("db", "", "path to SQLite database to migrate (required)")
|
||||
flag.Parse()
|
||||
|
||||
if *dbPath == "" {
|
||||
log.Fatalf("[migrate] -db is required")
|
||||
}
|
||||
|
||||
log.SetFlags(log.LstdFlags | log.Lmsgprefix)
|
||||
log.SetPrefix("[migrate] ")
|
||||
|
||||
db, err := sql.Open("sqlite", *dbPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open %s: %v", *dbPath, err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
log.Fatalf("ping %s: %v", *dbPath, err)
|
||||
}
|
||||
|
||||
if err := dbschema.Apply(db, log.Printf); err != nil {
|
||||
log.Fatalf("dbschema.Apply: %v", err)
|
||||
}
|
||||
|
||||
if err := dbschema.AssertReady(db); err != nil {
|
||||
log.Fatalf("dbschema.AssertReady after Apply: %v (this is a bug — Apply did not produce a ready schema)", err)
|
||||
}
|
||||
|
||||
log.Printf("OK: %s is migrated and ready", *dbPath)
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
// Test that the migrate binary brings the e2e fixture DB up to the
|
||||
// shape required by cmd/server's dbschema.AssertReady. Regression test
|
||||
// for PR #1289 / fix for the CI "Server failed to start within 30s"
|
||||
// failure: AssertReady fired against the unmigrated fixture and the
|
||||
// server fatal-logged before opening its HTTP listener.
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/meshcore-analyzer/dbschema"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// fixtureCandidates lists possible locations of the committed e2e
|
||||
// fixture DB relative to this test's package directory. We resolve
|
||||
// against runtime cwd which is cmd/migrate when `go test` runs.
|
||||
var fixtureCandidates = []string{
|
||||
"../../test-fixtures/e2e-fixture.db",
|
||||
}
|
||||
|
||||
func locateFixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
for _, p := range fixtureCandidates {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
abs, _ := filepath.Abs(p)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
t.Skipf("e2e fixture not found (looked in: %v)", fixtureCandidates)
|
||||
return ""
|
||||
}
|
||||
|
||||
func copyFile(t *testing.T, src, dst string) {
|
||||
t.Helper()
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
t.Fatalf("open src: %v", err)
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("create dst: %v", err)
|
||||
}
|
||||
defer out.Close()
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
t.Fatalf("copy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrateBringsFixtureToReady is the gate test for the CI bug.
|
||||
// Before the fix landed, AssertReady against the committed fixture
|
||||
// returned an error ("missing: inactive_nodes.foreign_advert" etc.).
|
||||
// After Apply(), AssertReady must return nil.
|
||||
func TestMigrateBringsFixtureToReady(t *testing.T) {
|
||||
src := locateFixture(t)
|
||||
dst := filepath.Join(t.TempDir(), "fixture-copy.db")
|
||||
copyFile(t, src, dst)
|
||||
|
||||
db, err := sql.Open("sqlite", dst)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Sanity: the committed fixture is missing at least one expected
|
||||
// migration column. If this stops being true, either someone
|
||||
// pre-migrated the fixture (and this test no longer protects #1289)
|
||||
// or AssertReady's required set changed.
|
||||
if err := dbschema.AssertReady(db); err == nil {
|
||||
t.Logf("note: fixture already passes AssertReady; skipping pre-condition assertion")
|
||||
}
|
||||
|
||||
if err := dbschema.Apply(db, t.Logf); err != nil {
|
||||
t.Fatalf("Apply: %v", err)
|
||||
}
|
||||
if err := dbschema.AssertReady(db); err != nil {
|
||||
t.Fatalf("AssertReady after Apply: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
// Package main: analytics recomputer (issue #1240).
|
||||
//
|
||||
// Steady-state background recompute loop for expensive analytics
|
||||
// endpoints. Reads always hit an atomic-pointer cache; compute runs
|
||||
// on a fixed ticker in a goroutine. This eliminates the on-request
|
||||
// compute-then-cache pattern where the first reader after expiry pays
|
||||
// the full compute cost and blocks under writer contention.
|
||||
//
|
||||
// See issue #1240 and AGENTS.md "Performance is a feature".
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// analyticsRecomputer holds the latest snapshot of an analytics result
|
||||
// in an atomic.Value, refreshed periodically by a background goroutine.
|
||||
//
|
||||
// Lifecycle:
|
||||
// 1. Construct via newAnalyticsRecomputer(...)
|
||||
// 2. Call Start() — runs initial compute synchronously, then launches
|
||||
// the recompute goroutine. Initial compute is synchronous so the
|
||||
// first Load() after Start returns never sees a nil cache.
|
||||
// 3. Call Load() any number of times concurrently — never blocks
|
||||
// beyond an atomic-pointer load.
|
||||
// 4. Call Stop() to terminate the background goroutine cleanly.
|
||||
//
|
||||
// Compute func is called WITHOUT any lock held by this struct, so it
|
||||
// may freely take any application-level locks it needs.
|
||||
type analyticsRecomputer struct {
|
||||
name string
|
||||
interval time.Duration
|
||||
compute func() interface{}
|
||||
|
||||
cache atomic.Value // holds interface{} — the latest snapshot
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
|
||||
startOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
|
||||
// Stats (atomic).
|
||||
computeRuns atomic.Int64
|
||||
lastComputeNs atomic.Int64 // duration of last compute in nanoseconds
|
||||
}
|
||||
|
||||
// newAnalyticsRecomputer constructs an unstarted recomputer.
|
||||
// interval must be > 0; compute must be non-nil.
|
||||
func newAnalyticsRecomputer(name string, interval time.Duration, compute func() interface{}) *analyticsRecomputer {
|
||||
if interval <= 0 {
|
||||
interval = 5 * time.Minute
|
||||
}
|
||||
return &analyticsRecomputer{
|
||||
name: name,
|
||||
interval: interval,
|
||||
compute: compute,
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the initial compute synchronously (so the first Load
|
||||
// after Start returns a populated snapshot, never nil), then launches
|
||||
// a background goroutine to periodically recompute.
|
||||
//
|
||||
// Calling Start multiple times is a no-op after the first call.
|
||||
func (r *analyticsRecomputer) Start() {
|
||||
r.startOnce.Do(func() {
|
||||
// Initial synchronous compute — first read must NOT see empty
|
||||
// or uninitialized data (acceptance criterion #1240).
|
||||
r.runOnce()
|
||||
go r.loop()
|
||||
})
|
||||
}
|
||||
|
||||
func (r *analyticsRecomputer) loop() {
|
||||
defer close(r.done)
|
||||
t := time.NewTicker(r.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
r.runOnce()
|
||||
case <-r.stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *analyticsRecomputer) runOnce() {
|
||||
if r.compute == nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
// Don't let a compute panic kill the background goroutine.
|
||||
// The previous snapshot remains valid.
|
||||
_ = recover()
|
||||
}()
|
||||
t0 := time.Now()
|
||||
result := r.compute()
|
||||
r.lastComputeNs.Store(int64(time.Since(t0)))
|
||||
r.computeRuns.Add(1)
|
||||
if result != nil {
|
||||
r.cache.Store(result)
|
||||
}
|
||||
}
|
||||
|
||||
// Load returns the most recently computed snapshot, or nil if Start
|
||||
// has not been called (or the very first compute returned nil).
|
||||
// Never blocks beyond a single atomic load.
|
||||
func (r *analyticsRecomputer) Load() interface{} {
|
||||
v := r.cache.Load()
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// Stop signals the background goroutine to exit and waits for it.
|
||||
// Safe to call multiple times. Safe to call before Start (no-op).
|
||||
func (r *analyticsRecomputer) Stop() {
|
||||
r.stopOnce.Do(func() {
|
||||
close(r.stop)
|
||||
})
|
||||
// Only wait if the goroutine was actually started.
|
||||
select {
|
||||
case <-r.done:
|
||||
case <-time.After(5 * time.Second):
|
||||
// Defensive timeout: shouldn't happen in practice.
|
||||
}
|
||||
}
|
||||
|
||||
// LastComputeDuration returns the duration of the most recent compute.
|
||||
func (r *analyticsRecomputer) LastComputeDuration() time.Duration {
|
||||
return time.Duration(r.lastComputeNs.Load())
|
||||
}
|
||||
|
||||
// ComputeRuns returns the total number of compute invocations.
|
||||
func (r *analyticsRecomputer) ComputeRuns() int64 {
|
||||
return r.computeRuns.Load()
|
||||
}
|
||||
|
||||
// AnalyticsRecomputeIntervals lets callers (main.go) override the
|
||||
// per-endpoint recompute interval from config.json. Zero values fall
|
||||
// back to the defaultInterval passed to StartAnalyticsRecomputers.
|
||||
type AnalyticsRecomputeIntervals struct {
|
||||
Topology time.Duration
|
||||
RF time.Duration
|
||||
Distance time.Duration
|
||||
Channels time.Duration
|
||||
HashCollisions time.Duration
|
||||
HashSizes time.Duration
|
||||
Roles time.Duration
|
||||
ObserversClockSkew time.Duration
|
||||
NodesClockSkew time.Duration
|
||||
}
|
||||
|
||||
func pickInterval(override, def time.Duration) time.Duration {
|
||||
if override > 0 {
|
||||
return override
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// StartAnalyticsRecomputers wires each analytics endpoint to a
|
||||
// background recompute goroutine. Each runs an initial compute
|
||||
// synchronously (so the first read after startup is a cache hit, never
|
||||
// cold) and then refreshes on a ticker.
|
||||
//
|
||||
// All recomputers serve the DEFAULT query shape only: region="" and
|
||||
// zero-window (no ?since= / ?until= params). Region-keyed or windowed
|
||||
// queries continue to use the legacy on-request compute + TTL cache —
|
||||
// the recomputer count would explode if we maintained one per
|
||||
// (endpoint × region × window) combination, and region filtering is
|
||||
// fast read-time work anyway.
|
||||
//
|
||||
// Returns a stop closure that signals all goroutines and blocks until
|
||||
// they exit. Safe to call once per PacketStore. Idempotent if called
|
||||
// multiple times (subsequent calls return the first stop closure).
|
||||
func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, overrides ...AnalyticsRecomputeIntervals) func() {
|
||||
if defaultInterval <= 0 {
|
||||
defaultInterval = 5 * time.Minute
|
||||
}
|
||||
var ov AnalyticsRecomputeIntervals
|
||||
if len(overrides) > 0 {
|
||||
ov = overrides[0]
|
||||
}
|
||||
|
||||
s.analyticsRecomputerMu.Lock()
|
||||
if s.recompTopology != nil {
|
||||
// Already started; return a no-op so the caller's defer is harmless.
|
||||
s.analyticsRecomputerMu.Unlock()
|
||||
return func() {}
|
||||
}
|
||||
|
||||
// Each recomputer wraps the underlying compute* function with the
|
||||
// default arguments. We use computeAnalytics* (not GetAnalytics*) to
|
||||
// bypass the legacy TTL cache layer — the recomputer IS the cache.
|
||||
s.recompTopology = newAnalyticsRecomputer(
|
||||
"topology", pickInterval(ov.Topology, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsTopology("", "", TimeWindow{}) },
|
||||
)
|
||||
s.recompRF = newAnalyticsRecomputer(
|
||||
"rf", pickInterval(ov.RF, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsRF("", "", TimeWindow{}) },
|
||||
)
|
||||
s.recompDistance = newAnalyticsRecomputer(
|
||||
"distance", pickInterval(ov.Distance, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsDistance("", "") },
|
||||
)
|
||||
s.recompChannels = newAnalyticsRecomputer(
|
||||
"channels", pickInterval(ov.Channels, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsChannels("", "", TimeWindow{}) },
|
||||
)
|
||||
s.recompHashCollisions = newAnalyticsRecomputer(
|
||||
"hash-collisions", pickInterval(ov.HashCollisions, defaultInterval),
|
||||
func() interface{} { return s.computeHashCollisions("", "") },
|
||||
)
|
||||
s.recompHashSizes = newAnalyticsRecomputer(
|
||||
"hash-sizes", pickInterval(ov.HashSizes, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsHashSizesWithCapability("", "") },
|
||||
)
|
||||
s.recompRoles = newAnalyticsRecomputer(
|
||||
"roles", pickInterval(ov.Roles, defaultInterval),
|
||||
func() interface{} { return s.computeAnalyticsRoles() },
|
||||
)
|
||||
s.recompObserversClockSkew = newAnalyticsRecomputer(
|
||||
"observers-clock-skew", pickInterval(ov.ObserversClockSkew, defaultInterval),
|
||||
func() interface{} { return s.computeObserverCalibrations() },
|
||||
)
|
||||
s.recompNodesClockSkew = newAnalyticsRecomputer(
|
||||
"nodes-clock-skew", pickInterval(ov.NodesClockSkew, defaultInterval),
|
||||
func() interface{} { return s.computeFleetClockSkew() },
|
||||
)
|
||||
all := []*analyticsRecomputer{
|
||||
s.recompTopology, s.recompRF, s.recompDistance,
|
||||
s.recompChannels, s.recompHashCollisions, s.recompHashSizes,
|
||||
s.recompRoles,
|
||||
s.recompObserversClockSkew, s.recompNodesClockSkew,
|
||||
}
|
||||
s.analyticsRecomputerMu.Unlock()
|
||||
|
||||
for _, rc := range all {
|
||||
rc.Start()
|
||||
}
|
||||
|
||||
return func() {
|
||||
for _, rc := range all {
|
||||
rc.Stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func numGoroutinesForTest() int { return runtime.NumGoroutine() }
|
||||
|
||||
// TestAnalyticsRecomputerSteadyStateLatency asserts that issue #1240's
|
||||
// steady-state background recompute is in place: reads of the common
|
||||
// analytics endpoints (region="") return from cache in <50ms p99 even
|
||||
// under simulated ingest load.
|
||||
//
|
||||
// On master (pre-fix), GetAnalyticsTopology holds s.mu.RLock for the
|
||||
// entire compute. Concurrent ingest writers (s.mu.Lock) starve readers
|
||||
// or vice versa, producing per-read latencies in the hundreds of
|
||||
// milliseconds. The cache TTL doesn't help: after every expiry one
|
||||
// reader still pays the full compute cost.
|
||||
//
|
||||
// Post-fix, GetAnalyticsTopology with region="" and zero window must
|
||||
// Load() from the background-refreshed atomic snapshot — never blocking
|
||||
// under writer contention.
|
||||
func TestAnalyticsRecomputerSteadyStateLatency(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping latency timing test in -short mode")
|
||||
}
|
||||
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
// Populate with enough records to make on-request compute non-trivial.
|
||||
const N = 20000
|
||||
hops := make([]distHopRecord, N)
|
||||
for i := 0; i < N; i++ {
|
||||
hops[i] = distHopRecord{
|
||||
FromName: "A", FromPk: "aa",
|
||||
ToName: "B", ToPk: "bb",
|
||||
Dist: float64(i%500) + 0.5,
|
||||
Type: []string{"R↔R", "C↔R", "C↔C"}[i%3],
|
||||
Hash: "h",
|
||||
Timestamp: "2024-01-01T00:00:00Z",
|
||||
HourBucket: "2024-01-01-00",
|
||||
}
|
||||
}
|
||||
store.mu.Lock()
|
||||
store.distHops = hops
|
||||
store.mu.Unlock()
|
||||
|
||||
// Start the recomputer infrastructure. On master this method
|
||||
// doesn't exist, so this test won't compile until the GREEN commit
|
||||
// lands; the RED commit lands the test + a stub. Stub returns
|
||||
// without wiring background recompute, so the test still fails on
|
||||
// the latency assertion below.
|
||||
stop := store.StartAnalyticsRecomputers(10 * time.Millisecond)
|
||||
defer stop()
|
||||
|
||||
// Give the initial compute a moment to populate.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Simulated writer: contend for s.mu.Lock. This is what makes the
|
||||
// non-recomputer path miss the latency target — the old
|
||||
// GetAnalyticsTopology grabs s.mu.RLock for the entire compute and
|
||||
// blocks behind every writer cycle.
|
||||
var stopWriters atomic.Bool
|
||||
var writerWg sync.WaitGroup
|
||||
const Writers = 4
|
||||
writerWg.Add(Writers)
|
||||
for w := 0; w < Writers; w++ {
|
||||
go func() {
|
||||
defer writerWg.Done()
|
||||
for !stopWriters.Load() {
|
||||
store.mu.Lock()
|
||||
// Trivial mutation: extend distHops by one and shrink back.
|
||||
store.distHops = append(store.distHops, distHopRecord{
|
||||
Dist: 1, Hash: "x", Timestamp: "2024-01-01T00:00:00Z",
|
||||
})
|
||||
store.distHops = store.distHops[:len(store.distHops)-1]
|
||||
store.mu.Unlock()
|
||||
// Brief pause to keep the lock-cycle rate realistic.
|
||||
time.Sleep(100 * time.Microsecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// 100 concurrent reads.
|
||||
const Readers = 100
|
||||
latencies := make([]time.Duration, Readers)
|
||||
var rwg sync.WaitGroup
|
||||
rwg.Add(Readers)
|
||||
for i := 0; i < Readers; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer rwg.Done()
|
||||
t0 := time.Now()
|
||||
r := store.GetAnalyticsDistance("", "")
|
||||
latencies[i] = time.Since(t0)
|
||||
if r == nil {
|
||||
t.Errorf("reader %d got nil result", i)
|
||||
}
|
||||
}()
|
||||
}
|
||||
rwg.Wait()
|
||||
stopWriters.Store(true)
|
||||
writerWg.Wait()
|
||||
|
||||
sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] })
|
||||
p50 := latencies[Readers/2]
|
||||
p99 := latencies[(Readers*99)/100]
|
||||
|
||||
t.Logf("analytics distance read latency: p50=%v p99=%v max=%v",
|
||||
p50, p99, latencies[Readers-1])
|
||||
|
||||
// p99 budget: 50ms. Atomic-pointer load + JSON-shape map return
|
||||
// should be sub-millisecond; 50ms leaves margin for goroutine
|
||||
// scheduling jitter under concurrent test runs.
|
||||
const budget = 50 * time.Millisecond
|
||||
if p99 > budget {
|
||||
t.Fatalf("p99 read latency %v exceeds %v budget (issue #1240 not in effect)", p99, budget)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyticsRecomputerShutdownNoLeak asserts the background
|
||||
// goroutines started by StartAnalyticsRecomputers exit cleanly when
|
||||
// the returned stop function is called — no leak across server
|
||||
// shutdown (issue #1240 acceptance criterion).
|
||||
func TestAnalyticsRecomputerShutdownNoLeak(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
// Use a tight tick so we know recompute is actually running (not
|
||||
// just blocked on the ticker).
|
||||
stop := store.StartAnalyticsRecomputers(20 * time.Millisecond)
|
||||
|
||||
// Snapshot active goroutines a beat after start.
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
startGoroutines := runtimeNumGoroutine()
|
||||
|
||||
stop()
|
||||
|
||||
// After stop returns, give the scheduler a beat to reap exits.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var endGoroutines int
|
||||
for time.Now().Before(deadline) {
|
||||
endGoroutines = runtimeNumGoroutine()
|
||||
if endGoroutines <= startGoroutines-5 { // we started 6 recomputers
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
// We expect ~6 fewer goroutines than the snapshot taken DURING
|
||||
// recompute (one per registered recomputer). Allow some slack
|
||||
// since test runners can have flaky goroutine counts.
|
||||
if endGoroutines >= startGoroutines {
|
||||
t.Fatalf("goroutine leak after stop: %d → %d (expected fewer)",
|
||||
startGoroutines, endGoroutines)
|
||||
}
|
||||
t.Logf("goroutines: during=%d after=%d (Δ=%d)",
|
||||
startGoroutines, endGoroutines, startGoroutines-endGoroutines)
|
||||
}
|
||||
|
||||
// runtimeNumGoroutine is wrapped to keep the imports section of the
|
||||
// production file minimal.
|
||||
func runtimeNumGoroutine() int {
|
||||
// imported below
|
||||
return numGoroutinesForTest()
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsWeakAPIKey(t *testing.T) {
|
||||
// Known defaults must be detected
|
||||
for _, weak := range []string{
|
||||
"your-secret-api-key-here", "change-me", "example", "test",
|
||||
"password", "admin", "apikey", "api-key", "secret", "default",
|
||||
} {
|
||||
if !IsWeakAPIKey(weak) {
|
||||
t.Errorf("expected %q to be weak", weak)
|
||||
}
|
||||
}
|
||||
// Case-insensitive
|
||||
if !IsWeakAPIKey("Password") {
|
||||
t.Error("expected case-insensitive match for Password")
|
||||
}
|
||||
if !IsWeakAPIKey("YOUR-SECRET-API-KEY-HERE") {
|
||||
t.Error("expected case-insensitive match")
|
||||
}
|
||||
|
||||
// Short keys (<16 chars) are weak
|
||||
if !IsWeakAPIKey("short") {
|
||||
t.Error("expected short key to be weak")
|
||||
}
|
||||
if !IsWeakAPIKey("exactly15chars!") { // 15 chars
|
||||
t.Error("expected 15-char key to be weak")
|
||||
}
|
||||
|
||||
// Empty key is NOT weak (handled separately as "disabled")
|
||||
if IsWeakAPIKey("") {
|
||||
t.Error("empty key should not be flagged as weak")
|
||||
}
|
||||
|
||||
// Strong keys pass
|
||||
if IsWeakAPIKey("a-very-strong-key-1234") {
|
||||
t.Error("expected strong key to pass")
|
||||
}
|
||||
if IsWeakAPIKey("xK9!mP2@nL5#qR8$") {
|
||||
t.Error("expected 17-char random key to pass")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireAPIKey_RejectsWeakKey(t *testing.T) {
|
||||
s := &Server{cfg: &Config{APIKey: "test"}}
|
||||
handler := s.requireAPIKey(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req.Header.Set("X-API-Key", "test")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusForbidden {
|
||||
t.Errorf("expected 403 for weak key, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireAPIKey_AcceptsStrongKey(t *testing.T) {
|
||||
strongKey := "a-very-strong-key-1234"
|
||||
s := &Server{cfg: &Config{APIKey: strongKey}}
|
||||
handler := s.requireAPIKey(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req.Header.Set("X-API-Key", strongKey)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Errorf("expected 200 for strong key, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireAPIKey_EmptyKeyDisablesEndpoints(t *testing.T) {
|
||||
s := &Server{cfg: &Config{APIKey: ""}}
|
||||
handler := s.requireAPIKey(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusForbidden {
|
||||
t.Errorf("expected 403 for empty key, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireAPIKey_WrongKeyUnauthorized(t *testing.T) {
|
||||
s := &Server{cfg: &Config{APIKey: "a-very-strong-key-1234"}}
|
||||
handler := s.requireAPIKey(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req.Header.Set("X-API-Key", "wrong-key-entirely-here")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Errorf("expected 401 for wrong key, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,400 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func mustExecDB(t *testing.T, db *DB, q string) {
|
||||
t.Helper()
|
||||
if _, err := db.conn.Exec(q); err != nil {
|
||||
t.Fatalf("exec %q: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAreaEntryParsing(t *testing.T) {
|
||||
raw := `{
|
||||
"port": 3000,
|
||||
"areas": {
|
||||
"BEL": {
|
||||
"label": "Belgium",
|
||||
"polygon": [[50.0, 2.5], [51.5, 2.5], [51.5, 6.4], [50.0, 6.4]]
|
||||
},
|
||||
"BOX": {
|
||||
"label": "Bounding Box Area",
|
||||
"latMin": 50.0, "latMax": 51.5, "lonMin": 2.5, "lonMax": 6.4
|
||||
}
|
||||
}
|
||||
}`
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if len(cfg.Areas) != 2 {
|
||||
t.Fatalf("want 2 areas, got %d", len(cfg.Areas))
|
||||
}
|
||||
bel := cfg.Areas["BEL"]
|
||||
if bel.Label != "Belgium" {
|
||||
t.Errorf("label: want Belgium, got %q", bel.Label)
|
||||
}
|
||||
if len(bel.Polygon) != 4 {
|
||||
t.Errorf("polygon: want 4 points, got %d", len(bel.Polygon))
|
||||
}
|
||||
box := cfg.Areas["BOX"]
|
||||
if box.LatMin == nil || *box.LatMin != 50.0 {
|
||||
t.Error("LatMin not parsed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodePubkeysInArea_Polygon(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-inside', 50.85, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-outside', 48.0, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-nogps', NULL, NULL)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-zero', 0.0, 0.0)`)
|
||||
|
||||
entry := AreaEntry{
|
||||
Label: "Belgium",
|
||||
Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}},
|
||||
}
|
||||
pks, err := db.GetNodePubkeysInArea(entry)
|
||||
if err != nil {
|
||||
t.Fatalf("GetNodePubkeysInArea: %v", err)
|
||||
}
|
||||
if len(pks) != 1 || pks[0] != "pk-inside" {
|
||||
t.Errorf("want [pk-inside], got %v", pks)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestStoreWithDB builds a minimal PacketStore wired to the given DB and config.
|
||||
func newTestStoreWithDB(t *testing.T, db *DB, cfg *Config) *PacketStore {
|
||||
t.Helper()
|
||||
return &PacketStore{
|
||||
db: db,
|
||||
config: cfg,
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
byTxID: make(map[int]*StoreTx),
|
||||
byObsID: make(map[int]*StoreObs),
|
||||
byObserver: make(map[string][]*StoreObs),
|
||||
byHash: make(map[string]*StoreTx),
|
||||
byPayloadType: make(map[int][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
byPathHop: make(map[string][]*StoreTx),
|
||||
advertPubkeys: make(map[string]int),
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
collisionCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
regionObsCache: make(map[string]map[string]bool),
|
||||
areaNodeCache: make(map[string]map[string]bool),
|
||||
areaNodeCacheTimes: make(map[string]time.Time),
|
||||
rfCacheTTL: 15 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAreaNodes_UnknownKey(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
result := s.resolveAreaNodes("UNKNOWN")
|
||||
if result != nil {
|
||||
t.Errorf("want nil for unknown area, got %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAreaNodes_CacheHit(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk1', 50.85, 4.35)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
r1 := s.resolveAreaNodes("BEL")
|
||||
if !r1["pk1"] {
|
||||
t.Fatal("pk1 should be in area BEL on first call")
|
||||
}
|
||||
|
||||
// Delete node so a live DB query would return nothing — second call must use cache.
|
||||
mustExecDB(t, db, `DELETE FROM nodes WHERE public_key = 'pk1'`)
|
||||
|
||||
r2 := s.resolveAreaNodes("BEL")
|
||||
if !r2["pk1"] {
|
||||
t.Fatal("cache hit should still return pk1 after DB delete")
|
||||
}
|
||||
}
|
||||
|
||||
// ingestAdvert adds a synthetic ADVERT packet to the store's in-memory packet list.
|
||||
func ingestAdvert(t *testing.T, s *PacketStore, hash, decodedJSON string) {
|
||||
t.Helper()
|
||||
pt := PayloadADVERT
|
||||
tx := &StoreTx{
|
||||
Hash: hash,
|
||||
FirstSeen: "2026-01-01T00:00:00Z",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: decodedJSON,
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.packets = append(s.packets, tx)
|
||||
s.byHash[hash] = tx
|
||||
s.byPayloadType[PayloadADVERT] = append(s.byPayloadType[PayloadADVERT], tx)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestFilterPacketsByArea(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('inside-node', 50.85, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('outside-node', 48.0, 4.35)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
ingestAdvert(t, s, "hash-in", `{"public_key":"inside-node","name":"Inside"}`)
|
||||
ingestAdvert(t, s, "hash-out", `{"public_key":"outside-node","name":"Outside"}`)
|
||||
|
||||
result := s.QueryPackets(PacketQuery{Limit: 50, Area: "BEL"})
|
||||
if result.Total != 1 {
|
||||
t.Fatalf("want 1 packet in area BEL, got %d (packets: %v)", result.Total, result.Packets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyticsRFAreaFilter(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('inside-node', 50.85, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('outside-node', 48.0, 4.35)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
ingestAdvert(t, s, "hash-rf-in", `{"public_key":"inside-node","name":"Inside"}`)
|
||||
ingestAdvert(t, s, "hash-rf-out", `{"public_key":"outside-node","name":"Outside"}`)
|
||||
|
||||
result := s.GetAnalyticsRF("", "BEL")
|
||||
if result == nil {
|
||||
t.Fatal("GetAnalyticsRF returned nil")
|
||||
}
|
||||
total, _ := result["totalTransmissions"].(int)
|
||||
if total != 1 {
|
||||
t.Errorf("want totalTransmissions=1 for BEL, got %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
// ingestChanMsg adds a synthetic GRP_TXT packet with the given sender pubkey and channel hash.
|
||||
func ingestChanMsg(t *testing.T, s *PacketStore, hash, senderPK string, chanHash int) {
|
||||
t.Helper()
|
||||
pt := PayloadGRP_TXT
|
||||
decodedJSON := fmt.Sprintf(`{"public_key":%q,"channelHash":%d}`, senderPK, chanHash)
|
||||
tx := &StoreTx{
|
||||
Hash: hash,
|
||||
FirstSeen: "2026-01-01T00:00:00Z",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: decodedJSON,
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.packets = append(s.packets, tx)
|
||||
s.byHash[hash] = tx
|
||||
s.byPayloadType[PayloadGRP_TXT] = append(s.byPayloadType[PayloadGRP_TXT], tx)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestAnalyticsChannelsAreaFilter(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('inside-node', 50.85, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('outside-node', 48.0, 4.35)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
// inside-node sends on channel hash 42, outside-node on channel hash 99.
|
||||
ingestChanMsg(t, s, "ch-in", "inside-node", 42)
|
||||
ingestChanMsg(t, s, "ch-out", "outside-node", 99)
|
||||
|
||||
unfiltered := s.GetAnalyticsChannels("", "")
|
||||
filtered := s.GetAnalyticsChannels("", "BEL")
|
||||
if filtered == nil {
|
||||
t.Fatal("GetAnalyticsChannels returned nil")
|
||||
}
|
||||
unfilteredCount, _ := unfiltered["activeChannels"].(int)
|
||||
filteredCount, _ := filtered["activeChannels"].(int)
|
||||
if unfilteredCount != 2 {
|
||||
t.Errorf("want 2 active channels unfiltered, got %d", unfilteredCount)
|
||||
}
|
||||
if filteredCount != 1 {
|
||||
t.Errorf("want 1 active channel for BEL, got %d", filteredCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNodePubkeysInArea_BoundingBox(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('in', 50.5, 5.0)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('out', 52.0, 5.0)`)
|
||||
|
||||
minLat, maxLat, minLon, maxLon := 50.0, 51.5, 2.5, 6.4
|
||||
entry := AreaEntry{LatMin: &minLat, LatMax: &maxLat, LonMin: &minLon, LonMax: &maxLon}
|
||||
pks, err := db.GetNodePubkeysInArea(entry)
|
||||
if err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if len(pks) != 1 || pks[0] != "in" {
|
||||
t.Errorf("want [in], got %v", pks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleConfigAreas(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
"MST": {Label: "Maastricht"},
|
||||
}}
|
||||
|
||||
r := mux.NewRouter()
|
||||
srv := &Server{db: db, cfg: cfg}
|
||||
r.HandleFunc("/api/config/areas", srv.handleConfigAreas).Methods("GET")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/config/areas", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("want 200, got %d", w.Code)
|
||||
}
|
||||
var result []map[string]string
|
||||
if err := json.NewDecoder(w.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(result) != 2 {
|
||||
t.Fatalf("want 2 areas, got %d", len(result))
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, entry := range result {
|
||||
keys[entry["key"]] = true
|
||||
if entry["label"] == "" {
|
||||
t.Errorf("missing label for key %q", entry["key"])
|
||||
}
|
||||
}
|
||||
if !keys["BEL"] || !keys["MST"] {
|
||||
t.Errorf("expected BEL and MST, got %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleConfigAreasEmpty(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
cfg := &Config{}
|
||||
|
||||
r := mux.NewRouter()
|
||||
srv := &Server{db: db, cfg: cfg}
|
||||
r.HandleFunc("/api/config/areas", srv.handleConfigAreas).Methods("GET")
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/config/areas", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
var result []interface{}
|
||||
if err := json.NewDecoder(w.Body).Decode(&result); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(result) != 0 {
|
||||
t.Errorf("want empty array, got %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAreaNodes_CalledBeforeRLock(t *testing.T) {
|
||||
// Verify resolveAreaNodes doesn't deadlock when called concurrently with writes.
|
||||
// This test catches the anti-pattern where resolveAreaNodes (which does a DB
|
||||
// query) is called while holding s.mu.RLock().
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('n1', 50.85, 4.35)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
ingestAdvert(t, s, "h1", `{"public_key":"n1","name":"N1"}`)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 5; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.GetBulkHealth(10, "", "BEL")
|
||||
}()
|
||||
}
|
||||
wg.Wait() // must not deadlock
|
||||
}
|
||||
|
||||
func TestResolveAreaNodes_PerKeyTTL(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('bel-node', 50.85, 4.35)`)
|
||||
mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('nl-node', 52.4, 4.9)`)
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
"NL": {Label: "Netherlands", Polygon: [][2]float64{{51.5, 3.4}, {53.6, 3.4}, {53.6, 7.2}, {51.5, 7.2}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
// Populate both keys into cache.
|
||||
r1 := s.resolveAreaNodes("BEL")
|
||||
if !r1["bel-node"] {
|
||||
t.Fatal("bel-node should be in BEL")
|
||||
}
|
||||
r2 := s.resolveAreaNodes("NL")
|
||||
if !r2["nl-node"] {
|
||||
t.Fatal("nl-node should be in NL")
|
||||
}
|
||||
|
||||
// Delete both nodes from DB to prove cache still serves them.
|
||||
mustExecDB(t, db, `DELETE FROM nodes`)
|
||||
|
||||
// BEL cache should still be warm (not evicted by NL query).
|
||||
r3 := s.resolveAreaNodes("BEL")
|
||||
if !r3["bel-node"] {
|
||||
t.Error("BEL cache was evicted by NL query (global TTL bug)")
|
||||
}
|
||||
// NL cache should still be warm too.
|
||||
r4 := s.resolveAreaNodes("NL")
|
||||
if !r4["nl-node"] {
|
||||
t.Error("NL cache was evicted unexpectedly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBulkHealth_AreaBypassesCap(t *testing.T) {
|
||||
db := setupTestDBv2(t)
|
||||
|
||||
// Insert 510 nodes inside BEL — all at 50.85, 4.35.
|
||||
for i := 0; i < 510; i++ {
|
||||
mustExecDB(t, db, fmt.Sprintf(
|
||||
`INSERT INTO nodes (public_key, lat, lon) VALUES ('node-%d', 50.85, 4.35)`, i,
|
||||
))
|
||||
}
|
||||
|
||||
cfg := &Config{Areas: map[string]AreaEntry{
|
||||
"BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}},
|
||||
}}
|
||||
s := newTestStoreWithDB(t, db, cfg)
|
||||
|
||||
// With limit=10 but area filter active, all 510 in-area nodes must be returned.
|
||||
result := s.GetBulkHealth(10, "", "BEL")
|
||||
if len(result) != 510 {
|
||||
t.Errorf("want 510 nodes from area BEL, got %d", len(result))
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// handleBackup streams a consistent SQLite snapshot of the analyzer DB.
|
||||
//
|
||||
// Requires API-key authentication (mounted via requireAPIKey in routes.go).
|
||||
//
|
||||
// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented
|
||||
// copy of the current database into a new file. It runs at READ ISOLATION
|
||||
// against the source DB (works on our read-only connection) and never
|
||||
// blocks concurrent writers — the ingestor keeps writing to the WAL while
|
||||
// the snapshot is taken from a consistent read transaction.
|
||||
//
|
||||
// Response:
|
||||
//
|
||||
// 200 OK
|
||||
// Content-Type: application/octet-stream
|
||||
// Content-Disposition: attachment; filename="corescope-backup-<unix>.db"
|
||||
// <body: complete SQLite database file>
|
||||
//
|
||||
// The temp file is removed after the response is fully written, regardless
|
||||
// of whether the client successfully consumed the stream.
|
||||
func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) {
|
||||
if s.db == nil || s.db.conn == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "database unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now().UTC().Unix()
|
||||
clientIP := r.Header.Get("X-Forwarded-For")
|
||||
if clientIP == "" {
|
||||
clientIP = r.RemoteAddr
|
||||
}
|
||||
log.Printf("[backup] generating backup for client %s", clientIP)
|
||||
|
||||
// Stage the snapshot in the OS temp dir so we never touch the live DB
|
||||
// directory (avoids confusing operators / accidental WAL clobber).
|
||||
tmpDir, err := os.MkdirTemp("", "corescope-backup-")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "tempdir failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if rmErr := os.RemoveAll(tmpDir); rmErr != nil {
|
||||
log.Printf("[backup] cleanup error: %v", rmErr)
|
||||
}
|
||||
}()
|
||||
|
||||
snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts))
|
||||
|
||||
// SQLite parses the path literal — escape any single quotes defensively.
|
||||
// (mkdtemp output won't contain quotes, but be paranoid for future-proofing.)
|
||||
escaped := strings.ReplaceAll(snapshotPath, "'", "''")
|
||||
if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(snapshotPath)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "open snapshot failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
stat, err := f.Stat()
|
||||
if err == nil {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
// Headers already flushed; just log. Client will see truncated stream.
|
||||
log.Printf("[backup] stream error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database.
|
||||
// See https://www.sqlite.org/fileformat.html#magic_header_string
|
||||
const sqliteMagic = "SQLite format 3\x00"
|
||||
|
||||
func TestBackupRequiresAPIKey(t *testing.T) {
|
||||
_, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough")
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/backup", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("expected 401 without API key, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) {
|
||||
const apiKey = "test-secret-key-strong-enough"
|
||||
_, router := setupTestServerWithAPIKey(t, apiKey)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/backup", nil)
|
||||
req.Header.Set("X-API-Key", apiKey)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
ct := w.Header().Get("Content-Type")
|
||||
if ct != "application/octet-stream" {
|
||||
t.Errorf("expected Content-Type application/octet-stream, got %q", ct)
|
||||
}
|
||||
|
||||
cd := w.Header().Get("Content-Disposition")
|
||||
if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") {
|
||||
t.Errorf("expected Content-Disposition attachment with corescope-backup-<ts>.db filename, got %q", cd)
|
||||
}
|
||||
|
||||
body := w.Body.Bytes()
|
||||
if len(body) < len(sqliteMagic) {
|
||||
t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body))
|
||||
}
|
||||
if got := string(body[:len(sqliteMagic)]); got != sqliteMagic {
|
||||
t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got)
|
||||
}
|
||||
}
|
||||
@@ -1,411 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// createTestDB creates a temporary SQLite database with N transmissions (1 obs each).
|
||||
func createTestDB(t *testing.T, numTx int) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
createTestDBAt(t, dbPath, numTx)
|
||||
return dbPath
|
||||
}
|
||||
|
||||
// loadStore creates a PacketStore from a test DB with given maxMemoryMB.
|
||||
func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore {
|
||||
t.Helper()
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: maxMemMB}
|
||||
store := NewPacketStore(db, cfg)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func TestBoundedLoad_LimitedMemory(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// Use 1MB budget — should load far fewer than 5000 packets
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
loaded := len(store.packets)
|
||||
if loaded >= 5000 {
|
||||
t.Errorf("expected bounded load to limit packets, got %d/5000", loaded)
|
||||
}
|
||||
if loaded < 1000 {
|
||||
t.Errorf("expected at least 1000 packets (minimum), got %d", loaded)
|
||||
}
|
||||
t.Logf("Loaded %d/5000 packets with 1MB budget", loaded)
|
||||
}
|
||||
|
||||
func TestBoundedLoad_NewestFirst(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
loaded := len(store.packets)
|
||||
if loaded >= 5000 {
|
||||
t.Skip("all packets loaded, can't verify newest-first")
|
||||
}
|
||||
|
||||
// The newest packet in DB has first_seen based on minute 5000.
|
||||
// The loaded packets should be the newest ones.
|
||||
// Last packet in store (sorted ASC) should be the newest in DB.
|
||||
last := store.packets[loaded-1]
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
newestExpected := base.Add(5000 * time.Minute).Format(time.RFC3339)
|
||||
if last.FirstSeen != newestExpected {
|
||||
t.Errorf("expected last packet to be newest (%s), got %s", newestExpected, last.FirstSeen)
|
||||
}
|
||||
|
||||
// First packet should NOT be the oldest in the DB (minute 1)
|
||||
first := store.packets[0]
|
||||
oldestAll := base.Add(1 * time.Minute).Format(time.RFC3339)
|
||||
if first.FirstSeen == oldestAll {
|
||||
t.Errorf("first loaded packet should not be the absolute oldest when bounded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedLoad_OldestLoadedSet(t *testing.T) {
|
||||
dbPath := createTestDB(t, 5000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if store.oldestLoaded == "" {
|
||||
t.Fatal("oldestLoaded should be set after bounded load")
|
||||
}
|
||||
if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen {
|
||||
t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen)
|
||||
}
|
||||
t.Logf("oldestLoaded = %s", store.oldestLoaded)
|
||||
}
|
||||
|
||||
func TestBoundedLoad_UnlimitedWithZero(t *testing.T) {
|
||||
dbPath := createTestDB(t, 200)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 0)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 200 {
|
||||
t.Errorf("expected all 200 packets with maxMemoryMB=0, got %d", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedLoad_AscendingOrder(t *testing.T) {
|
||||
dbPath := createTestDB(t, 3000)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
store := loadStore(t, dbPath, 1)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
// Verify packets are in ascending first_seen order
|
||||
for i := 1; i < len(store.packets); i++ {
|
||||
if store.packets[i].FirstSeen < store.packets[i-1].FirstSeen {
|
||||
t.Fatalf("packets not in ascending order at index %d: %s < %s",
|
||||
i, store.packets[i].FirstSeen, store.packets[i-1].FirstSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadStoreWithRetention creates a PacketStore with retentionHours set.
|
||||
func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore {
|
||||
t.Helper()
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{RetentionHours: retentionHours}
|
||||
store := NewPacketStore(db, cfg)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// createTestDBWithAgedPackets inserts numRecent packets with timestamps within
|
||||
// the last hour and numOld packets with timestamps 48 hours ago.
|
||||
func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(s string) {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
t.Fatalf("setup: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)
|
||||
execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`)
|
||||
execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`)
|
||||
execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`)
|
||||
execOrFail(`CREATE TABLE schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := 1
|
||||
// Insert old packets (48 hours ago)
|
||||
for i := 0; i < numOld; i++ {
|
||||
oldT := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second)
|
||||
ts := oldT.Format(time.RFC3339)
|
||||
conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`)
|
||||
// observations.timestamp is INTEGER (unix seconds) in production schema
|
||||
// — keep the fixture consistent so the RFC3339 subquery matches.
|
||||
conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "")
|
||||
id++
|
||||
}
|
||||
// Insert recent packets (within last hour)
|
||||
for i := 0; i < numRecent; i++ {
|
||||
newT := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second)
|
||||
ts := newT.Format(time.RFC3339)
|
||||
conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`)
|
||||
conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "")
|
||||
id++
|
||||
}
|
||||
return dbPath
|
||||
}
|
||||
|
||||
func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) {
|
||||
dbPath := createTestDBWithAgedPackets(t, 50, 100)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// retention = 2 hours — should load only the 50 recent packets, not the 100 old ones
|
||||
store := loadStoreWithRetention(t, dbPath, 2)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 50 {
|
||||
t.Errorf("expected 50 recent packets, got %d (old packets should be excluded by retentionHours)", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) {
|
||||
dbPath := createTestDBWithAgedPackets(t, 50, 100)
|
||||
defer os.RemoveAll(filepath.Dir(dbPath))
|
||||
|
||||
// retention = 0 (unlimited) — should load all 150 packets
|
||||
store := loadStoreWithRetention(t, dbPath, 0)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if len(store.packets) != 150 {
|
||||
t.Errorf("expected all 150 packets with retentionHours=0, got %d", len(store.packets))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEstimateStoreTxBytesTypical(t *testing.T) {
|
||||
est := estimateStoreTxBytesTypical(10)
|
||||
if est < 1000 {
|
||||
t.Errorf("typical estimate too low: %d", est)
|
||||
}
|
||||
// Should be roughly proportional to observation count
|
||||
est1 := estimateStoreTxBytesTypical(1)
|
||||
est20 := estimateStoreTxBytesTypical(20)
|
||||
if est20 <= est1 {
|
||||
t.Errorf("estimate should grow with observations: 1obs=%d, 20obs=%d", est1, est20)
|
||||
}
|
||||
t.Logf("Typical estimate: 1obs=%d, 10obs=%d, 20obs=%d bytes", est1, est, est20)
|
||||
}
|
||||
|
||||
func BenchmarkLoad_Bounded(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench.db")
|
||||
createTestDBAt(b, dbPath, 5000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 1}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLoad_Unlimited(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench.db")
|
||||
createTestDBAt(b, dbPath, 5000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions
|
||||
// and realistic observation counts (1–5 per transmission).
|
||||
func BenchmarkLoad_30K_Bounded(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench30k.db")
|
||||
createTestDBWithObs(b, dbPath, 30000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 50}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions
|
||||
// and realistic observation counts (1–5 per transmission).
|
||||
func BenchmarkLoad_30K_Unlimited(b *testing.B) {
|
||||
dir := b.TempDir()
|
||||
dbPath := filepath.Join(dir, "bench30k.db")
|
||||
createTestDBWithObs(b, dbPath, 30000)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
db, _ := OpenDB(dbPath)
|
||||
cfg := &PacketStoreConfig{MaxMemoryMB: 0}
|
||||
store := NewPacketStore(db, cfg)
|
||||
store.Load()
|
||||
db.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// createTestDBAt is like createTestDB but writes to a specific path.
|
||||
func createTestDBAt(tb testing.TB, dbPath string, numTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(sql string) {
|
||||
if _, err := conn.Exec(sql); err != nil {
|
||||
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER,
|
||||
payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare transmissions insert: %v", err)
|
||||
}
|
||||
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare observations insert: %v", err)
|
||||
}
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
|
||||
obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts)
|
||||
}
|
||||
}
|
||||
|
||||
// createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx).
|
||||
func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
execOrFail := func(sqlStr string) {
|
||||
if _, err := conn.Exec(sqlStr); err != nil {
|
||||
tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr)
|
||||
}
|
||||
}
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`)
|
||||
execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`)
|
||||
execOrFail(`INSERT INTO schema_version (version) VALUES (1)`)
|
||||
execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`)
|
||||
|
||||
txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare transmissions: %v", err)
|
||||
}
|
||||
obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
if err != nil {
|
||||
tb.Fatalf("test DB prepare observations: %v", err)
|
||||
}
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"}
|
||||
obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"}
|
||||
obsID := 1
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%06d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i))
|
||||
nObs := (i % 5) + 1 // 1–5 observations per transmission
|
||||
for j := 0; j < nObs; j++ {
|
||||
snr := -5.0 + float64(j)*2.5
|
||||
rssi := -90.0 + float64(j)*5.0
|
||||
obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts)
|
||||
obsID++
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestBridgeScore_HandleNodesSurface verifies that /api/nodes
|
||||
// includes a `bridge_score` field on repeater rows after the bridge
|
||||
// recomputer has run. Drives the line-graph A-B-C-D through the full
|
||||
// pipeline: insert nodes, populate the neighbor graph, force a
|
||||
// recompute, hit the handler, parse the response. Issue #672 axis 2.
|
||||
func TestBridgeScore_HandleNodesSurface(t *testing.T) {
|
||||
db := setupCapabilityTestDB(t)
|
||||
defer db.conn.Close()
|
||||
// handleNodes/db.GetNodes selects a foreign_advert column not in
|
||||
// the minimal capability-test schema.
|
||||
if _, err := db.conn.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Four repeater nodes in a line.
|
||||
pks := []string{
|
||||
"aaaa000000000000000000000000000000000000000000000000000000000000",
|
||||
"bbbb000000000000000000000000000000000000000000000000000000000000",
|
||||
"cccc000000000000000000000000000000000000000000000000000000000000",
|
||||
"dddd000000000000000000000000000000000000000000000000000000000000",
|
||||
}
|
||||
recent := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
for _, pk := range pks {
|
||||
if _, err := db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`,
|
||||
pk, "node-"+pk[:4], recent, recent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
// Build neighbor graph with the line A-B-C-D. Add each edge
|
||||
// `count` times so its time-decayed Score saturates.
|
||||
g := NewNeighborGraph()
|
||||
now := time.Now()
|
||||
obs := "obs-test"
|
||||
snr := 5.0
|
||||
for i := 0; i < 10; i++ {
|
||||
g.upsertEdge(pks[0], pks[1], "aa", obs, &snr, now)
|
||||
g.upsertEdge(pks[1], pks[2], "bb", obs, &snr, now)
|
||||
g.upsertEdge(pks[2], pks[3], "cc", obs, &snr, now)
|
||||
}
|
||||
store.graph.Store(g)
|
||||
|
||||
// Direct invocation of the recomputer's compute path — bypassing
|
||||
// StartBridgeScoreRecomputer's package-level once-flag (which is
|
||||
// problematic across tests).
|
||||
recomputeBridgeScoresSafe(store)
|
||||
|
||||
snap := store.GetBridgeScoreMap()
|
||||
if len(snap) == 0 {
|
||||
t.Fatalf("expected non-empty bridge score snapshot, got empty")
|
||||
}
|
||||
// Sanity: middle nodes b/c must be positive, ends must be zero.
|
||||
if snap[pks[1]] <= 0 || snap[pks[2]] <= 0 {
|
||||
t.Errorf("middle nodes should have positive bridge: b=%v c=%v",
|
||||
snap[pks[1]], snap[pks[2]])
|
||||
}
|
||||
if snap[pks[0]] != 0 || snap[pks[3]] != 0 {
|
||||
t.Errorf("end nodes should have zero bridge: a=%v d=%v",
|
||||
snap[pks[0]], snap[pks[3]])
|
||||
}
|
||||
|
||||
// Wire a Server, call handleNodes, parse the response.
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes?limit=100", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
router.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("handleNodes status: want 200, got %d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
Nodes []map[string]interface{} `json:"nodes"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v body=%s", err, rr.Body.String())
|
||||
}
|
||||
gotBy := map[string]map[string]interface{}{}
|
||||
for _, n := range resp.Nodes {
|
||||
if pk, _ := n["public_key"].(string); pk != "" {
|
||||
gotBy[pk] = n
|
||||
}
|
||||
}
|
||||
for _, pk := range pks {
|
||||
n, ok := gotBy[pk]
|
||||
if !ok {
|
||||
t.Errorf("node %s missing from response", pk[:4])
|
||||
continue
|
||||
}
|
||||
if _, has := n["bridge_score"]; !has {
|
||||
t.Errorf("node %s: bridge_score field absent from response", pk[:4])
|
||||
}
|
||||
}
|
||||
// Middle node B must report a non-zero bridge_score; end node A
|
||||
// must report exactly zero. These two assertions together prevent
|
||||
// a "field present but always 0" regression.
|
||||
if v, _ := gotBy[pks[1]]["bridge_score"].(float64); v <= 0 {
|
||||
t.Errorf("middle node B bridge_score in API response should be > 0, got %v", v)
|
||||
}
|
||||
if v, _ := gotBy[pks[0]]["bridge_score"].(float64); v != 0 {
|
||||
t.Errorf("end node A bridge_score in API response should be 0, got %v", v)
|
||||
}
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
// Package main: bridge-axis recomputer (issue #672 axis 2 of 4).
|
||||
//
|
||||
// Steady-state background loop that recomputes the per-pubkey bridge
|
||||
// centrality score over the in-memory NeighborGraph and stores the
|
||||
// resulting map atomically. handleNodes reads via a single atomic
|
||||
// load — no lock contention with ingest or with other recomputers
|
||||
// (same pattern as #1240 / #1248).
|
||||
//
|
||||
// Interval default: 5 minutes. The graph itself rebuilds asynchronously
|
||||
// on its own schedule (path_inspect.go); a 5-minute cadence here is
|
||||
// well within the freshness budget for a structural metric (centrality
|
||||
// changes slowly — a new edge or evicted node nudges scores by
|
||||
// fractions of a percent).
|
||||
//
|
||||
// Cost (Brandes + Dijkstra): O(V · (E + V log V)). Staging-scale ~600
|
||||
// nodes / ~2 000 edges ≈ ~4.8M ops, well under 100 ms in practice. On
|
||||
// host-fleet scale (5 000 nodes / 30 000 edges) it is still seconds,
|
||||
// running in a background goroutine off the request path.
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// bridgeRecomputerDefaultInterval is how often the bridge score map is
|
||||
// rebuilt. 5 minutes mirrors analytics_recomputer (#1240) and
|
||||
// repeater_enrich_recomputer (#1262); centrality is a slow-moving
|
||||
// structural signal and does not warrant tighter cadence.
|
||||
const bridgeRecomputerDefaultInterval = 5 * time.Minute
|
||||
|
||||
// bridgeRecompStartedMu serializes start of the bridge recomputer.
|
||||
// We do not currently expose Stop publicly — the goroutine lives for
|
||||
// the lifetime of the process — but keeping the started flag local
|
||||
// (instead of on PacketStore) avoids further field churn in store.go.
|
||||
var (
|
||||
bridgeRecompStartedMu sync.Mutex
|
||||
bridgeRecompStarted bool
|
||||
)
|
||||
|
||||
// StartBridgeScoreRecomputer launches the bridge-centrality recomputer
|
||||
// (issue #672 axis 2). It performs an initial synchronous compute so
|
||||
// that the very first /api/nodes after server start hits a populated
|
||||
// snapshot instead of returning bridge_score=0 for every node, then
|
||||
// reschedules every `interval` (default 5min if <= 0).
|
||||
//
|
||||
// Idempotent: subsequent calls are no-ops and return a no-op stop
|
||||
// closure.
|
||||
func (s *PacketStore) StartBridgeScoreRecomputer(interval time.Duration) func() {
|
||||
if interval <= 0 {
|
||||
interval = bridgeRecomputerDefaultInterval
|
||||
}
|
||||
|
||||
bridgeRecompStartedMu.Lock()
|
||||
if bridgeRecompStarted {
|
||||
bridgeRecompStartedMu.Unlock()
|
||||
return func() {}
|
||||
}
|
||||
bridgeRecompStarted = true
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
bridgeRecompStartedMu.Unlock()
|
||||
|
||||
// Initial synchronous prewarm — see comment above.
|
||||
recomputeBridgeScoresSafe(s)
|
||||
|
||||
var stopOnce sync.Once
|
||||
go func() {
|
||||
defer close(done)
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
recomputeBridgeScoresSafe(s)
|
||||
case <-stop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
stopOnce.Do(func() {
|
||||
close(stop)
|
||||
})
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recomputeBridgeScoresSafe runs ComputeBridgeScores over the current
|
||||
// neighbor graph and installs the result. Panics in compute are
|
||||
// swallowed (defensive) so the goroutine never dies; the previous
|
||||
// snapshot remains valid.
|
||||
func recomputeBridgeScoresSafe(s *PacketStore) {
|
||||
defer func() { _ = recover() }()
|
||||
graph := s.graph.Load()
|
||||
if graph == nil {
|
||||
// No graph yet — install an empty map so readers get a defined
|
||||
// zero rather than a nil sentinel (handleNodes treats both as
|
||||
// 0.0, but an explicit empty snapshot avoids "is this ready
|
||||
// yet?" confusion in operator-facing tooling).
|
||||
empty := map[string]float64{}
|
||||
s.bridgeScoreMap.Store(&empty)
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
edges := bridgeEdgesFromGraph(graph, now)
|
||||
scores := ComputeBridgeScores(edges)
|
||||
s.bridgeScoreMap.Store(&scores)
|
||||
}
|
||||
|
||||
// bridgeEdgesFromGraph snapshots the NeighborGraph into a flat slice
|
||||
// of BridgeEdge tuples with weight = Score(now) * Confidence(), per
|
||||
// the convention established by #1235. Edges with unresolved B
|
||||
// endpoints (no concrete pubkey yet — only a hop prefix) are skipped:
|
||||
// they contribute no betweenness signal because the second endpoint
|
||||
// is unknown.
|
||||
func bridgeEdgesFromGraph(graph *NeighborGraph, now time.Time) []BridgeEdge {
|
||||
all := graph.AllEdges()
|
||||
out := make([]BridgeEdge, 0, len(all))
|
||||
for _, e := range all {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if e.NodeA == "" || e.NodeB == "" {
|
||||
// Unresolved (prefix-only) — no defined second endpoint.
|
||||
continue
|
||||
}
|
||||
w := e.Score(now) * e.Confidence()
|
||||
if w < bridgeMinWeightEpsilon {
|
||||
continue
|
||||
}
|
||||
out = append(out, BridgeEdge{A: e.NodeA, B: e.NodeB, Weight: w})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetBridgeScore returns the bridge centrality score for a pubkey in
|
||||
// [0, 1], or 0 if the recomputer has not run yet or the pubkey is not
|
||||
// in the graph. Lookup is case-insensitive (the score map keys are
|
||||
// lowercase, matching byPathHop convention).
|
||||
func (s *PacketStore) GetBridgeScore(pubkey string) float64 {
|
||||
if pubkey == "" {
|
||||
return 0
|
||||
}
|
||||
snap := s.bridgeScoreMap.Load()
|
||||
if snap == nil {
|
||||
return 0
|
||||
}
|
||||
m := *snap
|
||||
if v, ok := m[pubkey]; ok {
|
||||
return v
|
||||
}
|
||||
// Try lowercase form.
|
||||
lc := pubkey
|
||||
for i := 0; i < len(lc); i++ {
|
||||
if lc[i] >= 'A' && lc[i] <= 'Z' {
|
||||
b := []byte(pubkey)
|
||||
for j := i; j < len(b); j++ {
|
||||
if b[j] >= 'A' && b[j] <= 'Z' {
|
||||
b[j] += 'a' - 'A'
|
||||
}
|
||||
}
|
||||
lc = string(b)
|
||||
break
|
||||
}
|
||||
}
|
||||
if v, ok := m[lc]; ok {
|
||||
return v
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// GetBridgeScoreMap returns a defensive copy-by-reference of the
|
||||
// current bridge score snapshot. Nil-safe: returns an empty map if
|
||||
// no snapshot has been installed yet. Map is read-only by convention
|
||||
// — callers MUST NOT mutate it (the snapshot is shared across all
|
||||
// concurrent readers).
|
||||
func (s *PacketStore) GetBridgeScoreMap() map[string]float64 {
|
||||
snap := s.bridgeScoreMap.Load()
|
||||
if snap == nil {
|
||||
return map[string]float64{}
|
||||
}
|
||||
return *snap
|
||||
}
|
||||
|
||||
// resetBridgeRecomputerForTest is a test-only helper to allow the
|
||||
// integration test to re-Start the recomputer in a fresh process
|
||||
// (which would otherwise be blocked by the package-level
|
||||
// bridgeRecompStarted flag). Production code must not call this.
|
||||
func resetBridgeRecomputerForTest() {
|
||||
bridgeRecompStartedMu.Lock()
|
||||
bridgeRecompStarted = false
|
||||
bridgeRecompStartedMu.Unlock()
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
// Package main: bridge axis of repeater usefulness score (issue #672,
|
||||
// axis 2 of 4). The "Bridge" signal is the betweenness centrality of a
|
||||
// node in the (undirected, weighted) neighbor graph: a high value means
|
||||
// the node lies on many shortest paths between other pairs and is hence
|
||||
// structurally important — removing it would force traffic around or
|
||||
// fragment the mesh.
|
||||
//
|
||||
// Algorithm: Brandes' algorithm (1) with Dijkstra for weighted
|
||||
// shortest paths. Complexity O(V · (E + V log V)). For the staging
|
||||
// graph (~600 nodes, ~2 000 edges) this is ~4.8M ops — trivial,
|
||||
// completes in milliseconds. We accumulate raw betweenness across all
|
||||
// sources, halve (an undirected pair is counted from each endpoint
|
||||
// once), then normalize by the max observed value so the per-node
|
||||
// score is in [0, 1].
|
||||
//
|
||||
// Edge weight follows the convention established by #1235: the
|
||||
// affinity score (count + recency decay) is multiplied by the
|
||||
// observer-diversity confidence — stronger, more corroborated
|
||||
// neighborships are preferred when there is a choice of paths.
|
||||
// Geo-rejected edges are already excluded from the input graph at
|
||||
// build time (#1230) so we don't have to re-filter here.
|
||||
//
|
||||
// For Dijkstra we need a DISTANCE (lower = better) not an affinity
|
||||
// (higher = better), so we convert: cost = 1 / max(epsilon, weight).
|
||||
// epsilon avoids divide-by-zero on a degenerate zero-weight edge.
|
||||
//
|
||||
// (1) Brandes, "A Faster Algorithm for Betweenness Centrality" (2001).
|
||||
package main
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BridgeEdge is the algorithm-facing edge tuple consumed by
|
||||
// ComputeBridgeScores. Endpoints A and B are pubkeys (case preserved
|
||||
// by caller; we lowercase internally for stable keying). Weight is
|
||||
// the affinity (higher = stronger connection). Edges with zero or
|
||||
// negative weight are skipped — they would break Dijkstra's
|
||||
// relaxation invariant.
|
||||
type BridgeEdge struct {
|
||||
A, B string
|
||||
Weight float64
|
||||
}
|
||||
|
||||
// bridgeMinWeightEpsilon is the floor applied to weights before we
|
||||
// invert them into Dijkstra distances. 1e-9 is small enough that any
|
||||
// real weight (Score in [0,1] times Confidence in [0,1]) dominates,
|
||||
// but large enough to avoid Inf when weight is exactly zero.
|
||||
const bridgeMinWeightEpsilon = 1e-9
|
||||
|
||||
// ComputeBridgeScores returns a map pubkey → bridge score in [0, 1]
|
||||
// computed via Brandes' weighted betweenness centrality on the
|
||||
// undirected graph defined by `edges`. Returned map is keyed by the
|
||||
// lowercase pubkey form (matching the byPathHop / persisted-edge
|
||||
// convention). Nodes appearing in the graph but with zero betweenness
|
||||
// are still present in the map with value 0.0.
|
||||
//
|
||||
// Self-loops (A == B) and edges with weight < epsilon are silently
|
||||
// skipped. Duplicate edges between the same pair keep the cheapest
|
||||
// (= the highest-weight) version — consistent with shortest-path
|
||||
// semantics.
|
||||
//
|
||||
// Pure (no global state, no locks); safe to call concurrently.
|
||||
// Cost: O(V · (E + V log V)).
|
||||
func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 {
|
||||
// 1. Build adjacency list with distance = 1/weight.
|
||||
adj := make(map[string]map[string]float64)
|
||||
addOrMerge := func(a, b string, dist float64) {
|
||||
m, ok := adj[a]
|
||||
if !ok {
|
||||
m = make(map[string]float64)
|
||||
adj[a] = m
|
||||
}
|
||||
if existing, has := m[b]; !has || dist < existing {
|
||||
m[b] = dist
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
a := strings.ToLower(strings.TrimSpace(e.A))
|
||||
b := strings.ToLower(strings.TrimSpace(e.B))
|
||||
if a == "" || b == "" || a == b {
|
||||
continue
|
||||
}
|
||||
w := e.Weight
|
||||
if w < bridgeMinWeightEpsilon {
|
||||
continue
|
||||
}
|
||||
dist := 1.0 / w
|
||||
addOrMerge(a, b, dist)
|
||||
addOrMerge(b, a, dist)
|
||||
}
|
||||
if len(adj) == 0 {
|
||||
return map[string]float64{}
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(adj))
|
||||
for n := range adj {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
|
||||
bc := make(map[string]float64, len(nodes))
|
||||
for _, n := range nodes {
|
||||
bc[n] = 0
|
||||
}
|
||||
|
||||
// 2. Brandes outer loop: one Dijkstra-based single-source shortest
|
||||
// path computation per source vertex.
|
||||
for _, s := range nodes {
|
||||
stack := make([]string, 0, len(nodes))
|
||||
pred := make(map[string][]string, len(nodes))
|
||||
sigma := make(map[string]float64, len(nodes))
|
||||
dist := make(map[string]float64, len(nodes))
|
||||
for _, n := range nodes {
|
||||
sigma[n] = 0
|
||||
dist[n] = math.Inf(1)
|
||||
}
|
||||
sigma[s] = 1
|
||||
dist[s] = 0
|
||||
|
||||
pq := &bridgePQ{}
|
||||
heap.Init(pq)
|
||||
heap.Push(pq, bridgePQItem{node: s, dist: 0})
|
||||
|
||||
visited := make(map[string]bool, len(nodes))
|
||||
for pq.Len() > 0 {
|
||||
top := heap.Pop(pq).(bridgePQItem)
|
||||
v := top.node
|
||||
if visited[v] {
|
||||
continue
|
||||
}
|
||||
visited[v] = true
|
||||
stack = append(stack, v)
|
||||
|
||||
for w, edgeDist := range adj[v] {
|
||||
alt := dist[v] + edgeDist
|
||||
if alt < dist[w]-1e-12 {
|
||||
dist[w] = alt
|
||||
sigma[w] = sigma[v]
|
||||
pred[w] = append(pred[w][:0], v)
|
||||
heap.Push(pq, bridgePQItem{node: w, dist: alt})
|
||||
} else if math.Abs(alt-dist[w]) <= 1e-12 {
|
||||
sigma[w] += sigma[v]
|
||||
pred[w] = append(pred[w], v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Back-propagation: walk the stack in reverse order.
|
||||
delta := make(map[string]float64, len(nodes))
|
||||
for i := len(stack) - 1; i >= 0; i-- {
|
||||
w := stack[i]
|
||||
for _, v := range pred[w] {
|
||||
if sigma[w] == 0 {
|
||||
continue
|
||||
}
|
||||
delta[v] += (sigma[v] / sigma[w]) * (1.0 + delta[w])
|
||||
}
|
||||
if w != s {
|
||||
bc[w] += delta[w]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Undirected graphs double-count each (s,t) pair, so halve.
|
||||
for k := range bc {
|
||||
bc[k] /= 2.0
|
||||
}
|
||||
|
||||
// 5. Normalize by max so scores live in [0, 1]. If max is 0
|
||||
// (clique or single edge) we leave everything at zero.
|
||||
maxBC := 0.0
|
||||
for _, v := range bc {
|
||||
if v > maxBC {
|
||||
maxBC = v
|
||||
}
|
||||
}
|
||||
if maxBC > 0 {
|
||||
for k, v := range bc {
|
||||
bc[k] = v / maxBC
|
||||
}
|
||||
}
|
||||
return bc
|
||||
}
|
||||
|
||||
// ─── min-heap for Dijkstra ─────────────────────────────────────────────────────
|
||||
|
||||
type bridgePQItem struct {
|
||||
node string
|
||||
dist float64
|
||||
}
|
||||
|
||||
type bridgePQ []bridgePQItem
|
||||
|
||||
func (h bridgePQ) Len() int { return len(h) }
|
||||
func (h bridgePQ) Less(i, j int) bool { return h[i].dist < h[j].dist }
|
||||
func (h bridgePQ) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *bridgePQ) Push(x interface{}) { *h = append(*h, x.(bridgePQItem)) }
|
||||
func (h *bridgePQ) Pop() interface{} {
|
||||
old := *h
|
||||
n := len(old)
|
||||
it := old[n-1]
|
||||
*h = old[:n-1]
|
||||
return it
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestComputeBridgeScores_LineGraph asserts the canonical property of
|
||||
// betweenness centrality on a 4-node line A-B-C-D: the two middle
|
||||
// nodes B and C have non-zero centrality (every path between an end
|
||||
// and a far end traverses them) while the two leaves A and D bridge
|
||||
// no pairs and score zero. This is the RED test for issue #672 bridge
|
||||
// axis — it fails on master where ComputeBridgeScores is a stub.
|
||||
func TestComputeBridgeScores_LineGraph(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "a", B: "b", Weight: 1.0},
|
||||
{A: "b", B: "c", Weight: 1.0},
|
||||
{A: "c", B: "d", Weight: 1.0},
|
||||
}
|
||||
scores := ComputeBridgeScores(edges)
|
||||
|
||||
for _, leaf := range []string{"a", "d"} {
|
||||
if v, ok := scores[leaf]; !ok || v != 0 {
|
||||
t.Errorf("leaf %q: want score 0 (present), got %v ok=%v", leaf, v, ok)
|
||||
}
|
||||
}
|
||||
for _, mid := range []string{"b", "c"} {
|
||||
v, ok := scores[mid]
|
||||
if !ok {
|
||||
t.Errorf("middle %q: missing from result map", mid)
|
||||
continue
|
||||
}
|
||||
if v <= 0 {
|
||||
t.Errorf("middle %q: want non-zero centrality, got %v", mid, v)
|
||||
}
|
||||
}
|
||||
// Normalization: max must equal 1.0 exactly when any node has
|
||||
// non-zero centrality.
|
||||
maxScore := 0.0
|
||||
for _, v := range scores {
|
||||
if v > maxScore {
|
||||
maxScore = v
|
||||
}
|
||||
}
|
||||
if math.Abs(maxScore-1.0) > 1e-9 {
|
||||
t.Errorf("max normalized score: want 1.0, got %v", maxScore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeBridgeScores_TriangleNoBridge: in a fully connected
|
||||
// triangle every node has at least one alternate path so betweenness
|
||||
// is zero everywhere. The map should still contain all three nodes
|
||||
// (so callers can distinguish "in graph but unimportant" from
|
||||
// "not in graph") with explicit zero values.
|
||||
func TestComputeBridgeScores_TriangleNoBridge(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "x", B: "y", Weight: 1.0},
|
||||
{A: "y", B: "z", Weight: 1.0},
|
||||
{A: "z", B: "x", Weight: 1.0},
|
||||
}
|
||||
scores := ComputeBridgeScores(edges)
|
||||
for _, n := range []string{"x", "y", "z"} {
|
||||
if v, ok := scores[n]; !ok || v != 0 {
|
||||
t.Errorf("triangle node %q: want 0 present, got %v ok=%v", n, v, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeBridgeScores_Empty: an empty edge list yields an empty
|
||||
// (non-nil) map. Defensive check so the recomputer can swap in an
|
||||
// empty result without crashing the lookup path.
|
||||
func TestComputeBridgeScores_Empty(t *testing.T) {
|
||||
scores := ComputeBridgeScores(nil)
|
||||
if scores == nil {
|
||||
t.Fatal("want non-nil empty map, got nil")
|
||||
}
|
||||
if len(scores) != 0 {
|
||||
t.Errorf("want empty map, got %d entries", len(scores))
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeBridgeScores_WeightSensitive verifies the algorithm uses
|
||||
// edge weights as affinity (higher = preferred). In a graph A-B-D and
|
||||
// A-C-D where the B-route has weight 1.0 and the C-route has weight
|
||||
// 0.1, shortest path (max-affinity = min 1/w) goes through B, so B
|
||||
// has positive centrality and C does not. This is the "mutation
|
||||
// test" — flip the cost formula (e.g., remove the 1/w inversion) and
|
||||
// this test inverts.
|
||||
func TestComputeBridgeScores_WeightSensitive(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "a", B: "b", Weight: 1.0},
|
||||
{A: "b", B: "d", Weight: 1.0},
|
||||
{A: "a", B: "c", Weight: 0.1},
|
||||
{A: "c", B: "d", Weight: 0.1},
|
||||
}
|
||||
scores := ComputeBridgeScores(edges)
|
||||
if scores["b"] <= scores["c"] {
|
||||
t.Errorf("stronger-weight intermediary b should outrank c: b=%v c=%v",
|
||||
scores["b"], scores["c"])
|
||||
}
|
||||
}
|
||||
@@ -9,15 +9,14 @@ import (
|
||||
func newTestStore(t *testing.T) *PacketStore {
|
||||
t.Helper()
|
||||
return &PacketStore{
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
collisionCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
rfCacheTTL: 15 * time.Second,
|
||||
invCooldown: 10 * time.Second,
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
rfCacheTTL: 15 * time.Second,
|
||||
invCooldown: 10 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +29,6 @@ func populateAllCaches(s *PacketStore) {
|
||||
s.rfCache["global"] = dummy
|
||||
s.topoCache["global"] = dummy
|
||||
s.hashCache["global"] = dummy
|
||||
s.collisionCache["global"] = dummy
|
||||
s.chanCache["global"] = dummy
|
||||
s.distCache["global"] = dummy
|
||||
s.subpathCache["global"] = dummy
|
||||
@@ -41,13 +39,12 @@ func cachePopulated(s *PacketStore) map[string]bool {
|
||||
s.cacheMu.Lock()
|
||||
defer s.cacheMu.Unlock()
|
||||
return map[string]bool{
|
||||
"rf": len(s.rfCache) > 0,
|
||||
"topo": len(s.topoCache) > 0,
|
||||
"hash": len(s.hashCache) > 0,
|
||||
"collision": len(s.collisionCache) > 0,
|
||||
"chan": len(s.chanCache) > 0,
|
||||
"dist": len(s.distCache) > 0,
|
||||
"subpath": len(s.subpathCache) > 0,
|
||||
"rf": len(s.rfCache) > 0,
|
||||
"topo": len(s.topoCache) > 0,
|
||||
"hash": len(s.hashCache) > 0,
|
||||
"chan": len(s.chanCache) > 0,
|
||||
"dist": len(s.distCache) > 0,
|
||||
"subpath": len(s.subpathCache) > 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,8 +90,7 @@ func TestInvalidateCachesFor_NewTransmissionsOnly(t *testing.T) {
|
||||
if pop["hash"] {
|
||||
t.Error("hash cache should be cleared on new transmissions")
|
||||
}
|
||||
// collisionCache should NOT be cleared by transmissions alone (only by hasNewNodes)
|
||||
for _, name := range []string{"rf", "topo", "collision", "chan", "dist", "subpath"} {
|
||||
for _, name := range []string{"rf", "topo", "chan", "dist", "subpath"} {
|
||||
if !pop[name] {
|
||||
t.Errorf("%s cache should NOT be cleared on transmission-only ingest", name)
|
||||
}
|
||||
@@ -335,180 +331,3 @@ func BenchmarkCacheHitDuringIngestion(b *testing.B) {
|
||||
}
|
||||
b.ReportMetric(float64(hits)/float64(hits+misses)*100, "hit%")
|
||||
}
|
||||
|
||||
// TestInvCooldownFromConfig verifies that invalidationDebounce from config
|
||||
// is wired to invCooldown on PacketStore.
|
||||
func TestInvCooldownFromConfig(t *testing.T) {
|
||||
// Default without config
|
||||
ps := NewPacketStore(nil, nil)
|
||||
if ps.invCooldown != 300*time.Second {
|
||||
t.Errorf("default invCooldown = %v, want 300s", ps.invCooldown)
|
||||
}
|
||||
|
||||
// With config override
|
||||
ct := map[string]interface{}{"invalidationDebounce": float64(60)}
|
||||
ps2 := NewPacketStore(nil, nil, ct)
|
||||
if ps2.invCooldown != 60*time.Second {
|
||||
t.Errorf("configured invCooldown = %v, want 60s", ps2.invCooldown)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollisionCacheNotClearedByTransmissions verifies that collisionCache
|
||||
// is only cleared by hasNewNodes, not hasNewTransmissions (fixes #720).
|
||||
func TestCollisionCacheNotClearedByTransmissions(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
populateAllCaches(s)
|
||||
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewTransmissions: true})
|
||||
|
||||
pop := cachePopulated(s)
|
||||
if !pop["collision"] {
|
||||
t.Error("collisionCache should NOT be cleared by hasNewTransmissions alone")
|
||||
}
|
||||
if pop["hash"] {
|
||||
t.Error("hashCache should be cleared by hasNewTransmissions")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollisionCacheClearedByNewNodes verifies that collisionCache IS cleared
|
||||
// when genuinely new nodes are discovered.
|
||||
func TestCollisionCacheClearedByNewNodes(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
populateAllCaches(s)
|
||||
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewNodes: true})
|
||||
|
||||
pop := cachePopulated(s)
|
||||
if pop["collision"] {
|
||||
t.Error("collisionCache should be cleared by hasNewNodes")
|
||||
}
|
||||
// Other caches should survive
|
||||
for _, name := range []string{"rf", "topo", "hash", "chan", "dist", "subpath"} {
|
||||
if !pop[name] {
|
||||
t.Errorf("%s cache should NOT be cleared on new-nodes-only ingest", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCacheSurvivesMultipleIngestCyclesWithinCooldown verifies that caches
|
||||
// survive repeated ingest cycles during the cooldown period.
|
||||
func TestCacheSurvivesMultipleIngestCyclesWithinCooldown(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.invCooldown = 200 * time.Millisecond
|
||||
|
||||
// First invalidation goes through (starts cooldown)
|
||||
populateAllCaches(s)
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
|
||||
pop := cachePopulated(s)
|
||||
if pop["rf"] {
|
||||
t.Error("rf should be cleared on first invalidation")
|
||||
}
|
||||
|
||||
// Repopulate and simulate 5 rapid ingest cycles
|
||||
populateAllCaches(s)
|
||||
for i := 0; i < 5; i++ {
|
||||
s.invalidateCachesFor(cacheInvalidation{
|
||||
hasNewObservations: true,
|
||||
hasNewTransmissions: true,
|
||||
hasNewPaths: true,
|
||||
})
|
||||
}
|
||||
|
||||
// All caches should survive during cooldown
|
||||
pop = cachePopulated(s)
|
||||
for name, has := range pop {
|
||||
if !has {
|
||||
t.Errorf("%s cache should survive during cooldown period (ingest cycle %d)", name, 5)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewNodesAccumulatedDuringCooldown verifies that hasNewNodes flags
|
||||
// accumulated during cooldown are applied when cooldown expires.
|
||||
func TestNewNodesAccumulatedDuringCooldown(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
s.invCooldown = 100 * time.Millisecond
|
||||
|
||||
// First call starts cooldown
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
|
||||
|
||||
// During cooldown, accumulate hasNewNodes
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewNodes: true})
|
||||
|
||||
// Verify accumulated
|
||||
s.cacheMu.Lock()
|
||||
if s.pendingInv == nil || !s.pendingInv.hasNewNodes {
|
||||
t.Error("hasNewNodes should be accumulated in pendingInv")
|
||||
}
|
||||
s.cacheMu.Unlock()
|
||||
|
||||
// Wait for cooldown
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
|
||||
// Trigger flush
|
||||
populateAllCaches(s)
|
||||
s.invalidateCachesFor(cacheInvalidation{})
|
||||
|
||||
pop := cachePopulated(s)
|
||||
if pop["collision"] {
|
||||
t.Error("collisionCache should be cleared after pending hasNewNodes is flushed")
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkAnalyticsLatencyCacheHitVsMiss benchmarks cache hit vs miss
|
||||
// for analytics endpoints to demonstrate the performance impact.
|
||||
func BenchmarkAnalyticsLatencyCacheHitVsMiss(b *testing.B) {
|
||||
s := &PacketStore{
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
collisionCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
rfCacheTTL: 1800 * time.Second,
|
||||
invCooldown: 300 * time.Second,
|
||||
}
|
||||
|
||||
// Pre-populate cache
|
||||
s.cacheMu.Lock()
|
||||
s.rfCache["global"] = &cachedResult{
|
||||
data: map[string]interface{}{"bins": make([]int, 100)},
|
||||
expiresAt: time.Now().Add(time.Hour),
|
||||
}
|
||||
s.cacheMu.Unlock()
|
||||
|
||||
// Trigger initial invalidation to start cooldown
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
|
||||
|
||||
var hits, misses int64
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Re-populate (simulates query filling cache)
|
||||
s.cacheMu.Lock()
|
||||
if len(s.rfCache) == 0 {
|
||||
s.rfCache["global"] = &cachedResult{
|
||||
data: map[string]interface{}{"bins": make([]int, 100)},
|
||||
expiresAt: time.Now().Add(time.Hour),
|
||||
}
|
||||
}
|
||||
s.cacheMu.Unlock()
|
||||
|
||||
// Simulate ingest (rate-limited)
|
||||
s.invalidateCachesFor(cacheInvalidation{hasNewObservations: true})
|
||||
|
||||
// Check hit
|
||||
s.cacheMu.Lock()
|
||||
if len(s.rfCache) > 0 {
|
||||
hits++
|
||||
} else {
|
||||
misses++
|
||||
}
|
||||
s.cacheMu.Unlock()
|
||||
}
|
||||
|
||||
hitRate := float64(hits) / float64(hits+misses) * 100
|
||||
b.ReportMetric(hitRate, "hit%")
|
||||
if hitRate < 50 {
|
||||
b.Errorf("hit rate %.1f%% is below 50%% target", hitRate)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ = time.Second // suppress unused import
|
||||
|
||||
// Helper to create a minimal PacketStore with GRP_TXT packets for channel analytics testing.
|
||||
func newChannelTestStore(packets []*StoreTx) *PacketStore {
|
||||
ps := &PacketStore{
|
||||
packets: packets,
|
||||
byHash: make(map[string]*StoreTx),
|
||||
byTxID: make(map[int]*StoreTx),
|
||||
byObsID: make(map[int]*StoreObs),
|
||||
byObserver: make(map[string][]*StoreObs),
|
||||
byNode: make(map[string][]*StoreTx),
|
||||
byPathHop: make(map[string][]*StoreTx),
|
||||
nodeHashes: make(map[string]map[string]bool),
|
||||
byPayloadType: make(map[int][]*StoreTx),
|
||||
rfCache: make(map[string]*cachedResult),
|
||||
topoCache: make(map[string]*cachedResult),
|
||||
hashCache: make(map[string]*cachedResult),
|
||||
collisionCache: make(map[string]*cachedResult),
|
||||
chanCache: make(map[string]*cachedResult),
|
||||
distCache: make(map[string]*cachedResult),
|
||||
subpathCache: make(map[string]*cachedResult),
|
||||
spIndex: make(map[string]int),
|
||||
spTxIndex: make(map[string][]*StoreTx),
|
||||
advertPubkeys: make(map[string]int),
|
||||
lastSeenTouched: make(map[string]time.Time),
|
||||
clockSkew: NewClockSkewEngine(),
|
||||
}
|
||||
ps.byPayloadType[5] = packets
|
||||
return ps
|
||||
}
|
||||
|
||||
func makeGrpTx(channelHash int, channel, text, sender string) *StoreTx {
|
||||
decoded := map[string]interface{}{
|
||||
"type": "CHAN",
|
||||
"channelHash": float64(channelHash),
|
||||
"channel": channel,
|
||||
"text": text,
|
||||
"sender": sender,
|
||||
}
|
||||
b, _ := json.Marshal(decoded)
|
||||
pt := 5
|
||||
return &StoreTx{
|
||||
ID: 1,
|
||||
DecodedJSON: string(b),
|
||||
FirstSeen: "2026-05-01T12:00:00Z",
|
||||
PayloadType: &pt,
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_MergesEncryptedAndDecrypted verifies that packets
|
||||
// with the same hash byte but different decryption status merge into ONE bucket.
|
||||
func TestComputeAnalyticsChannels_MergesEncryptedAndDecrypted(t *testing.T) {
|
||||
// Hash 129 is the real hash for #wardriving: SHA256(SHA256("#wardriving")[:16])[0] = 129
|
||||
// Some packets are decrypted (have channel name), some are not (encrypted)
|
||||
packets := []*StoreTx{
|
||||
makeGrpTx(129, "#wardriving", "hello", "alice"),
|
||||
makeGrpTx(129, "#wardriving", "world", "bob"),
|
||||
makeGrpTx(129, "", "", ""), // encrypted — no channel name
|
||||
makeGrpTx(129, "", "", ""), // encrypted
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", "", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected 1 channel bucket, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
ch := channels[0]
|
||||
if ch["name"] != "#wardriving" {
|
||||
t.Errorf("expected name '#wardriving', got %q", ch["name"])
|
||||
}
|
||||
if ch["messages"] != 4 {
|
||||
t.Errorf("expected 4 messages, got %v", ch["messages"])
|
||||
}
|
||||
if ch["encrypted"] != false {
|
||||
t.Errorf("expected encrypted=false (some packets decrypted), got %v", ch["encrypted"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_RejectsRainbowTableMismatch verifies that a packet
|
||||
// with channelHash=72 but channel="#wardriving" (mismatch) does NOT create a
|
||||
// "#wardriving" bucket — it falls into "ch72" instead.
|
||||
func TestComputeAnalyticsChannels_RejectsRainbowTableMismatch(t *testing.T) {
|
||||
// Hash 72 is NOT the correct hash for #wardriving (which is 129).
|
||||
// This simulates a rainbow-table collision/mismatch.
|
||||
packets := []*StoreTx{
|
||||
makeGrpTx(72, "#wardriving", "ghost", "eve"), // mismatch: hash 72 != wardriving's real hash
|
||||
makeGrpTx(129, "#wardriving", "real", "alice"), // correct match
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", "", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 2 {
|
||||
t.Fatalf("expected 2 channel buckets, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
|
||||
// Find the buckets
|
||||
var ch72, ch129 map[string]interface{}
|
||||
for _, ch := range channels {
|
||||
if ch["hash"] == "72" {
|
||||
ch72 = ch
|
||||
} else if ch["hash"] == "129" {
|
||||
ch129 = ch
|
||||
}
|
||||
}
|
||||
|
||||
if ch72 == nil {
|
||||
t.Fatal("expected a bucket for hash 72")
|
||||
}
|
||||
if ch129 == nil {
|
||||
t.Fatal("expected a bucket for hash 129")
|
||||
}
|
||||
|
||||
// ch72 should NOT be named "#wardriving" — it should be the placeholder
|
||||
if ch72["name"] == "#wardriving" {
|
||||
t.Errorf("hash 72 bucket should NOT be named '#wardriving' (rainbow-table mismatch rejected)")
|
||||
}
|
||||
if ch72["name"] != "ch72" {
|
||||
t.Errorf("expected hash 72 bucket named 'ch72', got %q", ch72["name"])
|
||||
}
|
||||
|
||||
// ch129 should be named "#wardriving"
|
||||
if ch129["name"] != "#wardriving" {
|
||||
t.Errorf("expected hash 129 bucket named '#wardriving', got %q", ch129["name"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelNameMatchesHash verifies the hash validation function.
|
||||
func TestChannelNameMatchesHash(t *testing.T) {
|
||||
// #wardriving hashes to 129
|
||||
if !channelNameMatchesHash("#wardriving", "129") {
|
||||
t.Error("expected #wardriving to match hash 129")
|
||||
}
|
||||
if channelNameMatchesHash("#wardriving", "72") {
|
||||
t.Error("expected #wardriving to NOT match hash 72")
|
||||
}
|
||||
// Without leading # should also work
|
||||
if !channelNameMatchesHash("wardriving", "129") {
|
||||
t.Error("expected wardriving (without #) to match hash 129")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsPlaceholderName verifies placeholder detection.
|
||||
func TestIsPlaceholderName(t *testing.T) {
|
||||
if !isPlaceholderName("ch129") {
|
||||
t.Error("ch129 should be placeholder")
|
||||
}
|
||||
if !isPlaceholderName("ch0") {
|
||||
t.Error("ch0 should be placeholder")
|
||||
}
|
||||
if isPlaceholderName("#wardriving") {
|
||||
t.Error("#wardriving should NOT be placeholder")
|
||||
}
|
||||
if isPlaceholderName("Public") {
|
||||
t.Error("Public should NOT be placeholder")
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestPacketsChannelFilter verifies /api/packets?channel=... actually filters
|
||||
// (regression test for #812).
|
||||
func TestPacketsChannelFilter(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
get := func(url string) map[string]interface{} {
|
||||
req := httptest.NewRequest("GET", url, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s: expected 200, got %d", url, w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode %s: %v", url, err)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
all := get("/api/packets?limit=50")
|
||||
allTotal := int(all["total"].(float64))
|
||||
if allTotal < 2 {
|
||||
t.Fatalf("expected baseline >= 2 packets, got %d", allTotal)
|
||||
}
|
||||
|
||||
test := get("/api/packets?limit=50&channel=%23test")
|
||||
testTotal := int(test["total"].(float64))
|
||||
if testTotal == 0 {
|
||||
t.Fatalf("channel=#test: expected >= 1 match, got 0 (filter ignored?)")
|
||||
}
|
||||
if testTotal >= allTotal {
|
||||
t.Fatalf("channel=#test: expected fewer packets than baseline (%d), got %d", allTotal, testTotal)
|
||||
}
|
||||
|
||||
// Every returned packet must be a CHAN/GRP_TXT (payload_type=5) on #test.
|
||||
pkts, _ := test["packets"].([]interface{})
|
||||
for _, p := range pkts {
|
||||
m := p.(map[string]interface{})
|
||||
if pt, _ := m["payload_type"].(float64); int(pt) != 5 {
|
||||
t.Errorf("channel=#test: returned non-GRP_TXT packet (payload_type=%v)", m["payload_type"])
|
||||
}
|
||||
}
|
||||
|
||||
none := get("/api/packets?limit=50&channel=nonexistentchannel")
|
||||
if int(none["total"].(float64)) != 0 {
|
||||
t.Fatalf("channel=nonexistentchannel: expected total=0, got %v", none["total"])
|
||||
}
|
||||
}
|
||||
@@ -1,354 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression tests for issue #1366: Channel view shows stale timestamps
|
||||
// because GetChannelMessages emits tx.FirstSeen (first-observation time)
|
||||
// when the operator-visible expectation is the latest observation time
|
||||
// (tx.LatestSeen). For repeated heartbeat-style messages whose tx.Hash is
|
||||
// stable, FirstSeen stays pinned to the very first observation while the
|
||||
// real-world transmission keeps repeating, producing a multi-hour gap
|
||||
// between the channel view and the operator's live MeshCore client.
|
||||
//
|
||||
// Server-side UTC clocks are trusted; client-reported sender_timestamp
|
||||
// is NOT (firmware lacks reliable wall-clock on many builds). Therefore
|
||||
// the fix uses tx.LatestSeen (== max observation timestamp), NOT
|
||||
// sender_timestamp. sender_timestamp remains exposed in the response
|
||||
// for debug surfaces but MUST NOT be the rendered field.
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestChannelMessages_TimestampUsesLatestSeen: a CHAN tx with multiple
|
||||
// observations spanning hours must render with the LATEST observation
|
||||
// timestamp, not the first-seen ingest time.
|
||||
func TestChannelMessages_TimestampUsesLatestSeen(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
firstSeen := now.Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
firstSeenEpoch := now.Add(-7 * time.Hour).Unix()
|
||||
laterEpoch := now.Add(-5 * time.Minute).Unix()
|
||||
_ = laterEpoch
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsA', 'ObsA', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen)
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsB', 'ObsB', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen)
|
||||
|
||||
// One transmission with two observations: T0 (7h ago) and T1 (5m ago).
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AA01', 'hash_repeated_msg', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#test","text":"Heartbeat: ping","sender":"Heartbeat","sender_timestamp":` +
|
||||
strconv.FormatInt(firstSeenEpoch, 10) + `}',
|
||||
'#test')`, firstSeen)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 2, 11.0, -88, '["bb"]', ?)`, laterEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
msgs, total := store.GetChannelMessages("#test", 10, 0)
|
||||
if total != 1 {
|
||||
t.Fatalf("want 1 msg, got %d (msgs=%+v)", total, msgs)
|
||||
}
|
||||
got, _ := msgs[0]["timestamp"].(string)
|
||||
gotParsed, err := time.Parse(time.RFC3339, got)
|
||||
if err != nil {
|
||||
// Try the milli-second precision form that SQLite strftime emits.
|
||||
gotParsed, err = time.Parse("2006-01-02T15:04:05.000Z", got)
|
||||
if err != nil {
|
||||
gotParsed, err = time.Parse("2006-01-02T15:04:05.000Z07:00", got)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("timestamp not parseable: %q (%v)", got, err)
|
||||
}
|
||||
// LatestSeen should equal the laterEpoch observation (±1s).
|
||||
if delta := gotParsed.Unix() - laterEpoch; delta < -1 || delta > 1 {
|
||||
t.Errorf("timestamp: want ~%s (LatestSeen, observation at T-5m), got %q (Δ=%ds — likely FirstSeen, issue #1366)",
|
||||
time.Unix(laterEpoch, 0).UTC().Format(time.RFC3339), got, delta)
|
||||
}
|
||||
|
||||
// first_seen MUST also be exposed separately so the UI/debug can see
|
||||
// when the analyzer first heard the packet (older than `timestamp`).
|
||||
fs, _ := msgs[0]["first_seen"].(string)
|
||||
if fs == "" {
|
||||
t.Errorf("first_seen field must be exposed alongside timestamp; got empty")
|
||||
}
|
||||
if fs == got {
|
||||
t.Errorf("first_seen should differ from latest-seen timestamp (both = %q)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelMessages_TimestampNotSenderTimestamp: a CHAN tx whose
|
||||
// decoded sender_timestamp is wildly off (e.g. client with bad RTC)
|
||||
// must NOT cause the rendered timestamp to drift. Rendered timestamp
|
||||
// must remain server UTC (LatestSeen/FirstSeen), regardless of what
|
||||
// the client claimed.
|
||||
func TestChannelMessages_TimestampNotSenderTimestamp(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
firstSeen := now.Add(-10 * time.Minute).Format(time.RFC3339)
|
||||
firstSeenEpoch := now.Add(-10 * time.Minute).Unix()
|
||||
|
||||
// Client claims it sent the message in year 2000 (bad RTC).
|
||||
badSenderTs := int64(946684800) // 2000-01-01 UTC
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsX', 'ObsX', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, firstSeen)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BB01', 'hash_bad_clock', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#bad","text":"Alice: ping","sender":"Alice","sender_timestamp":` +
|
||||
strconv.FormatInt(badSenderTs, 10) + `}',
|
||||
'#bad')`, firstSeen)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
msgs, total := store.GetChannelMessages("#bad", 10, 0)
|
||||
if total != 1 {
|
||||
t.Fatalf("want 1 msg, got %d", total)
|
||||
}
|
||||
got, _ := msgs[0]["timestamp"].(string)
|
||||
// MUST be the server-side observation time, parseable as RFC3339, and
|
||||
// within ~1h of now — NOT the year-2000 client value.
|
||||
parsed, err := time.Parse(time.RFC3339, got)
|
||||
if err != nil {
|
||||
t.Fatalf("timestamp not RFC3339: %q (%v)", got, err)
|
||||
}
|
||||
if parsed.Year() < now.Year() {
|
||||
t.Errorf("rendered timestamp %q took on the client's bad sender_timestamp (year %d) instead of server UTC",
|
||||
got, parsed.Year())
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelMessages_TimestampIsUTCZ: rendered timestamp MUST end with
|
||||
// 'Z' (or +00:00) so the browser does NOT interpret it as a local-zone
|
||||
// string and shift by the operator's TZ offset.
|
||||
func TestChannelMessages_TimestampIsUTCZ(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
fs := now.Add(-30 * time.Minute).Format(time.RFC3339)
|
||||
ep := now.Add(-30 * time.Minute).Unix()
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsZ', 'ObsZ', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, fs)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('ZZ01', 'hash_zone_check', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#zone","text":"Carol: ping","sender":"Carol"}',
|
||||
'#zone')`, fs)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 11.0, -89, '["zz"]', ?)`, ep)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
msgs, _ := store.GetChannelMessages("#zone", 10, 0)
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("want 1 msg, got %d", len(msgs))
|
||||
}
|
||||
ts, _ := msgs[0]["timestamp"].(string)
|
||||
if ts == "" {
|
||||
t.Fatal("empty timestamp")
|
||||
}
|
||||
n := len(ts)
|
||||
if !(ts[n-1] == 'Z' || (n >= 6 && ts[n-6:] == "+00:00")) {
|
||||
t.Errorf("timestamp not UTC-suffixed (Z/+00:00): %q", ts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChannelMessages_OrderedByLatestSeen: adversarial follow-up to #1366
|
||||
// (PR #1368). The earlier fix only adjusted the rendered `timestamp`
|
||||
// field; page SELECTION and SORT ORDER on both the in-memory and DB
|
||||
// paths still used FirstSeen. This test pins the contract:
|
||||
//
|
||||
// - tx-A: FirstSeen 24h ago, LatestSeen NOW (via a fresh observation).
|
||||
// - tx-B: FirstSeen 1h ago, LatestSeen 1h ago (single observation).
|
||||
//
|
||||
// Both paths MUST:
|
||||
// 1. Return BOTH transmissions in a small (limit=10) page — tx-A must
|
||||
// not be excluded because its FirstSeen is old.
|
||||
// 2. Return tx-A AFTER tx-B (newest-LatestSeen-LAST), matching the
|
||||
// tail-of-msgOrder convention used by the rest of the API and
|
||||
// the frontend's scrollToBottom().
|
||||
func TestChannelMessages_OrderedByLatestSeen_InMemory(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
tOld := now.Add(-24 * time.Hour)
|
||||
tMid := now.Add(-1 * time.Hour)
|
||||
tNewest := now.Add(-30 * time.Minute)
|
||||
tFresh := now.Add(-1 * time.Minute)
|
||||
|
||||
tOldStr := tOld.Format(time.RFC3339)
|
||||
tMidStr := tMid.Format(time.RFC3339)
|
||||
tNewestStr := tNewest.Format(time.RFC3339)
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsO', 'ObsO', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr)
|
||||
|
||||
// tx-A: FirstSeen 24h ago, LatestSeen NOW (T-1m). Old insertion order.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AAAA', 'order_hash_a', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ord","text":"Alpha: hb","sender":"Alpha"}', '#ord')`, tOldStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix())
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix())
|
||||
|
||||
// tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BBBB', 'order_hash_b', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ord","text":"Bravo: msg","sender":"Bravo"}', '#ord')`, tMidStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix())
|
||||
|
||||
// tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('CCCC', 'order_hash_c', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ord","text":"Charlie: msg","sender":"Charlie"}', '#ord')`, tNewestStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix())
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
// Full-page: ordering check (fix #1 gates this — without sort,
|
||||
// msgOrder is insertion order and Alpha lands FIRST, not LAST).
|
||||
msgsAll, totalAll := store.GetChannelMessages("#ord", 10, 0)
|
||||
if totalAll != 3 {
|
||||
t.Fatalf("in-memory: want total=3, got %d", totalAll)
|
||||
}
|
||||
if len(msgsAll) != 3 {
|
||||
t.Fatalf("in-memory: want 3 msgs, got %d", len(msgsAll))
|
||||
}
|
||||
wantOrder := []string{"Bravo", "Charlie", "Alpha"}
|
||||
for i, want := range wantOrder {
|
||||
got, _ := msgsAll[i]["sender"].(string)
|
||||
if got != want {
|
||||
t.Errorf("in-memory: msg[%d] want sender=%q, got %q (LatestSeen ASC, fix #1)", i, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Small page (limit=2): tx-A (Alpha) MUST be included because its
|
||||
// LatestSeen is freshest, even though FirstSeen is oldest. Without
|
||||
// fix #1, the in-memory path takes msgOrder[total-2:] which would
|
||||
// drop Alpha (it sits at msgOrder[0] by insertion order).
|
||||
msgsPage, _ := store.GetChannelMessages("#ord", 2, 0)
|
||||
if len(msgsPage) != 2 {
|
||||
t.Fatalf("in-memory: want 2 msgs at limit=2, got %d", len(msgsPage))
|
||||
}
|
||||
hasAlpha := false
|
||||
for _, m := range msgsPage {
|
||||
if s, _ := m["sender"].(string); s == "Alpha" {
|
||||
hasAlpha = true
|
||||
}
|
||||
}
|
||||
if !hasAlpha {
|
||||
t.Errorf("in-memory: tx-A (Alpha) excluded from limit=2 page — FirstSeen-based tail selection bug (fix #1 reverted?)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChannelMessages_OrderedByLatestSeen_DB(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
tOld := now.Add(-24 * time.Hour)
|
||||
tMid := now.Add(-1 * time.Hour)
|
||||
tNewest := now.Add(-30 * time.Minute)
|
||||
tFresh := now.Add(-1 * time.Minute)
|
||||
|
||||
tOldStr := tOld.Format(time.RFC3339)
|
||||
tMidStr := tMid.Format(time.RFC3339)
|
||||
tNewestStr := tNewest.Format(time.RFC3339)
|
||||
|
||||
db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count)
|
||||
VALUES ('obsD', 'ObsD', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr)
|
||||
|
||||
// tx-A: FirstSeen 24h ago, observations at T-24h and T-1m (LatestSeen
|
||||
// = T-1m, the FRESHEST). Despite the freshest LatestSeen, a
|
||||
// FirstSeen-DESC selection would push it OFF a small page.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('AADB', 'order_db_hash_a', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ordb","text":"Alpha: hb","sender":"Alpha"}', '#ordb')`, tOldStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix())
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix())
|
||||
|
||||
// tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST LatestSeen.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('BBDB', 'order_db_hash_b', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ordb","text":"Bravo: msg","sender":"Bravo"}', '#ordb')`, tMidStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix())
|
||||
|
||||
// tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle LatestSeen.
|
||||
// With FirstSeen-DESC selection + limit=2, page = [tx-C, tx-B] and
|
||||
// tx-A is EXCLUDED — that's the selection bug fix #2 gates.
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES ('CCDB', 'order_db_hash_c', ?, 1, 5,
|
||||
'{"type":"CHAN","channel":"#ordb","text":"Charlie: msg","sender":"Charlie"}', '#ordb')`, tNewestStr)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix())
|
||||
|
||||
msgs, total, err := db.GetChannelMessages("#ordb", 2, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != 3 {
|
||||
t.Fatalf("DB: want total=3, got %d", total)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("DB: want 2 msgs in page (limit=2), got %d", len(msgs))
|
||||
}
|
||||
// Selection (fix #2): the page MUST include tx-A (Alpha) because its
|
||||
// LatestSeen is the newest — even though its FirstSeen is the OLDEST.
|
||||
// With limit=2 + LatestSeen-DESC selection, page = [Alpha, Charlie].
|
||||
// Returned ASC by LatestSeen (newest LAST, fix #3) = [Charlie, Alpha].
|
||||
sender0, _ := msgs[0]["sender"].(string)
|
||||
sender1, _ := msgs[1]["sender"].(string)
|
||||
if sender0 != "Charlie" || sender1 != "Alpha" {
|
||||
t.Errorf("DB: want order [Charlie, Alpha] (page selected by LatestSeen DESC, returned ASC, fix #2+#3), got [%q, %q]",
|
||||
sender0, sender1)
|
||||
}
|
||||
hasAlpha := false
|
||||
for _, m := range msgs {
|
||||
if s, _ := m["sender"].(string); s == "Alpha" {
|
||||
hasAlpha = true
|
||||
}
|
||||
}
|
||||
if !hasAlpha {
|
||||
t.Errorf("DB: tx-A (Alpha) excluded from page — FirstSeen-based selection bug (fix #2 reverted?)")
|
||||
}
|
||||
|
||||
// Also exercise large-page case (limit > total): ordering-only check.
|
||||
msgsAll, totalAll, err := db.GetChannelMessages("#ordb", 10, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if totalAll != 3 || len(msgsAll) != 3 {
|
||||
t.Fatalf("DB: want all 3 msgs at limit=10, got total=%d len=%d", totalAll, len(msgsAll))
|
||||
}
|
||||
// Expected ASC by LatestSeen: Bravo (T-1h), Charlie (T-30m), Alpha (T-1m).
|
||||
wantOrder := []string{"Bravo", "Charlie", "Alpha"}
|
||||
for i, want := range wantOrder {
|
||||
got, _ := msgsAll[i]["sender"].(string)
|
||||
if got != want {
|
||||
t.Errorf("DB: msg[%d] want sender=%q, got %q (full order: must be LatestSeen ASC, fix #3)", i, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue #1373: /api/channels emits a ghost "unknown" bucket for encrypted GRP_TXT
|
||||
// packets whose decoded JSON sets channel="" (server has no PSK to decrypt).
|
||||
// Fix A (cosmetic): drop the "unknown" bucket from the response so users only
|
||||
// see real channels. Encrypted-no-key packets are still observable via the
|
||||
// encrypted-channels analytics, just not as a fake "unknown" channel.
|
||||
//
|
||||
// This test seeds 5 GRP_TXT with Channel="" (encrypted-no-key) + 3 with
|
||||
// Channel="#real" and asserts GetChannels returns exactly one entry, #real —
|
||||
// no "unknown" bucket.
|
||||
|
||||
func TestGetChannels_NoUnknownBucket_1373(t *testing.T) {
|
||||
packets := []*StoreTx{
|
||||
makeGrpTx(129, "", "", ""),
|
||||
makeGrpTx(129, "", "", ""),
|
||||
makeGrpTx(129, "", "", ""),
|
||||
makeGrpTx(129, "", "", ""),
|
||||
makeGrpTx(129, "", "", ""),
|
||||
makeGrpTx(72, "#real", "hello", "alice"),
|
||||
makeGrpTx(72, "#real", "world", "bob"),
|
||||
makeGrpTx(72, "#real", "third", "carol"),
|
||||
}
|
||||
store := newChannelTestStore(packets)
|
||||
|
||||
channels := store.GetChannels("")
|
||||
|
||||
var gotNames []string
|
||||
for _, ch := range channels {
|
||||
name, _ := ch["name"].(string)
|
||||
gotNames = append(gotNames, name)
|
||||
if name == "unknown" {
|
||||
t.Errorf("GetChannels emitted ghost 'unknown' bucket (issue #1373): %+v", ch)
|
||||
}
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected exactly 1 channel (#real), got %d: %v", len(channels), gotNames)
|
||||
}
|
||||
if name, _ := channels[0]["name"].(string); name != "#real" {
|
||||
t.Errorf("expected channel name '#real', got %q", name)
|
||||
}
|
||||
if mc, _ := channels[0]["messageCount"].(int); mc != 3 {
|
||||
t.Errorf("expected messageCount=3 for #real, got %v", channels[0]["messageCount"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetChannels_DB_NoUnknownBucket_1373 mirrors the in-memory test against
|
||||
// the DB-backed GetChannels path in cmd/server/db.go. It seeds GRP_TXT rows
|
||||
// with channel_hash NULL (encrypted, no PSK known to ingestor) + rows with
|
||||
// channel_hash="#real" and asserts the response contains only #real.
|
||||
//
|
||||
// Note: the DB path already filters NULL channel_hash via the SELECT (`channel_hash IS NOT NULL`),
|
||||
// AND nullStr("")==empty triggers `continue` in the loop. This test pins that
|
||||
// contract so a future refactor can't reintroduce an "unknown" bucket on the
|
||||
// DB side either.
|
||||
func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
// Seed 5 encrypted GRP_TXT rows with channel_hash NULL (server had no PSK).
|
||||
for i := 0; i < 5; i++ {
|
||||
_, err := db.conn.Exec(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"","text":"","sender":""}', NULL)`,
|
||||
"AA", sqlHashFor(i))
|
||||
if err != nil {
|
||||
t.Fatalf("seed encrypted row %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Seed 3 decrypted GRP_TXT rows with channel_hash="#real".
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err := db.conn.Exec(`INSERT INTO transmissions
|
||||
(raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash)
|
||||
VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5,
|
||||
'{"type":"CHAN","channel":"#real","text":"Alice: hi","sender":"Alice"}', '#real')`,
|
||||
"BB", sqlHashFor(100+i))
|
||||
if err != nil {
|
||||
t.Fatalf("seed real row %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
channels, err := db.GetChannels()
|
||||
if err != nil {
|
||||
t.Fatalf("GetChannels: %v", err)
|
||||
}
|
||||
|
||||
var gotNames []string
|
||||
for _, ch := range channels {
|
||||
name, _ := ch["name"].(string)
|
||||
gotNames = append(gotNames, name)
|
||||
if name == "unknown" {
|
||||
t.Errorf("DB GetChannels emitted ghost 'unknown' bucket (issue #1373): %+v", ch)
|
||||
}
|
||||
if name == "" {
|
||||
t.Errorf("DB GetChannels emitted empty-name channel bucket (issue #1373): %+v", ch)
|
||||
}
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected exactly 1 channel (#real), got %d: %v", len(channels), gotNames)
|
||||
}
|
||||
if name, _ := channels[0]["name"].(string); name != "#real" {
|
||||
t.Errorf("expected channel name '#real', got %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
// sqlHashFor returns a unique 16-char hex string per index for the
|
||||
// `hash` UNIQUE column in transmissions.
|
||||
func sqlHashFor(i int) string {
|
||||
return fmt.Sprintf("%016x", uint64(0x1373_0000_0000_0000)+uint64(i))
|
||||
}
|
||||
|
||||
// silence unused-import warning when the file is reduced.
|
||||
var _ = sql.ErrNoRows
|
||||
@@ -1,963 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── Clock Skew Severity ────────────────────────────────────────────────────────
|
||||
|
||||
type SkewSeverity string
|
||||
|
||||
const (
|
||||
SkewOK SkewSeverity = "ok" // < 5 min
|
||||
SkewWarning SkewSeverity = "warning" // 5 min – 1 hour
|
||||
SkewCritical SkewSeverity = "critical" // 1 hour – 30 days
|
||||
SkewAbsurd SkewSeverity = "absurd" // > 30 days
|
||||
SkewNoClock SkewSeverity = "no_clock" // > 365 days — uninitialized RTC
|
||||
SkewBimodalClock SkewSeverity = "bimodal_clock" // mixed good+bad recent samples (flaky RTC)
|
||||
)
|
||||
|
||||
// Default thresholds in seconds.
|
||||
const (
|
||||
skewThresholdWarnSec = 5 * 60 // 5 minutes
|
||||
skewThresholdCriticalSec = 60 * 60 // 1 hour
|
||||
skewThresholdAbsurdSec = 30 * 24 * 3600 // 30 days
|
||||
skewThresholdNoClockSec = 365 * 24 * 3600 // 365 days — uninitialized RTC
|
||||
|
||||
// minDriftSamples is the minimum number of advert transmissions needed
|
||||
// to compute a meaningful linear drift rate.
|
||||
minDriftSamples = 5
|
||||
|
||||
// maxReasonableDriftPerDay caps drift display. Physically impossible
|
||||
// drift rates (> 1 day/day) indicate insufficient or outlier samples.
|
||||
maxReasonableDriftPerDay = 86400.0
|
||||
|
||||
// recentSkewWindowCount is the number of most-recent advert samples
|
||||
// used to derive the "current" skew for severity classification (see
|
||||
// issue #789). The all-time median is poisoned by historical bad
|
||||
// samples (e.g. a node that was off and then GPS-corrected); severity
|
||||
// must reflect current health, not lifetime statistics.
|
||||
recentSkewWindowCount = 5
|
||||
|
||||
// recentSkewWindowSec bounds the recent-window in time as well: only
|
||||
// samples from the last N seconds count as "recent" for severity.
|
||||
// The effective window is min(recentSkewWindowCount, samples in 1h).
|
||||
recentSkewWindowSec = 3600
|
||||
|
||||
// bimodalSkewThresholdSec is the absolute skew threshold (1 hour)
|
||||
// above which a sample is considered "bad" — likely firmware emitting
|
||||
// a nonsense timestamp from an uninitialized RTC, not real drift.
|
||||
// Chosen to match the warning/critical severity boundary: real clock
|
||||
// drift rarely exceeds 1 hour, while epoch-0 RTCs produce ~1.7B sec.
|
||||
bimodalSkewThresholdSec = 3600.0
|
||||
|
||||
// rtcResetOutlierThresholdSec is the absolute skew above which a
|
||||
// sample is treated as obvious sensor garbage — an RTC-reset advert
|
||||
// where the firmware emitted its factory timestamp (typically off by
|
||||
// months/years). These samples are excluded from the recent-window
|
||||
// "good/bad" split (bug #1285 — single RTC-reset advert among 30
|
||||
// healthy adverts must not flip a node to bimodal_clock) and from the
|
||||
// per-hash evidence median (a 700-day median is not actionable for
|
||||
// operators). They remain in the raw sample stream and the RTC-reset
|
||||
// badge logic which surfaces them separately. 24h is a generous floor:
|
||||
// real drift is fractions of a sec/advert, real clock-skew tops out
|
||||
// in the hours range; anything above a day is structurally not a
|
||||
// drift signal.
|
||||
rtcResetOutlierThresholdSec = 24 * 3600.0
|
||||
|
||||
// maxPlausibleSkewJumpSec is the largest skew change between
|
||||
// consecutive samples that we treat as physical drift. Anything larger
|
||||
// (e.g. a GPS sync that jumps the clock by minutes/days) is rejected
|
||||
// as an outlier when computing drift. Real microcontroller drift is
|
||||
// fractions of a second per advert; 60s is a generous safety factor.
|
||||
maxPlausibleSkewJumpSec = 60.0
|
||||
|
||||
// theilSenMaxPoints caps the number of points fed to Theil-Sen
|
||||
// regression (O(n²) in pairs). For nodes with thousands of samples we
|
||||
// keep the most-recent points, which are also the most relevant for
|
||||
// current drift.
|
||||
theilSenMaxPoints = 200
|
||||
)
|
||||
|
||||
// classifySkew maps absolute skew (seconds) to a severity level.
|
||||
// Float64 comparison is safe: inputs are rounded to 1 decimal via round(),
|
||||
// and thresholds are integer multiples of 60 — no rounding artifacts.
|
||||
func classifySkew(absSkewSec float64) SkewSeverity {
|
||||
switch {
|
||||
case absSkewSec >= skewThresholdNoClockSec:
|
||||
return SkewNoClock
|
||||
case absSkewSec >= skewThresholdAbsurdSec:
|
||||
return SkewAbsurd
|
||||
case absSkewSec >= skewThresholdCriticalSec:
|
||||
return SkewCritical
|
||||
case absSkewSec >= skewThresholdWarnSec:
|
||||
return SkewWarning
|
||||
default:
|
||||
return SkewOK
|
||||
}
|
||||
}
|
||||
|
||||
// ── Data Types ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// skewSample is a single raw skew measurement from one advert observation.
|
||||
type skewSample struct {
|
||||
advertTS int64 // node's advert Unix timestamp
|
||||
observedTS int64 // observation Unix timestamp
|
||||
observerID string // which observer saw this
|
||||
hash string // transmission hash (for multi-observer grouping)
|
||||
}
|
||||
|
||||
// ObserverCalibration holds the computed clock offset for an observer.
|
||||
type ObserverCalibration struct {
|
||||
ObserverID string `json:"observerID"`
|
||||
OffsetSec float64 `json:"offsetSec"` // positive = observer clock ahead
|
||||
Samples int `json:"samples"` // number of multi-observer packets used
|
||||
}
|
||||
|
||||
// NodeClockSkew is the API response for a single node's clock skew data.
|
||||
type NodeClockSkew struct {
|
||||
Pubkey string `json:"pubkey"`
|
||||
MeanSkewSec float64 `json:"meanSkewSec"` // corrected mean skew (positive = node ahead)
|
||||
MedianSkewSec float64 `json:"medianSkewSec"` // corrected median skew
|
||||
LastSkewSec float64 `json:"lastSkewSec"` // most recent corrected skew
|
||||
RecentMedianSkewSec float64 `json:"recentMedianSkewSec"` // median across most-recent samples (drives severity, see #789)
|
||||
DriftPerDaySec float64 `json:"driftPerDaySec"` // linear drift rate (sec/day)
|
||||
Severity SkewSeverity `json:"severity"`
|
||||
SampleCount int `json:"sampleCount"`
|
||||
Calibrated bool `json:"calibrated"` // true if observer calibration was applied
|
||||
LastAdvertTS int64 `json:"lastAdvertTS"` // most recent advert timestamp
|
||||
LastObservedTS int64 `json:"lastObservedTS"` // most recent observation timestamp
|
||||
Samples []SkewSample `json:"samples,omitempty"` // time-series for sparklines
|
||||
GoodFraction float64 `json:"goodFraction"` // fraction of recent samples with |skew| <= 1h
|
||||
RecentBadSampleCount int `json:"recentBadSampleCount"` // count of recent samples with |skew| > 1h
|
||||
RecentSampleCount int `json:"recentSampleCount"` // total recent samples in window
|
||||
RecentHashEvidence []HashEvidence `json:"recentHashEvidence,omitempty"`
|
||||
CalibrationSummary *CalibrationSummary `json:"calibrationSummary,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"` // populated in fleet responses
|
||||
NodeRole string `json:"nodeRole,omitempty"` // populated in fleet responses
|
||||
}
|
||||
|
||||
// SkewSample is a single (timestamp, skew) point for sparkline rendering.
|
||||
type SkewSample struct {
|
||||
Timestamp int64 `json:"ts"` // Unix epoch of observation
|
||||
SkewSec float64 `json:"skew"` // corrected skew in seconds
|
||||
}
|
||||
|
||||
// HashEvidenceObserver is one observer's contribution to a per-hash evidence entry.
|
||||
type HashEvidenceObserver struct {
|
||||
ObserverID string `json:"observerID"`
|
||||
ObserverName string `json:"observerName"`
|
||||
RawSkewSec float64 `json:"rawSkewSec"`
|
||||
CorrectedSkewSec float64 `json:"correctedSkewSec"`
|
||||
ObserverOffsetSec float64 `json:"observerOffsetSec"`
|
||||
Calibrated bool `json:"calibrated"`
|
||||
}
|
||||
|
||||
// HashEvidence is per-hash clock skew evidence showing individual observer contributions.
|
||||
type HashEvidence struct {
|
||||
Hash string `json:"hash"`
|
||||
Observers []HashEvidenceObserver `json:"observers"`
|
||||
MedianCorrectedSkewSec float64 `json:"medianCorrectedSkewSec"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// CalibrationSummary counts how many samples were corrected via observer calibration.
|
||||
type CalibrationSummary struct {
|
||||
TotalSamples int `json:"totalSamples"`
|
||||
CalibratedSamples int `json:"calibratedSamples"`
|
||||
UncalibratedSamples int `json:"uncalibratedSamples"`
|
||||
}
|
||||
|
||||
// txSkewResult maps tx hash → per-transmission skew stats. This is an
|
||||
// intermediate result keyed by hash (not pubkey); the store maps hash → pubkey
|
||||
// when building the final per-node view.
|
||||
type txSkewResult = map[string]*NodeClockSkew
|
||||
|
||||
// ── Clock Skew Engine ──────────────────────────────────────────────────────────
|
||||
|
||||
// ClockSkewEngine computes and caches clock skew data for nodes and observers.
|
||||
type ClockSkewEngine struct {
|
||||
mu sync.RWMutex
|
||||
observerOffsets map[string]float64 // observerID → calibrated offset (seconds)
|
||||
observerSamples map[string]int // observerID → number of multi-observer packets used
|
||||
nodeSkew txSkewResult
|
||||
hashEvidence map[string][]hashEvidenceEntry // hash → per-observer raw/corrected data
|
||||
lastComputed time.Time
|
||||
computeInterval time.Duration
|
||||
}
|
||||
|
||||
// hashEvidenceEntry stores raw evidence per observer per hash, cached during Recompute.
|
||||
type hashEvidenceEntry struct {
|
||||
observerID string
|
||||
rawSkew float64
|
||||
corrected float64
|
||||
offset float64
|
||||
calibrated bool
|
||||
observedTS int64
|
||||
}
|
||||
|
||||
func NewClockSkewEngine() *ClockSkewEngine {
|
||||
return &ClockSkewEngine{
|
||||
observerOffsets: make(map[string]float64),
|
||||
observerSamples: make(map[string]int),
|
||||
nodeSkew: make(txSkewResult),
|
||||
hashEvidence: make(map[string][]hashEvidenceEntry),
|
||||
computeInterval: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Recompute recalculates all clock skew data from the packet store.
|
||||
// Called periodically or on demand. Holds store RLock externally.
|
||||
// Uses read-copy-update: heavy computation runs outside the write lock,
|
||||
// then results are swapped in under a brief lock.
|
||||
func (e *ClockSkewEngine) Recompute(store *PacketStore) {
|
||||
// Fast path: check under read lock if recompute is needed.
|
||||
e.mu.RLock()
|
||||
fresh := time.Since(e.lastComputed) < e.computeInterval
|
||||
e.mu.RUnlock()
|
||||
if fresh {
|
||||
return
|
||||
}
|
||||
|
||||
// Phase 1: Collect skew samples from ADVERT packets (store RLock held by caller).
|
||||
samples := collectSamples(store)
|
||||
|
||||
// Phase 2–3: Compute outside the write lock.
|
||||
var newOffsets map[string]float64
|
||||
var newSamples map[string]int
|
||||
var newNodeSkew txSkewResult
|
||||
var newHashEvidence map[string][]hashEvidenceEntry
|
||||
|
||||
if len(samples) > 0 {
|
||||
newOffsets, newSamples = calibrateObservers(samples)
|
||||
newNodeSkew, newHashEvidence = computeNodeSkew(samples, newOffsets)
|
||||
} else {
|
||||
newOffsets = make(map[string]float64)
|
||||
newSamples = make(map[string]int)
|
||||
newNodeSkew = make(txSkewResult)
|
||||
newHashEvidence = make(map[string][]hashEvidenceEntry)
|
||||
}
|
||||
|
||||
// Swap results under brief write lock.
|
||||
e.mu.Lock()
|
||||
// Re-check: another goroutine may have computed while we were working.
|
||||
if time.Since(e.lastComputed) < e.computeInterval {
|
||||
e.mu.Unlock()
|
||||
return
|
||||
}
|
||||
e.observerOffsets = newOffsets
|
||||
e.observerSamples = newSamples
|
||||
e.nodeSkew = newNodeSkew
|
||||
e.hashEvidence = newHashEvidence
|
||||
e.lastComputed = time.Now()
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// collectSamples extracts skew samples from ADVERT packets in the store.
|
||||
// Must be called with store.mu held (at least RLock).
|
||||
func collectSamples(store *PacketStore) []skewSample {
|
||||
adverts := store.byPayloadType[PayloadADVERT]
|
||||
if len(adverts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
samples := make([]skewSample, 0, len(adverts)*2)
|
||||
for _, tx := range adverts {
|
||||
decoded := tx.ParsedDecoded()
|
||||
if decoded == nil {
|
||||
continue
|
||||
}
|
||||
// Extract advert timestamp from decoded JSON.
|
||||
advertTS := extractTimestamp(decoded)
|
||||
if advertTS <= 0 {
|
||||
continue
|
||||
}
|
||||
// Sanity: skip timestamps before year 2020 or after year 2100.
|
||||
if advertTS < 1577836800 || advertTS > 4102444800 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, obs := range tx.Observations {
|
||||
obsTS := parseISO(obs.Timestamp)
|
||||
if obsTS <= 0 {
|
||||
continue
|
||||
}
|
||||
samples = append(samples, skewSample{
|
||||
advertTS: advertTS,
|
||||
observedTS: obsTS,
|
||||
observerID: obs.ObserverID,
|
||||
hash: tx.Hash,
|
||||
})
|
||||
}
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
// extractTimestamp gets the Unix timestamp from a decoded ADVERT payload.
|
||||
func extractTimestamp(decoded map[string]interface{}) int64 {
|
||||
// Try payload.timestamp first (nested in "payload" key).
|
||||
if payload, ok := decoded["payload"]; ok {
|
||||
if pm, ok := payload.(map[string]interface{}); ok {
|
||||
if ts := jsonNumber(pm, "timestamp"); ts > 0 {
|
||||
return ts
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: top-level timestamp.
|
||||
if ts := jsonNumber(decoded, "timestamp"); ts > 0 {
|
||||
return ts
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// jsonNumber extracts an int64 from a JSON-parsed map (handles float64 and json.Number).
|
||||
func jsonNumber(m map[string]interface{}, key string) int64 {
|
||||
v, ok := m[key]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int64(n)
|
||||
case int64:
|
||||
return n
|
||||
case int:
|
||||
return int64(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseISO parses an ISO 8601 timestamp string to Unix seconds.
|
||||
func parseISO(s string) int64 {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
// Try with fractional seconds.
|
||||
t, err = time.Parse("2006-01-02T15:04:05.999999999Z07:00", s)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ── Phase 2: Observer Calibration ──────────────────────────────────────────────
|
||||
|
||||
// calibrateObservers computes each observer's clock offset using multi-observer
|
||||
// packets. Returns offset map and sample count map.
|
||||
func calibrateObservers(samples []skewSample) (map[string]float64, map[string]int) {
|
||||
// Group observations by packet hash.
|
||||
byHash := make(map[string][]skewSample)
|
||||
for _, s := range samples {
|
||||
byHash[s.hash] = append(byHash[s.hash], s)
|
||||
}
|
||||
|
||||
// For each multi-observer packet, compute per-observer deviation from median.
|
||||
deviations := make(map[string][]float64) // observerID → list of deviations
|
||||
for _, group := range byHash {
|
||||
if len(group) < 2 {
|
||||
continue // single-observer packet, can't calibrate
|
||||
}
|
||||
// Compute median observation timestamp for this packet.
|
||||
obsTimes := make([]float64, len(group))
|
||||
for i, s := range group {
|
||||
obsTimes[i] = float64(s.observedTS)
|
||||
}
|
||||
medianObs := median(obsTimes)
|
||||
for _, s := range group {
|
||||
dev := float64(s.observedTS) - medianObs
|
||||
deviations[s.observerID] = append(deviations[s.observerID], dev)
|
||||
}
|
||||
}
|
||||
|
||||
// Each observer's offset = median of its deviations.
|
||||
offsets := make(map[string]float64, len(deviations))
|
||||
counts := make(map[string]int, len(deviations))
|
||||
for obsID, devs := range deviations {
|
||||
offsets[obsID] = median(devs)
|
||||
counts[obsID] = len(devs)
|
||||
}
|
||||
return offsets, counts
|
||||
}
|
||||
|
||||
// ── Phase 3: Per-Node Skew ─────────────────────────────────────────────────────
|
||||
|
||||
// computeNodeSkew calculates corrected skew statistics for each node.
|
||||
func computeNodeSkew(samples []skewSample, obsOffsets map[string]float64) (txSkewResult, map[string][]hashEvidenceEntry) {
|
||||
// Compute corrected skew per sample, grouped by hash (each hash = one
|
||||
// node's advert transmission). The caller maps hash → pubkey via byNode.
|
||||
type correctedSample struct {
|
||||
skew float64
|
||||
observedTS int64
|
||||
calibrated bool
|
||||
}
|
||||
|
||||
byHash := make(map[string][]correctedSample)
|
||||
hashAdvertTS := make(map[string]int64)
|
||||
evidence := make(map[string][]hashEvidenceEntry) // hash → per-observer evidence
|
||||
|
||||
for _, s := range samples {
|
||||
obsOffset, hasCal := obsOffsets[s.observerID]
|
||||
rawSkew := float64(s.advertTS - s.observedTS)
|
||||
corrected := rawSkew
|
||||
if hasCal {
|
||||
// Observer offset = obs_ts - median(all_obs_ts). If observer is ahead,
|
||||
// its obs_ts is inflated, making raw_skew too low. Add offset to correct.
|
||||
corrected = rawSkew + obsOffset
|
||||
}
|
||||
byHash[s.hash] = append(byHash[s.hash], correctedSample{
|
||||
skew: corrected,
|
||||
observedTS: s.observedTS,
|
||||
calibrated: hasCal,
|
||||
})
|
||||
hashAdvertTS[s.hash] = s.advertTS
|
||||
evidence[s.hash] = append(evidence[s.hash], hashEvidenceEntry{
|
||||
observerID: s.observerID,
|
||||
rawSkew: round(rawSkew, 1),
|
||||
corrected: round(corrected, 1),
|
||||
offset: round(obsOffset, 1),
|
||||
calibrated: hasCal,
|
||||
observedTS: s.observedTS,
|
||||
})
|
||||
}
|
||||
|
||||
// Each hash represents one advert from one node. Compute median corrected
|
||||
// skew per hash (across multiple observers).
|
||||
|
||||
result := make(map[string]*NodeClockSkew) // keyed by hash for now
|
||||
for hash, cs := range byHash {
|
||||
skews := make([]float64, len(cs))
|
||||
for i, c := range cs {
|
||||
skews[i] = c.skew
|
||||
}
|
||||
medSkew := median(skews)
|
||||
meanSkew := mean(skews)
|
||||
|
||||
// Find latest observation.
|
||||
var latestObsTS int64
|
||||
var anyCal bool
|
||||
for _, c := range cs {
|
||||
if c.observedTS > latestObsTS {
|
||||
latestObsTS = c.observedTS
|
||||
}
|
||||
if c.calibrated {
|
||||
anyCal = true
|
||||
}
|
||||
}
|
||||
|
||||
absMedian := math.Abs(medSkew)
|
||||
result[hash] = &NodeClockSkew{
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(cs[len(cs)-1].skew, 1),
|
||||
Severity: classifySkew(absMedian),
|
||||
SampleCount: len(cs),
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: hashAdvertTS[hash],
|
||||
LastObservedTS: latestObsTS,
|
||||
}
|
||||
}
|
||||
return result, evidence
|
||||
}
|
||||
|
||||
// ── Integration with PacketStore ───────────────────────────────────────────────
|
||||
|
||||
// GetNodeClockSkew returns the clock skew data for a specific node (acquires RLock).
|
||||
func (s *PacketStore) GetNodeClockSkew(pubkey string) *NodeClockSkew {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.getNodeClockSkewLocked(pubkey)
|
||||
}
|
||||
|
||||
// getNodeClockSkewLocked returns clock skew for a node.
|
||||
// Must be called with s.mu held (at least RLock).
|
||||
func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
s.clockSkew.Recompute(s)
|
||||
|
||||
txs := s.byNode[pubkey]
|
||||
if len(txs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.clockSkew.mu.RLock()
|
||||
defer s.clockSkew.mu.RUnlock()
|
||||
|
||||
var allSkews []float64
|
||||
var lastSkew float64
|
||||
var lastObsTS, lastAdvTS int64
|
||||
var totalSamples int
|
||||
var anyCal bool
|
||||
var tsSkews []tsSkewPair
|
||||
|
||||
for _, tx := range txs {
|
||||
if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
|
||||
continue
|
||||
}
|
||||
cs, ok := s.clockSkew.nodeSkew[tx.Hash]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
allSkews = append(allSkews, cs.MedianSkewSec)
|
||||
totalSamples += cs.SampleCount
|
||||
if cs.Calibrated {
|
||||
anyCal = true
|
||||
}
|
||||
if cs.LastObservedTS > lastObsTS {
|
||||
lastObsTS = cs.LastObservedTS
|
||||
lastSkew = cs.LastSkewSec
|
||||
lastAdvTS = cs.LastAdvertTS
|
||||
}
|
||||
tsSkews = append(tsSkews, tsSkewPair{ts: cs.LastObservedTS, skew: cs.MedianSkewSec})
|
||||
}
|
||||
|
||||
if len(allSkews) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
medSkew := median(allSkews)
|
||||
meanSkew := mean(allSkews)
|
||||
|
||||
// Severity is derived from RECENT samples only (issue #789). The
|
||||
// all-time median is poisoned by historical bad data — a node that
|
||||
// was off for hours and then GPS-corrected can have median = -59M sec
|
||||
// while its current skew is -0.8s. Operators need severity to reflect
|
||||
// current health, so they trust the dashboard.
|
||||
//
|
||||
// Sort tsSkews by time and take the last recentSkewWindowCount samples
|
||||
// (or all samples within recentSkewWindowSec of the latest, whichever
|
||||
// gives FEWER samples — we want the more-current view; a chatty node
|
||||
// can fit dozens of samples in 1h, in which case the count cap wins).
|
||||
sort.Slice(tsSkews, func(i, j int) bool { return tsSkews[i].ts < tsSkews[j].ts })
|
||||
|
||||
recentSkew := lastSkew
|
||||
var recentVals []float64
|
||||
if n := len(tsSkews); n > 0 {
|
||||
latestTS := tsSkews[n-1].ts
|
||||
// Index-based window: last K samples.
|
||||
startByCount := n - recentSkewWindowCount
|
||||
if startByCount < 0 {
|
||||
startByCount = 0
|
||||
}
|
||||
// Time-based window: samples newer than latestTS - windowSec.
|
||||
startByTime := n - 1
|
||||
for i := n - 1; i >= 0; i-- {
|
||||
if latestTS-tsSkews[i].ts <= recentSkewWindowSec {
|
||||
startByTime = i
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Pick the narrower (larger-index) of the two windows — the most
|
||||
// current view of the node's clock health.
|
||||
start := startByCount
|
||||
if startByTime > start {
|
||||
start = startByTime
|
||||
}
|
||||
recentVals = make([]float64, 0, n-start)
|
||||
for i := start; i < n; i++ {
|
||||
recentVals = append(recentVals, tsSkews[i].skew)
|
||||
}
|
||||
if len(recentVals) > 0 {
|
||||
recentSkew = median(recentVals)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Bimodal detection (#845) ─────────────────────────────────────────
|
||||
// Split recent samples into "good" (|skew| <= 1h, real clock) and
|
||||
// "bad" (|skew| > 1h, firmware nonsense from uninitialized RTC).
|
||||
// Classification order (first match wins):
|
||||
// no_clock — goodFraction < 0.10 (essentially no real clock)
|
||||
// bimodal_clock — 0.10 <= goodFraction < 0.80 AND badCount > 0
|
||||
// ok/warn/etc. — goodFraction >= 0.80 (normal, outliers filtered)
|
||||
//
|
||||
// RTC-reset outliers (|skew| > 24h — single advert where the firmware
|
||||
// emitted its factory timestamp) are EXCLUDED from this split (bug
|
||||
// #1285): they're not "bimodal-bad real-but-large skew" but obvious
|
||||
// sensor garbage, surfaced separately via the RTC-reset badge. Counting
|
||||
// them as bimodal-bad produces a false-alarm warning ("3 of last 5
|
||||
// adverts had nonsense timestamps") on otherwise-healthy nodes.
|
||||
var goodSamples []float64
|
||||
var rtcResetCount int
|
||||
for _, v := range recentVals {
|
||||
absV := math.Abs(v)
|
||||
switch {
|
||||
case absV > rtcResetOutlierThresholdSec:
|
||||
rtcResetCount++ // ignored for good/bad classification
|
||||
case absV <= bimodalSkewThresholdSec:
|
||||
goodSamples = append(goodSamples, v)
|
||||
}
|
||||
}
|
||||
recentSampleCount := len(recentVals) - rtcResetCount
|
||||
recentBadCount := recentSampleCount - len(goodSamples)
|
||||
var goodFraction float64
|
||||
if recentSampleCount > 0 {
|
||||
goodFraction = float64(len(goodSamples)) / float64(recentSampleCount)
|
||||
}
|
||||
|
||||
var severity SkewSeverity
|
||||
if goodFraction < 0.10 {
|
||||
// Essentially no real clock — classify as no_clock regardless
|
||||
// of the raw skew magnitude.
|
||||
severity = SkewNoClock
|
||||
} else if goodFraction < 0.80 && recentBadCount > 0 {
|
||||
// Bimodal: use median of GOOD samples as the "real" skew.
|
||||
severity = SkewBimodalClock
|
||||
if len(goodSamples) > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
} else {
|
||||
// Normal path: if there are good samples, use their median
|
||||
// (filters out rare outliers in ≥80% good case, and rejects
|
||||
// RTC-reset outliers regardless of bimodal/bad counts — #1285).
|
||||
if len(goodSamples) > 0 {
|
||||
recentSkew = median(goodSamples)
|
||||
}
|
||||
severity = classifySkew(math.Abs(recentSkew))
|
||||
}
|
||||
|
||||
// For no_clock / bimodal_clock nodes, skip drift when data is unreliable.
|
||||
var drift float64
|
||||
if severity != SkewNoClock && severity != SkewBimodalClock && len(tsSkews) >= minDriftSamples {
|
||||
drift = computeDrift(tsSkews)
|
||||
// Cap physically impossible drift rates.
|
||||
if math.Abs(drift) > maxReasonableDriftPerDay {
|
||||
drift = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Build sparkline samples from tsSkews (already sorted by time above).
|
||||
samples := make([]SkewSample, len(tsSkews))
|
||||
for i, p := range tsSkews {
|
||||
samples[i] = SkewSample{Timestamp: p.ts, SkewSec: round(p.skew, 1)}
|
||||
}
|
||||
|
||||
// Build per-hash evidence (most recent 10 hashes with ≥1 observer).
|
||||
// Observer name lookup from store observations.
|
||||
obsNameMap := make(map[string]string)
|
||||
type hashMeta struct {
|
||||
hash string
|
||||
ts int64
|
||||
}
|
||||
var evidenceHashes []hashMeta
|
||||
for _, tx := range txs {
|
||||
if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT {
|
||||
continue
|
||||
}
|
||||
ev, ok := s.clockSkew.hashEvidence[tx.Hash]
|
||||
if !ok || len(ev) == 0 {
|
||||
continue
|
||||
}
|
||||
// Collect observer names from tx observations.
|
||||
for _, obs := range tx.Observations {
|
||||
if obs.ObserverID != "" && obs.ObserverName != "" {
|
||||
obsNameMap[obs.ObserverID] = obs.ObserverName
|
||||
}
|
||||
}
|
||||
evidenceHashes = append(evidenceHashes, hashMeta{hash: tx.Hash, ts: ev[0].observedTS})
|
||||
}
|
||||
// Sort by timestamp descending, take most recent 10.
|
||||
sort.Slice(evidenceHashes, func(i, j int) bool { return evidenceHashes[i].ts > evidenceHashes[j].ts })
|
||||
if len(evidenceHashes) > 10 {
|
||||
evidenceHashes = evidenceHashes[:10]
|
||||
}
|
||||
var recentEvidence []HashEvidence
|
||||
var calSummary CalibrationSummary
|
||||
for _, eh := range evidenceHashes {
|
||||
entries := s.clockSkew.hashEvidence[eh.hash]
|
||||
var observers []HashEvidenceObserver
|
||||
var corrSkews []float64
|
||||
for _, e := range entries {
|
||||
name := obsNameMap[e.observerID]
|
||||
if name == "" {
|
||||
name = e.observerID
|
||||
}
|
||||
observers = append(observers, HashEvidenceObserver{
|
||||
ObserverID: e.observerID,
|
||||
ObserverName: name,
|
||||
RawSkewSec: e.rawSkew,
|
||||
CorrectedSkewSec: e.corrected,
|
||||
ObserverOffsetSec: e.offset,
|
||||
Calibrated: e.calibrated,
|
||||
})
|
||||
corrSkews = append(corrSkews, e.corrected)
|
||||
calSummary.TotalSamples++
|
||||
if e.calibrated {
|
||||
calSummary.CalibratedSamples++
|
||||
} else {
|
||||
calSummary.UncalibratedSamples++
|
||||
}
|
||||
}
|
||||
recentEvidence = append(recentEvidence, HashEvidence{
|
||||
Hash: eh.hash,
|
||||
Observers: observers,
|
||||
MedianCorrectedSkewSec: round(hashEvidenceMedian(corrSkews), 1),
|
||||
Timestamp: eh.ts,
|
||||
})
|
||||
}
|
||||
|
||||
return &NodeClockSkew{
|
||||
Pubkey: pubkey,
|
||||
MeanSkewSec: round(meanSkew, 1),
|
||||
MedianSkewSec: round(medSkew, 1),
|
||||
LastSkewSec: round(lastSkew, 1),
|
||||
RecentMedianSkewSec: round(recentSkew, 1),
|
||||
DriftPerDaySec: round(drift, 2),
|
||||
Severity: severity,
|
||||
SampleCount: totalSamples,
|
||||
Calibrated: anyCal,
|
||||
LastAdvertTS: lastAdvTS,
|
||||
LastObservedTS: lastObsTS,
|
||||
Samples: samples,
|
||||
GoodFraction: round(goodFraction, 2),
|
||||
RecentBadSampleCount: recentBadCount,
|
||||
RecentSampleCount: recentSampleCount,
|
||||
RecentHashEvidence: recentEvidence,
|
||||
CalibrationSummary: &calSummary,
|
||||
}
|
||||
}
|
||||
|
||||
// GetFleetClockSkew returns clock skew data for all nodes, optionally
|
||||
// filtered to area. With no area, prefers the steady-state recomputer
|
||||
// snapshot (issue #1265). Must NOT be called with s.mu held.
|
||||
func (s *PacketStore) GetFleetClockSkew(area string) []*NodeClockSkew {
|
||||
if area == "" {
|
||||
s.analyticsRecomputerMu.RLock()
|
||||
rc := s.recompNodesClockSkew
|
||||
s.analyticsRecomputerMu.RUnlock()
|
||||
if rc != nil {
|
||||
if v := rc.Load(); v != nil {
|
||||
if r, ok := v.([]*NodeClockSkew); ok {
|
||||
return r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.computeFleetClockSkewForArea(area)
|
||||
}
|
||||
|
||||
// computeFleetClockSkew wraps computeFleetClockSkewForArea with no area
|
||||
// filter; called by the steady-state recomputer. Must NOT be called with
|
||||
// s.mu held.
|
||||
func (s *PacketStore) computeFleetClockSkew() []*NodeClockSkew {
|
||||
return s.computeFleetClockSkewForArea("")
|
||||
}
|
||||
|
||||
// computeFleetClockSkewForArea is the underlying compute. Must NOT be
|
||||
// called with s.mu held.
|
||||
func (s *PacketStore) computeFleetClockSkewForArea(area string) []*NodeClockSkew {
|
||||
var areaNodes map[string]bool
|
||||
if area != "" {
|
||||
areaNodes = s.resolveAreaNodes(area)
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
// Build name/role lookup from DB cache (requires s.mu held).
|
||||
allNodes, _ := s.getCachedNodesAndPM()
|
||||
nameMap := make(map[string]nodeInfo, len(allNodes))
|
||||
for _, ni := range allNodes {
|
||||
nameMap[ni.PublicKey] = ni
|
||||
}
|
||||
|
||||
var results = []*NodeClockSkew{}
|
||||
for pubkey := range s.byNode {
|
||||
if areaNodes != nil && !areaNodes[pubkey] {
|
||||
continue
|
||||
}
|
||||
cs := s.getNodeClockSkewLocked(pubkey)
|
||||
if cs == nil {
|
||||
continue
|
||||
}
|
||||
// Enrich with node name/role.
|
||||
if ni, ok := nameMap[pubkey]; ok {
|
||||
cs.NodeName = ni.Name
|
||||
cs.NodeRole = ni.Role
|
||||
}
|
||||
// Omit samples and evidence in fleet response (too much data).
|
||||
cs.Samples = nil
|
||||
cs.RecentHashEvidence = nil
|
||||
cs.CalibrationSummary = nil
|
||||
results = append(results, cs)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// GetObserverCalibrations returns the current observer clock offsets,
|
||||
// preferring the steady-state recomputer snapshot (issue #1265). Falls
|
||||
// back to an on-request compute when the recomputer is not running.
|
||||
func (s *PacketStore) GetObserverCalibrations() []ObserverCalibration {
|
||||
s.analyticsRecomputerMu.RLock()
|
||||
rc := s.recompObserversClockSkew
|
||||
s.analyticsRecomputerMu.RUnlock()
|
||||
if rc != nil {
|
||||
if v := rc.Load(); v != nil {
|
||||
if r, ok := v.([]ObserverCalibration); ok {
|
||||
return r
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.computeObserverCalibrations()
|
||||
}
|
||||
|
||||
// computeObserverCalibrations is the underlying compute used by the
|
||||
// recomputer and on-request fallback. Must NOT be called with s.mu held.
|
||||
func (s *PacketStore) computeObserverCalibrations() []ObserverCalibration {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.clockSkew.Recompute(s)
|
||||
|
||||
s.clockSkew.mu.RLock()
|
||||
defer s.clockSkew.mu.RUnlock()
|
||||
|
||||
result := make([]ObserverCalibration, 0, len(s.clockSkew.observerOffsets))
|
||||
for obsID, offset := range s.clockSkew.observerOffsets {
|
||||
result = append(result, ObserverCalibration{
|
||||
ObserverID: obsID,
|
||||
OffsetSec: round(offset, 1),
|
||||
Samples: s.clockSkew.observerSamples[obsID],
|
||||
})
|
||||
}
|
||||
// Sort by absolute offset descending.
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return math.Abs(result[i].OffsetSec) > math.Abs(result[j].OffsetSec)
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Math Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
func median(vals []float64) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
sorted := make([]float64, len(vals))
|
||||
copy(sorted, vals)
|
||||
sort.Float64s(sorted)
|
||||
n := len(sorted)
|
||||
if n%2 == 0 {
|
||||
return (sorted[n/2-1] + sorted[n/2]) / 2
|
||||
}
|
||||
return sorted[n/2]
|
||||
}
|
||||
|
||||
// hashEvidenceMedian returns the median corrected skew for a single
|
||||
// transmission hash, filtering out RTC-reset outliers (|skew| > 24h —
|
||||
// firmware emitting factory timestamp). Issue #1285: a single outlier
|
||||
// observer was dragging the displayed median to ~-704d on an otherwise
|
||||
// healthy node. If filtering leaves zero usable samples (every observer
|
||||
// of this hash saw a reset-shaped advert), return 0 so the UI can render
|
||||
// "insufficient data" rather than the garbage outlier value.
|
||||
func hashEvidenceMedian(vals []float64) float64 {
|
||||
clean := vals[:0:0]
|
||||
for _, v := range vals {
|
||||
if math.Abs(v) <= rtcResetOutlierThresholdSec {
|
||||
clean = append(clean, v)
|
||||
}
|
||||
}
|
||||
return median(clean)
|
||||
}
|
||||
|
||||
func mean(vals []float64) float64 {
|
||||
if len(vals) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0.0
|
||||
for _, v := range vals {
|
||||
sum += v
|
||||
}
|
||||
return sum / float64(len(vals))
|
||||
}
|
||||
|
||||
// tsSkewPair is a (timestamp, skew) pair for drift estimation.
|
||||
type tsSkewPair struct {
|
||||
ts int64
|
||||
skew float64
|
||||
}
|
||||
|
||||
// computeDrift estimates linear drift in seconds per day from time-ordered
|
||||
// (timestamp, skew) pairs. Issue #789: a single GPS-correction event (huge
|
||||
// skew jump in seconds) used to dominate ordinary least squares and produce
|
||||
// absurd drift like 1.7M sec/day. We now:
|
||||
//
|
||||
// 1. Drop pairs whose consecutive skew jump exceeds maxPlausibleSkewJumpSec
|
||||
// (clock corrections, not physical drift). This protects both OLS-style
|
||||
// consumers and Theil-Sen.
|
||||
// 2. Use Theil-Sen regression — the slope is the median of all pairwise
|
||||
// slopes, naturally robust to remaining outliers (breakdown point ~29%).
|
||||
//
|
||||
// For very small samples after filtering we fall back to a simple slope
|
||||
// between first and last calibrated samples.
|
||||
func computeDrift(pairs []tsSkewPair) float64 {
|
||||
if len(pairs) < 2 {
|
||||
return 0
|
||||
}
|
||||
// Sort by timestamp.
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
return pairs[i].ts < pairs[j].ts
|
||||
})
|
||||
|
||||
// Time span too short? Skip.
|
||||
spanSec := float64(pairs[len(pairs)-1].ts - pairs[0].ts)
|
||||
if spanSec < 3600 { // need at least 1 hour of data
|
||||
return 0
|
||||
}
|
||||
|
||||
// Outlier filter: drop samples where the skew jumps more than
|
||||
// maxPlausibleSkewJumpSec from the running "stable" baseline.
|
||||
// We anchor on the first sample, then accept each subsequent point
|
||||
// that's within the threshold of the most recent accepted point —
|
||||
// this preserves a slow drift while rejecting correction events.
|
||||
filtered := make([]tsSkewPair, 0, len(pairs))
|
||||
filtered = append(filtered, pairs[0])
|
||||
for i := 1; i < len(pairs); i++ {
|
||||
prev := filtered[len(filtered)-1]
|
||||
if math.Abs(pairs[i].skew-prev.skew) <= maxPlausibleSkewJumpSec {
|
||||
filtered = append(filtered, pairs[i])
|
||||
}
|
||||
}
|
||||
// If the filter killed too much (e.g. unstable node), fall back to the
|
||||
// raw series so we at least produce *something* — it'll be capped by
|
||||
// maxReasonableDriftPerDay downstream.
|
||||
if len(filtered) < 2 || float64(filtered[len(filtered)-1].ts-filtered[0].ts) < 3600 {
|
||||
filtered = pairs
|
||||
}
|
||||
|
||||
// Cap point count for Theil-Sen (O(n²) on pairs). Keep most-recent.
|
||||
if len(filtered) > theilSenMaxPoints {
|
||||
filtered = filtered[len(filtered)-theilSenMaxPoints:]
|
||||
}
|
||||
|
||||
return theilSenSlope(filtered) * 86400 // sec/sec → sec/day
|
||||
}
|
||||
|
||||
// theilSenSlope returns the Theil-Sen estimator: median of all pairwise
|
||||
// slopes (yj - yi) / (tj - ti) for i < j. Naturally robust to outliers.
|
||||
// Pairs must be sorted by timestamp ascending.
|
||||
func theilSenSlope(pairs []tsSkewPair) float64 {
|
||||
n := len(pairs)
|
||||
if n < 2 {
|
||||
return 0
|
||||
}
|
||||
// Pre-allocate: n*(n-1)/2 pairs.
|
||||
slopes := make([]float64, 0, n*(n-1)/2)
|
||||
for i := 0; i < n; i++ {
|
||||
for j := i + 1; j < n; j++ {
|
||||
dt := float64(pairs[j].ts - pairs[i].ts)
|
||||
if dt <= 0 {
|
||||
continue
|
||||
}
|
||||
slopes = append(slopes, (pairs[j].skew-pairs[i].skew)/dt)
|
||||
}
|
||||
}
|
||||
if len(slopes) == 0 {
|
||||
return 0
|
||||
}
|
||||
return median(slopes)
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression tests for #1285:
|
||||
//
|
||||
// Bug A: per-hash evidence's MedianCorrectedSkewSec includes 700-day RTC-reset
|
||||
// outliers, dragging the displayed median into "−704d 18h" garbage even
|
||||
// though every recent sample is small (< 30s).
|
||||
//
|
||||
// Bug B: RecentBadSampleCount counts samples outside the *displayed* recent
|
||||
// window (or counts against raw skew, not corrected) → "3 of last 5
|
||||
// adverts had nonsense timestamps" warning fires on healthy nodes.
|
||||
//
|
||||
// Both must pass without disturbing the existing bimodal / no-clock logic.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Synthesizes the repro from issue #1285:
|
||||
// 30 healthy adverts (skew ~−20s) + 1 historical advert with an RTC reset
|
||||
// (advertTS = 2024-06-13, observed today → −705d skew).
|
||||
// Returns the populated store.
|
||||
func seedIssue1285Repro(t *testing.T) *PacketStore {
|
||||
t.Helper()
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4 // ADVERT
|
||||
|
||||
const pubkey = "RTCRESET"
|
||||
const skewSec = int64(-20) // node clock is 20s BEHIND wall-clock
|
||||
|
||||
baseObs := int64(1779000000) // ~mid-2026
|
||||
rtcResetAdv := int64(1718281640) // 2024-06-13 (from the issue repro)
|
||||
|
||||
var txs []*StoreTx
|
||||
|
||||
// 30 healthy adverts spanning the older end of the recent window.
|
||||
for i := 0; i < 30; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS + skewSec
|
||||
tx := &StoreTx{
|
||||
Hash: "healthy-" + formatInt64(int64(i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
}
|
||||
|
||||
// One RTC-reset packet observed MOST RECENTLY (so it sits in the
|
||||
// per-hash evidence list AND is included in the recent-window count
|
||||
// on master). Its advertTS is from 2024 → corrected skew ≈ -60M sec.
|
||||
rtcResetObs := baseObs + int64(30*60) + 60
|
||||
rtcTx := &StoreTx{
|
||||
Hash: "rtc-reset-0001",
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(rtcResetAdv) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(rtcResetObs, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
}
|
||||
txs = append(txs, rtcTx)
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode[pubkey] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
return ps
|
||||
}
|
||||
|
||||
// Bug A — per-hash evidence median must EXCLUDE the 705-day RTC-reset outlier.
|
||||
// On master this asserts on the RTC-reset hash's MedianCorrectedSkewSec being
|
||||
// ≈ −60M sec ("−704d 18h"); after the fix the field is suppressed (0) or
|
||||
// otherwise marked insufficient, never displayed as the garbage value.
|
||||
func TestIssue1285_HashEvidence_OutlierExcludedFromMedian(t *testing.T) {
|
||||
ps := seedIssue1285Repro(t)
|
||||
r := ps.GetNodeClockSkew("RTCRESET")
|
||||
if r == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
|
||||
// The recent-hash evidence list is the source of the "median corrected:
|
||||
// −704d 18h" string in the UI. After the fix, NO entry in this list
|
||||
// should report a |median| above the 24h sanity threshold — the fix is
|
||||
// to drop outlier samples (or flag the hash as insufficient-data) before
|
||||
// publishing the median.
|
||||
const maxSaneAbsSec = float64(24 * 3600)
|
||||
for _, ev := range r.RecentHashEvidence {
|
||||
if abs(ev.MedianCorrectedSkewSec) > maxSaneAbsSec {
|
||||
t.Errorf("hash %s exposes outlier-dominated median %.0fs (~%.1fd); "+
|
||||
"expected entry to be filtered out or marked insufficient (|median| <= %.0fs)",
|
||||
ev.Hash, ev.MedianCorrectedSkewSec,
|
||||
ev.MedianCorrectedSkewSec/86400, maxSaneAbsSec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bug B — RecentBadSampleCount must be 0 when every sample in the recent
|
||||
// window is healthy (<30s |corrected skew|). On master this fires because
|
||||
// "recent" is computed over the wrong set (or against raw skew).
|
||||
func TestIssue1285_RecentBadCount_NotPollutedByOldOutlier(t *testing.T) {
|
||||
ps := seedIssue1285Repro(t)
|
||||
r := ps.GetNodeClockSkew("RTCRESET")
|
||||
if r == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
if r.RecentBadSampleCount != 0 {
|
||||
t.Errorf("RecentBadSampleCount = %d, want 0 — recent samples are all "+
|
||||
"~−20s (healthy); the historical RTC-reset outlier is outside the "+
|
||||
"recent window and must not be counted", r.RecentBadSampleCount)
|
||||
}
|
||||
if r.Severity == SkewBimodalClock || r.Severity == SkewNoClock {
|
||||
t.Errorf("severity = %v, want ok/warning — recent samples are all "+
|
||||
"healthy (~−20s skew), one historical outlier must not flip the node "+
|
||||
"to bimodal/no-clock", r.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func abs(v float64) float64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue #1265: /api/observers/clock-skew (3.3s) and /api/nodes/clock-skew (8.9s)
|
||||
// must be wired into the steady-state analytics recomputer so reads serve
|
||||
// from an atomic-pointer snapshot in <100ms p99 under concurrent load.
|
||||
|
||||
func TestClockSkewRecomputersRegistered(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
|
||||
stop := store.StartAnalyticsRecomputers(50 * time.Millisecond)
|
||||
defer stop()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
store.analyticsRecomputerMu.RLock()
|
||||
rcObs := store.recompObserversClockSkew
|
||||
rcNodes := store.recompNodesClockSkew
|
||||
store.analyticsRecomputerMu.RUnlock()
|
||||
|
||||
if rcObs == nil {
|
||||
t.Fatalf("recompObserversClockSkew not registered after StartAnalyticsRecomputers (issue #1265 not fixed)")
|
||||
}
|
||||
if rcNodes == nil {
|
||||
t.Fatalf("recompNodesClockSkew not registered after StartAnalyticsRecomputers (issue #1265 not fixed)")
|
||||
}
|
||||
if rcObs.Load() == nil {
|
||||
t.Fatalf("recompObserversClockSkew snapshot is nil after initial compute")
|
||||
}
|
||||
if rcNodes.Load() == nil {
|
||||
t.Fatalf("recompNodesClockSkew snapshot is nil after initial compute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClockSkewHandlersSteadyStateLatency(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
stop := store.StartAnalyticsRecomputers(50 * time.Millisecond)
|
||||
defer stop()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
s := &Server{store: store}
|
||||
|
||||
endpoints := []struct {
|
||||
name string
|
||||
path string
|
||||
handler http.HandlerFunc
|
||||
}{
|
||||
{"observers", "/api/observers/clock-skew", s.handleObserverClockSkew},
|
||||
{"nodes", "/api/nodes/clock-skew", s.handleFleetClockSkew},
|
||||
}
|
||||
|
||||
for _, ep := range endpoints {
|
||||
ep := ep
|
||||
t.Run(ep.name, func(t *testing.T) {
|
||||
const readers = 8
|
||||
const perReader = 25
|
||||
var (
|
||||
mu sync.Mutex
|
||||
samples []time.Duration
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
wg.Add(readers)
|
||||
for r := 0; r < readers; r++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < perReader; i++ {
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, ep.path, nil)
|
||||
t0 := time.Now()
|
||||
ep.handler(rr, req)
|
||||
dt := time.Since(t0)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Errorf("%s status = %d, want 200", ep.path, rr.Code)
|
||||
}
|
||||
mu.Lock()
|
||||
samples = append(samples, dt)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] })
|
||||
p99 := samples[int(float64(len(samples))*0.99)]
|
||||
if p99 > 100*time.Millisecond {
|
||||
t.Fatalf("%s p99 latency = %v over %d reqs, want <100ms (recomputer snapshot)", ep.path, p99, len(samples))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCollisionDetailsIncludeNodePairs verifies that collision details contain
|
||||
// the correct prefix and matching node pairs (#757).
|
||||
func TestCollisionDetailsIncludeNodePairs(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Insert two repeater nodes with the same 3-byte prefix "AABB11"
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Node Alpha', 'repeater')`)
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11eeff334455', 'Node Beta', 'repeater')`)
|
||||
|
||||
// Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3)
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
result := store.GetAnalyticsHashCollisions("", "")
|
||||
bySize, ok := result["by_size"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size map")
|
||||
}
|
||||
|
||||
size3, ok := bySize["3"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size[3] map")
|
||||
}
|
||||
|
||||
collisions, ok := size3["collisions"].([]collisionEntry)
|
||||
if !ok {
|
||||
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
|
||||
}
|
||||
|
||||
// Find our collision
|
||||
var found *collisionEntry
|
||||
for i := range collisions {
|
||||
if collisions[i].Prefix == "AABB11" {
|
||||
found = &collisions[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatal("expected collision with prefix AABB11")
|
||||
}
|
||||
if found.Appearances != 2 {
|
||||
t.Errorf("expected 2 appearances, got %d", found.Appearances)
|
||||
}
|
||||
if len(found.Nodes) != 2 {
|
||||
t.Fatalf("expected 2 nodes in collision, got %d", len(found.Nodes))
|
||||
}
|
||||
|
||||
// Verify node pairs
|
||||
pubkeys := map[string]bool{}
|
||||
names := map[string]bool{}
|
||||
for _, n := range found.Nodes {
|
||||
pubkeys[n.PublicKey] = true
|
||||
names[n.Name] = true
|
||||
}
|
||||
if !pubkeys["aabb11ccdd001122"] {
|
||||
t.Error("expected node aabb11ccdd001122 in collision")
|
||||
}
|
||||
if !pubkeys["aabb11eeff334455"] {
|
||||
t.Error("expected node aabb11eeff334455 in collision")
|
||||
}
|
||||
if !names["Node Alpha"] {
|
||||
t.Error("expected Node Alpha in collision")
|
||||
}
|
||||
if !names["Node Beta"] {
|
||||
t.Error("expected Node Beta in collision")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollisionDetailsEmptyWhenNoCollisions verifies that collision details are
|
||||
// empty when there are no collisions (#757).
|
||||
func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
now := time.Now().UTC()
|
||||
recent := now.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
recentEpoch := now.Add(-1 * time.Hour).Unix()
|
||||
|
||||
// Insert one repeater node with 3-byte hash
|
||||
db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`)
|
||||
|
||||
db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json)
|
||||
VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent)
|
||||
db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch)
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
|
||||
result := store.GetAnalyticsHashCollisions("", "")
|
||||
bySize, ok := result["by_size"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size map")
|
||||
}
|
||||
|
||||
size3, ok := bySize["3"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected by_size[3] map")
|
||||
}
|
||||
|
||||
collisions, ok := size3["collisions"].([]collisionEntry)
|
||||
if !ok {
|
||||
t.Fatalf("expected collisions as []collisionEntry, got %T", size3["collisions"])
|
||||
}
|
||||
|
||||
if len(collisions) != 0 {
|
||||
t.Errorf("expected 0 collisions, got %d", len(collisions))
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// gzipWriterPool pools *gzip.Writer instances to avoid the ~256KB sliding
|
||||
// window allocation on every compressed response. Writers are Reset() to the
|
||||
// new underlying writer on Get and returned via gzipPut after Close.
|
||||
//
|
||||
// We use a bounded buffered channel rather than sync.Pool because sync.Pool
|
||||
// is aggressively reaped by the GC (full clear after two GC cycles), which
|
||||
// makes it lose its pooled entries under any workload that triggers GC —
|
||||
// notably the -race-enabled test suite where allocations are inflated ~8x
|
||||
// and GC fires repeatedly during a 200-request loop. A channel keeps the
|
||||
// gzip.Writer instances live across GC cycles, which is exactly the
|
||||
// guarantee `TestGZipMiddleware_PoolReusesWriters` asserts.
|
||||
const gzipPoolCapacity = 64
|
||||
|
||||
var gzipWriterPool = make(chan *gzip.Writer, gzipPoolCapacity)
|
||||
|
||||
func gzipGet() *gzip.Writer {
|
||||
select {
|
||||
case gz := <-gzipWriterPool:
|
||||
return gz
|
||||
default:
|
||||
// gzip.NewWriterLevel only errors on invalid level; DefaultCompression
|
||||
// is always valid, so the error branch is unreachable. Fall back to
|
||||
// the default writer (same level) so we always return a usable writer.
|
||||
gz, err := gzip.NewWriterLevel(discardWriter{}, gzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return gzip.NewWriter(discardWriter{})
|
||||
}
|
||||
return gz
|
||||
}
|
||||
}
|
||||
|
||||
func gzipPut(gz *gzip.Writer) {
|
||||
// Reset to a no-op writer so the pooled instance does not retain a
|
||||
// reference to the previous http.ResponseWriter (which would defeat GC
|
||||
// of the request's allocations).
|
||||
gz.Reset(discardWriter{})
|
||||
select {
|
||||
case gzipWriterPool <- gz:
|
||||
default:
|
||||
// Pool full; drop the writer and let GC reclaim it.
|
||||
}
|
||||
}
|
||||
|
||||
type discardWriter struct{}
|
||||
|
||||
func (discardWriter) Write(p []byte) (int, error) { return len(p), nil }
|
||||
|
||||
// defaultCompressibleTypes is the conservative allow-list of MIME types the
|
||||
// middleware will gzip-encode. Anything already compressed (images, video,
|
||||
// fonts, octet-stream, x-gzip, …) bypasses the encoder entirely.
|
||||
var defaultCompressibleTypes = []string{
|
||||
"application/json",
|
||||
"application/javascript",
|
||||
"application/x-javascript",
|
||||
"application/xml",
|
||||
"text/html",
|
||||
"text/css",
|
||||
"text/plain",
|
||||
"text/xml",
|
||||
"image/svg+xml",
|
||||
}
|
||||
|
||||
// gzipResponseWriter wraps http.ResponseWriter and compresses Write() output
|
||||
// only when the response Content-Type matches the configured allow-list and
|
||||
// no upstream handler has already set Content-Encoding. It also propagates
|
||||
// Flush / Hijack to the underlying writer (required for SSE and WebSocket).
|
||||
type gzipResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
gz *gzip.Writer
|
||||
level int
|
||||
allowedTypes []string
|
||||
wroteHeader bool
|
||||
compressActive bool
|
||||
}
|
||||
|
||||
// init lazily decides per response whether to compress, based on the response
|
||||
// headers the inner handler has set. We must defer this until WriteHeader (or
|
||||
// the first Write call) because Content-Type is set by the handler, not the
|
||||
// middleware.
|
||||
func (g *gzipResponseWriter) init() {
|
||||
if g.wroteHeader {
|
||||
return
|
||||
}
|
||||
g.wroteHeader = true
|
||||
|
||||
h := g.ResponseWriter.Header()
|
||||
// Don't double-encode.
|
||||
if h.Get("Content-Encoding") != "" {
|
||||
g.compressActive = false
|
||||
return
|
||||
}
|
||||
if !isCompressibleContentType(h.Get("Content-Type"), g.allowedTypes) {
|
||||
g.compressActive = false
|
||||
return
|
||||
}
|
||||
|
||||
// Lease a writer from the pool and rebind it to the real ResponseWriter.
|
||||
gz := gzipGet()
|
||||
gz.Reset(g.ResponseWriter)
|
||||
g.gz = gz
|
||||
g.compressActive = true
|
||||
|
||||
h.Set("Content-Encoding", "gzip")
|
||||
h.Add("Vary", "Accept-Encoding")
|
||||
// gzip stream length is unknown — strip any precomputed length.
|
||||
h.Del("Content-Length")
|
||||
}
|
||||
|
||||
func (g *gzipResponseWriter) WriteHeader(code int) {
|
||||
g.init()
|
||||
g.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (g *gzipResponseWriter) Write(b []byte) (int, error) {
|
||||
g.init()
|
||||
if !g.compressActive {
|
||||
return g.ResponseWriter.Write(b)
|
||||
}
|
||||
return g.gz.Write(b)
|
||||
}
|
||||
|
||||
// Flush propagates to the underlying writer so SSE / streaming handlers can
|
||||
// push chunks to the client immediately. We must also flush the gzip writer
|
||||
// when active, otherwise the buffered DEFLATE block never reaches the wire.
|
||||
func (g *gzipResponseWriter) Flush() {
|
||||
if g.compressActive && g.gz != nil {
|
||||
_ = g.gz.Flush()
|
||||
}
|
||||
if f, ok := g.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Hijack delegates to the underlying writer's Hijacker. We refuse to hijack a
|
||||
// connection that has already started a gzip stream — that would leave the
|
||||
// caller with a half-written DEFLATE block.
|
||||
func (g *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
if h, ok := g.ResponseWriter.(http.Hijacker); ok {
|
||||
return h.Hijack()
|
||||
}
|
||||
return nil, nil, http.ErrNotSupported
|
||||
}
|
||||
|
||||
// close releases the pooled gzip.Writer back to the pool.
|
||||
func (g *gzipResponseWriter) close() {
|
||||
if g.gz == nil {
|
||||
return
|
||||
}
|
||||
_ = g.gz.Close()
|
||||
gzipPut(g.gz)
|
||||
g.gz = nil
|
||||
}
|
||||
|
||||
// isCompressibleContentType returns true if ct matches one of allow (which
|
||||
// is the configured allow-list, or defaultCompressibleTypes). Matching is
|
||||
// done on the bare MIME type, ignoring any "; charset=..." parameters.
|
||||
func isCompressibleContentType(ct string, allow []string) bool {
|
||||
if ct == "" {
|
||||
// No content-type set → handler hasn't decided yet. Refuse to
|
||||
// compress; we cannot guess. Most real handlers set Content-Type
|
||||
// before the first Write.
|
||||
return false
|
||||
}
|
||||
mt := ct
|
||||
if idx := strings.Index(mt, ";"); idx >= 0 {
|
||||
mt = mt[:idx]
|
||||
}
|
||||
mt = strings.TrimSpace(strings.ToLower(mt))
|
||||
|
||||
// Hard skip: anything that is already compressed.
|
||||
if strings.HasPrefix(mt, "image/") && mt != "image/svg+xml" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(mt, "video/") || strings.HasPrefix(mt, "audio/") {
|
||||
return false
|
||||
}
|
||||
switch mt {
|
||||
case "application/x-gzip", "application/gzip", "application/zip",
|
||||
"application/x-bzip2", "application/x-7z-compressed",
|
||||
"application/x-rar-compressed", "application/x-zstd",
|
||||
"application/octet-stream", "application/pdf":
|
||||
return false
|
||||
}
|
||||
|
||||
if len(allow) == 0 {
|
||||
allow = defaultCompressibleTypes
|
||||
}
|
||||
for _, a := range allow {
|
||||
if strings.EqualFold(mt, a) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// gzipMiddleware compresses HTTP responses when the client supports gzip and
|
||||
// the response Content-Type is in the allow-list. WebSocket upgrade requests
|
||||
// pass through unmodified. The middleware uses the default allow-list and
|
||||
// gzip.DefaultCompression — for configurable behaviour use
|
||||
// gzipMiddlewareWithConfig.
|
||||
func gzipMiddleware(next http.Handler) http.Handler {
|
||||
return gzipMiddlewareWithConfig(nil, next)
|
||||
}
|
||||
|
||||
// gzipMiddlewareWithConfig is the configurable form of gzipMiddleware. When
|
||||
// cfg is nil, defaults (gzip.DefaultCompression, defaultCompressibleTypes)
|
||||
// are used.
|
||||
func gzipMiddlewareWithConfig(cfg *CompressionConfig, next http.Handler) http.Handler {
|
||||
level := gzip.DefaultCompression
|
||||
var allow []string
|
||||
if cfg != nil {
|
||||
if cfg.Level >= gzip.BestSpeed && cfg.Level <= gzip.BestCompression {
|
||||
level = cfg.Level
|
||||
}
|
||||
if len(cfg.ContentTypes) > 0 {
|
||||
allow = cfg.ContentTypes
|
||||
}
|
||||
}
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if strings.EqualFold(r.Header.Get("Upgrade"), "websocket") {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
grw := &gzipResponseWriter{
|
||||
ResponseWriter: w,
|
||||
level: level,
|
||||
allowedTypes: allow,
|
||||
}
|
||||
defer grw.close()
|
||||
next.ServeHTTP(grw, r)
|
||||
})
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests added in response to PR #934 review findings. These tests demonstrate
|
||||
// the four behaviors the original implementation lacked:
|
||||
//
|
||||
// 1. gzipResponseWriter must implement http.Flusher (SSE / streaming).
|
||||
// 2. gzipResponseWriter must implement http.Hijacker (WebSocket / raw conn).
|
||||
// 3. gzip.Writer instances must be pooled (sync.Pool) to avoid the
|
||||
// ~256KB window allocation per request.
|
||||
// 4. A content-type allow-list must skip already-compressed payloads
|
||||
// (images, video, application/x-gzip, …) and must skip responses
|
||||
// whose handler already set its own Content-Encoding header.
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGZipResponseWriter_ImplementsFlusher(t *testing.T) {
|
||||
seen := false
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := w.(http.Flusher); ok {
|
||||
seen = true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
req := httptest.NewRequest("GET", "/api/events", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
if !seen {
|
||||
t.Error("gzipResponseWriter must implement http.Flusher (required for SSE / streaming endpoints)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipResponseWriter_ImplementsHijacker(t *testing.T) {
|
||||
seen := false
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := w.(http.Hijacker); ok {
|
||||
seen = true
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{}`))
|
||||
}))
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
req, _ := http.NewRequest("GET", srv.URL+"/api/x", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if !seen {
|
||||
t.Error("gzipResponseWriter must implement http.Hijacker (required for raw conn / WebSocket upgrade)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddleware_SkipsImageContentType(t *testing.T) {
|
||||
payload := strings.Repeat("\x89PNGfakebinary", 64)
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/png")
|
||||
w.Write([]byte(payload))
|
||||
}))
|
||||
req := httptest.NewRequest("GET", "/tiles/1.png", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if got := rr.Header().Get("Content-Encoding"); got == "gzip" {
|
||||
t.Errorf("image/png responses must NOT be gzip-encoded, got Content-Encoding=%q", got)
|
||||
}
|
||||
if rr.Body.String() != payload {
|
||||
t.Errorf("image body was mutated; expected pass-through")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddleware_SkipsAlreadyEncodedResponses(t *testing.T) {
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Content-Encoding", "br")
|
||||
w.Write([]byte("alreadybrotlied"))
|
||||
}))
|
||||
req := httptest.NewRequest("GET", "/api/x", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if got := rr.Header().Get("Content-Encoding"); got != "br" {
|
||||
t.Errorf("handler-set Content-Encoding must be preserved, got %q (gzip middleware double-wrapped)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddleware_AllowsJSON(t *testing.T) {
|
||||
body := `{"nodes":[{"id":"abc"}]}`
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Write([]byte(body))
|
||||
}))
|
||||
req := httptest.NewRequest("GET", "/api/nodes", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
if rr.Header().Get("Content-Encoding") != "gzip" {
|
||||
t.Fatalf("application/json must still be compressed, got %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
gz, err := gzip.NewReader(rr.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("invalid gzip: %v", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
decoded, _ := io.ReadAll(gz)
|
||||
if string(decoded) != body {
|
||||
t.Errorf("decoded=%q, want %q", string(decoded), body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddleware_PoolReusesWriters(t *testing.T) {
|
||||
body := strings.Repeat("x", 1024)
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(body))
|
||||
}))
|
||||
// Warm the pool: first N requests pay the one-time allocation cost.
|
||||
for i := 0; i < 16; i++ {
|
||||
req := httptest.NewRequest("GET", "/api", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
}
|
||||
runtime.GC()
|
||||
var before runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
|
||||
const N = 200
|
||||
for i := 0; i < N; i++ {
|
||||
req := httptest.NewRequest("GET", "/api", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
}
|
||||
var after runtime.MemStats
|
||||
runtime.ReadMemStats(&after)
|
||||
allocBytes := after.TotalAlloc - before.TotalAlloc
|
||||
|
||||
// Each gzip.Writer carries a ~256KB sliding window. Without a sync.Pool,
|
||||
// N=200 requests allocate roughly N * 256KB = 50MB. With a pool the
|
||||
// per-request alloc footprint should be a tiny fraction of that.
|
||||
// 10MB ceiling gives generous headroom for testing.AllocsPerRun noise
|
||||
// while still catching a regression to the unpooled implementation.
|
||||
if allocBytes > 10*1024*1024 {
|
||||
t.Errorf("gzip.Writer not pooled: %d bytes allocated across %d requests (expected ≤10MB)", allocBytes, N)
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCompressionConfigDefaults(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if cfg.GZipEnabled() {
|
||||
t.Error("GZipEnabled should be false when compression is nil")
|
||||
}
|
||||
if cfg.WSCompressionEnabled() {
|
||||
t.Error("WSCompressionEnabled should be false when compression is nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressionConfigExplicitFalse(t *testing.T) {
|
||||
cfg := &Config{Compression: &CompressionConfig{GZip: false, Websocket: false}}
|
||||
if cfg.GZipEnabled() {
|
||||
t.Error("GZipEnabled should be false")
|
||||
}
|
||||
if cfg.WSCompressionEnabled() {
|
||||
t.Error("WSCompressionEnabled should be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressionConfigEnabled(t *testing.T) {
|
||||
cfg := &Config{Compression: &CompressionConfig{GZip: true, Websocket: true}}
|
||||
if !cfg.GZipEnabled() {
|
||||
t.Error("GZipEnabled should be true")
|
||||
}
|
||||
if !cfg.WSCompressionEnabled() {
|
||||
t.Error("WSCompressionEnabled should be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddlewareCompresses(t *testing.T) {
|
||||
body := `{"nodes":[{"id":"abc"}]}`
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(body))
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Header().Get("Content-Encoding") != "gzip" {
|
||||
t.Errorf("expected Content-Encoding: gzip, got %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
if rr.Header().Get("Vary") != "Accept-Encoding" {
|
||||
t.Errorf("expected Vary: Accept-Encoding, got %q", rr.Header().Get("Vary"))
|
||||
}
|
||||
gz, err := gzip.NewReader(rr.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("response is not valid gzip: %v", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
decoded, err := io.ReadAll(gz)
|
||||
if err != nil {
|
||||
t.Fatalf("reading gzip: %v", err)
|
||||
}
|
||||
if string(decoded) != body {
|
||||
t.Errorf("decompressed body = %q, want %q", string(decoded), body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddlewareSkipsNoAcceptEncoding(t *testing.T) {
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("hello"))
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Header().Get("Content-Encoding") != "" {
|
||||
t.Errorf("expected no Content-Encoding, got %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
if rr.Body.String() != "hello" {
|
||||
t.Errorf("expected plain body, got %q", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGZipMiddlewareSkipsWebSocket(t *testing.T) {
|
||||
called := false
|
||||
handler := gzipMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
w.Write([]byte("ws"))
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("GET", "/ws", nil)
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
req.Header.Set("Upgrade", "websocket")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if !called {
|
||||
t.Error("expected next handler to be called")
|
||||
}
|
||||
if rr.Header().Get("Content-Encoding") != "" {
|
||||
t.Errorf("WebSocket should not be gzip-encoded, got %q", rr.Header().Get("Content-Encoding"))
|
||||
}
|
||||
}
|
||||
+4
-365
@@ -2,45 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
)
|
||||
|
||||
// AreaEntry defines a geographic area by polygon or bounding box.
|
||||
type AreaEntry struct {
|
||||
Label string `json:"label"`
|
||||
Polygon [][2]float64 `json:"polygon,omitempty"`
|
||||
LatMin *float64 `json:"latMin,omitempty"`
|
||||
LatMax *float64 `json:"latMax,omitempty"`
|
||||
LonMin *float64 `json:"lonMin,omitempty"`
|
||||
LonMax *float64 `json:"lonMax,omitempty"`
|
||||
}
|
||||
|
||||
// Config mirrors the Node.js config.json structure (read-only fields).
|
||||
type Config struct {
|
||||
Port int `json:"port"`
|
||||
APIKey string `json:"apiKey"`
|
||||
DBPath string `json:"dbPath"`
|
||||
|
||||
// NodeBlacklist is a list of public keys to exclude from all API responses.
|
||||
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
|
||||
// Use this to filter out trolls, nodes with offensive names, or nodes
|
||||
// reporting deliberately false data (e.g. wrong GPS position) that the
|
||||
// operator refuses to fix.
|
||||
NodeBlacklist []string `json:"nodeBlacklist"`
|
||||
|
||||
// blacklistSetCached is the lazily-built set version of NodeBlacklist.
|
||||
blacklistSetCached map[string]bool
|
||||
blacklistOnce sync.Once
|
||||
|
||||
Branding map[string]interface{} `json:"branding"`
|
||||
Theme map[string]interface{} `json:"theme"`
|
||||
ThemeDark map[string]interface{} `json:"themeDark"`
|
||||
@@ -75,187 +50,29 @@ type Config struct {
|
||||
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
|
||||
PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
|
||||
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
|
||||
Areas map[string]AreaEntry `json:"areas,omitempty"`
|
||||
|
||||
Timestamps *TimestampConfig `json:"timestamps,omitempty"`
|
||||
|
||||
// CORSAllowedOrigins is the list of origins permitted to make cross-origin
|
||||
// requests. When empty (default), no Access-Control-* headers are sent,
|
||||
// so browsers enforce same-origin policy. Set to ["*"] to allow all origins.
|
||||
CORSAllowedOrigins []string `json:"corsAllowedOrigins,omitempty"`
|
||||
|
||||
DebugAffinity bool `json:"debugAffinity,omitempty"`
|
||||
|
||||
// MapDarkTileProvider selects the default dark-mode basemap provider for
|
||||
// new visitors. The client may override per-browser via the customizer
|
||||
// (persisted to localStorage). Allowed values: "carto-dark" (default),
|
||||
// "esri-darkgray-labels", "voyager-inverted", "positron-inverted". See
|
||||
// public/map-tile-providers.js for the registry. #1420.
|
||||
MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"`
|
||||
|
||||
// ObserverBlacklist is a list of observer public keys to exclude from API
|
||||
// responses (defense in depth — ingestor drops at ingest, server filters
|
||||
// any that slipped through from a prior unblocked window).
|
||||
ObserverBlacklist []string `json:"observerBlacklist,omitempty"`
|
||||
|
||||
// obsBlacklistSetCached is the lazily-built set version of ObserverBlacklist.
|
||||
obsBlacklistSetCached map[string]bool
|
||||
obsBlacklistOnce sync.Once
|
||||
|
||||
Compression *CompressionConfig `json:"compression,omitempty"`
|
||||
ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"`
|
||||
NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"`
|
||||
|
||||
// Analytics steady-state background recompute (issue #1240).
|
||||
Analytics *AnalyticsConfig `json:"analytics,omitempty"`
|
||||
|
||||
// BatteryThresholds: voltage cutoffs for low/critical alerts (#663).
|
||||
BatteryThresholds *BatteryThresholdsConfig `json:"batteryThresholds,omitempty"`
|
||||
}
|
||||
|
||||
// weakAPIKeys is the blocklist of known default/example API keys that must be rejected.
|
||||
var weakAPIKeys = map[string]bool{
|
||||
"your-secret-api-key-here": true,
|
||||
"change-me": true,
|
||||
"example": true,
|
||||
"test": true,
|
||||
"password": true,
|
||||
"admin": true,
|
||||
"apikey": true,
|
||||
"api-key": true,
|
||||
"secret": true,
|
||||
"default": true,
|
||||
}
|
||||
|
||||
// IsWeakAPIKey returns true if the key is in the blocklist or shorter than 16 characters.
|
||||
func IsWeakAPIKey(key string) bool {
|
||||
if key == "" {
|
||||
return false // empty is handled separately (endpoints disabled)
|
||||
}
|
||||
if weakAPIKeys[strings.ToLower(key)] {
|
||||
return true
|
||||
}
|
||||
if len(key) < 16 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CompressionConfig controls HTTP gzip and WebSocket permessage-deflate compression.
|
||||
// Both are disabled by default — enable only when the upstream proxy does not already compress.
|
||||
type CompressionConfig struct {
|
||||
GZip bool `json:"gzip"`
|
||||
Websocket bool `json:"websocket"`
|
||||
|
||||
// Level is the gzip compression level (1=BestSpeed … 9=BestCompression).
|
||||
// 0 / out-of-range means "use compress/gzip.DefaultCompression".
|
||||
Level int `json:"level,omitempty"`
|
||||
|
||||
// MinSizeBytes is an advisory minimum response size below which gzip
|
||||
// would not pay off. Currently informational — kept here so operators
|
||||
// can express intent and so future small-body fast-paths can use it.
|
||||
MinSizeBytes int `json:"minSizeBytes,omitempty"`
|
||||
|
||||
// ContentTypes overrides the default compressible-MIME allow-list. When
|
||||
// empty, a conservative default (application/json, text/html, text/css,
|
||||
// application/javascript, text/plain, image/svg+xml, application/xml)
|
||||
// is used. Already-compressed types (image/*, video/*, application/zip,
|
||||
// application/x-gzip, …) are always skipped.
|
||||
ContentTypes []string `json:"contentTypes,omitempty"`
|
||||
}
|
||||
|
||||
// GZipEnabled returns true when HTTP gzip compression is explicitly enabled.
|
||||
func (c *Config) GZipEnabled() bool {
|
||||
return c.Compression != nil && c.Compression.GZip
|
||||
}
|
||||
|
||||
// WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled.
|
||||
func (c *Config) WSCompressionEnabled() bool {
|
||||
return c.Compression != nil && c.Compression.Websocket
|
||||
}
|
||||
|
||||
// ResolvedPathConfig controls async backfill behavior.
|
||||
type ResolvedPathConfig struct {
|
||||
BackfillHours int `json:"backfillHours"` // how far back (hours) to scan for NULL resolved_path (default 24)
|
||||
}
|
||||
|
||||
// NeighborGraphConfig controls neighbor edge pruning.
|
||||
type NeighborGraphConfig struct {
|
||||
MaxAgeDays int `json:"maxAgeDays"` // edges older than this are pruned (default 5)
|
||||
MaxEdgeKm float64 `json:"maxEdgeKm"` // geo-implausibility threshold (km); 0 = default 500; negative disables (#1228)
|
||||
}
|
||||
|
||||
// PacketStoreConfig controls in-memory packet store limits.
|
||||
type PacketStoreConfig struct {
|
||||
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
MaxResolvedPubkeyIndexEntries int `json:"maxResolvedPubkeyIndexEntries"` // warning threshold for index size (0 = 5M default)
|
||||
HotStartupHours float64 `json:"hotStartupHours"` // load only this many hours synchronously; 0 = disabled
|
||||
RetentionHours float64 `json:"retentionHours"` // max age of packets in hours (0 = unlimited)
|
||||
MaxMemoryMB int `json:"maxMemoryMB"` // hard memory ceiling in MB (0 = unlimited)
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
type GeoFilterConfig = geofilter.Config
|
||||
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
NodeDays int `json:"nodeDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
type DBConfig = dbconfig.DBConfig
|
||||
|
||||
// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default.
|
||||
func (c *Config) IncrementalVacuumPages() int {
|
||||
if c.DB != nil && c.DB.IncrementalVacuumPages > 0 {
|
||||
return c.DB.IncrementalVacuumPages
|
||||
}
|
||||
return 1024
|
||||
}
|
||||
|
||||
// MetricsRetentionDays returns configured metrics retention or 30 days default.
|
||||
func (c *Config) MetricsRetentionDays() int {
|
||||
if c.Retention != nil && c.Retention.MetricsDays > 0 {
|
||||
return c.Retention.MetricsDays
|
||||
}
|
||||
return 30
|
||||
}
|
||||
|
||||
// BackfillHours returns configured backfill window or 24h default.
|
||||
func (c *Config) BackfillHours() int {
|
||||
if c.ResolvedPath != nil && c.ResolvedPath.BackfillHours > 0 {
|
||||
return c.ResolvedPath.BackfillHours
|
||||
}
|
||||
return 24
|
||||
}
|
||||
|
||||
// NeighborMaxAgeDays returns configured max edge age or 30 days default.
|
||||
func (c *Config) NeighborMaxAgeDays() int {
|
||||
if c.NeighborGraph != nil && c.NeighborGraph.MaxAgeDays > 0 {
|
||||
return c.NeighborGraph.MaxAgeDays
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// NeighborMaxEdgeKm returns the geo-implausibility threshold in km.
|
||||
// 0 (unset) → DefaultMaxEdgeKm (500). Negative → 0 (filter disabled).
|
||||
// See issue #1228.
|
||||
func (c *Config) NeighborMaxEdgeKm() float64 {
|
||||
if c == nil || c.NeighborGraph == nil || c.NeighborGraph.MaxEdgeKm == 0 {
|
||||
return DefaultMaxEdgeKm
|
||||
}
|
||||
if c.NeighborGraph.MaxEdgeKm < 0 {
|
||||
return 0
|
||||
}
|
||||
return c.NeighborGraph.MaxEdgeKm
|
||||
}
|
||||
|
||||
type TimestampConfig struct {
|
||||
DefaultMode string `json:"defaultMode"` // "ago" | "absolute"
|
||||
@@ -283,24 +100,11 @@ func (c *Config) NodeDaysOrDefault() int {
|
||||
return 7
|
||||
}
|
||||
|
||||
// ObserverDaysOrDefault returns the configured retention.observerDays or 14 if not set.
|
||||
// A value of -1 means observers are never removed.
|
||||
func (c *Config) ObserverDaysOrDefault() int {
|
||||
if c.Retention != nil && c.Retention.ObserverDays != 0 {
|
||||
return c.Retention.ObserverDays
|
||||
}
|
||||
return 14
|
||||
}
|
||||
|
||||
type HealthThresholds struct {
|
||||
InfraDegradedHours float64 `json:"infraDegradedHours"`
|
||||
InfraSilentHours float64 `json:"infraSilentHours"`
|
||||
NodeDegradedHours float64 `json:"nodeDegradedHours"`
|
||||
NodeSilentHours float64 `json:"nodeSilentHours"`
|
||||
// RelayActiveHours: how recent a path-hop appearance must be for a
|
||||
// repeater to be considered "actively relaying" vs only "alive
|
||||
// (advert-only)". See issue #662. Defaults to 24h.
|
||||
RelayActiveHours float64 `json:"relayActiveHours"`
|
||||
}
|
||||
|
||||
// ThemeFile mirrors theme.json overlay.
|
||||
@@ -369,7 +173,6 @@ func (c *Config) GetHealthThresholds() HealthThresholds {
|
||||
InfraSilentHours: 72,
|
||||
NodeDegradedHours: 1,
|
||||
NodeSilentHours: 24,
|
||||
RelayActiveHours: 24,
|
||||
}
|
||||
if c.HealthThresholds != nil {
|
||||
if c.HealthThresholds.InfraDegradedHours > 0 {
|
||||
@@ -384,9 +187,6 @@ func (c *Config) GetHealthThresholds() HealthThresholds {
|
||||
if c.HealthThresholds.NodeSilentHours > 0 {
|
||||
h.NodeSilentHours = c.HealthThresholds.NodeSilentHours
|
||||
}
|
||||
if c.HealthThresholds.RelayActiveHours > 0 {
|
||||
h.RelayActiveHours = c.HealthThresholds.RelayActiveHours
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -473,164 +273,3 @@ func (c *Config) PropagationBufferMs() int {
|
||||
}
|
||||
return 5000
|
||||
}
|
||||
|
||||
// blacklistSet lazily builds and caches the nodeBlacklist as a set for O(1) lookups.
|
||||
// Uses sync.Once to eliminate the data race on first concurrent access.
|
||||
func (c *Config) blacklistSet() map[string]bool {
|
||||
c.blacklistOnce.Do(func() {
|
||||
if len(c.NodeBlacklist) == 0 {
|
||||
return
|
||||
}
|
||||
m := make(map[string]bool, len(c.NodeBlacklist))
|
||||
for _, pk := range c.NodeBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.blacklistSetCached = m
|
||||
})
|
||||
return c.blacklistSetCached
|
||||
}
|
||||
|
||||
// IsBlacklisted returns true if the given public key is in the nodeBlacklist.
|
||||
func (c *Config) IsBlacklisted(pubkey string) bool {
|
||||
if c == nil || len(c.NodeBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
|
||||
}
|
||||
|
||||
// SaveGeoFilter writes the geo_filter section back to config.json on disk.
|
||||
// Pass gf=nil to remove the filter. The rest of config.json is preserved as-is.
|
||||
func SaveGeoFilter(configDir string, gf *GeoFilterConfig) error {
|
||||
var configPath string
|
||||
for _, p := range []string{
|
||||
filepath.Join(configDir, "config.json"),
|
||||
filepath.Join(configDir, "data", "config.json"),
|
||||
} {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
configPath = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if configPath == "" {
|
||||
return fmt.Errorf("config.json not found in %s", configDir)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read config: %w", err)
|
||||
}
|
||||
|
||||
// Parse as a raw map so non-struct fields (_comment, etc.) are preserved.
|
||||
var raw map[string]interface{}
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
|
||||
if gf == nil || len(gf.Polygon) == 0 {
|
||||
delete(raw, "geo_filter")
|
||||
} else {
|
||||
// Round-trip through JSON to get a plain interface{} value.
|
||||
b, _ := json.Marshal(gf)
|
||||
var v interface{}
|
||||
_ = json.Unmarshal(b, &v)
|
||||
raw["geo_filter"] = v
|
||||
}
|
||||
|
||||
out, err := json.MarshalIndent(raw, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal config: %w", err)
|
||||
}
|
||||
out = append(out, '\n')
|
||||
|
||||
// Atomic write: temp file + rename.
|
||||
tmp := configPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, out, 0644); err != nil {
|
||||
return fmt.Errorf("write config: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, configPath); err != nil {
|
||||
os.Remove(tmp)
|
||||
return fmt.Errorf("rename config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// obsBlacklistSet lazily builds and caches the observerBlacklist as a set for O(1) lookups.
|
||||
func (c *Config) obsBlacklistSet() map[string]bool {
|
||||
c.obsBlacklistOnce.Do(func() {
|
||||
if len(c.ObserverBlacklist) == 0 {
|
||||
return
|
||||
}
|
||||
m := make(map[string]bool, len(c.ObserverBlacklist))
|
||||
for _, pk := range c.ObserverBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.obsBlacklistSetCached = m
|
||||
})
|
||||
return c.obsBlacklistSetCached
|
||||
}
|
||||
|
||||
// IsObserverBlacklisted returns true if the given observer ID is in the observerBlacklist.
|
||||
func (c *Config) IsObserverBlacklisted(id string) bool {
|
||||
if c == nil || len(c.ObserverBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
return c.obsBlacklistSet()[strings.ToLower(strings.TrimSpace(id))]
|
||||
}
|
||||
|
||||
// AnalyticsConfig controls steady-state background recompute of
|
||||
// analytics endpoints (issue #1240).
|
||||
//
|
||||
// DefaultIntervalSeconds applies to every endpoint that does not have
|
||||
// an explicit per-endpoint override in RecomputeIntervalSeconds. The
|
||||
// project default is 300 (5 minutes): the operator's guiding principle
|
||||
// is "serving slightly stale data quickly is better than real-time
|
||||
// data slowly." Lower values give fresher data at higher CPU cost.
|
||||
//
|
||||
// RecomputeIntervalSeconds keys (all optional):
|
||||
// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew
|
||||
type AnalyticsConfig struct {
|
||||
DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"`
|
||||
RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// AnalyticsDefaultRecomputeInterval returns the configured default
|
||||
// recompute interval, or 5 minutes if unset/invalid.
|
||||
func (c *Config) AnalyticsDefaultRecomputeInterval() time.Duration {
|
||||
if c != nil && c.Analytics != nil && c.Analytics.DefaultIntervalSeconds > 0 {
|
||||
return time.Duration(c.Analytics.DefaultIntervalSeconds) * time.Second
|
||||
}
|
||||
return 5 * time.Minute
|
||||
}
|
||||
|
||||
// AnalyticsRecomputeIntervals returns the per-endpoint override map.
|
||||
// Returns the zero value (all defaults) if the analytics block is
|
||||
// absent or empty.
|
||||
func (c *Config) AnalyticsRecomputeIntervals() AnalyticsRecomputeIntervals {
|
||||
out := AnalyticsRecomputeIntervals{}
|
||||
if c == nil || c.Analytics == nil || c.Analytics.RecomputeIntervalSeconds == nil {
|
||||
return out
|
||||
}
|
||||
get := func(key string) time.Duration {
|
||||
v, ok := c.Analytics.RecomputeIntervalSeconds[key]
|
||||
if !ok || v <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(v) * time.Second
|
||||
}
|
||||
out.Topology = get("topology")
|
||||
out.RF = get("rf")
|
||||
out.Distance = get("distance")
|
||||
out.Channels = get("channels")
|
||||
out.HashCollisions = get("hashCollisions")
|
||||
out.HashSizes = get("hashSizes")
|
||||
out.Roles = get("roles")
|
||||
out.ObserversClockSkew = get("observersClockSkew")
|
||||
out.NodesClockSkew = get("nodesClockSkew")
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBackfillHoursDefault(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if got := cfg.BackfillHours(); got != 24 {
|
||||
t.Errorf("BackfillHours() = %d, want 24", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillHoursConfigured(t *testing.T) {
|
||||
cfg := &Config{ResolvedPath: &ResolvedPathConfig{BackfillHours: 48}}
|
||||
if got := cfg.BackfillHours(); got != 48 {
|
||||
t.Errorf("BackfillHours() = %d, want 48", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillHoursZeroFallsBack(t *testing.T) {
|
||||
cfg := &Config{ResolvedPath: &ResolvedPathConfig{BackfillHours: 0}}
|
||||
if got := cfg.BackfillHours(); got != 24 {
|
||||
t.Errorf("BackfillHours() = %d, want 24 (default for zero)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeighborMaxAgeDaysDefault(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
if got := cfg.NeighborMaxAgeDays(); got != 5 {
|
||||
t.Errorf("NeighborMaxAgeDays() = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeighborMaxAgeDaysConfigured(t *testing.T) {
|
||||
cfg := &Config{NeighborGraph: &NeighborGraphConfig{MaxAgeDays: 7}}
|
||||
if got := cfg.NeighborMaxAgeDays(); got != 7 {
|
||||
t.Errorf("NeighborMaxAgeDays() = %d, want 7", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphPruneOlderThan(t *testing.T) {
|
||||
g := NewNeighborGraph()
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Add a recent edge
|
||||
g.upsertEdge("aaa", "bbb", "bb", "obs1", nil, now)
|
||||
// Add an old edge
|
||||
g.upsertEdge("ccc", "ddd", "dd", "obs1", nil, now.Add(-60*24*time.Hour))
|
||||
|
||||
if len(g.AllEdges()) != 2 {
|
||||
t.Fatalf("expected 2 edges, got %d", len(g.AllEdges()))
|
||||
}
|
||||
|
||||
cutoff := now.Add(-30 * 24 * time.Hour)
|
||||
pruned := g.PruneOlderThan(cutoff)
|
||||
if pruned != 1 {
|
||||
t.Errorf("PruneOlderThan pruned %d, want 1", pruned)
|
||||
}
|
||||
|
||||
edges := g.AllEdges()
|
||||
if len(edges) != 1 {
|
||||
t.Fatalf("expected 1 edge after prune, got %d", len(edges))
|
||||
}
|
||||
if edges[0].NodeA != "aaa" && edges[0].NodeB != "aaa" {
|
||||
t.Errorf("wrong edge survived prune: %+v", edges[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,25 +365,3 @@ func TestPropagationBufferMs(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestObserverDaysOrDefault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *Config
|
||||
want int
|
||||
}{
|
||||
{"nil retention", &Config{}, 14},
|
||||
{"zero observer days", &Config{Retention: &RetentionConfig{ObserverDays: 0}}, 14},
|
||||
{"positive value", &Config{Retention: &RetentionConfig{ObserverDays: 30}}, 30},
|
||||
{"keep forever", &Config{Retention: &RetentionConfig{ObserverDays: -1}}, -1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.cfg.ObserverDaysOrDefault()
|
||||
if got != tt.want {
|
||||
t.Errorf("ObserverDaysOrDefault() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import "net/http"
|
||||
|
||||
// corsMiddleware returns a middleware that sets CORS headers based on the
|
||||
// configured allowed origins. When CORSAllowedOrigins is empty (default),
|
||||
// no Access-Control-* headers are added, preserving browser same-origin policy.
|
||||
func (s *Server) corsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
origins := s.cfg.CORSAllowedOrigins
|
||||
if len(origins) == 0 {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
reqOrigin := r.Header.Get("Origin")
|
||||
if reqOrigin == "" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if origin is allowed
|
||||
allowed := false
|
||||
wildcard := false
|
||||
for _, o := range origins {
|
||||
if o == "*" {
|
||||
allowed = true
|
||||
wildcard = true
|
||||
break
|
||||
}
|
||||
if o == reqOrigin {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !allowed {
|
||||
// Origin not in allowlist — don't add CORS headers
|
||||
if r.Method == http.MethodOptions {
|
||||
// Still reject preflight with 403
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Set CORS headers
|
||||
if wildcard {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
} else {
|
||||
w.Header().Set("Access-Control-Allow-Origin", reqOrigin)
|
||||
w.Header().Set("Vary", "Origin")
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key")
|
||||
|
||||
// Handle preflight
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newTestServerWithCORS creates a minimal Server with the given CORS config.
|
||||
func newTestServerWithCORS(origins []string) *Server {
|
||||
cfg := &Config{CORSAllowedOrigins: origins}
|
||||
srv := &Server{cfg: cfg}
|
||||
return srv
|
||||
}
|
||||
|
||||
// dummyHandler is a simple handler that writes 200 OK.
|
||||
var dummyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
func TestCORS_DefaultNoHeaders(t *testing.T) {
|
||||
srv := newTestServerWithCORS(nil)
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO header, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_AllowlistMatch(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://good.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "https://good.example" {
|
||||
t.Fatalf("expected origin echo, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Methods"); v != "GET, POST, OPTIONS" {
|
||||
t.Fatalf("expected methods header, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Headers"); v != "Content-Type, X-API-Key" {
|
||||
t.Fatalf("expected headers header, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Vary"); v != "Origin" {
|
||||
t.Fatalf("expected Vary: Origin, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_AllowlistNoMatch(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO header for non-matching origin, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_PreflightAllowed(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("OPTIONS", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://good.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("expected 204, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "https://good.example" {
|
||||
t.Fatalf("expected origin echo, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_PreflightRejected(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("OPTIONS", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://evil.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected 403, got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_Wildcard(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"*"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://anything.example")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "*" {
|
||||
t.Fatalf("expected *, got %q", v)
|
||||
}
|
||||
// Wildcard should NOT set Vary: Origin
|
||||
if v := rr.Header().Get("Vary"); v == "Origin" {
|
||||
t.Fatalf("wildcard should not set Vary: Origin")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCORS_NoOriginHeader(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://good.example"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
// No Origin header
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", rr.Code)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "" {
|
||||
t.Fatalf("expected no ACAO without Origin header, got %q", v)
|
||||
}
|
||||
}
|
||||
+42
-814
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user