mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-08-07 15:59:42 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Kpa-clawbot
|
070dea8507 |
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"821 passed","color":"brightgreen"}
|
||||
{"schemaVersion":1,"label":"e2e tests","message":"717 passed","color":"brightgreen"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"36.64%","color":"red"}
|
||||
{"schemaVersion":1,"label":"frontend coverage","message":"37.73%","color":"red"}
|
||||
|
||||
+2
-11
@@ -42,7 +42,6 @@
|
||||
"PULL_THRESHOLD_PX": "readonly",
|
||||
"PacketFilter": "readonly",
|
||||
"PathInspector": "readonly",
|
||||
"PrefixReserved": "readonly",
|
||||
"QRCode": "readonly",
|
||||
"ROLE_COLORS": "readonly",
|
||||
"ROLE_EMOJI": "readonly",
|
||||
@@ -52,7 +51,6 @@
|
||||
"ROLE_STYLE": "readonly",
|
||||
"ROUTE_TYPES": "readonly",
|
||||
"RegionFilter": "readonly",
|
||||
"RegionShowAll": "readonly",
|
||||
"SITE_CONFIG": "readonly",
|
||||
"SKEW_SEVERITY_COLORS": "readonly",
|
||||
"SKEW_SEVERITY_LABELS": "readonly",
|
||||
@@ -61,12 +59,6 @@
|
||||
"SlideOver": "readonly",
|
||||
"TILE_DARK": "readonly",
|
||||
"TILE_LIGHT": "readonly",
|
||||
"MC_TILE_PROVIDERS": "readonly",
|
||||
"MC_setDarkTileProvider": "readonly",
|
||||
"MC_getDarkTileProvider": "readonly",
|
||||
"MC_setServerDefaultTileProvider": "readonly",
|
||||
"MC_applyTileFilter": "readonly",
|
||||
"MC_DARK_TILE_DEFAULT": "readonly",
|
||||
"TYPE_COLORS": "readonly",
|
||||
"TableResponsive": "readonly",
|
||||
"TableSort": "readonly",
|
||||
@@ -209,19 +201,20 @@
|
||||
"escapeHtml": "readonly",
|
||||
"exports": "readonly",
|
||||
"favStar": "readonly",
|
||||
"fetchAllNodes": "readonly",
|
||||
"filterPacketsByRoute": "readonly",
|
||||
"formatAbsoluteTimestamp": "readonly",
|
||||
"formatChartAxisLabel": "readonly",
|
||||
"formatDistance": "readonly",
|
||||
"formatDistanceRound": "readonly",
|
||||
"formatDrift": "readonly",
|
||||
"formatEngineBadge": "readonly",
|
||||
"formatHex": "readonly",
|
||||
"formatIsoLike": "readonly",
|
||||
"formatSkew": "readonly",
|
||||
"formatTimestamp": "readonly",
|
||||
"formatTimestampCustom": "readonly",
|
||||
"formatTimestampWithTooltip": "readonly",
|
||||
"formatVersionBadge": "readonly",
|
||||
"getDistanceUnit": "readonly",
|
||||
"getFavorites": "readonly",
|
||||
"getHashParams": "readonly",
|
||||
@@ -253,14 +246,12 @@
|
||||
"pad2": "readonly",
|
||||
"pad3": "readonly",
|
||||
"pages": "readonly",
|
||||
"parseViewportHash": "readonly",
|
||||
"payloadTypeColor": "readonly",
|
||||
"payloadTypeName": "readonly",
|
||||
"process": "readonly",
|
||||
"pullReconnect": "readonly",
|
||||
"qrcode": "readonly",
|
||||
"registerPage": "readonly",
|
||||
"renderVersionCard": "readonly",
|
||||
"renderSkewBadge": "readonly",
|
||||
"renderSkewSparkline": "readonly",
|
||||
"require": "readonly",
|
||||
|
||||
@@ -3,6 +3,7 @@ name: CI/CD Pipeline
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
tags: ['v*']
|
||||
pull_request:
|
||||
branches: [master]
|
||||
workflow_dispatch:
|
||||
@@ -13,7 +14,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
@@ -56,7 +57,7 @@ jobs:
|
||||
go build .
|
||||
# -race gates PR #1208's atomic.Pointer migration: the race-detector
|
||||
# is what makes path_inspect_atomic_race_test.go actually assert.
|
||||
go test -timeout 20m -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log
|
||||
go test -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log
|
||||
echo "--- Go Server Coverage ---"
|
||||
go tool cover -func=server-coverage.out | tail -1
|
||||
|
||||
@@ -65,7 +66,7 @@ jobs:
|
||||
set -e -o pipefail
|
||||
cd cmd/ingestor
|
||||
go build .
|
||||
go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log
|
||||
go test -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log
|
||||
echo "--- Go Ingestor Coverage ---"
|
||||
go tool cover -func=ingestor-coverage.out | tail -1
|
||||
|
||||
@@ -80,12 +81,6 @@ jobs:
|
||||
go test ./...
|
||||
echo "--- Decrypt CLI tests passed ---"
|
||||
|
||||
- name: Verify Dockerfile COPY invariants (issue #1316)
|
||||
run: bash scripts/check-dockerfile-internal-pkgs.sh
|
||||
|
||||
- name: Staging disk-monitor unit tests (issue #1684)
|
||||
run: bash scripts/staging/test-disk-monitor.sh
|
||||
|
||||
- name: Lint CSS variables (issue #1128)
|
||||
run: |
|
||||
set -e
|
||||
@@ -97,10 +92,6 @@ jobs:
|
||||
set -e
|
||||
node test-packet-filter.js
|
||||
node test-packet-filter-time.js
|
||||
node test-confidence-indicator.js
|
||||
node test-1659-analytics-warmup.js
|
||||
node test-channels-merge-1498-unit.js
|
||||
node test-issue-1518-home-url.js
|
||||
node test-channel-decrypt-insecure-context.js
|
||||
node test-live-region-filter.js
|
||||
node test-issue-1136-observer-iata-map.js
|
||||
@@ -108,7 +99,6 @@ jobs:
|
||||
node test-channel-qr-wiring.js
|
||||
node test-channel-modal-ux.js
|
||||
node test-channel-issue-1087.js
|
||||
node test-issue-1409-no-encrypted-flood.js
|
||||
node test-channel-issue-1101.js
|
||||
node test-observer-iata-1188.js
|
||||
node test-pull-to-reconnect-1091.js
|
||||
@@ -121,63 +111,7 @@ jobs:
|
||||
node test-issue-1364-pill-no-clamp.js
|
||||
node test-issue-1375-scope-stats-fetch.js
|
||||
node test-issue-1361-cb-presets.js
|
||||
node test-issue-1380-cb-sim-overlay.js
|
||||
node test-issue-1380-cb-reset-button.js
|
||||
node test-issue-1407-cb-preset-propagation.js
|
||||
node test-issue-1412-customizer-no-override.js
|
||||
node test-issue-1418-raw-hex-extraction.js
|
||||
node test-issue-1418-edge-weights.js
|
||||
node test-issue-1418-cb-preset-ramp.js
|
||||
node test-issue-1418-spider-fan.js
|
||||
node test-issue-1418-deeplink-hops-channels.js
|
||||
node test-issue-1418-polish-review.js
|
||||
node test-issue-1420-tile-providers.js
|
||||
node test-issue-1614-tile-url-function.js
|
||||
node test-issue-1438-marker-css-vars.js
|
||||
node test-issue-1846-observers-width.js
|
||||
node test-issue-1562-observers-summary.js
|
||||
node test-issue-1509-nav-active-bg.js
|
||||
node test-issue-1509-detect-preset.js
|
||||
node test-live.js
|
||||
node test-coverage-gate.js
|
||||
node test-node-reach-coverage.js
|
||||
node test-issue-1107-live-layout.js
|
||||
node test-issue-1532-live-fullscreen.js
|
||||
node test-issue-1619-feed-detail-card-draggable.js
|
||||
node test-xss-escape-sinks.js
|
||||
node test-preflight-xss-gate.js
|
||||
node test-traces.js
|
||||
node test-issue-1648-m4-emoji-scan.js
|
||||
node test-issue-1753-copy-url-slash.js
|
||||
node test-issue-1668-m3-typography.js
|
||||
node test-mqtt-status-panel.js
|
||||
node test-issue-1697-mqtt-mobile-e2e.js
|
||||
node test-warmup-banner.js
|
||||
node test-issue-1633-hide-1byte-hops.js
|
||||
node test-issue-1668-m4-per-route.js
|
||||
node test-a11y-axe-1668-selftest.js
|
||||
node test-a11y-1716-rf-range-btn-active.js
|
||||
node test-issue-1705-subpath-contrast.js
|
||||
node test-issue-1770-mobile-row-clamp.js
|
||||
node test-a11y-axe-routes-coverage.js
|
||||
|
||||
- name: 🛡️ Preflight XSS gate — actual --diff check (PR only)
|
||||
# The fixture self-test above (test-preflight-xss-gate.js) only
|
||||
# asserts the script's behavior against fixtures. It does NOT scan
|
||||
# the PR's own changes. This step closes that gap by running the
|
||||
# gate against added lines in public/**/*.{js,html} on the PR.
|
||||
# Gate is PR-scoped only (per djb finding: merge commits would
|
||||
# slip an opt-out otherwise). Master pushes skip this step.
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
PREFLIGHT_PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }}
|
||||
run: |
|
||||
set -e
|
||||
git fetch origin master --depth=50 2>&1 | tail -3 || true
|
||||
# Materialize PR body to a file for the opt-out parser.
|
||||
printf '%s' "$PR_BODY" > /tmp/pr-body.md
|
||||
PREFLIGHT_PR_BODY=/tmp/pr-body.md bash scripts/check-xss-sinks.sh --diff origin/master
|
||||
|
||||
- name: 🧹 Frontend lint (eslint no-undef) — issue #1342
|
||||
run: |
|
||||
@@ -293,75 +227,6 @@ jobs:
|
||||
- name: Freshen fixture timestamps
|
||||
run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db
|
||||
|
||||
- name: Seed grouped-packet row for #1486 collapse test
|
||||
# The committed fixture has 499 packets, each with exactly ONE
|
||||
# observation, so the packets-page renders only flat
|
||||
# (select-hash) rows. The #1486 repro needs at least one grouped
|
||||
# (toggle-select) row. Insert a NEW transmission with 3
|
||||
# observations.
|
||||
#
|
||||
# The server's async hash-migrate (cmd/server/hash_migrate.go)
|
||||
# recomputes `transmissions.hash` from `raw_hex` via
|
||||
# ComputeContentHash(), so the inserted hash MUST equal that
|
||||
# function's output for the chosen raw_hex — otherwise the row
|
||||
# gets relabelled and the E2E can't find it.
|
||||
#
|
||||
# raw_hex 15000102030405060708090a0b0c0d0e0f
|
||||
# → header=0x15 (route_type=1, payload_type=5)
|
||||
# → ComputeContentHash(...) = fae0c9e6d357a814
|
||||
#
|
||||
# The first_seen / observation timestamps are pinned to a date
|
||||
# within retentionHours but outside the default 15-min UI
|
||||
# window so the row is hidden in the default view (keeping
|
||||
# test-e2e-playwright's first-10-rows hex-pane test
|
||||
# unaffected) and reachable via the explicit ?timeWindow=0
|
||||
# deep-link the #1486 test uses.
|
||||
run: |
|
||||
sqlite3 test-fixtures/e2e-fixture.db <<'SQL'
|
||||
-- Sort the seeded row LAST in BOTH default packets views:
|
||||
-- • flat view sorts by transmissions.id DESC → id=0 puts it last
|
||||
-- • grouped view (#default for the packets page) sorts by
|
||||
-- MAX(observations.timestamp) DESC → we must keep our obs
|
||||
-- timestamps OLDER than every other fixture observation.
|
||||
-- Fixture (after freshen) has obs timestamps spanning
|
||||
-- 2026-05-17 16:01:39Z .. 2026-05-28 00:00:00Z (max).
|
||||
-- Note: freshen only shifts transmissions.first_seen forward
|
||||
-- to ~now; observation.timestamp is left alone except for
|
||||
-- the timestamp=0 case.
|
||||
-- Use 2026-05-15 (~2 days older than the oldest fixture obs)
|
||||
-- so our row sorts LAST in the grouped view too, keeping
|
||||
-- test-e2e-playwright's first-10-rows hex-pane test
|
||||
-- unaffected. The #1486 test still reaches the row via the
|
||||
-- explicit hash + ?timeWindow=0 deep-link.
|
||||
INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey)
|
||||
VALUES (0,'15000102030405060708090a0b0c0d0e0f','fae0c9e6d357a814','2026-05-15T00:00:00Z',1,5,0,'{"type":"CHAN","channel":"#test","text":"#1486 fixture"}',NULL,NULL);
|
||||
INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,resolved_path) VALUES
|
||||
(0,1,'rx',5.0,-95,0,'["AA"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["aa00000000000000000000000000000000000000000000000000000000000000"]'),
|
||||
(0,2,'rx',5.5,-92,0,'["BB"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["bb00000000000000000000000000000000000000000000000000000000000000"]'),
|
||||
(0,3,'rx',6.0,-90,0,'["CC"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["cc00000000000000000000000000000000000000000000000000000000000000"]');
|
||||
-- #1791 fixture: a single GRP_DATA (payload_type=6) packet so the
|
||||
-- E2E "Group Data filter" test has at least one row to filter on.
|
||||
-- Use an obs timestamp within the default UI window so the row
|
||||
-- appears with no time-window override.
|
||||
--
|
||||
-- raw_hex header byte 0x19 = bits 5-2 (payload)=0110=6 (GRP_DATA),
|
||||
-- bits 1-0 (route)=01=1 (FLOOD).
|
||||
-- path_len byte 0x00 = hash_size=1, hash_count=0 (zero-hop on-wire,
|
||||
-- typical GRP_DATA going FLOOD). path_json/resolved_path are kept
|
||||
-- EMPTY so the rendered hop-row count matches the hex-path byte
|
||||
-- count (a prior fixture used path_json=["AA"] but raw_hex
|
||||
-- path_len=0, which broke the "hex strip Path range matches hop
|
||||
-- row count" E2E).
|
||||
--
|
||||
-- Note: id=-1000000 is a deliberately out-of-band sentinel id so
|
||||
-- this synthetic fixture row cannot collide with real ingested
|
||||
-- transmissions (real ids are positive autoincrement values).
|
||||
INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey)
|
||||
VALUES (-1000000,'19000102030405060708090a0b0c0d0e0f','17910000deadbeef',strftime('%Y-%m-%dT%H:%M:%SZ','now'),1,6,0,'{"type":"GRP_DATA","channel":"#test","raw":"deadbeef"}',NULL,NULL);
|
||||
INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,resolved_path) VALUES
|
||||
(-1000000,1,'rx',7.0,-88,0,'[]',CAST(strftime('%s','now') AS INTEGER),'[]');
|
||||
SQL
|
||||
|
||||
- name: Migrate fixture DB to current schema (#1287)
|
||||
# Server now ASSERTs schema is migrated and refuses to start
|
||||
# otherwise (cmd/server/main.go: dbschema.AssertReady). In prod
|
||||
@@ -392,31 +257,15 @@ jobs:
|
||||
- name: Run Playwright E2E tests (fail-fast)
|
||||
run: |
|
||||
BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt
|
||||
# M5+M6 of #1668 — axe-core CI gate.
|
||||
# M5: color-contrast on desktop dark+light.
|
||||
# M6: expanded ruleset (image-alt, label, aria-required-attr,
|
||||
# aria-valid-attr, aria-valid-attr-value, landmark-one-main,
|
||||
# region, button-name, link-name, document-title, html-has-lang,
|
||||
# duplicate-id) AND adds 375x812 mobile viewport (with
|
||||
# color-contrast on mobile too).
|
||||
# Allowlist: tests/a11y-allowlist.yaml (0 entries — hard pass policy).
|
||||
# Per-viewport summary printed at the end; any net>0 fails the build.
|
||||
BASE_URL=http://localhost:13581 AXE_SCREENSHOT_DIR=/tmp/axe-1668 \
|
||||
node test-a11y-axe-1668.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1639-observers-sort-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1758-ng-filter-rerenders-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
@@ -427,20 +276,16 @@ jobs:
|
||||
BASE_URL=http://localhost:13581 node test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1692-packets-init-parallel-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-live-multibyte-only-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 node test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
@@ -452,14 +297,6 @@ jobs:
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1510-live-nav-pin-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-fullscreen-1572-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1599-replay-freeze-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m1-icons-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m2-icons-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m3-icons-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m4-icons-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1657-analytics-channels-group-sprites-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
@@ -467,7 +304,6 @@ jobs:
|
||||
BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-issue-1799-label-vocab-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
@@ -482,39 +318,8 @@ jobs:
|
||||
BASE_URL=http://localhost:13581 node test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
BASE_URL=http://localhost:13581 node test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1567-corner-clears-drag-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-list-render-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-selection-flow-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-add-modal-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-share-color-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-batch-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt
|
||||
|
||||
# #1616: slide-over focus-restore flake-gate. Runs the slide-over
|
||||
# E2E 20 consecutive times against the SAME backend instance so
|
||||
# the Chromium-headless focus race documented in #1172/#1616 has
|
||||
# a 20× shot at firing. Any single non-zero exit aborts. This is
|
||||
# the architectural-fix gate — if it ever turns red post-merge,
|
||||
# the focused-but-hidden state has crept back in.
|
||||
#
|
||||
# PERMANENT step. Adds ~3-4 min to the e2e-test job in exchange
|
||||
# for closing out a flake family that was blocking ~8 unrelated
|
||||
# PRs at a time. If profiling pressures the budget later, drop
|
||||
# repeat count first; do not delete.
|
||||
- name: Slide-over E2E flake-gate (#1616, --repeat-each=3)
|
||||
run: |
|
||||
set -e
|
||||
for i in $(seq 1 3); do
|
||||
echo "--- slide-over E2E run $i/20 ---"
|
||||
BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt
|
||||
done
|
||||
echo "3 passed"
|
||||
|
||||
- name: Collect frontend coverage (parallel)
|
||||
if: success() && github.event_name == 'push'
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
name: Release Fast-Path
|
||||
|
||||
# Issue #1677: re-tag :edge as :vX.Y.Z when the tag SHA matches :edge's
|
||||
# org.opencontainers.image.revision label. Skips ~30 min of Go test +
|
||||
# Playwright + Docker rebuild because the bytes are identical — only the
|
||||
# manifest name changes. Falls back to deploy.yml when SHAs differ so
|
||||
# tags on older commits still go through full validation.
|
||||
#
|
||||
# This workflow is the SOLE consumer of push.tags. deploy.yml's tag
|
||||
# trigger has been removed to prevent double-fire.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v[0-9]+.[0-9]+.[0-9]+']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
actions: write # issue #1702: required so the fallback `gh workflow run deploy.yml` dispatch is allowed
|
||||
|
||||
concurrency:
|
||||
group: release-fast-path-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
retag-or-fallback:
|
||||
name: "🏷️ Re-tag :edge → :vX.Y.Z (fast) or dispatch deploy.yml (fallback)"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
uses: imjasonh/setup-crane@v0.4
|
||||
|
||||
- name: Parse semver from tag
|
||||
id: semver
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${GITHUB_REF#refs/tags/}"
|
||||
# Expect vMAJOR.MINOR.PATCH (workflow trigger already enforces this).
|
||||
if [[ ! "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
|
||||
echo "Tag $TAG does not match vMAJOR.MINOR.PATCH" >&2
|
||||
exit 1
|
||||
fi
|
||||
MAJOR="${BASH_REMATCH[1]}"
|
||||
MINOR="${BASH_REMATCH[2]}"
|
||||
{
|
||||
echo "tag=$TAG"
|
||||
echo "vMajor=v$MAJOR"
|
||||
echo "vMajorMinor=v$MAJOR.$MINOR"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "Parsed: $TAG → v$MAJOR / v$MAJOR.$MINOR / $TAG"
|
||||
|
||||
- name: Inspect :edge revision label
|
||||
id: edge
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE="ghcr.io/kpa-clawbot/corescope"
|
||||
EDGE_REF="${IMAGE}:edge"
|
||||
# crane config returns the OCI image config JSON; the revision label
|
||||
# is set by docker/metadata-action on the master-edge build.
|
||||
# If :edge doesn't exist yet (first run on a fresh registry), fall
|
||||
# through to the slow path.
|
||||
if ! CONFIG="$(crane config "$EDGE_REF" 2>/dev/null)"; then
|
||||
echo "edge_revision=" >> "$GITHUB_OUTPUT"
|
||||
echo "no_edge=true" >> "$GITHUB_OUTPUT"
|
||||
echo ":edge not found in registry — will use fallback path"
|
||||
exit 0
|
||||
fi
|
||||
REV="$(echo "$CONFIG" | jq -r '.config.Labels["org.opencontainers.image.revision"] // ""')"
|
||||
echo "edge_revision=$REV" >> "$GITHUB_OUTPUT"
|
||||
echo "no_edge=false" >> "$GITHUB_OUTPUT"
|
||||
echo ":edge org.opencontainers.image.revision = $REV"
|
||||
echo "tag SHA (github.sha) = ${{ github.sha }}"
|
||||
|
||||
# ─────────── FAST PATH: SHAs match, metadata-only retag ───────────
|
||||
- name: Re-tag :edge → :vX.Y.Z + :vX.Y + :vX + :latest (fast path)
|
||||
if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == github.sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE="ghcr.io/kpa-clawbot/corescope"
|
||||
SRC="${IMAGE}:edge"
|
||||
echo "SHA match — fast-path re-tag from $SRC"
|
||||
for NEW_TAG in \
|
||||
"${{ steps.semver.outputs.tag }}" \
|
||||
"${{ steps.semver.outputs.vMajorMinor }}" \
|
||||
"${{ steps.semver.outputs.vMajor }}" \
|
||||
"latest"; do
|
||||
echo " crane tag $SRC $NEW_TAG"
|
||||
crane tag "$SRC" "$NEW_TAG"
|
||||
done
|
||||
echo "Fast-path complete — all tags point at the :edge manifest digest."
|
||||
|
||||
# ─────────── FALLBACK: SHAs differ, run the full pipeline ───────────
|
||||
- name: Dispatch full deploy.yml pipeline (fallback)
|
||||
if: steps.edge.outputs.no_edge == 'true' || steps.edge.outputs.edge_revision != github.sha
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "SHA mismatch (or no :edge) — falling back to full pipeline"
|
||||
echo " :edge revision = '${{ steps.edge.outputs.edge_revision }}'"
|
||||
echo " tag SHA = '${{ github.sha }}'"
|
||||
gh workflow run deploy.yml \
|
||||
--repo "${{ github.repository }}" \
|
||||
--ref "${{ github.ref }}"
|
||||
echo "Dispatched deploy.yml against ${{ github.ref }}"
|
||||
@@ -33,7 +33,3 @@ corescope-server
|
||||
cmd/server/server
|
||||
# Local-only planning and design files
|
||||
docs/superpowers/
|
||||
|
||||
# Environment-specific deploy scripts — live only on the deploy host, not tracked
|
||||
deploy-live.sh
|
||||
deploy-staging.sh
|
||||
|
||||
+1
-26
@@ -2,32 +2,7 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [3.9.1] — 2026-06-12
|
||||
|
||||
Patch release on top of v3.9.0 — v3.9.0's container image never published (Playwright flake gated Docker build). See [docs/release-notes/v3.9.1.md](docs/release-notes/v3.9.1.md).
|
||||
|
||||
### 🎨 Accessibility
|
||||
- **WCAG AA contrast pass** (#1676, f0addfda) — two-tier CSS palette; muted-text ≥4.5:1 in both themes; unknown-repeater chip fixed (2.75:1 → 4.95:1). Closes #1671. Partial fix for #1668.
|
||||
|
||||
### 🧪 Test stability
|
||||
- **Slideover E2E flake fix** (#1663+followups, f06359d7) — tightened selectors, bumped data-row wait. Fixes #1662.
|
||||
|
||||
## [3.9.0] — 2026-06-12
|
||||
|
||||
See [docs/release-notes/v3.9.0.md](docs/release-notes/v3.9.0.md) for the full notes. 257 commits since v3.8.3 (72 substantive + 185 coverage bumps).
|
||||
|
||||
### ✨ Highlights
|
||||
- **Relay timelines survive an ingestor restart** (#1643) — relay-hop attribution is rebuilt from `path_json` on cold load.
|
||||
- **Observer Compare is first-class** (#1642, #1645, #1647) — three new entry points + Tufte-grade compare page with state-preserving multi-select.
|
||||
- **Emoji → Phosphor icon migration** (#1648, #1649–#1654) — every UI emoji replaced with theme-tinted Phosphor sprites, lint-gated.
|
||||
- **Per-node Reach page + API** (#1627) — `GET /api/nodes/{pubkey}/reach` with cache invalidation on blacklist changes (#1636).
|
||||
- **Hashtag channels catalogue integration** (#1656) — public hashtag channels appear without manual config.
|
||||
- **Operator-customizable name-prefix hiding** (#1655) — new `hiddenNamePrefixes` config (default `["🚫"]`).
|
||||
|
||||
### ⚙️ Config
|
||||
- New: `hiddenNamePrefixes`, `liveMap.maxNodes`, `runtime.maxMemoryMB`, configurable observer-health thresholds, `branding.homeUrl`, customizer disabled-tabs.
|
||||
|
||||
### 📝 Documentation Corrections (carried from prior [Unreleased])
|
||||
### 📝 Documentation Corrections
|
||||
- **PR #1324 historical record correction** (#1387) — the merged PR #1324 body referenced four tests that do NOT exist in master: `TestMultibyteCapPersistRoundTrip`, `TestMultibyteCapPersistSkipsUnknown`, `TestMaybePersistCoalesces`, and a `TryLock` coalescing test. The actual tests that landed are `TestRunMultibyteCapPersist_AppliesSnapshot` and `TestRunMultibyteCapPersist_NoSnapshot_NoOp`. See issue #1386 for the corrective test additions (round-trip, unknown-key skip, coalescing).
|
||||
|
||||
## [3.7.2] — 2026-05-06
|
||||
|
||||
@@ -41,7 +41,7 @@ Settings can be overridden via environment variables:
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `DISABLE_CADDY` | `false` | Skip internal Caddy (set `true` behind a reverse proxy) |
|
||||
| `DISABLE_MOSQUITTO` | `true` in `docker-compose.staging.yml`; `false` elsewhere | Skip internal MQTT broker. Default flipped to `true` for the staging deploy in v3.7+ because a standalone `mqtt-broker` container owns MQTT on that host — see "Standalone MQTT broker (staging)" below. |
|
||||
| `DISABLE_MOSQUITTO` | `false` | Skip internal MQTT broker (use external) |
|
||||
| `HTTP_PORT` | `80` | Host port mapping |
|
||||
| `DATA_DIR` | `./data` | Host path for persistent data |
|
||||
|
||||
@@ -71,65 +71,6 @@ Option B — **Built-in Caddy**: Mount a custom Caddyfile at `/etc/caddy/Caddyfi
|
||||
|
||||
---
|
||||
|
||||
## Standalone MQTT broker (staging)
|
||||
|
||||
Starting in v3.7, `docker-compose.staging.yml` assumes a **standalone
|
||||
`mqtt-broker` container** (image: `eclipse-mosquitto:2`) already runs
|
||||
on the staging VM, out-of-band from this repo. That container:
|
||||
|
||||
- Owns port `8883` externally (TLS-terminated MQTT for real observers).
|
||||
- Is attached to a shared docker network named `meshcore-net`.
|
||||
- Is operator-managed state — it is **not** defined in any compose
|
||||
file in this repository. Its config, TLS certs, and ACLs live on the
|
||||
host, outside git.
|
||||
|
||||
`corescope-staging-go` reaches it in-network at `mqtt-broker:1883` via
|
||||
docker DNS (no host port hop). To make that work, `docker-compose.staging.yml`
|
||||
joins the external `meshcore-net` network and defaults `DISABLE_MOSQUITTO=true`
|
||||
so the built-in mosquitto stays off.
|
||||
|
||||
### Prereq — one-time provisioning on the staging host
|
||||
|
||||
```bash
|
||||
docker network create meshcore-net
|
||||
# ...then bring up the operator-managed mqtt-broker container on that
|
||||
# network (not covered here; that's operator state). THEN:
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
```
|
||||
|
||||
If `meshcore-net` doesn't exist when compose starts, docker will refuse
|
||||
to bring `staging-go` up (`external: true` — compose won't create it).
|
||||
|
||||
### Reverting to the old single-container behaviour
|
||||
|
||||
Third-party operators cloning this repo who want the legacy shape
|
||||
(in-container mosquitto + `1883:1883` on the host, no external broker)
|
||||
should override both the env default and re-add the port mapping.
|
||||
|
||||
In `.env` (or the shell):
|
||||
|
||||
```
|
||||
DISABLE_MOSQUITTO=false
|
||||
```
|
||||
|
||||
And in `docker-compose.staging.yml`, restore the `1883:1883` mapping
|
||||
under `services.staging-go.ports`:
|
||||
|
||||
```yaml
|
||||
ports:
|
||||
- "${STAGING_GO_HTTP_PORT:-80}:80"
|
||||
- "${STAGING_GO_MQTT_PORT:-1883}:1883" # ← re-added
|
||||
- "6060:6060"
|
||||
- "6061:6061"
|
||||
```
|
||||
|
||||
That gives you back the pre-v3.7 self-contained staging shape. In that
|
||||
mode you do **not** need `meshcore-net`, but note the compose file still
|
||||
declares it as `external: true`, so either remove that declaration in
|
||||
your fork or ensure the network exists.
|
||||
|
||||
---
|
||||
|
||||
## Migrating from manage.sh (existing admins)
|
||||
|
||||
If you're currently deploying with `manage.sh` (git clone + local build), you have two options going forward:
|
||||
@@ -188,98 +129,3 @@ docker compose pull && docker compose up -d
|
||||
| `./manage.sh setup` | Copy `docker-compose.example.yml`, edit env vars |
|
||||
|
||||
`manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments.
|
||||
|
||||
## Staging VM — disk-usage monitor & cleanup (#1684)
|
||||
|
||||
The staging VM ran out of disk during a hot-patch (#1684). To prevent
|
||||
repeats, two scripts live in `scripts/staging/`:
|
||||
|
||||
- `disk-monitor.sh <mount>` — reads `df -P`, classifies usage against
|
||||
`<80 ok / >=80 warn / >=90 error / >=95 alert`, emits to stderr +
|
||||
journald (via `logger`). Returns non-zero on `error|alert` so
|
||||
systemd surfaces the unit as failed.
|
||||
- `disk-cleanup.sh` — removes `/tmp` snapshot files (`*.db`,
|
||||
`staging-snap.*`, `cs-*`, `node-compile-cache`) older than 7 days
|
||||
and runs `docker builder prune` + `docker image prune` with
|
||||
`--filter "until=72h" --filter "label!=keep"`. Set
|
||||
`CORESCOPE_CLEANUP_DRY_RUN=1` to log without deleting.
|
||||
|
||||
### Install on the staging host
|
||||
|
||||
SSH to `<STAGING_HOST>` as the staging operator user and:
|
||||
|
||||
```bash
|
||||
sudo install -m 0755 scripts/staging/disk-monitor.sh /usr/local/bin/corescope-disk-monitor
|
||||
sudo install -m 0755 scripts/staging/disk-cleanup.sh /usr/local/bin/corescope-disk-cleanup
|
||||
|
||||
# 15-minute monitor
|
||||
sudo tee /etc/systemd/system/corescope-disk-monitor.service >/dev/null <<'UNIT'
|
||||
[Unit]
|
||||
Description=CoreScope staging disk-usage monitor (issue #1684)
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/corescope-disk-monitor /
|
||||
UNIT
|
||||
|
||||
sudo tee /etc/systemd/system/corescope-disk-monitor.timer >/dev/null <<'UNIT'
|
||||
[Unit]
|
||||
Description=Run CoreScope disk-usage monitor every 15 minutes
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=15min
|
||||
Unit=corescope-disk-monitor.service
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
UNIT
|
||||
|
||||
# Daily cleanup at 03:30 local
|
||||
sudo tee /etc/systemd/system/corescope-disk-cleanup.service >/dev/null <<'UNIT'
|
||||
[Unit]
|
||||
Description=CoreScope staging disk cleanup (issue #1684)
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/corescope-disk-cleanup
|
||||
UNIT
|
||||
|
||||
sudo tee /etc/systemd/system/corescope-disk-cleanup.timer >/dev/null <<'UNIT'
|
||||
[Unit]
|
||||
Description=Run CoreScope disk cleanup daily at off-peak
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:30:00
|
||||
Persistent=true
|
||||
Unit=corescope-disk-cleanup.service
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
UNIT
|
||||
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now corescope-disk-monitor.timer corescope-disk-cleanup.timer
|
||||
```
|
||||
|
||||
`<STAGING_HOST>` is the staging VM hostname/IP — operator supplies it,
|
||||
not committed to the repo.
|
||||
|
||||
### Inspecting alerts
|
||||
|
||||
```bash
|
||||
journalctl -t corescope-disk-monitor --since '-1d'
|
||||
journalctl -t corescope-disk-cleanup --since '-7d'
|
||||
systemctl list-timers | grep corescope-disk
|
||||
```
|
||||
|
||||
`logger` priorities map: `ok→info`, `warn→warning`, `error→err`,
|
||||
`alert→alert` (syslog severity 1, the highest level). Wire
|
||||
`journalctl -p alert ...` to whatever ops channel the operator
|
||||
prefers; use `-p err` to also catch the `error` tier.
|
||||
|
||||
### Notes on `staging-snap.db` root cause (#1684 phase 3)
|
||||
|
||||
`grep -rn staging-snap.db cmd/ public/ scripts/` returns **zero**
|
||||
hits in the repo. The 4.4 GB orphan was a manual debugging artifact,
|
||||
not produced by any committed code. The `disk-cleanup.sh` retention
|
||||
rule (anything matching `staging-snap.*` in `/tmp` older than 7 days)
|
||||
prevents recurrence without needing source-side TTL changes.
|
||||
|
||||
If a future feature legitimately needs persistent snapshot DBs, put
|
||||
them under `/var/lib/corescope/snapshots/` with explicit rotation —
|
||||
not in `/tmp`, which is ephemeral by definition.
|
||||
|
||||
@@ -23,7 +23,6 @@ COPY internal/dbschema/ ../../internal/dbschema/
|
||||
COPY internal/prunequeue/ ../../internal/prunequeue/
|
||||
COPY internal/perfio/ ../../internal/perfio/
|
||||
COPY internal/mbcapqueue/ ../../internal/mbcapqueue/
|
||||
COPY internal/lora/ ../../internal/lora/
|
||||
RUN go mod download
|
||||
COPY cmd/server/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
|
||||
-142
@@ -1,142 +0,0 @@
|
||||
# MIGRATIONS — async vs sync policy
|
||||
|
||||
CoreScope's ingestor applies schema/data migrations inline at boot in
|
||||
`cmd/ingestor/db.go`. Every migration that runs synchronously blocks the
|
||||
ingestor from accepting packets until it returns. On a dev DB that's
|
||||
milliseconds; at prod scale (1.9M+ observations, 80K+ adverts, 2600+ nodes
|
||||
on Cascadia) it can pin the boot for minutes and trigger restart loops —
|
||||
the "upgrade broke prod" failure class (#791, #1483, and others).
|
||||
|
||||
## The rule
|
||||
|
||||
**Any new `CREATE INDEX`, `ALTER TABLE`, or data-rewriting `UPDATE`/`DELETE`
|
||||
in a migration file MUST do ONE of the following:**
|
||||
|
||||
### Option 1 — Run via `Store.RunAsyncMigration` (preferred for backfills)
|
||||
|
||||
```go
|
||||
// Scheduled in OpenStore() AFTER the *Store is constructed.
|
||||
if err := s.RunAsyncMigration(ctx, "my_migration_v1",
|
||||
func(ctx context.Context, db *sql.DB) error {
|
||||
_, err := db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS ...`)
|
||||
return err
|
||||
}); err != nil {
|
||||
log.Printf("[migration/async] scheduling failed: %v", err)
|
||||
}
|
||||
```
|
||||
|
||||
- The migration is recorded as `pending_async` in the `_async_migrations`
|
||||
table **immediately** — the ingestor boots and starts ingesting.
|
||||
- `fn` runs in a goroutine; the WaitGroup is shared with the rest of the
|
||||
ingestor (`Store.WaitForAsyncMigrations()` waits for everything).
|
||||
- On success the row flips to `done`; on error/panic to `failed` with the
|
||||
error message captured.
|
||||
- Idempotent: rows in `done` state short-circuit; `failed`/`pending_async`
|
||||
rows are retried on the next boot.
|
||||
|
||||
Reference implementations: `Store.BackfillPathJSONAsync` (path_json
|
||||
backfill) and the converted `obs_observer_ts_idx_v1` index build in
|
||||
`OpenStore`.
|
||||
|
||||
### Option 2 — Annotate as preflight-cheap
|
||||
|
||||
Some migrations are genuinely cheap at any scale (e.g. `ALTER TABLE ADD
|
||||
COLUMN`, `CREATE INDEX` on a table you know is bounded to a few thousand
|
||||
rows). Annotate the migration block with a comment **on the line
|
||||
immediately above the migration block** so the preflight gate recognises
|
||||
the opt-out:
|
||||
|
||||
```go
|
||||
// PREFLIGHT: async=true reason="ALTER ADD COLUMN — O(1) sqlite operation"
|
||||
if r := db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foo_v1'"); ...
|
||||
```
|
||||
|
||||
The reason MUST be a real one-line justification you can defend in
|
||||
review. "It's fine" is not a reason.
|
||||
|
||||
### Option 3 — Opt out per PR
|
||||
|
||||
If the migration is genuinely safe and you don't want to add an inline
|
||||
annotation, put a single line in the PR body:
|
||||
|
||||
```
|
||||
PREFLIGHT-MIGRATION-SCALE: <30s N=80K verified on Cascadia staging snapshot
|
||||
```
|
||||
|
||||
This must include both `<30s` and `N=<some scale>` so a reviewer can
|
||||
challenge the measurement.
|
||||
|
||||
## The gate
|
||||
|
||||
`~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh` runs
|
||||
on every PR via the preflight orchestrator. It greps the diff for new or
|
||||
modified migration blocks (files matching `cmd/ingestor/db.go`,
|
||||
`cmd/ingestor/maintenance.go`, `internal/dbschema/**`, `**/migrations/**`,
|
||||
`**/*.sql`, plus any Go file touching `CREATE INDEX` / `ALTER TABLE` /
|
||||
`CREATE UNIQUE INDEX`). For each hit it requires one of the three
|
||||
opt-outs above. Hard-fail (exit 1) — no warning-only mode.
|
||||
|
||||
## Concurrency model
|
||||
|
||||
CoreScope runs **one ingestor process** per deployment (`cmd/ingestor/`,
|
||||
single binary, single `*Store`). There is no cluster mode, no leader
|
||||
election, no second writer. SQLite is opened with `SetMaxOpenConns(1)`
|
||||
and a 5s `busy_timeout`; all writes (live MQTT ingest + async migration
|
||||
goroutines + maintenance backfills) serialize through the one connection
|
||||
in a single process.
|
||||
|
||||
What this means for async migrations:
|
||||
|
||||
- **No cross-process race** to worry about. Two ingestor instances
|
||||
running against the same DB is not a supported deployment shape.
|
||||
- **Within a single process**, concurrent `RunAsyncMigration(name=X)`
|
||||
callers race the initial `SELECT status` → `UPDATE/INSERT` step. The
|
||||
current implementation re-schedules `fn` on a pending/failed row so a
|
||||
duplicate caller may legitimately re-run it; once status is `done` all
|
||||
further calls short-circuit. See
|
||||
`TestRunAsyncMigration_ConcurrentSameNameSerialized` for the contract.
|
||||
- **`fn` runs concurrently with live ingest writers.** Because
|
||||
`MaxOpenConns=1`, a long `CREATE INDEX` will serialize behind / ahead
|
||||
of insert batches via SQLite's busy-timeout. This is acceptable for
|
||||
index builds (the boot path is unblocked, which was the whole point),
|
||||
but it means long migrations DO add latency to live writes. Document
|
||||
expected runtime in the `reason=` annotation and prefer batched/chunked
|
||||
fn implementations for multi-minute work (see `BackfillPathJSONAsync`
|
||||
for the canonical batched pattern with inter-batch `time.Sleep`).
|
||||
|
||||
## Scale budgets
|
||||
|
||||
Per-migration target: **<30s** at current prod scale (Cascadia: ~2,600
|
||||
nodes, ~80K observations; previous prod snapshot: ~1.9M observations).
|
||||
|
||||
Worked example (#1483, `obs_observer_ts_idx_v1`): composite index build
|
||||
on `observations(observer_idx, timestamp)`. At ~1.9M rows the sync build
|
||||
pinned ingestor boot for several minutes → restart loop. Converted to
|
||||
async via `RunAsyncMigration` in `OpenStore` so boot returns immediately
|
||||
and the index materializes in the background; the existing `_migrations`
|
||||
short-circuit at the top of the migration block ensures DBs that already
|
||||
completed the sync v3.8.3 build do NOT re-run it through the goroutine
|
||||
path on subsequent boots.
|
||||
|
||||
If you cannot meet the <30s budget, document the expected upper bound
|
||||
and operator runbook expectation (e.g. "index build expected ~10 min on
|
||||
a 5M-row table; ingestor remains responsive; monitor via
|
||||
`SELECT status, error FROM _async_migrations WHERE name = ...`").
|
||||
|
||||
## Why this exists
|
||||
|
||||
Pattern that keeps repeating:
|
||||
|
||||
1. Author writes `CREATE INDEX foo ON observations(...)` in a migration.
|
||||
2. Local dev DB has ~100 rows. Migration returns in 1ms. CI is green.
|
||||
3. Reviewer focuses on plan correctness, not scale.
|
||||
4. Ship.
|
||||
5. Prod boots, sqlite scans 1.9M rows, the ingestor sits at `[migration]
|
||||
Adding index...` for 8 minutes, healthcheck times out, container
|
||||
restarts, loops.
|
||||
6. Operator pages. Hotfix. Apology.
|
||||
|
||||
The gate doesn't try to detect table size (undecidable from a diff). It
|
||||
enforces **annotation discipline**: every author who adds a migration
|
||||
must consciously decide which bucket it falls into and write that down.
|
||||
That is the cheapest possible intervention that breaks the cycle.
|
||||
@@ -21,7 +21,6 @@ The Go backend serves all 40+ API endpoints from an in-memory packet store with
|
||||
| Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) |
|
||||
| WebSocket broadcast | **Real-time** to all connected browsers |
|
||||
| Channel decryption | **AES-128-ECB** with rainbow table |
|
||||
| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). |
|
||||
|
||||
See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks.
|
||||
|
||||
@@ -270,4 +269,4 @@ Contributions welcome. Please read [AGENTS.md](AGENTS.md) for coding conventions
|
||||
|
||||
## License
|
||||
|
||||
GPL-3.0-or-later
|
||||
MIT
|
||||
|
||||
@@ -294,6 +294,5 @@
|
||||
"#colombia": "bea223a8c1d13ed9638ee000ea3a6aca",
|
||||
"#bogota": "6d0864985b64350ce4cbfebf4979e970",
|
||||
"#peru": "7e6fc347bf29a4c128ac3156865bd521",
|
||||
"#lima": "5f167ce354eca08ab742463df10ef255",
|
||||
"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72"
|
||||
}
|
||||
"#lima": "5f167ce354eca08ab742463df10ef255"
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
// Async migration helper — runs schema/backfill work that may take minutes on
|
||||
// large prod tables WITHOUT blocking ingestor startup.
|
||||
//
|
||||
// MIGRATION ANNOTATION CONVENTION (read this before touching migrations):
|
||||
//
|
||||
// Sync schema/data migrations (CREATE INDEX, ALTER TABLE, UPDATE ... WHERE)
|
||||
// that run inline during OpenStore() block the ingestor from accepting
|
||||
// packets until they finish. On an empty dev DB they return in milliseconds;
|
||||
// at prod scale (1.9M+ observations, 80K+ adverts) they can pin the boot
|
||||
// for minutes and trigger restart loops. This regression class has bitten us
|
||||
// repeatedly (#791 resolved_path backfill, #1483 obs_observer_ts_idx_v1).
|
||||
//
|
||||
// ANY new CREATE INDEX / ALTER TABLE / data-rewrite migration MUST EITHER:
|
||||
// 1. Run via Store.RunAsyncMigration(...) below (preferred for backfills
|
||||
// and any work that may touch >1K rows). The migration is recorded as
|
||||
// `pending_async` immediately, returns to the caller (boot proceeds),
|
||||
// and completes in a goroutine. Status flips to `done` (or `failed`
|
||||
// with an error message) when fn returns.
|
||||
// 2. Carry the preflight annotation comment immediately above the
|
||||
// migration block, e.g.
|
||||
// // PREFLIGHT: async=true reason="<one-line justification>"
|
||||
// Use this for migrations that are genuinely cheap at any scale
|
||||
// (e.g. ALTER TABLE ADD COLUMN, CREATE INDEX on a known-bounded
|
||||
// table). The annotation is grepped by
|
||||
// ~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh
|
||||
// — its absence on a touched migration block is a hard-fail gate.
|
||||
//
|
||||
// See MIGRATIONS.md in the repo root for the full policy and examples.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
// ensureAsyncMigrationsTable creates the bookkeeping table used by
|
||||
// RunAsyncMigration / AsyncMigrationStatus. Idempotent.
|
||||
func ensureAsyncMigrationsTable(db *sql.DB) error {
|
||||
_, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS _async_migrations (
|
||||
name TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL, -- pending_async | done | failed
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
ended_at TEXT,
|
||||
error TEXT
|
||||
)
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// RunAsyncMigration registers `name` as a pending async migration and
|
||||
// schedules `fn` to run in a background goroutine. It returns to the caller
|
||||
// immediately so the ingestor can keep booting.
|
||||
//
|
||||
// Contract (pinned by async_migration_test.go):
|
||||
// - status is `pending_async` IMMEDIATELY after this returns.
|
||||
// - fn runs in a goroutine; on success status becomes `done`, on error or
|
||||
// panic status becomes `failed` and the error is recorded.
|
||||
// - Idempotent: if a row with the same name already exists in `done`
|
||||
// state, fn is NOT re-run. If in `failed` or `pending_async` state,
|
||||
// fn IS re-scheduled (a previous run may have crashed mid-flight).
|
||||
// - The caller's WaitGroup tracks the goroutine so tests/shutdown can
|
||||
// wait via Store.WaitForAsyncMigrations().
|
||||
func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(context.Context, *sql.DB) error) error {
|
||||
if err := ensureAsyncMigrationsTable(s.db); err != nil {
|
||||
return fmt.Errorf("ensure _async_migrations: %w", err)
|
||||
}
|
||||
|
||||
var existing string
|
||||
row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name)
|
||||
switch err := row.Scan(&existing); err {
|
||||
case nil:
|
||||
if existing == "done" {
|
||||
return nil // already complete, nothing to do
|
||||
}
|
||||
// pending_async or failed → reset and retry.
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE _async_migrations
|
||||
SET status = 'pending_async', started_at = datetime('now'), ended_at = NULL, error = NULL
|
||||
WHERE name = ?`, name); err != nil {
|
||||
return fmt.Errorf("reset async migration %q: %w", name, err)
|
||||
}
|
||||
case sql.ErrNoRows:
|
||||
if _, err := s.db.Exec(`
|
||||
INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`,
|
||||
name); err != nil {
|
||||
return fmt.Errorf("register async migration %q: %w", name, err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("lookup async migration %q: %w", name, err)
|
||||
}
|
||||
|
||||
s.backfillWg.Add(1)
|
||||
go func() {
|
||||
defer s.backfillWg.Done()
|
||||
var runErr error
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
runErr = fmt.Errorf("panic: %v", r)
|
||||
log.Printf("[async-migration] %q panic recovered: %v", name, r)
|
||||
}
|
||||
if runErr != nil {
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE _async_migrations
|
||||
SET status = 'failed', ended_at = datetime('now'), error = ?
|
||||
WHERE name = ?`, runErr.Error(), name); err != nil {
|
||||
log.Printf("[async-migration] failed to record failure for %q: %v", name, err)
|
||||
}
|
||||
log.Printf("[async-migration] %q FAILED: %v", name, runErr)
|
||||
return
|
||||
}
|
||||
if _, err := s.db.Exec(`
|
||||
UPDATE _async_migrations
|
||||
SET status = 'done', ended_at = datetime('now'), error = NULL
|
||||
WHERE name = ?`, name); err != nil {
|
||||
log.Printf("[async-migration] failed to mark %q done: %v", name, err)
|
||||
return
|
||||
}
|
||||
log.Printf("[async-migration] %q done", name)
|
||||
}()
|
||||
log.Printf("[async-migration] %q starting (boot continues)", name)
|
||||
runErr = fn(ctx, s.db)
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AsyncMigrationStatus returns the current status of an async migration
|
||||
// (one of "pending_async", "done", "failed") or sql.ErrNoRows if no such
|
||||
// migration has been registered.
|
||||
func (s *Store) AsyncMigrationStatus(name string) (string, error) {
|
||||
if err := ensureAsyncMigrationsTable(s.db); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var status string
|
||||
err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name).Scan(&status)
|
||||
return status, err
|
||||
}
|
||||
|
||||
// WaitForAsyncMigrations blocks until all currently-scheduled async migrations
|
||||
// finish. Intended for tests + graceful shutdown; production boot path does NOT
|
||||
// call this (that's the whole point).
|
||||
func (s *Store) WaitForAsyncMigrations() {
|
||||
s.backfillWg.Wait()
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// waitForStatus polls AsyncMigrationStatus until it matches `want` or `deadline` passes.
|
||||
func waitForStatus(t *testing.T, s *Store, name, want string, timeout time.Duration) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var status string
|
||||
var err error
|
||||
for time.Now().Before(deadline) {
|
||||
status, err = s.AsyncMigrationStatus(name)
|
||||
if err == nil && status == want {
|
||||
return status
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("status never reached %q within %s: got %q (err=%v)", want, timeout, status, err)
|
||||
return status
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_PendingThenDone pins the contract for RunAsyncMigration:
|
||||
//
|
||||
// 1. After calling, the migration name MUST be queryable in the migrations
|
||||
// table with status `pending_async` IMMEDIATELY (no waiting for fn).
|
||||
// 2. After fn returns, the status MUST transition to `done`.
|
||||
// 3. RunAsyncMigration MUST return without blocking on fn.
|
||||
//
|
||||
// This is the regression test for the recurring "sync migration on large
|
||||
// table blocks ingestor startup" class (#791, #1483, ...). If this test
|
||||
// fails the contract is broken — do not relax it; fix the runner.
|
||||
func TestRunAsyncMigration_PendingThenDone(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
const name = "test_async_migration_v1"
|
||||
if err := s.RunAsyncMigration(ctx, name, func(ctx context.Context, db *sql.DB) error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RunAsyncMigration returned error: %v", err)
|
||||
}
|
||||
|
||||
// Wait for the goroutine to actually start before checking status; this
|
||||
// proves RunAsyncMigration did not block on fn and that fn is running
|
||||
// concurrently.
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("async migration fn did not start within 2s — RunAsyncMigration may have blocked or never scheduled")
|
||||
}
|
||||
|
||||
status, err := s.AsyncMigrationStatus(name)
|
||||
if err != nil {
|
||||
t.Fatalf("AsyncMigrationStatus while running: %v", err)
|
||||
}
|
||||
if status != "pending_async" {
|
||||
t.Fatalf("status while fn running: got %q, want %q", status, "pending_async")
|
||||
}
|
||||
|
||||
close(release)
|
||||
|
||||
// Poll for transition to done.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
status, err = s.AsyncMigrationStatus(name)
|
||||
if err == nil && status == "done" {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("status never transitioned to done within 2s: got %q (err=%v)", status, err)
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_PanicCapture proves that a panic inside fn does NOT
|
||||
// leak past the recover, AND that the migration row transitions to
|
||||
// "failed" with the panic message captured — NOT silently to "done".
|
||||
// Operator visibility into mid-migration crashes is the whole point.
|
||||
func TestRunAsyncMigration_PanicCapture(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_panic_capture_v1"
|
||||
|
||||
if err := s.RunAsyncMigration(context.Background(), name,
|
||||
func(ctx context.Context, db *sql.DB) error {
|
||||
panic("synthetic boom")
|
||||
}); err != nil {
|
||||
t.Fatalf("RunAsyncMigration returned error: %v", err)
|
||||
}
|
||||
|
||||
s.WaitForAsyncMigrations()
|
||||
|
||||
status, err := s.AsyncMigrationStatus(name)
|
||||
if err != nil {
|
||||
t.Fatalf("status lookup: %v", err)
|
||||
}
|
||||
if status != "failed" {
|
||||
t.Fatalf("status after panic: got %q, want %q (silent-done would be catastrophic)", status, "failed")
|
||||
}
|
||||
|
||||
var errMsg sql.NullString
|
||||
if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errMsg); err != nil {
|
||||
t.Fatalf("error column lookup: %v", err)
|
||||
}
|
||||
if !errMsg.Valid || errMsg.String == "" {
|
||||
t.Fatalf("error column empty after panic — operator has no clue what failed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_IdempotentSecondCallNoOps verifies that calling
|
||||
// RunAsyncMigration a second time with the same name AFTER it has reached
|
||||
// "done" status does NOT re-run fn. This protects the prod path: ingestor
|
||||
// restarts must not rebuild already-built indexes.
|
||||
func TestRunAsyncMigration_IdempotentSecondCallNoOps(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_idempotent_v1"
|
||||
|
||||
var calls int32
|
||||
fn := func(ctx context.Context, db *sql.DB) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.RunAsyncMigration(context.Background(), name, fn); err != nil {
|
||||
t.Fatalf("first call: %v", err)
|
||||
}
|
||||
s.WaitForAsyncMigrations()
|
||||
waitForStatus(t, s, name, "done", 2*time.Second)
|
||||
|
||||
// Second call must short-circuit; fn must not be invoked again.
|
||||
if err := s.RunAsyncMigration(context.Background(), name, fn); err != nil {
|
||||
t.Fatalf("second call: %v", err)
|
||||
}
|
||||
s.WaitForAsyncMigrations()
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Fatalf("fn invoked %d times, want 1 (done-state row must short-circuit)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_RestartSafetyFailedIsRetried simulates a crashed
|
||||
// previous run: a row exists in `failed` state from a prior boot. The next
|
||||
// RunAsyncMigration call MUST re-schedule fn (reset to pending_async, then
|
||||
// run it), not leave the migration stuck in `failed` forever.
|
||||
func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_restart_failed_v1"
|
||||
|
||||
if err := ensureAsyncMigrationsTable(s.db); err != nil {
|
||||
t.Fatalf("ensure table: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES (?, 'failed', 'simulated prior crash')`, name); err != nil {
|
||||
t.Fatalf("seed failed row: %v", err)
|
||||
}
|
||||
|
||||
var calls int32
|
||||
if err := s.RunAsyncMigration(context.Background(), name,
|
||||
func(ctx context.Context, db *sql.DB) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RunAsyncMigration on failed row: %v", err)
|
||||
}
|
||||
s.WaitForAsyncMigrations()
|
||||
waitForStatus(t, s, name, "done", 2*time.Second)
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Fatalf("fn invoked %d times, want 1 (failed-state row must be retried)", got)
|
||||
}
|
||||
|
||||
// And the error column must be cleared on success.
|
||||
var errCol sql.NullString
|
||||
if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil {
|
||||
t.Fatalf("error col: %v", err)
|
||||
}
|
||||
if errCol.Valid && errCol.String != "" {
|
||||
t.Fatalf("error column not cleared on retry success: %q", errCol.String)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_RestartSafetyPendingIsRetried simulates the
|
||||
// ingestor crashing while a migration was still in `pending_async` (the
|
||||
// goroutine never finished). On next boot the migration MUST be re-picked-up
|
||||
// — leaving it stuck in pending forever would be a silent prod outage.
|
||||
func TestRunAsyncMigration_RestartSafetyPendingIsRetried(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_restart_pending_v1"
|
||||
|
||||
if err := ensureAsyncMigrationsTable(s.db); err != nil {
|
||||
t.Fatalf("ensure table: %v", err)
|
||||
}
|
||||
if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, name); err != nil {
|
||||
t.Fatalf("seed pending row: %v", err)
|
||||
}
|
||||
|
||||
var calls int32
|
||||
if err := s.RunAsyncMigration(context.Background(), name,
|
||||
func(ctx context.Context, db *sql.DB) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RunAsyncMigration on pending row: %v", err)
|
||||
}
|
||||
s.WaitForAsyncMigrations()
|
||||
waitForStatus(t, s, name, "done", 2*time.Second)
|
||||
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Fatalf("fn invoked %d times, want 1 (pending row must be retried after crash)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_FnErrorRecorded covers the non-panic failure path:
|
||||
// fn returns an error → status MUST be "failed" with the error captured.
|
||||
func TestRunAsyncMigration_FnErrorRecorded(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_fn_error_v1"
|
||||
|
||||
if err := s.RunAsyncMigration(context.Background(), name,
|
||||
func(ctx context.Context, db *sql.DB) error {
|
||||
return fmt.Errorf("simulated migration error")
|
||||
}); err != nil {
|
||||
t.Fatalf("RunAsyncMigration: %v", err)
|
||||
}
|
||||
s.WaitForAsyncMigrations()
|
||||
|
||||
status, err := s.AsyncMigrationStatus(name)
|
||||
if err != nil {
|
||||
t.Fatalf("status: %v", err)
|
||||
}
|
||||
if status != "failed" {
|
||||
t.Fatalf("status: got %q, want failed", status)
|
||||
}
|
||||
|
||||
var errCol sql.NullString
|
||||
if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil {
|
||||
t.Fatalf("error col: %v", err)
|
||||
}
|
||||
if !errCol.Valid || errCol.String == "" {
|
||||
t.Fatalf("error column empty after fn error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAsyncMigration_ConcurrentSameNameSerialized validates the
|
||||
// single-process-instance assumption: ingestor has only one *Store, and
|
||||
// concurrent RunAsyncMigration(name=X) calls on the SAME *Store must not
|
||||
// execute fn more than once for a given name. (CoreScope does not support
|
||||
// multi-ingestor / cluster mode — see MIGRATIONS.md "Concurrency" note —
|
||||
// so cross-process races are out of scope.)
|
||||
func TestRunAsyncMigration_ConcurrentSameNameSerialized(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const name = "test_concurrent_serialize_v1"
|
||||
|
||||
var calls int32
|
||||
fn := func(ctx context.Context, db *sql.DB) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 5; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// All concurrent callers use the SAME name. Each is allowed
|
||||
// to either no-op (status==done short-circuit) or schedule
|
||||
// a re-run; the invariant is "fn never runs more than once
|
||||
// concurrently and on second-call-after-done it does not
|
||||
// re-execute."
|
||||
_ = s.RunAsyncMigration(context.Background(), name, fn)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
s.WaitForAsyncMigrations()
|
||||
waitForStatus(t, s, name, "done", 2*time.Second)
|
||||
|
||||
// The contract per the helper's docstring + Idempotent test is: once
|
||||
// status is `done`, subsequent calls short-circuit. Concurrent calls
|
||||
// that lose the race to set up the pending_async row may legitimately
|
||||
// re-schedule fn (the comment "previous run may have crashed
|
||||
// mid-flight" justifies retry on pending_async). The hard bound is
|
||||
// "fn runs at most ONCE PER pending->done transition" — for this
|
||||
// test we assert fn ran at least once and at most a small bounded
|
||||
// number (5 callers, each may have scheduled before any reached done).
|
||||
if got := atomic.LoadInt32(&calls); got < 1 || got > 5 {
|
||||
t.Fatalf("fn invoked %d times, want 1..5 inclusive (bounded by caller count)", got)
|
||||
}
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
)
|
||||
|
||||
// clientPubkeyRe validates the companion pubkey taken from the MQTT topic
|
||||
// (meshcore/client/<PUBLIC_KEY>/packets). A no-ACL broker would let a client
|
||||
// publish under an arbitrary topic segment (e.g. "!@#$"), so we reject anything
|
||||
// that is not lowercase hex before it reaches client_receptions/client_observers.
|
||||
// Mirrors the server-side hexPrefixRe (cmd/server/node_resolve.go).
|
||||
var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`)
|
||||
|
||||
// handleClientPacket processes a packet from the mobile client RX topic
|
||||
// (meshcore/client/{PUBLIC_KEY}/packets). Unlike observer packets, a roaming
|
||||
// companion reports WHERE it directly heard a node, so we write a
|
||||
// client_receptions row and never touch the observers/observations tables.
|
||||
// rxPubkey is the companion pubkey from the topic (ACL-bound by the broker).
|
||||
func handleClientPacket(store *Store, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string) {
|
||||
// The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex
|
||||
// topic segments so a no-ACL broker can't pollute the coverage tables, and
|
||||
// never fall back to a payload-supplied id (that would defeat the ACL trust
|
||||
// model — see docs/client-rx-coverage.md).
|
||||
rxPubkey = strings.ToLower(strings.TrimSpace(rxPubkey))
|
||||
if !clientPubkeyRe.MatchString(rxPubkey) {
|
||||
log.Printf("MQTT [%s] client: invalid pubkey %.8q, dropping", tag, rxPubkey)
|
||||
return
|
||||
}
|
||||
rawHex, _ := msg["raw"].(string)
|
||||
if rawHex == "" {
|
||||
return
|
||||
}
|
||||
gps, ok := msg["gps"].(map[string]interface{})
|
||||
if !ok {
|
||||
return // a client packet without a GPS fix is not coverage; drop
|
||||
}
|
||||
lat, latOK := toFloat64(gps["lat"])
|
||||
lon, lonOK := toFloat64(gps["lon"])
|
||||
if !latOK || !lonOK {
|
||||
return
|
||||
}
|
||||
var accPtr *float64
|
||||
if acc, ok := toFloat64(gps["acc_m"]); ok {
|
||||
accPtr = &acc
|
||||
}
|
||||
|
||||
decoded, err := DecodePacket(rawHex, channelKeys, false)
|
||||
if err != nil {
|
||||
log.Printf("MQTT [%s] client decode error: %v", tag, err)
|
||||
return
|
||||
}
|
||||
|
||||
direction := ""
|
||||
if v, ok := msg["direction"].(string); ok {
|
||||
direction = v
|
||||
} else if v, ok := msg["Direction"].(string); ok {
|
||||
direction = v
|
||||
}
|
||||
|
||||
var snrPtr *float64
|
||||
if f, ok := toFloat64(firstPresent(msg, "SNR", "snr")); ok {
|
||||
snrPtr = &f
|
||||
}
|
||||
var rssiPtr *int
|
||||
if f, ok := toFloat64(firstPresent(msg, "RSSI", "rssi")); ok {
|
||||
v := int(f)
|
||||
rssiPtr = &v
|
||||
}
|
||||
|
||||
rxAt, _ := resolveRxTime(msg, tag)
|
||||
isAdvert := decoded.Header.PayloadTypeName == "ADVERT"
|
||||
|
||||
rec, ok := buildClientReception(
|
||||
rxPubkey,
|
||||
direction, decoded.Header.RouteType, decoded.Path.Hops, decoded.Payload.PubKey, isAdvert,
|
||||
snrPtr, rssiPtr, lat, lon, accPtr, rxAt, time.Now().UTC().Format(time.RFC3339),
|
||||
)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, err := store.InsertClientReception(rec); err != nil {
|
||||
log.Printf("MQTT [%s] client_reception insert: %v", tag, err)
|
||||
}
|
||||
// Remember the companion's self-reported name (sent as "origin") so the
|
||||
// leaderboard can show a name even if this companion never advertised.
|
||||
if name := stringField(msg, "origin"); name != "" {
|
||||
if err := store.UpsertClientObserver(rec.RxPubkey, name, time.Now().UTC().Format(time.RFC3339)); err != nil {
|
||||
log.Printf("MQTT [%s] client_observer upsert: %v", tag, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UpsertClientObserver records/updates a mobile client's self-reported name.
|
||||
// All writes live in the ingestor (read/write invariant #1283).
|
||||
func (s *Store) UpsertClientObserver(pubkey, name, ts string) error {
|
||||
if pubkey == "" || name == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?)
|
||||
ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`,
|
||||
strings.ToLower(pubkey), name, ts)
|
||||
return err
|
||||
}
|
||||
|
||||
// firstPresent returns the first present value among the given keys.
|
||||
func firstPresent(msg map[string]interface{}, keys ...string) interface{} {
|
||||
for _, k := range keys {
|
||||
if v, ok := msg[k]; ok {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stringField returns msg[key] as a string, or "" if absent/not a string.
|
||||
func stringField(msg map[string]interface{}, key string) string {
|
||||
if v, ok := msg[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ClientReception is one mobile RX coverage point: a companion (RxPubkey)
|
||||
// directly heard a node (HeardKey) at a GPS position. Hex binning is done
|
||||
// server-side from Lat/Lon at query time, so no cell id is stored here.
|
||||
type ClientReception struct {
|
||||
RxPubkey string
|
||||
HeardKey string
|
||||
HeardKeyLen int
|
||||
RSSI *int
|
||||
SNR *float64
|
||||
Lat float64
|
||||
Lon float64
|
||||
PosAccM *float64
|
||||
RxAt string
|
||||
IngestedAt string
|
||||
Src string
|
||||
}
|
||||
|
||||
// deriveHeardKey applies the RX capture HARD RULE: record only what the
|
||||
// companion heard itself and directly.
|
||||
// - direction must be "rx".
|
||||
// - hops present AND a FLOOD route → the directly-heard node is the LAST hop
|
||||
// (path[len-1] = the forwarder that just transmitted; each FLOOD forwarder
|
||||
// appends its hash to the end). 1-byte (2 hex char) prefixes are rejected.
|
||||
// - hops present on a DIRECT route → NOT attributable: direct forwarders
|
||||
// consume the next hop from the FRONT (firmware Mesh.cpp removeSelfFromPath),
|
||||
// so path[len-1] is the route's destination-side end, not who was heard.
|
||||
// - hops empty + isAdvert → the 0-hop advertiser, by its full pubkey.
|
||||
// - otherwise → not attributable (ok=false).
|
||||
//
|
||||
// Returns (heardKey lowercased, keylenBytes, src, ok).
|
||||
func deriveHeardKey(direction string, routeType int, hops []string, advertPubkey string, isAdvert bool) (string, int, string, bool) {
|
||||
if !strings.EqualFold(direction, "rx") {
|
||||
return "", 0, "", false
|
||||
}
|
||||
if len(hops) > 0 {
|
||||
// FLOOD routes (TRANSPORT_FLOOD 0, FLOOD 1) APPEND each forwarder's hash to
|
||||
// the END of the path, so path[last] is the immediate RF transmitter. DIRECT
|
||||
// routes (2, 3) consume the next hop from the FRONT, so path[last] is the
|
||||
// route's destination-side end, NOT who was heard.
|
||||
if routeType != packetpath.RouteTransportFlood && routeType != packetpath.RouteFlood { // direct route: path[last] is not the transmitter
|
||||
return "", 0, "", false
|
||||
}
|
||||
last := strings.ToLower(strings.TrimSpace(hops[len(hops)-1]))
|
||||
keylen := len(last) / 2
|
||||
if keylen < 2 { // exclude 1-byte (collision-prone), matching Reach
|
||||
return "", 0, "", false
|
||||
}
|
||||
return last, keylen, "rxlog", true
|
||||
}
|
||||
if isAdvert && advertPubkey != "" {
|
||||
pk := strings.ToLower(strings.TrimSpace(advertPubkey))
|
||||
return pk, len(pk) / 2, "advert", true
|
||||
}
|
||||
return "", 0, "", false
|
||||
}
|
||||
|
||||
// buildClientReception validates inputs and assembles a ClientReception, or
|
||||
// returns ok=false when the packet is not attributable / out of range.
|
||||
func buildClientReception(
|
||||
rxPubkey, direction string, routeType int, hops []string, advertPubkey string, isAdvert bool,
|
||||
snr *float64, rssi *int, lat, lon float64, posAccM *float64, rxAt, ingestedAt string,
|
||||
) (*ClientReception, bool) {
|
||||
if rxPubkey == "" || rxAt == "" {
|
||||
return nil, false
|
||||
}
|
||||
if lat < -90 || lat > 90 || lon < -180 || lon > 180 {
|
||||
return nil, false
|
||||
}
|
||||
heardKey, keylen, src, ok := deriveHeardKey(direction, routeType, hops, advertPubkey, isAdvert)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return &ClientReception{
|
||||
RxPubkey: strings.ToLower(rxPubkey), HeardKey: heardKey, HeardKeyLen: keylen,
|
||||
RSSI: rssi, SNR: snr, Lat: lat, Lon: lon, PosAccM: posAccM,
|
||||
RxAt: rxAt, IngestedAt: ingestedAt, Src: src,
|
||||
}, true
|
||||
}
|
||||
|
||||
// InsertClientReception writes one coverage row. Idempotent via the
|
||||
// UNIQUE(rx_pubkey, heard_key, rx_at) constraint; returns ins=false when the
|
||||
// row already existed. All writes live in the ingestor (read/write invariant #1283).
|
||||
func (s *Store) InsertClientReception(r *ClientReception) (bool, error) {
|
||||
res, err := s.db.Exec(`
|
||||
INSERT INTO client_receptions
|
||||
(rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`,
|
||||
r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n > 0, nil
|
||||
}
|
||||
@@ -1,334 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
)
|
||||
|
||||
// TestPruneOldClientReceptions verifies the retention reaper bounds the coverage
|
||||
// tables: rows older than the window (and stale companion names) are deleted,
|
||||
// recent ones kept, and days=0 disables it.
|
||||
func TestPruneOldClientReceptions(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
now := time.Now().UTC()
|
||||
recent := now.AddDate(0, 0, -1).Format(time.RFC3339)
|
||||
old := now.AddDate(0, 0, -40).Format(time.RFC3339)
|
||||
const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3"
|
||||
|
||||
s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: recent, IngestedAt: "x", Src: "rxlog"})
|
||||
s.InsertClientReception(&ClientReception{RxPubkey: testCompanionPK, HeardKey: "aabbcc", HeardKeyLen: 3, Lat: 51, Lon: 3.7, RxAt: old, IngestedAt: "x", Src: "rxlog"})
|
||||
s.UpsertClientObserver(testCompanionPK, "Fresh", recent)
|
||||
s.UpsertClientObserver(companion2, "Stale", old)
|
||||
|
||||
if n, _ := s.PruneOldClientReceptions(0); n != 0 {
|
||||
t.Fatalf("days=0 must be a no-op, got %d", n)
|
||||
}
|
||||
n, err := s.PruneOldClientReceptions(7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("expected 1 old reception pruned, got %d", n)
|
||||
}
|
||||
var recN, obsN int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&recN)
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&obsN)
|
||||
if recN != 1 {
|
||||
t.Fatalf("expected 1 reception remaining (recent), got %d", recN)
|
||||
}
|
||||
if obsN != 1 {
|
||||
t.Fatalf("expected 1 observer remaining (fresh), got %d", obsN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientReceptionsTableExists(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
cols := map[string]bool{}
|
||||
rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`)
|
||||
if err != nil {
|
||||
t.Fatalf("PRAGMA failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull, pk int
|
||||
var dflt any
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cols[name] = true
|
||||
}
|
||||
for _, want := range []string{"id", "rx_pubkey", "heard_key", "heard_keylen", "rssi", "snr", "lat", "lon", "pos_acc_m", "rx_at", "ingested_at", "src"} {
|
||||
if !cols[want] {
|
||||
t.Errorf("missing column %q in client_receptions", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func crF(f float64) *float64 { return &f }
|
||||
func crI(i int) *int { return &i }
|
||||
|
||||
// TestClientReceptionsCoverageQueryUsesIndex verifies #5/#18: the dominant
|
||||
// per-node coverage query (sargable heard_key IN-list + bbox, mirroring
|
||||
// cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index
|
||||
// rather than scanning the table. Without idx_client_recept_heard_geo the plan
|
||||
// is "SCAN client_receptions".
|
||||
func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at
|
||||
FROM client_receptions
|
||||
WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`
|
||||
rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
plan := ""
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail + "\n"
|
||||
}
|
||||
if !strings.Contains(plan, "USING INDEX idx_client_recept") {
|
||||
t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan)
|
||||
}
|
||||
if strings.Contains(plan, "SCAN client_receptions") {
|
||||
t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientReceptionsRetentionUsesRxAtIndex verifies the retention reaper's
|
||||
// DELETE ... WHERE rx_at < ? (and the leaderboard's rx_at window) seek the rx_at
|
||||
// index rather than full-scanning under the writer lock (polish review).
|
||||
func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
plan := ""
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail + "\n"
|
||||
}
|
||||
if !strings.Contains(plan, "idx_client_recept_rxat") {
|
||||
t.Fatalf("retention DELETE should use idx_client_recept_rxat, plan was:\n%s", plan)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRxLeaderboardQueryIsIndexBacked pins the planner choice for the leaderboard
|
||||
// SELECT (the rx_at-windowed, rx_pubkey-grouped query in cmd/server/rx_dashboard.go).
|
||||
// SQLite serves it from the UNIQUE(rx_pubkey,heard_key,rx_at) constraint index as a
|
||||
// COVERING scan (not idx_client_recept_rxat, and not a table-heap scan). The table
|
||||
// is retention-bounded, so a covering scan is acceptable; this test guards against a
|
||||
// silent regression to a bare table scan under the writer lock when the schema is
|
||||
// next tweaked. Representative form (no JOINs — they don't change whether `cr` is
|
||||
// index-backed).
|
||||
func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
rows, err := s.db.Query(`EXPLAIN QUERY PLAN
|
||||
SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key)
|
||||
FROM client_receptions cr
|
||||
WHERE cr.rx_at >= ?
|
||||
GROUP BY cr.rx_pubkey
|
||||
ORDER BY COUNT(*) DESC
|
||||
LIMIT ?`, "2026-01-01T00:00:00Z", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
plan := ""
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail + "\n"
|
||||
}
|
||||
t.Logf("leaderboard plan:\n%s", plan)
|
||||
// The concern is a bare table-heap scan, not which specific index wins. The
|
||||
// plan must stay index-backed (covering or search) — a regression to a bare
|
||||
// "SCAN cr" without an index fails here.
|
||||
if !strings.Contains(plan, "INDEX") {
|
||||
t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveHeardKey(t *testing.T) {
|
||||
full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
|
||||
k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, nil, strings.ToUpper(full), true)
|
||||
if !ok || l != 32 || src != "advert" || k != full {
|
||||
t.Fatalf("0-hop advert: got k=%q l=%d src=%q ok=%v", k, l, src, ok)
|
||||
}
|
||||
k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false)
|
||||
if !ok || k != "bbccdd" || l != 3 || src != "rxlog" {
|
||||
t.Fatalf("flood path: got k=%q l=%d src=%q ok=%v", k, l, src, ok)
|
||||
}
|
||||
// DIRECT route: path[last] is the route's far end, not the transmitter — must be rejected.
|
||||
if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, []string{"aa", "bbccdd"}, "", false); ok {
|
||||
t.Fatalf("direct-route path must be rejected")
|
||||
}
|
||||
if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteTransportDirect, []string{"aa", "bbccdd"}, "", false); ok {
|
||||
t.Fatalf("transport-direct-route path must be rejected")
|
||||
}
|
||||
if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, []string{"aa", "bb"}, "", false); ok {
|
||||
t.Fatalf("1-byte last hop should be rejected")
|
||||
}
|
||||
if _, _, _, ok = deriveHeardKey("tx", packetpath.RouteFlood, []string{"aabbcc"}, "", false); ok {
|
||||
t.Fatalf("tx must be rejected")
|
||||
}
|
||||
if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, nil, "", false); ok {
|
||||
t.Fatalf("no hops + non-advert must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildClientReception(t *testing.T) {
|
||||
acc := 8.0
|
||||
rec, ok := buildClientReception("companionpk", "rx", packetpath.RouteFlood, []string{"aa", "bbccdd"}, "", false,
|
||||
crF(-7.5), crI(-92), 51.05, 3.72, &acc, "2026-06-09T12:00:00Z", "2026-06-09T12:00:01Z")
|
||||
if !ok || rec.HeardKey != "bbccdd" || rec.HeardKeyLen != 3 || rec.Src != "rxlog" {
|
||||
t.Fatalf("bad reception: %+v ok=%v", rec, ok)
|
||||
}
|
||||
if _, ok := buildClientReception("c", "rx", packetpath.RouteDirect, []string{"bbccdd"}, "", false,
|
||||
crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); ok {
|
||||
t.Fatal("direct-route path must be rejected (not the transmitter)")
|
||||
}
|
||||
if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, []string{"bbccdd"}, "", false, nil, nil, 99.0, 3.72, nil, "t", "t"); ok {
|
||||
t.Fatal("out-of-range lat must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertClientReceptionRoundTripAndIdempotent(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
rec := &ClientReception{
|
||||
RxPubkey: "companionpk", HeardKey: "bbccdd", HeardKeyLen: 3, RSSI: crI(-92),
|
||||
Lat: 51.05, Lon: 3.72, RxAt: "2026-06-09T12:00:00Z", IngestedAt: "2026-06-09T12:00:01Z", Src: "rxlog",
|
||||
}
|
||||
if ins, err := s.InsertClientReception(rec); err != nil || !ins {
|
||||
t.Fatalf("first insert: ins=%v err=%v", ins, err)
|
||||
}
|
||||
if ins, err := s.InsertClientReception(rec); err != nil || ins {
|
||||
t.Fatalf("second insert should be a no-op: ins=%v err=%v", ins, err)
|
||||
}
|
||||
var n int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n)
|
||||
if n != 1 {
|
||||
t.Fatalf("expected 1 row, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
|
||||
msg := map[string]interface{}{
|
||||
"raw": advertHex,
|
||||
"direction": "rx",
|
||||
"timestamp": "2026-06-09T12:00:00Z",
|
||||
"origin": "MyMob",
|
||||
"SNR": -7.0,
|
||||
"RSSI": -92.0,
|
||||
"gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0},
|
||||
}
|
||||
handleClientPacket(s, "test", testCompanionPK, msg, nil)
|
||||
|
||||
var obsName string
|
||||
s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName)
|
||||
if obsName != "MyMob" {
|
||||
t.Fatalf("expected client_observers name 'MyMob', got %q", obsName)
|
||||
}
|
||||
|
||||
// This fixture is a relayed advert (non-empty path), so by the capture HARD
|
||||
// RULE we record the directly-heard LAST hop (multibyte), not the originator.
|
||||
// The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey.
|
||||
var n, keylen int
|
||||
var src string
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 || keylen < 2 || src != "rxlog" {
|
||||
t.Fatalf("expected 1 rxlog reception (multibyte last hop), got n=%d keylen=%d src=%q", n, keylen, src)
|
||||
}
|
||||
|
||||
// No GPS → no row.
|
||||
const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3"
|
||||
handleClientPacket(s, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil)
|
||||
var n2 int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2)
|
||||
if n2 != 0 {
|
||||
t.Fatalf("packet without gps must be dropped, got %d rows", n2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleClientPacketZeroHopAdvertWritesReception covers the #9 gap: the
|
||||
// advert fixture used above is a RELAYED advert (non-empty path), so it exercises
|
||||
// the rxlog last-hop branch, not the 0-hop src='advert' branch. Here we rebuild
|
||||
// the same advert with zero hops — header (FLOOD ADVERT) + "00" (0 hops) + the
|
||||
// same advert payload — so handleClientPacket stores the advertiser by its full
|
||||
// pubkey with src='advert', and we assert gps/snr were captured too.
|
||||
func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
relayed := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
|
||||
// relayed = header(2) + path-descriptor(2) + 5*2-byte hops(20) + payload.
|
||||
payload := relayed[24:]
|
||||
zeroHop := "1100" + payload
|
||||
advertPubkey := strings.ToLower(payload[:64]) // advert payload starts with the 32-byte pubkey
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"raw": zeroHop, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z",
|
||||
"origin": "MyMob", "SNR": -7.0, "RSSI": -92.0,
|
||||
"gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0},
|
||||
}
|
||||
handleClientPacket(s, "test", testCompanionPK, msg, nil)
|
||||
|
||||
var heardKey, src string
|
||||
var keylen int
|
||||
var snr sql.NullFloat64
|
||||
var lat, lon float64
|
||||
if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).
|
||||
Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil {
|
||||
t.Fatalf("expected a 0-hop advert reception: %v", err)
|
||||
}
|
||||
if src != "advert" || keylen != 32 || heardKey != advertPubkey {
|
||||
t.Fatalf("0-hop advert: want advert/32/%s, got %s/%d/%s", advertPubkey, src, keylen, heardKey)
|
||||
}
|
||||
if !snr.Valid || snr.Float64 != -7 || lat != 51.05 || lon != 3.72 {
|
||||
t.Fatalf("gps/snr not captured: snr=%v lat=%f lon=%f", snr, lat, lon)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleClientPacketRejectsNonHexPubkey verifies the #2 fix: a companion
|
||||
// pubkey from the topic that isn't lowercase hex (a no-ACL broker could publish
|
||||
// meshcore/client/!@#$/packets) writes nothing to either coverage table. Without
|
||||
// the clientPubkeyRe guard this fixture would insert a polluting row.
|
||||
func TestHandleClientPacketRejectsNonHexPubkey(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
|
||||
for _, bad := range []string{"!@#$", "companionpk", "", "g0g0", "xyz"} {
|
||||
msg := map[string]interface{}{
|
||||
"raw": advertHex, "direction": "rx", "timestamp": "2026-06-09T12:00:00Z",
|
||||
"origin": "Spoof", "SNR": -7.0, "RSSI": -92.0,
|
||||
"gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0},
|
||||
}
|
||||
handleClientPacket(s, "test", bad, msg, nil)
|
||||
}
|
||||
var nRecept, nObs int
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&nRecept)
|
||||
s.db.QueryRow(`SELECT COUNT(*) FROM client_observers`).Scan(&nObs)
|
||||
if nRecept != 0 || nObs != 0 {
|
||||
t.Fatalf("non-hex pubkey must write nothing, got %d receptions, %d observers", nRecept, nObs)
|
||||
}
|
||||
}
|
||||
+15
-76
@@ -43,22 +43,20 @@ type MQTTLegacy struct {
|
||||
|
||||
// Config holds the ingestor configuration, compatible with the Node.js config.json format.
|
||||
type Config struct {
|
||||
DBPath string `json:"dbPath"`
|
||||
MQTT *MQTTLegacy `json:"mqtt,omitempty"`
|
||||
MQTTSources []MQTTSource `json:"mqttSources,omitempty"`
|
||||
LogLevel string `json:"logLevel,omitempty"`
|
||||
ChannelKeysPath string `json:"channelKeysPath,omitempty"`
|
||||
ChannelKeys map[string]string `json:"channelKeys,omitempty"`
|
||||
HashChannels []string `json:"hashChannels,omitempty"`
|
||||
HashRegions []string `json:"hashRegions,omitempty"`
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
Runtime *RuntimeConfig `json:"runtime,omitempty"`
|
||||
ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"`
|
||||
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
DBPath string `json:"dbPath"`
|
||||
MQTT *MQTTLegacy `json:"mqtt,omitempty"`
|
||||
MQTTSources []MQTTSource `json:"mqttSources,omitempty"`
|
||||
LogLevel string `json:"logLevel,omitempty"`
|
||||
ChannelKeysPath string `json:"channelKeysPath,omitempty"`
|
||||
ChannelKeys map[string]string `json:"channelKeys,omitempty"`
|
||||
HashChannels []string `json:"hashChannels,omitempty"`
|
||||
HashRegions []string `json:"hashRegions,omitempty"`
|
||||
Retention *RetentionConfig `json:"retention,omitempty"`
|
||||
Metrics *MetricsConfig `json:"metrics,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
ForeignAdverts *ForeignAdvertConfig `json:"foreignAdverts,omitempty"`
|
||||
ValidateSignatures *bool `json:"validateSignatures,omitempty"`
|
||||
DB *DBConfig `json:"db,omitempty"`
|
||||
|
||||
// ObserverIATAWhitelist restricts which observer IATA regions are processed.
|
||||
// When non-empty, only observers whose IATA code (from the MQTT topic) matches
|
||||
@@ -82,12 +80,6 @@ type Config struct {
|
||||
// NeighborEdgesMaxAgeDays controls neighbor_edges row retention
|
||||
// (#1287 — moved from cmd/server). 0 = default 5.
|
||||
NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"`
|
||||
|
||||
// IngestBufferSize caps the in-memory queue (number of MQTT messages) held
|
||||
// while the single SQLite writer is blocked by startup migrations/prunes
|
||||
// (#1608). Received messages are drained once the write path is ready.
|
||||
// 0 / unset => default. Bounded memory.
|
||||
IngestBufferSize int `json:"ingestBufferSize,omitempty"`
|
||||
}
|
||||
|
||||
// NeighborEdgesDaysOrDefault returns the configured pruning window or 5.
|
||||
@@ -98,17 +90,6 @@ func (c *Config) NeighborEdgesDaysOrDefault() int {
|
||||
return c.NeighborEdgesMaxAgeDays
|
||||
}
|
||||
|
||||
// IngestBufferSizeOrDefault returns the ingest buffer capacity. Default 50000:
|
||||
// at typical mesh rates (~1-2 msg/s) that is many minutes of headroom while a
|
||||
// startup migration holds the writer; each queued item is a small closure, so
|
||||
// worst-case memory stays in the tens of MB.
|
||||
func (c *Config) IngestBufferSizeOrDefault() int {
|
||||
if c.IngestBufferSize > 0 {
|
||||
return c.IngestBufferSize
|
||||
}
|
||||
return 50000
|
||||
}
|
||||
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
type GeoFilterConfig = geofilter.Config
|
||||
|
||||
@@ -129,17 +110,6 @@ func (f *ForeignAdvertConfig) IsDropMode() bool {
|
||||
return strings.EqualFold(strings.TrimSpace(f.Mode), "drop")
|
||||
}
|
||||
|
||||
// ClientRxCoverageConfig controls the opt-in mobile client-RX coverage feature.
|
||||
type ClientRxCoverageConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage
|
||||
// feature is on. Absent/nil ⇒ off (the safe default).
|
||||
func (c *Config) ClientRxCoverageEnabled() bool {
|
||||
return c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled
|
||||
}
|
||||
|
||||
// RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes.
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
@@ -148,10 +118,6 @@ type RetentionConfig struct {
|
||||
// PacketDays is the retention window for transmissions (#1283).
|
||||
// Ownership moved from cmd/server to cmd/ingestor; 0 disables.
|
||||
PacketDays int `json:"packetDays"`
|
||||
// ClientRxDays is the retention window (by rx_at) for mobile client-RX
|
||||
// coverage rows in client_receptions / client_observers; 0 disables. Bounds
|
||||
// the table the opt-in coverage feature would otherwise grow without limit.
|
||||
ClientRxDays int `json:"clientRxDays"`
|
||||
}
|
||||
|
||||
// PacketDaysOrZero returns the configured retention.packetDays or 0
|
||||
@@ -163,29 +129,11 @@ func (c *Config) PacketDaysOrZero() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// ClientRxDaysOrZero returns the configured retention.clientRxDays or 0
|
||||
// (disabled) if not set.
|
||||
func (c *Config) ClientRxDaysOrZero() int {
|
||||
if c.Retention != nil && c.Retention.ClientRxDays > 0 {
|
||||
return c.Retention.ClientRxDays
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// MetricsConfig controls observer metrics collection.
|
||||
type MetricsConfig struct {
|
||||
SampleIntervalSec int `json:"sampleIntervalSec"`
|
||||
}
|
||||
|
||||
// RuntimeConfig holds Go runtime tuning knobs (#1010).
|
||||
type RuntimeConfig struct {
|
||||
// MaxMemoryMB is the soft memory limit (GOMEMLIMIT) in MiB applied via
|
||||
// runtime/debug.SetMemoryLimit at startup. The GOMEMLIMIT environment
|
||||
// variable, when set, takes precedence over this value. 0/unset means
|
||||
// no limit is applied and default Go runtime behavior is preserved.
|
||||
MaxMemoryMB int `json:"maxMemoryMB"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
type DBConfig = dbconfig.DBConfig
|
||||
|
||||
@@ -338,24 +286,15 @@ func LoadConfig(path string) (*Config, error) {
|
||||
}
|
||||
|
||||
// ResolvedSources returns the final list of MQTT sources to connect to.
|
||||
//
|
||||
// Scheme mapping:
|
||||
//
|
||||
// mqtt:// → tcp:// (paho plain TCP)
|
||||
// mqtts:// → ssl:// (paho TLS over TCP)
|
||||
// ws:// (paho WebSocket — passed through, no mapping needed)
|
||||
// wss:// (paho WebSocket TLS — passed through, no mapping needed)
|
||||
func (c *Config) ResolvedSources() []MQTTSource {
|
||||
for i := range c.MQTTSources {
|
||||
// paho uses tcp:// and ssl:// for plain MQTT; ws:// and wss:// are accepted natively.
|
||||
// paho uses tcp:// and ssl:// not mqtt:// and mqtts://
|
||||
b := c.MQTTSources[i].Broker
|
||||
if strings.HasPrefix(b, "mqtt://") {
|
||||
c.MQTTSources[i].Broker = "tcp://" + b[7:]
|
||||
} else if strings.HasPrefix(b, "mqtts://") {
|
||||
c.MQTTSources[i].Broker = "ssl://" + b[8:]
|
||||
}
|
||||
// ws:// and wss:// pass through unchanged — paho handles WebSocket
|
||||
// connections natively via gorilla/websocket.
|
||||
}
|
||||
return c.MQTTSources
|
||||
}
|
||||
|
||||
@@ -394,105 +394,3 @@ func TestMQTTSourceRegionField(t *testing.T) {
|
||||
t.Fatalf("expected region PDX, got %q", cfg.MQTTSources[0].Region)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvedSourcesSchemeMapping verifies that mqtt:// and mqtts:// are translated
|
||||
// to the paho-native tcp:// and ssl:// schemes, while ws:// and wss:// pass through
|
||||
// unchanged (paho handles WebSocket connections natively).
|
||||
func TestResolvedSourcesSchemeMapping(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"mqtt://host:1883", "tcp://host:1883"},
|
||||
{"mqtts://host:8883", "ssl://host:8883"},
|
||||
{"tcp://host:1883", "tcp://host:1883"},
|
||||
{"ssl://host:8883", "ssl://host:8883"},
|
||||
{"ws://host:9001", "ws://host:9001"},
|
||||
{"wss://host:9001", "wss://host:9001"},
|
||||
{"ws://host:9001/mqtt", "ws://host:9001/mqtt"},
|
||||
{"wss://host:9001/mqtt", "wss://host:9001/mqtt"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
cfg := &Config{
|
||||
MQTTSources: []MQTTSource{
|
||||
{Name: "test", Broker: tt.input, Topics: []string{"meshcore/#"}},
|
||||
},
|
||||
}
|
||||
sources := cfg.ResolvedSources()
|
||||
if got := sources[0].Broker; got != tt.want {
|
||||
t.Errorf("ResolvedSources(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadConfigWSSource verifies that a WebSocket MQTT source round-trips through
|
||||
// LoadConfig correctly — username/password preserved, scheme unchanged.
|
||||
func TestLoadConfigWSSource(t *testing.T) {
|
||||
t.Setenv("DB_PATH", "")
|
||||
t.Setenv("MQTT_BROKER", "")
|
||||
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
os.WriteFile(cfgPath, []byte(`{
|
||||
"dbPath": "test.db",
|
||||
"mqttSources": [
|
||||
{
|
||||
"name": "local-tcp",
|
||||
"broker": "mqtt://localhost:1883",
|
||||
"topics": ["meshcore/#"]
|
||||
},
|
||||
{
|
||||
"name": "wsmqtt-ws",
|
||||
"broker": "wss://wsmqtt.example.com/mqtt",
|
||||
"username": "corescope",
|
||||
"password": "s3cr3t",
|
||||
"topics": ["meshcore/#"]
|
||||
}
|
||||
]
|
||||
}`), 0o644)
|
||||
|
||||
cfg, err := LoadConfig(cfgPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.MQTTSources) != 2 {
|
||||
t.Fatalf("mqttSources len=%d, want 2", len(cfg.MQTTSources))
|
||||
}
|
||||
|
||||
tcp := cfg.MQTTSources[0]
|
||||
if tcp.Name != "local-tcp" {
|
||||
t.Errorf("name=%s, want local-tcp", tcp.Name)
|
||||
}
|
||||
|
||||
ws := cfg.MQTTSources[1]
|
||||
if ws.Name != "wsmqtt-ws" {
|
||||
t.Errorf("name=%s, want wsmqtt-ws", ws.Name)
|
||||
}
|
||||
if ws.Broker != "wss://wsmqtt.example.com/mqtt" {
|
||||
t.Errorf("broker=%s, want wss://wsmqtt.example.com/mqtt", ws.Broker)
|
||||
}
|
||||
if ws.Username != "corescope" {
|
||||
t.Errorf("username=%s, want corescope", ws.Username)
|
||||
}
|
||||
if ws.Password != "s3cr3t" {
|
||||
t.Errorf("password=%s, want s3cr3t", ws.Password)
|
||||
}
|
||||
|
||||
sources := cfg.ResolvedSources()
|
||||
if sources[1].Broker != "wss://wsmqtt.example.com/mqtt" {
|
||||
t.Errorf("ResolvedSources wss broker=%s, want unchanged", sources[1].Broker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBufferSizeOrDefault(t *testing.T) {
|
||||
if got := (&Config{}).IngestBufferSizeOrDefault(); got != 50000 {
|
||||
t.Fatalf("default: want 50000, got %d", got)
|
||||
}
|
||||
if got := (&Config{IngestBufferSize: 10}).IngestBufferSizeOrDefault(); got != 10 {
|
||||
t.Fatalf("override: want 10, got %d", got)
|
||||
}
|
||||
if got := (&Config{IngestBufferSize: -5}).IngestBufferSizeOrDefault(); got != 50000 {
|
||||
t.Fatalf("invalid negative should fall back to default, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// testCompanionPK is a valid lowercase-hex companion pubkey for coverage tests.
|
||||
// The topic segment must be hex (clientPubkeyRe) or handleClientPacket drops it.
|
||||
const testCompanionPK = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2"
|
||||
|
||||
// clientCoverageMsg builds a valid mobile client-RX coverage message on the
|
||||
// dedicated topic meshcore/client/<pubkey>/packets. The raw hex is a relayed
|
||||
// advert with GPS, so handleClientPacket would write exactly one
|
||||
// client_receptions row when the feature is enabled (see
|
||||
// TestHandleClientPacketAdvertWritesReception).
|
||||
func clientCoverageMsg() *mockMessage {
|
||||
advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172"
|
||||
payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","SNR":-7.0,"RSSI":-92.0,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`)
|
||||
return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/packets", payload: payload}
|
||||
}
|
||||
|
||||
func clientReceptionCount(t *testing.T, s *Store) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// TestClientRxCoverageEnabledDefault verifies the gate helper defaults OFF for
|
||||
// nil/absent config and is only true when explicitly enabled.
|
||||
func TestClientRxCoverageEnabledDefault(t *testing.T) {
|
||||
if (&Config{}).ClientRxCoverageEnabled() {
|
||||
t.Fatal("nil ClientRxCoverage must report disabled")
|
||||
}
|
||||
if (&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: false}}).ClientRxCoverageEnabled() {
|
||||
t.Fatal("Enabled:false must report disabled")
|
||||
}
|
||||
if !(&Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}).ClientRxCoverageEnabled() {
|
||||
t.Fatal("Enabled:true must report enabled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientRxCoverageGateOff drives handleMessage with the feature OFF: the
|
||||
// client-topic message must fall through and write no client_receptions rows.
|
||||
func TestClientRxCoverageGateOff(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{} // ClientRxCoverage nil ⇒ disabled
|
||||
|
||||
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
|
||||
|
||||
if n := clientReceptionCount(t, store); n != 0 {
|
||||
t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientRxCoverageGateOn drives handleMessage with the feature ON: the
|
||||
// client-topic message must be dispatched and write exactly one row.
|
||||
func TestClientRxCoverageGateOn(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}
|
||||
|
||||
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
|
||||
|
||||
if n := clientReceptionCount(t, store); n != 1 {
|
||||
t.Fatalf("feature ON: expected 1 client_receptions row, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClientRxCoverageBlacklistedDropped verifies the #1 fix: a blacklisted
|
||||
// operator cannot skirt the observer blacklist via the client topic. With the
|
||||
// feature ON but the companion pubkey blacklisted, no row is written. Without
|
||||
// the gate the client dispatch runs before the blacklist check and inserts.
|
||||
func TestClientRxCoverageBlacklistedDropped(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
cfg := &Config{
|
||||
ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true},
|
||||
ObserverBlacklist: []string{testCompanionPK},
|
||||
}
|
||||
|
||||
handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg)
|
||||
|
||||
if n := clientReceptionCount(t, store); n != 0 {
|
||||
t.Fatalf("blacklisted companion: expected 0 client_receptions rows, got %d", n)
|
||||
}
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// coverageBenchSQL is the dominant per-node coverage query (mirrors
|
||||
// cmd/server queryCoverageRows): a bbox range plus a full-key/2-3-byte-prefix
|
||||
// match on the heard node.
|
||||
const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at
|
||||
FROM client_receptions
|
||||
WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?
|
||||
AND ( (heard_keylen = 32 AND heard_key = ?)
|
||||
OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )`
|
||||
|
||||
// BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times
|
||||
// the coverage query with the indexes (#5/#18) versus a forced full table scan.
|
||||
// Run: go test -run x -bench BenchmarkCoverageQuery -benchtime 20x ./cmd/ingestor
|
||||
func BenchmarkCoverageQuery(b *testing.B) {
|
||||
const n = 1_000_000
|
||||
const prefixPool = 2000 // distinct 3-byte heard_key prefixes
|
||||
|
||||
dir := b.TempDir()
|
||||
s, err := OpenStore(dir + "/bench.db")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
stmt, err := tx.Prepare(`INSERT INTO client_receptions
|
||||
(rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
hk := fmt.Sprintf("%06x", rng.Intn(prefixPool))
|
||||
lat := 51.0 + rng.Float64()*0.4 // ~44 km metro span
|
||||
lon := 3.5 + rng.Float64()*0.4
|
||||
rxpk := fmt.Sprintf("%064x", rng.Intn(500))
|
||||
// rx_at carries i so (rx_pubkey,heard_key,rx_at) stays unique.
|
||||
if _, err := stmt.Exec(rxpk, hk, 3, -6.0, lat, lon, fmt.Sprintf("t%d", i), "x", "rxlog"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
stmt.Close()
|
||||
if err := tx.Commit(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
// Target node whose 3-byte prefix (0003e8 = 1000) is in the pool, queried
|
||||
// over a sub-bbox of the metro area.
|
||||
target := "0003e8" + strings.Repeat("ab", 29) // 6 + 58 = 64 hex
|
||||
|
||||
// OR/substr query (original shape): bbox range OR'd with a non-sargable
|
||||
// substr prefix match.
|
||||
runOR := func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
rows, err := s.db.Query(coverageBenchSQL, 51.1, 51.3, 3.6, 3.8, target, target)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// IN-list query (sargable): the heard node's candidate keys are exactly the
|
||||
// full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the
|
||||
// heard_key-leading composite instead of scanning the bbox.
|
||||
inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions
|
||||
WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`
|
||||
runIN := func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for rows.Next() {
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
}
|
||||
|
||||
b.Run("or_query_indexed", runOR)
|
||||
b.Run("inlist_query_indexed", runIN)
|
||||
|
||||
// Drop the coverage indexes to measure the full-scan baseline.
|
||||
for _, idx := range []string{"idx_client_recept_heard_geo", "idx_client_recept_latlon", "idx_client_recept_rxpk"} {
|
||||
if _, err := s.db.Exec("DROP INDEX IF EXISTS " + idx); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.Run("or_query_table_scan", runOR)
|
||||
}
|
||||
+30
-623
@@ -1,14 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -71,7 +69,6 @@ type Store struct {
|
||||
stmtGetTxByHash *sql.Stmt
|
||||
stmtInsertTransmission *sql.Stmt
|
||||
stmtUpdateTxFirstSeen *sql.Stmt
|
||||
stmtBumpTxLastSeen *sql.Stmt
|
||||
stmtInsertObservation *sql.Stmt
|
||||
stmtUpsertNode *sql.Stmt
|
||||
stmtIncrementAdvertCount *sql.Stmt
|
||||
@@ -83,16 +80,6 @@ type Store struct {
|
||||
|
||||
sampleIntervalSec int
|
||||
backfillWg sync.WaitGroup
|
||||
|
||||
// prefixIdx holds the prefix → pubkey index used by the
|
||||
// resolved_path writer (#1547). Rebuilt on startup and once per
|
||||
// neighbor-edges builder tick (60s).
|
||||
prefixIdx prefixIdxHolder
|
||||
|
||||
// neighborGraph holds the in-memory NeighborGraph snapshot used
|
||||
// by the context-aware resolver (#1560). Rebuilt on startup and
|
||||
// once per neighbor-edges builder tick (60s).
|
||||
neighborGraph neighborGraphHolder
|
||||
}
|
||||
|
||||
// OpenStore opens or creates a SQLite DB at the given path, applying the
|
||||
@@ -137,53 +124,6 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
|
||||
return nil, fmt.Errorf("preparing statements: %w", err)
|
||||
}
|
||||
|
||||
// Schedule async migrations. These must NOT block boot. See
|
||||
// async_migration.go for the convention.
|
||||
// PREFLIGHT: async=true reason="composite index build on observations (1.9M+ rows in prod) — converted from sync after v3.8.3"
|
||||
var idxDone int
|
||||
if s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_observer_ts_idx_v1'").Scan(&idxDone) != nil {
|
||||
if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1",
|
||||
func(ctx context.Context, d *sql.DB) error {
|
||||
log.Println("[migration/async] Building (observer_idx, timestamp) composite index on observations...")
|
||||
if _, err := d.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('obs_observer_ts_idx_v1')`); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("[migration/async] observations(observer_idx, timestamp) index created")
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("[migration/async] scheduling obs_observer_ts_idx_v1 failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// #1690: backfill transmissions.last_seen from MAX(observations.timestamp)
|
||||
// per transmission. The column is added inline by dbschema.Apply (cheap
|
||||
// metadata-only ALTER); the populate query is potentially expensive
|
||||
// (full obs scan + group) so we run it async. Subsequent observation
|
||||
// inserts maintain the column inline (see InsertTransmission below).
|
||||
// PREFLIGHT: async=true reason="full-table backfill JOIN (1.9M+ obs × 86k+ tx in prod) — must not block ingestor boot"
|
||||
if err := s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1",
|
||||
func(ctx context.Context, d *sql.DB) error {
|
||||
log.Println("[migration/async] Backfilling transmissions.last_seen from MAX(observations.timestamp)...")
|
||||
res, err := d.ExecContext(ctx, `
|
||||
UPDATE transmissions
|
||||
SET last_seen = COALESCE((
|
||||
SELECT MAX(timestamp) FROM observations WHERE transmission_id = transmissions.id
|
||||
), last_seen)
|
||||
WHERE last_seen = 0
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
log.Printf("[migration/async] transmissions.last_seen backfill complete: %d rows updated", n)
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -221,12 +161,7 @@ func applySchema(db *sql.DB) error {
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT DEFAULT NULL,
|
||||
clock_skew_seconds INTEGER DEFAULT NULL,
|
||||
clock_skew_count_24h INTEGER DEFAULT 0,
|
||||
clock_last_naive_at TEXT DEFAULT NULL,
|
||||
can_relay INTEGER DEFAULT 1,
|
||||
can_relay_seen INTEGER DEFAULT 0
|
||||
last_packet_at TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen);
|
||||
@@ -258,7 +193,6 @@ func applySchema(db *sql.DB) error {
|
||||
payload_version INTEGER,
|
||||
decoded_json TEXT,
|
||||
from_pubkey TEXT,
|
||||
last_seen INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
@@ -267,55 +201,6 @@ func applySchema(db *sql.DB) error {
|
||||
CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type);
|
||||
-- idx_transmissions_from_pubkey is created by the from_pubkey_v1
|
||||
-- migration after the column is added on legacy DBs (#1143).
|
||||
-- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by
|
||||
-- dbschema.Apply after ensuring the last_seen column exists (#1690,
|
||||
-- partial-index swap #1740) — keep it OUT of this base schema block
|
||||
-- so legacy DBs (table-exists, column-missing) don't trip on the
|
||||
-- CREATE INDEX before the ALTER runs.
|
||||
|
||||
-- Mobile client RX coverage: a roaming companion = a mobile observer
|
||||
-- with a moving GPS position, so it gets its own table rather than
|
||||
-- observations (which assumes a fixed observer/location).
|
||||
CREATE TABLE IF NOT EXISTS client_receptions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rx_pubkey TEXT NOT NULL,
|
||||
heard_key TEXT NOT NULL,
|
||||
heard_keylen INTEGER NOT NULL,
|
||||
rssi INTEGER,
|
||||
snr REAL,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
pos_acc_m REAL,
|
||||
rx_at TEXT NOT NULL,
|
||||
ingested_at TEXT NOT NULL,
|
||||
src TEXT NOT NULL,
|
||||
UNIQUE(rx_pubkey, heard_key, rx_at)
|
||||
);
|
||||
-- Coverage queries filter by bbox AND match the heard node either by full
|
||||
-- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite
|
||||
-- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and
|
||||
-- carries lat/lon so the bbox range is satisfied from the index; it also
|
||||
-- supersedes the old single-column heard_key index. idx_client_recept_latlon
|
||||
-- lets the planner instead drive from a selective bbox. (#5, #18)
|
||||
CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon);
|
||||
CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon);
|
||||
-- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the
|
||||
-- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per
|
||||
-- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without
|
||||
-- this index either would full-scan the table under the writer lock
|
||||
-- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays
|
||||
-- redundant — the leaderboard no longer groups by rx_pubkey in SQL.
|
||||
CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at);
|
||||
DROP INDEX IF EXISTS idx_client_recept_rxpk;
|
||||
|
||||
-- Self-reported name of each mobile client (companion), from the SELF_INFO
|
||||
-- name the app sends as "origin". Lets the leaderboard show a name even
|
||||
-- when the companion never advertised (so it isn't in the nodes table).
|
||||
CREATE TABLE IF NOT EXISTS client_observers (
|
||||
pubkey TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
last_seen TEXT
|
||||
);
|
||||
`
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
return fmt.Errorf("base schema: %w", err)
|
||||
@@ -475,39 +360,6 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] observations timestamp index created")
|
||||
}
|
||||
|
||||
// #1481 P0-3: covering index for GetObserverPacketCounts. The query
|
||||
// joins observations → observers and GROUP BYs observer_idx with a
|
||||
// timestamp WHERE filter; a composite (observer_idx, timestamp)
|
||||
// index lets SQLite resolve the grouping + range filter from the
|
||||
// index alone instead of a 1.9M-row scan.
|
||||
//
|
||||
// CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling
|
||||
// happens in OpenStore() once the real *Store exists so the
|
||||
// backfill WaitGroup is shared with the rest of the ingestor.
|
||||
// The legacy `_migrations` gate is preserved by the async fn so
|
||||
// DBs that already completed the sync build stay no-op.
|
||||
|
||||
// #1483: normalize nodes.public_key to lowercase. The server's
|
||||
// GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf
|
||||
// (#1481 P0-3) and now relies on stored keys being lowercase. The
|
||||
// decoder writes lowercase today, but legacy/admin/API inserts may
|
||||
// have left mixed-case rows. Idempotent: counts and lowers any
|
||||
// non-lowercase rows on every boot, runs once via _migrations gate
|
||||
// for the bulk fix. Re-running stays cheap because subsequent
|
||||
// passes match zero rows.
|
||||
if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil {
|
||||
var n int64
|
||||
_ = r.Scan(&n)
|
||||
if n > 0 {
|
||||
log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n)
|
||||
if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil {
|
||||
log.Printf("[migration] public_key lowercase normalize failed: %v", err)
|
||||
} else {
|
||||
log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// observer_metrics table for RF health dashboard
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
@@ -645,28 +497,6 @@ func applySchema(db *sql.DB) error {
|
||||
log.Println("[migration] observers.last_packet_at column added")
|
||||
}
|
||||
|
||||
// Migration: per-observer naive-clock skew tracking (#1478).
|
||||
// When the ingestor clamps a packet's envelope timestamp because the
|
||||
// observer emitted a zone-less local-time string off from UTC by >15min
|
||||
// (resolveRxTime in main.go), we record the event here so the UI can
|
||||
// surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep.
|
||||
row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'")
|
||||
if row.Scan(&migDone) != nil {
|
||||
log.Println("[migration] Adding clock-naive columns to observers (#1478)...")
|
||||
// Each ALTER is independent — ignore "duplicate column" so reruns are safe.
|
||||
for _, stmt := range []string{
|
||||
`ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`,
|
||||
`ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`,
|
||||
`ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`,
|
||||
} {
|
||||
if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") {
|
||||
return fmt.Errorf("clock_naive migration: %w", err)
|
||||
}
|
||||
}
|
||||
db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`)
|
||||
log.Println("[migration] observers.clock_naive columns added")
|
||||
}
|
||||
|
||||
// Migration: backfill observations.path_json from raw_hex (#888)
|
||||
// NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup.
|
||||
// See staging outage where ~502K rows blocked ingest for 15+ hours.
|
||||
@@ -758,8 +588,8 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtInsertTransmission, err = s.db.Prepare(`
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey, last_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -770,29 +600,14 @@ func (s *Store) prepareStatements() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// #1690: bump transmissions.last_seen to MAX(current, ?) on every
|
||||
// observation insert so cold-load can filter on effective recency.
|
||||
// This is NOT a migration — it's the steady-state writer path. The
|
||||
// one-time backfill (BackfillPathJSONAsync-shaped) runs via
|
||||
// RunAsyncMigration above; this prepared-statement UPDATE is the
|
||||
// per-row maintenance that keeps the column current after the
|
||||
// backfill completes. Recorded in _migrations under
|
||||
// "tx_last_seen_backfill_v1".
|
||||
// PREFLIGHT: async=true reason="prepared-statement row-level UPDATE BY PRIMARY KEY (transmissions.id) — single-row touch per observation, indexed by PK, constant-time at any scale. Not a migration."
|
||||
s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = ? WHERE id = ? AND last_seen < ?")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.stmtInsertObservation, err = s.db.Prepare(`
|
||||
INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET
|
||||
snr = COALESCE(excluded.snr, snr),
|
||||
rssi = COALESCE(excluded.rssi, rssi),
|
||||
score = COALESCE(excluded.score, score),
|
||||
raw_hex = COALESCE(excluded.raw_hex, raw_hex),
|
||||
resolved_path = COALESCE(excluded.resolved_path, resolved_path)
|
||||
snr = COALESCE(excluded.snr, snr),
|
||||
rssi = COALESCE(excluded.rssi, rssi),
|
||||
score = COALESCE(excluded.score, score),
|
||||
raw_hex = COALESCE(excluded.raw_hex, raw_hex)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -820,8 +635,8 @@ func (s *Store) prepareStatements() error {
|
||||
}
|
||||
|
||||
s.stmtUpsertObserver, err = s.db.Prepare(`
|
||||
INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen)
|
||||
VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, COALESCE(?, 1), CASE WHEN ? IS NULL THEN 0 ELSE 1 END)
|
||||
INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor)
|
||||
VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = COALESCE(?, name),
|
||||
iata = COALESCE(?, iata),
|
||||
@@ -833,9 +648,7 @@ func (s *Store) prepareStatements() error {
|
||||
radio = COALESCE(?, radio),
|
||||
battery_mv = COALESCE(?, battery_mv),
|
||||
uptime_secs = COALESCE(?, uptime_secs),
|
||||
noise_floor = COALESCE(?, noise_floor),
|
||||
can_relay = COALESCE(?, can_relay),
|
||||
can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END
|
||||
noise_floor = COALESCE(?, noise_floor)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -887,21 +700,6 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Wait/hold instrumentation (#1340). The hot path uses prepared
|
||||
// statements that auto-commit; gate the whole function under
|
||||
// writerMu so concurrent mqtt_handler inserts queue behind any
|
||||
// other writer (vacuum, prune, neighbor-builder) and the wait is
|
||||
// Go-visible.
|
||||
mqttWaitStart := time.Now()
|
||||
writerMu.Lock()
|
||||
mqttWait := time.Since(mqttWaitStart)
|
||||
mqttHoldStart := time.Now()
|
||||
defer func() {
|
||||
mqttHold := time.Since(mqttHoldStart)
|
||||
writerMu.Unlock()
|
||||
recordWriterTiming("mqtt_handler", mqttWait, mqttHold, "InsertTransmission")
|
||||
}()
|
||||
|
||||
rxTime := data.Timestamp
|
||||
ingestNow := time.Now().UTC().Format(time.RFC3339)
|
||||
if rxTime == "" {
|
||||
@@ -930,7 +728,6 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
data.DecodedJSON, nilIfEmpty(data.ChannelHash),
|
||||
scopeNameForDB(data),
|
||||
nilIfEmpty(data.FromPubkey),
|
||||
epochSecondsForLastSeen(rxTime),
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
@@ -951,11 +748,9 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
err := s.stmtGetObserverRowid.QueryRow(data.ObserverID).Scan(&rowid)
|
||||
if err == nil {
|
||||
observerIdx = &rowid
|
||||
// observer.last_seen and last_packet_at answer "when did the analyzer
|
||||
// last hear from this observer" — both are ingest-time questions.
|
||||
// Per-packet rxTime is stored separately on observations/transmissions
|
||||
// using envelope time (see InsertTransmission above). See #1465.
|
||||
_, _ = s.stmtUpdateObserverLastSeen.Exec(ingestNow, ingestNow, ingestNow, ingestNow, rowid)
|
||||
// Update observer last_seen and last_packet_at on every packet to prevent
|
||||
// low-traffic observers from appearing offline (#463)
|
||||
_, _ = s.stmtUpdateObserverLastSeen.Exec(ingestNow, rxTime, ingestNow, rxTime, rowid)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -965,37 +760,16 @@ func (s *Store) InsertTransmission(data *PacketData) (bool, error) {
|
||||
epochTs = t.Unix()
|
||||
}
|
||||
|
||||
// Resolve hop prefixes to full pubkeys for `observations.resolved_path`.
|
||||
// Per #1547: this writer was lost in the #1289 refactor and lives in
|
||||
// the ingestor now. Per #1560: use the context-aware resolver so
|
||||
// 1-byte prefix collisions are disambiguated via NeighborGraph
|
||||
// adjacency (anchored on from_pubkey for ADVERTs, previous hop
|
||||
// otherwise). Empty resolved JSON → NULL via nilIfEmpty.
|
||||
resolved := resolvePathWithContext(
|
||||
parsePathArray(data.PathJSON),
|
||||
strings.ToLower(data.FromPubkey),
|
||||
s.neighborGraph.load(),
|
||||
s.prefixIdx.load(),
|
||||
)
|
||||
resolvedJSON := marshalResolvedPath(resolved)
|
||||
|
||||
_, err = s.stmtInsertObservation.Exec(
|
||||
txID, observerIdx, data.Direction,
|
||||
data.SNR, data.RSSI, data.Score,
|
||||
data.PathJSON, epochTs, nilIfEmpty(data.RawHex),
|
||||
nilIfEmpty(resolvedJSON),
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
log.Printf("[db] observation insert (non-fatal): %v", err)
|
||||
} else {
|
||||
s.Stats.ObservationsInserted.Add(1)
|
||||
// #1690: bump transmissions.last_seen so cold-load can filter on
|
||||
// effective recency. Conditional `last_seen < ?` so we never go
|
||||
// backwards on out-of-order ingest.
|
||||
if _, err := s.stmtBumpTxLastSeen.Exec(epochTs, txID, epochTs); err != nil {
|
||||
log.Printf("[db] tx last_seen bump (non-fatal): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Each prepared-stmt Exec auto-commits. Count one WAL commit per
|
||||
@@ -1075,13 +849,6 @@ type ObserverMeta struct {
|
||||
RecvErrors *int // cumulative CRC/decode failures since boot
|
||||
PacketsSent *int // cumulative packets sent since boot
|
||||
PacketsRecv *int // cumulative packets received since boot
|
||||
// CanRelay reflects the firmware 1.16 /status `repeat` flag (#1290).
|
||||
// nil means the firmware did not send the field — caller must
|
||||
// preserve the existing observers.can_relay value (default 1).
|
||||
// true → relay-capable (`repeat:on`); false → listener-only
|
||||
// (`repeat:off`), which causes the server-side disambiguator to
|
||||
// exclude this observer's pubkey from path-hop candidate sets.
|
||||
CanRelay *bool
|
||||
}
|
||||
|
||||
// UpsertObserver inserts or updates an observer using the current wall-clock
|
||||
@@ -1104,7 +871,7 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last
|
||||
normalizedIATA := strings.TrimSpace(strings.ToUpper(iata))
|
||||
|
||||
var model, firmware, clientVersion, radio interface{}
|
||||
var batteryMv, uptimeSecs, noiseFloor, canRelay interface{}
|
||||
var batteryMv, uptimeSecs, noiseFloor interface{}
|
||||
if meta != nil {
|
||||
if meta.Model != nil {
|
||||
model = *meta.Model
|
||||
@@ -1127,22 +894,11 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last
|
||||
if meta.NoiseFloor != nil {
|
||||
noiseFloor = *meta.NoiseFloor
|
||||
}
|
||||
// Issue #1290: nil → leave DB column unchanged (COALESCE in
|
||||
// the prepared stmt); 0/1 written when firmware provided
|
||||
// the `repeat` field. INSERT branch defaults to 1 via the
|
||||
// COALESCE in the VALUES clause.
|
||||
if meta.CanRelay != nil {
|
||||
if *meta.CanRelay {
|
||||
canRelay = 1
|
||||
} else {
|
||||
canRelay = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.stmtUpsertObserver.Exec(
|
||||
id, name, normalizedIATA, lastSeen, lastSeen, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay,
|
||||
name, normalizedIATA, ingestNow, lastSeen, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay,
|
||||
id, name, normalizedIATA, lastSeen, lastSeen, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor,
|
||||
name, normalizedIATA, ingestNow, lastSeen, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor,
|
||||
)
|
||||
if err != nil {
|
||||
s.Stats.WriteErrors.Add(1)
|
||||
@@ -1224,8 +980,7 @@ func (s *Store) InsertMetrics(data *MetricsData) error {
|
||||
// PruneOldMetrics deletes observer_metrics rows older than retentionDays.
|
||||
func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) {
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339)
|
||||
// Tagged for /api/perf writer-lock visibility (#1340).
|
||||
result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff)
|
||||
result, err := s.db.Exec(`DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune metrics: %w", err)
|
||||
}
|
||||
@@ -1266,11 +1021,11 @@ func (s *Store) CheckAutoVacuum(cfg *Config) {
|
||||
log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...")
|
||||
start := time.Now()
|
||||
|
||||
if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
|
||||
if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err)
|
||||
return
|
||||
}
|
||||
if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil {
|
||||
if _, err := s.db.Exec("VACUUM"); err != nil {
|
||||
log.Printf("[db] VACUUM failed: %v", err)
|
||||
return
|
||||
}
|
||||
@@ -1283,26 +1038,19 @@ func (s *Store) CheckAutoVacuum(cfg *Config) {
|
||||
// RunIncrementalVacuum returns free pages to the OS (#919).
|
||||
// Safe to call on auto_vacuum=NONE databases (noop).
|
||||
func (s *Store) RunIncrementalVacuum(pages int) {
|
||||
// Tagged for /api/perf writer-lock visibility (#1340).
|
||||
if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
|
||||
if _, err := s.db.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil {
|
||||
log.Printf("[vacuum] incremental_vacuum error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoint runs a WAL checkpoint (TRUNCATE mode).
|
||||
// Returns the number of WAL frames checkpointed (0 if WAL was already empty).
|
||||
// TRUNCATE resets the WAL file to zero bytes when all frames are checkpointed;
|
||||
// if active readers hold frames, it checkpoints what it can and leaves the rest.
|
||||
func (s *Store) Checkpoint() int {
|
||||
var busy, walFrames, checkpointed int
|
||||
if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil {
|
||||
// Checkpoint forces a WAL checkpoint to release the WAL lock file,
|
||||
// preventing lock contention with a new process starting up.
|
||||
func (s *Store) Checkpoint() {
|
||||
if _, err := s.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
|
||||
log.Printf("[db] WAL checkpoint error: %v", err)
|
||||
return 0
|
||||
} else {
|
||||
log.Println("[db] WAL checkpoint complete")
|
||||
}
|
||||
if walFrames > 0 {
|
||||
log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0)
|
||||
}
|
||||
return checkpointed
|
||||
}
|
||||
|
||||
// BackfillPathJSONAsync launches the path_json backfill in a background goroutine.
|
||||
@@ -1499,15 +1247,14 @@ func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) {
|
||||
return 0, nil // keep forever
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339)
|
||||
// Tagged for /api/perf writer-lock visibility (#1340).
|
||||
result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
|
||||
result, err := s.db.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mark stale observers inactive: %w", err)
|
||||
}
|
||||
removed, _ := result.RowsAffected()
|
||||
if removed > 0 {
|
||||
// Clean up orphaned metrics for now-inactive observers
|
||||
_, _ = s.instrumentedExec("prune_observers", `DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
|
||||
s.db.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`)
|
||||
log.Printf("Marked %d observer(s) as inactive (not seen in %d days)", removed, observerDays)
|
||||
}
|
||||
return removed, nil
|
||||
@@ -1602,15 +1349,7 @@ func scopeNameForDB(data *PacketData) *string {
|
||||
// node. Skips the UPDATE when the stored value already matches to avoid
|
||||
// redundant writes on the hot MQTT ingest path. Updates both nodes and
|
||||
// inactive_nodes to stay consistent.
|
||||
//
|
||||
// Defense-in-depth (#1534): an empty scope is treated as a no-op. The call
|
||||
// site at handleMessage is the primary guard (shouldUpdateDefaultScope),
|
||||
// but this layer refuses the invalid write so a future caller cannot
|
||||
// reintroduce the bug by passing "" directly.
|
||||
func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error {
|
||||
if scope == "" {
|
||||
return nil
|
||||
}
|
||||
// Short-circuit: skip if already stored.
|
||||
var cur sql.NullString
|
||||
row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey)
|
||||
@@ -1625,39 +1364,6 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordNaiveSkew is called when resolveRxTime() clamps a packet's envelope
|
||||
// timestamp because the observer is emitting a zone-less local-time string
|
||||
// off from UTC by more than 15 min (issue #1478). Stamps the observer's
|
||||
// clock_skew_seconds / clock_skew_count_24h / clock_last_naive_at so the
|
||||
// server can surface a ⚠️ chip + banner in the UI.
|
||||
//
|
||||
// The count is reset to 1 (not incremented) if no event has been recorded in
|
||||
// the past 24h, otherwise incremented. deltaSec is signed: negative = observer
|
||||
// clock is behind UTC, positive = ahead.
|
||||
func (s *Store) RecordNaiveSkew(observerID string, deltaSec int64, now time.Time) error {
|
||||
if observerID == "" {
|
||||
return nil
|
||||
}
|
||||
nowStr := now.UTC().Format(time.RFC3339)
|
||||
cutoff := now.Add(-24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
// One INSERT-or-UPDATE round trip. ON CONFLICT path resets the rolling
|
||||
// counter when the previous event is older than the 24h window, otherwise
|
||||
// increments it.
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO observers (id, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
clock_skew_seconds = excluded.clock_skew_seconds,
|
||||
clock_last_naive_at = excluded.clock_last_naive_at,
|
||||
clock_skew_count_24h = CASE
|
||||
WHEN clock_last_naive_at IS NULL OR clock_last_naive_at < ?
|
||||
THEN 1
|
||||
ELSE COALESCE(clock_skew_count_24h, 0) + 1
|
||||
END
|
||||
`, observerID, deltaSec, nowStr, cutoff)
|
||||
return err
|
||||
}
|
||||
|
||||
// MQTTPacketMessage is the JSON payload from an MQTT raw packet message.
|
||||
type MQTTPacketMessage struct {
|
||||
Raw string `json:"raw"`
|
||||
@@ -1747,302 +1453,3 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID,
|
||||
|
||||
return pd
|
||||
}
|
||||
|
||||
// ─── Writer-lock instrumentation (issue #1340) ────────────────────────────
|
||||
//
|
||||
// Make SQLite writer-lock starvation visible to operators. Per-component
|
||||
// wait_ms / hold_ms / contention_total histograms, surfaced via
|
||||
// /api/perf/write-sources under the "writer_perf" key. Component tags:
|
||||
// neighbor_builder, mqtt_handler, prune_packets, prune_observers,
|
||||
// prune_metrics, mbcap_persist (deferred — see PR body), vacuum.
|
||||
//
|
||||
// The single writer connection (SetMaxOpenConns(1)) means writes serialise
|
||||
// inside the driver and the wait is invisible to Go. writerMu measures the
|
||||
// wait Go can see (everyone queueing behind the current holder) by gating
|
||||
// every wrapped call site through the same package-level mutex.
|
||||
|
||||
// WriterStatsSnapshot is a per-component wait/hold latency snapshot
|
||||
// surfaced via /api/perf to make SQLite writer-lock starvation visible
|
||||
// to operators (issue #1340). Times are in milliseconds.
|
||||
type WriterStatsSnapshot struct {
|
||||
Count int64 `json:"count"`
|
||||
ContentionTotal int64 `json:"contention_total"`
|
||||
WaitMsP50 float64 `json:"wait_ms_p50"`
|
||||
WaitMsP95 float64 `json:"wait_ms_p95"`
|
||||
WaitMsP99 float64 `json:"wait_ms_p99"`
|
||||
WaitMsMax float64 `json:"wait_ms_max"`
|
||||
HoldMsP50 float64 `json:"hold_ms_p50"`
|
||||
HoldMsP95 float64 `json:"hold_ms_p95"`
|
||||
HoldMsP99 float64 `json:"hold_ms_p99"`
|
||||
HoldMsMax float64 `json:"hold_ms_max"`
|
||||
}
|
||||
|
||||
const (
|
||||
// writerSampleWindow bounds the per-component rolling window so a
|
||||
// long-running ingestor doesn't grow this unbounded.
|
||||
writerSampleWindow = 1024
|
||||
// contentionThresholdMs: wait_ms above this counts as a "contended"
|
||||
// write (per #1340 spec).
|
||||
contentionThresholdMs = 100.0
|
||||
defaultSlowWriterMs = 500.0
|
||||
)
|
||||
|
||||
// slowWriterThresholdMsAtomic — hold_ms threshold above which writes
|
||||
// emit a [db-slow-writer] log line. Read on the hot path; written once
|
||||
// at startup by SetSlowWriterThresholdMs.
|
||||
var slowWriterThresholdMsAtomic atomic.Uint64
|
||||
|
||||
// SetSlowWriterThresholdMs sets the [db-slow-writer] log threshold.
|
||||
// ms<=0 restores the 500ms default. Operators can also set
|
||||
// CORESCOPE_DB_SLOW_WRITER_MS at process start — see initSlowWriterFromEnv.
|
||||
func SetSlowWriterThresholdMs(ms float64) {
|
||||
if ms <= 0 {
|
||||
ms = defaultSlowWriterMs
|
||||
}
|
||||
slowWriterThresholdMsAtomic.Store(uint64(ms))
|
||||
}
|
||||
|
||||
func getSlowWriterThresholdMs() float64 {
|
||||
v := slowWriterThresholdMsAtomic.Load()
|
||||
if v == 0 {
|
||||
return defaultSlowWriterMs
|
||||
}
|
||||
return float64(v)
|
||||
}
|
||||
|
||||
// initSlowWriterFromEnv is called once from package init so operators can
|
||||
// override the threshold via CORESCOPE_DB_SLOW_WRITER_MS without a
|
||||
// Go-side Config change.
|
||||
func initSlowWriterFromEnv() {
|
||||
v := os.Getenv("CORESCOPE_DB_SLOW_WRITER_MS")
|
||||
if v == "" {
|
||||
return
|
||||
}
|
||||
var ms float64
|
||||
if _, err := fmt.Sscanf(v, "%f", &ms); err == nil && ms > 0 {
|
||||
SetSlowWriterThresholdMs(ms)
|
||||
}
|
||||
}
|
||||
|
||||
func init() { initSlowWriterFromEnv() }
|
||||
|
||||
type writerComponentStats struct {
|
||||
mu sync.Mutex
|
||||
count int64
|
||||
contentionTotal int64
|
||||
waitMs []float64
|
||||
holdMs []float64
|
||||
waitMax float64
|
||||
holdMax float64
|
||||
}
|
||||
|
||||
func (c *writerComponentStats) record(waitMs, holdMs float64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.count++
|
||||
if waitMs > contentionThresholdMs {
|
||||
c.contentionTotal++
|
||||
}
|
||||
if waitMs > c.waitMax {
|
||||
c.waitMax = waitMs
|
||||
}
|
||||
if holdMs > c.holdMax {
|
||||
c.holdMax = holdMs
|
||||
}
|
||||
c.waitMs = appendBoundedFloat(c.waitMs, waitMs, writerSampleWindow)
|
||||
c.holdMs = appendBoundedFloat(c.holdMs, holdMs, writerSampleWindow)
|
||||
}
|
||||
|
||||
func appendBoundedFloat(s []float64, v float64, max int) []float64 {
|
||||
if len(s) < max {
|
||||
return append(s, v)
|
||||
}
|
||||
copy(s, s[1:])
|
||||
s[len(s)-1] = v
|
||||
return s
|
||||
}
|
||||
|
||||
func (c *writerComponentStats) snapshot() WriterStatsSnapshot {
|
||||
c.mu.Lock()
|
||||
wait := append([]float64(nil), c.waitMs...)
|
||||
hold := append([]float64(nil), c.holdMs...)
|
||||
snap := WriterStatsSnapshot{
|
||||
Count: c.count,
|
||||
ContentionTotal: c.contentionTotal,
|
||||
WaitMsMax: c.waitMax,
|
||||
HoldMsMax: c.holdMax,
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Float64s(wait)
|
||||
sort.Float64s(hold)
|
||||
snap.WaitMsP50 = nearestRankPercentile(wait, 0.50)
|
||||
snap.WaitMsP95 = nearestRankPercentile(wait, 0.95)
|
||||
snap.WaitMsP99 = nearestRankPercentile(wait, 0.99)
|
||||
snap.HoldMsP50 = nearestRankPercentile(hold, 0.50)
|
||||
snap.HoldMsP95 = nearestRankPercentile(hold, 0.95)
|
||||
snap.HoldMsP99 = nearestRankPercentile(hold, 0.99)
|
||||
return snap
|
||||
}
|
||||
|
||||
func nearestRankPercentile(sorted []float64, p float64) float64 {
|
||||
n := len(sorted)
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
if n == 1 {
|
||||
return sorted[0]
|
||||
}
|
||||
idx := int(p*float64(n-1) + 0.5)
|
||||
if idx < 0 {
|
||||
idx = 0
|
||||
}
|
||||
if idx >= n {
|
||||
idx = n - 1
|
||||
}
|
||||
return sorted[idx]
|
||||
}
|
||||
|
||||
type writerStatsAggregator struct {
|
||||
mu sync.Mutex
|
||||
components map[string]*writerComponentStats
|
||||
}
|
||||
|
||||
var writerStatsAgg = &writerStatsAggregator{
|
||||
components: make(map[string]*writerComponentStats),
|
||||
}
|
||||
|
||||
func (a *writerStatsAggregator) get(component string) *writerComponentStats {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
c, ok := a.components[component]
|
||||
if !ok {
|
||||
c = &writerComponentStats{}
|
||||
a.components[component] = c
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// reset clears all per-component samples. Test-only: lets a single
|
||||
// scenario assert against a clean aggregator without prior-test noise
|
||||
// in the same package run (TestWriterStarvationVisibleInPerf would
|
||||
// otherwise mix this run's 5 starved samples with thousands of fast
|
||||
// InsertTransmission samples from earlier tests and the p99 would
|
||||
// collapse below the 50s threshold).
|
||||
func (a *writerStatsAggregator) reset() {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.components = make(map[string]*writerComponentStats)
|
||||
}
|
||||
|
||||
// ResetWriterStatsForTest wipes the per-component writer stats
|
||||
// aggregator. Test-only; not safe to call from production code paths.
|
||||
func ResetWriterStatsForTest() { writerStatsAgg.reset() }
|
||||
|
||||
func (a *writerStatsAggregator) snapshot() map[string]WriterStatsSnapshot {
|
||||
a.mu.Lock()
|
||||
keys := make([]string, 0, len(a.components))
|
||||
stats := make([]*writerComponentStats, 0, len(a.components))
|
||||
for k, v := range a.components {
|
||||
keys = append(keys, k)
|
||||
stats = append(stats, v)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
out := make(map[string]WriterStatsSnapshot, len(keys))
|
||||
for i, k := range keys {
|
||||
out[k] = stats[i].snapshot()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// WriterStatsSnapshot returns a per-component wait/hold/contention
|
||||
// snapshot for exposure on /api/perf/write-sources (issue #1340).
|
||||
func (s *Store) WriterStatsSnapshot() map[string]WriterStatsSnapshot {
|
||||
return writerStatsAgg.snapshot()
|
||||
}
|
||||
|
||||
// recordWriterTiming aggregates a single sample under component and
|
||||
// emits [db-slow-writer] if hold_ms > configured threshold (default
|
||||
// 500ms). queryForLog is truncated to 200 chars.
|
||||
func recordWriterTiming(component string, wait, hold time.Duration, queryForLog string) {
|
||||
waitMs := float64(wait.Nanoseconds()) / 1e6
|
||||
holdMs := float64(hold.Nanoseconds()) / 1e6
|
||||
writerStatsAgg.get(component).record(waitMs, holdMs)
|
||||
if holdMs > getSlowWriterThresholdMs() {
|
||||
q := queryForLog
|
||||
if len(q) > 200 {
|
||||
q = q[:200]
|
||||
}
|
||||
log.Printf("[db-slow-writer] component=%s duration=%.1fms query=%s", component, holdMs, q)
|
||||
}
|
||||
}
|
||||
|
||||
// writerMu serialises every wrapped writer call so the wait the next
|
||||
// caller sees is the wait the perf snapshot can attribute. The
|
||||
// SQLite driver also enforces serial writes (SetMaxOpenConns(1)),
|
||||
// but the wait inside the driver is invisible to Go — writerMu makes
|
||||
// it Go-visible.
|
||||
var writerMu sync.Mutex
|
||||
|
||||
// WriterExec wraps s.db.Exec with per-component wait/hold/contention
|
||||
// instrumentation (issue #1340).
|
||||
func (s *Store) WriterExec(component, query string, args ...interface{}) (sql.Result, error) {
|
||||
waitStart := time.Now()
|
||||
writerMu.Lock()
|
||||
wait := time.Since(waitStart)
|
||||
holdStart := time.Now()
|
||||
res, err := s.db.Exec(query, args...)
|
||||
hold := time.Since(holdStart)
|
||||
writerMu.Unlock()
|
||||
recordWriterTiming(component, wait, hold, query)
|
||||
return res, err
|
||||
}
|
||||
|
||||
// WriterTx wraps Begin → fn → Commit under component tagging.
|
||||
// hold_ms covers the whole tx so a slow body counts against its owner.
|
||||
func (s *Store) WriterTx(component string, fn func(*sql.Tx) error) error {
|
||||
waitStart := time.Now()
|
||||
writerMu.Lock()
|
||||
wait := time.Since(waitStart)
|
||||
holdStart := time.Now()
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
hold := time.Since(holdStart)
|
||||
writerMu.Unlock()
|
||||
recordWriterTiming(component, wait, hold, "BEGIN")
|
||||
return err
|
||||
}
|
||||
if err := fn(tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
hold := time.Since(holdStart)
|
||||
writerMu.Unlock()
|
||||
recordWriterTiming(component, wait, hold, "tx-body")
|
||||
return err
|
||||
}
|
||||
err = tx.Commit()
|
||||
hold := time.Since(holdStart)
|
||||
writerMu.Unlock()
|
||||
recordWriterTiming(component, wait, hold, "COMMIT")
|
||||
return err
|
||||
}
|
||||
|
||||
// Wrap helpers below tag existing call sites with the canonical
|
||||
// component names so the call sites read naturally. These keep the
|
||||
// instrumentation out of the hot-path business logic.
|
||||
|
||||
// instrumentedExec is the package-internal pass-through used by call
|
||||
// sites already inside db.go (PruneOldMetrics, RemoveStaleObservers,
|
||||
// vacuum). Equivalent to WriterExec, kept short for readability.
|
||||
func (s *Store) instrumentedExec(component, query string, args ...interface{}) (sql.Result, error) {
|
||||
return s.WriterExec(component, query, args...)
|
||||
}
|
||||
|
||||
// epochSecondsForLastSeen parses an RFC3339 timestamp to a unix-second
|
||||
// value for the transmissions.last_seen denormalized column (#1690).
|
||||
// Falls back to the current time on parse failure so the column is
|
||||
// never seeded with 0 for a brand-new row.
|
||||
func epochSecondsForLastSeen(rfc3339 string) int64 {
|
||||
if t, err := time.Parse(time.RFC3339, rfc3339); err == nil {
|
||||
return t.Unix()
|
||||
}
|
||||
return time.Now().UTC().Unix()
|
||||
}
|
||||
|
||||
+7
-66
@@ -554,26 +554,18 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) {
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
}
|
||||
before := time.Now().Unix()
|
||||
if _, err := s.InsertTransmission(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := time.Now().Unix()
|
||||
|
||||
// Verify last_seen was updated to INGEST time, not envelope time (#1465).
|
||||
// Verify last_seen was updated
|
||||
var lastSeenAfter string
|
||||
s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenAfter)
|
||||
if lastSeenAfter == oldTime {
|
||||
t.Error("observer last_seen was NOT updated after packet insertion — low-traffic observers will appear offline")
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeenAfter)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeenAfter, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("expected last_seen ≈ server now (in [%d, %d]), got %s (epoch %d). "+
|
||||
"observer.last_seen must use ingest time, not envelope time (#1465).",
|
||||
before, after, lastSeenAfter, ls.Unix())
|
||||
if lastSeenAfter != "2026-03-25T01:00:00Z" {
|
||||
t.Errorf("expected last_seen=2026-03-25T01:00:00Z, got %s", lastSeenAfter)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -606,26 +598,18 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) {
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
}
|
||||
before := time.Now().Unix()
|
||||
if _, err := s.InsertTransmission(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := time.Now().Unix()
|
||||
|
||||
s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt)
|
||||
if !lastPacketAt.Valid {
|
||||
t.Fatal("expected last_packet_at to be non-NULL after InsertTransmission")
|
||||
}
|
||||
// last_packet_at, like last_seen, is "when did the analyzer last receive a
|
||||
// packet from this observer" — an ingest-time question, independent of the
|
||||
// envelope timestamp. See #1465.
|
||||
lp, err := time.Parse(time.RFC3339, lastPacketAt.String)
|
||||
if err != nil {
|
||||
t.Fatalf("last_packet_at %q not RFC3339: %v", lastPacketAt.String, err)
|
||||
}
|
||||
if lp.Unix() < before-5 || lp.Unix() > after+5 {
|
||||
t.Errorf("expected last_packet_at ≈ server now (in [%d, %d]), got %s (epoch %d)",
|
||||
before, after, lastPacketAt.String, lp.Unix())
|
||||
// InsertTransmission uses `now = data.Timestamp || time.Now()`, so last_packet_at
|
||||
// should match the packet's Timestamp when provided (same source-of-truth as last_seen).
|
||||
if lastPacketAt.String != "2026-04-24T12:00:00Z" {
|
||||
t.Errorf("expected last_packet_at=2026-04-24T12:00:00Z, got %s", lastPacketAt.String)
|
||||
}
|
||||
|
||||
// UpsertObserver again (status path) — last_packet_at should NOT change
|
||||
@@ -2917,46 +2901,3 @@ func TestSchemaMultibyteSupColumns(t *testing.T) {
|
||||
}
|
||||
store2.Close()
|
||||
}
|
||||
|
||||
// TestUpdateNodeDefaultScope_EmptyScopeIsNoop is the DB-layer defense-in-depth
|
||||
// regression test for #1534. Even if the call-site guard at main.go:720 is
|
||||
// later removed or refactored, the DB function MUST refuse to overwrite a
|
||||
// previously-correct default_scope with the empty string. This is the
|
||||
// belt-and-braces guard recommended by adversarial review (MAJOR-2) and
|
||||
// dijkstra review (MINOR-2).
|
||||
func TestUpdateNodeDefaultScope_EmptyScopeIsNoop(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "test.db")
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ('pk1', 'Node1', '#belgium')`); err != nil {
|
||||
t.Fatalf("insert node: %v", err)
|
||||
}
|
||||
if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name, default_scope) VALUES ('pk1', 'Node1', '#belgium')`); err != nil {
|
||||
t.Fatalf("insert inactive node: %v", err)
|
||||
}
|
||||
|
||||
// Empty-scope call must be a silent no-op (return nil), NOT overwrite.
|
||||
if err := store.UpdateNodeDefaultScope("pk1", ""); err != nil {
|
||||
t.Fatalf("UpdateNodeDefaultScope(\"\") returned error: %v (want nil)", err)
|
||||
}
|
||||
|
||||
var got string
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = 'pk1'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read nodes.default_scope: %v", err)
|
||||
}
|
||||
if got != "#belgium" {
|
||||
t.Errorf("nodes.default_scope after empty-scope call = %q, want #belgium (DB-layer guard missing — #1534)", got)
|
||||
}
|
||||
var gotInactive string
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM inactive_nodes WHERE public_key = 'pk1'`).Scan(&gotInactive); err != nil {
|
||||
t.Fatalf("read inactive_nodes.default_scope: %v", err)
|
||||
}
|
||||
if gotInactive != "#belgium" {
|
||||
t.Errorf("inactive_nodes.default_scope after empty-scope call = %q, want #belgium (DB-layer guard missing — #1534)", gotInactive)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestWriterStarvationVisibleInPerf reproduces the #1339 class of bug:
|
||||
// one component (neighbor_builder) holds the writer connection for an
|
||||
// extended period; a second component (mqtt_handler) firing concurrent
|
||||
// writes must show observable wait_ms in the perf snapshot.
|
||||
//
|
||||
// This is the gate test for issue #1340: SQLite write-lock instrumentation
|
||||
// per component. If the wait_ms percentile collapses to zero, the
|
||||
// observability gap remains and the regression class is invisible again.
|
||||
//
|
||||
// Runs ~60s — guarded by testing.Short() so fast unit-test passes can
|
||||
// skip it locally, but CI runs `go test ./...` without -short.
|
||||
func TestWriterStarvationVisibleInPerf(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping 60s starvation test in short mode")
|
||||
}
|
||||
|
||||
// Isolate from samples accumulated by earlier tests in the same
|
||||
// package run — without this the mqtt_handler component already
|
||||
// has ~thousand fast InsertTransmission samples and the 5 slow
|
||||
// follower samples can't move p99 above 50s.
|
||||
ResetWriterStatsForTest()
|
||||
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
const blockDur = 60 * time.Second
|
||||
|
||||
// Blocker: acquire the writer via the wrapped Tx path, tag as
|
||||
// neighbor_builder, sleep 60s while holding the single conn,
|
||||
// then commit. This monopolises the writer for the duration.
|
||||
blockStarted := make(chan struct{})
|
||||
blockerDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(blockerDone)
|
||||
err := s.WriterTx("neighbor_builder", func(tx *sql.Tx) error {
|
||||
if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE 0`); err != nil {
|
||||
return err
|
||||
}
|
||||
close(blockStarted)
|
||||
time.Sleep(blockDur)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("blocker tx: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for the blocker to be inside its transaction.
|
||||
<-blockStarted
|
||||
// Small safety margin so the blocker is firmly holding the conn.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Now fire several mqtt_handler writes. Each will block on the
|
||||
// single writer connection until the blocker commits.
|
||||
const followers = 5
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(followers)
|
||||
for i := 0; i < followers; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, err := s.WriterExec(
|
||||
"mqtt_handler",
|
||||
`INSERT OR IGNORE INTO _migrations (name) VALUES (?)`,
|
||||
fmt.Sprintf("writer_starvation_test_%d", i),
|
||||
)
|
||||
if err != nil {
|
||||
t.Errorf("mqtt follower %d: %v", i, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
<-blockerDone
|
||||
|
||||
snap := s.WriterStatsSnapshot()
|
||||
mqtt, ok := snap["mqtt_handler"]
|
||||
if !ok {
|
||||
t.Fatalf("no perf snapshot for mqtt_handler component (got components: %v)", componentKeys(snap))
|
||||
}
|
||||
if mqtt.Count < followers {
|
||||
t.Fatalf("expected at least %d mqtt_handler samples, got %d", followers, mqtt.Count)
|
||||
}
|
||||
// This is the gate assertion. With instrumentation present the
|
||||
// follower writes should each register ~60s of wait_ms; p99 must
|
||||
// be well above 50_000ms. With instrumentation missing or broken
|
||||
// the percentile collapses to zero and this fails — which is the
|
||||
// exact regression class #1340 is meant to prevent.
|
||||
if mqtt.WaitMsP99 <= 50_000 {
|
||||
t.Fatalf("mqtt_handler wait_ms p99 = %.1fms, want > 50000ms; "+
|
||||
"writer starvation is invisible to /api/perf — issue #1340 not fixed",
|
||||
mqtt.WaitMsP99)
|
||||
}
|
||||
}
|
||||
|
||||
func componentKeys(m map[string]WriterStatsSnapshot) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
+4
-113
@@ -109,15 +109,6 @@ type Payload struct {
|
||||
MAC string `json:"mac,omitempty"`
|
||||
EncryptedData string `json:"encryptedData,omitempty"`
|
||||
ExtraHash string `json:"extraHash,omitempty"`
|
||||
// Extended ACK fields per firmware 1.16.0 (issue #1610) —
|
||||
// firmware/src/helpers/BaseChatMesh.cpp:218-234. ACK payloads grew from
|
||||
// always-4 bytes to 4/5/6 (4-byte truncated sha256 CRC, optional 1-byte
|
||||
// attempt counter, optional 1-byte RNG byte added in commit a130a95a).
|
||||
// AckLen is the wire payload length; AckAttempt/AckRand are surfaced
|
||||
// only when the sender included them (legacy 4-byte ACKs leave them nil).
|
||||
AckLen *int `json:"ackLen,omitempty"`
|
||||
AckAttempt *int `json:"ackAttempt,omitempty"`
|
||||
AckRand *int `json:"ackRand,omitempty"`
|
||||
PubKey string `json:"pubKey,omitempty"`
|
||||
Timestamp uint32 `json:"timestamp,omitempty"`
|
||||
TimestampISO string `json:"timestampISO,omitempty"`
|
||||
@@ -157,30 +148,12 @@ type Payload struct {
|
||||
InnerType *int `json:"innerType,omitempty"`
|
||||
InnerTypeName string `json:"innerTypeName,omitempty"`
|
||||
InnerAckCrc string `json:"innerAckCrc,omitempty"`
|
||||
// Extended ACK inner fields (issue #1610) — when the multipart inner
|
||||
// blob is a v1.16+ extended ACK (5 or 6 bytes after the byte0 header),
|
||||
// surface the same attempt/rand bytes as the top-level decoder.
|
||||
InnerAckLen *int `json:"innerAckLen,omitempty"`
|
||||
InnerAckAttempt *int `json:"innerAckAttempt,omitempty"`
|
||||
InnerAckRand *int `json:"innerAckRand,omitempty"`
|
||||
InnerPayload string `json:"innerPayload,omitempty"`
|
||||
// CONTROL (PAYLOAD_TYPE_CONTROL=0x0B) byte0 flags, per
|
||||
// firmware/src/Mesh.cpp:69 — byte0 high-bit marks zero-hop direct subset.
|
||||
CtrlFlags string `json:"ctrlFlags,omitempty"`
|
||||
CtrlZeroHop *bool `json:"ctrlZeroHop,omitempty"`
|
||||
CtrlLength *int `json:"ctrlLength,omitempty"`
|
||||
// CONTROL DISCOVER_REQ / DISCOVER_RESP body fields (#1802). Subtype is
|
||||
// "DISCOVER_REQ" | "DISCOVER_RESP" | "UNKNOWN". For REQ: filter, tag, and
|
||||
// optional since. For RESP: node_type (low nibble of byte0), snr, tag, and
|
||||
// pubkey (hex; 32 bytes or 8 bytes when prefix_only). All optional —
|
||||
// emitted only when the body length is sufficient.
|
||||
CtrlSubtype string `json:"ctrlSubtype,omitempty"`
|
||||
CtrlFilter *int `json:"ctrlFilter,omitempty"`
|
||||
CtrlTag *uint32 `json:"ctrlTag,omitempty"`
|
||||
CtrlSince *uint32 `json:"ctrlSince,omitempty"`
|
||||
CtrlNodeType *int `json:"ctrlNodeType,omitempty"`
|
||||
CtrlSNR *int `json:"ctrlSNR,omitempty"`
|
||||
CtrlPubKey string `json:"ctrlPubKey,omitempty"`
|
||||
// RAW_CUSTOM (PAYLOAD_TYPE_RAW_CUSTOM=0x0F) — application-defined per
|
||||
// firmware/src/Mesh.cpp:577 (createRawData). Exposes the bare envelope
|
||||
// shape (length + leading tag) so consumers can triage by app id.
|
||||
@@ -293,27 +266,10 @@ func decodeAck(buf []byte) Payload {
|
||||
return Payload{Type: "ACK", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
checksum := binary.LittleEndian.Uint32(buf[0:4])
|
||||
ackLen := len(buf)
|
||||
if ackLen > 6 {
|
||||
ackLen = 6
|
||||
}
|
||||
p := Payload{
|
||||
return Payload{
|
||||
Type: "ACK",
|
||||
ExtraHash: fmt.Sprintf("%08x", checksum),
|
||||
AckLen: &ackLen,
|
||||
}
|
||||
// Firmware 1.16.0 extended ACK (issue #1610): 5th byte is the attempt
|
||||
// counter (commit f6e6fdaa), 6th byte is a random byte added so identical
|
||||
// attempts still hash uniquely (commit a130a95a).
|
||||
if len(buf) >= 5 {
|
||||
attempt := int(buf[4])
|
||||
p.AckAttempt = &attempt
|
||||
}
|
||||
if len(buf) >= 6 {
|
||||
rnd := int(buf[5])
|
||||
p.AckRand = &rnd
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
@@ -708,92 +664,27 @@ func decodeMultipart(buf []byte) Payload {
|
||||
// to match decodeAck's extraHash convention.
|
||||
crc := binary.LittleEndian.Uint32(buf[1:5])
|
||||
p.InnerAckCrc = fmt.Sprintf("%08x", crc)
|
||||
// Firmware 1.16.0 extended ACK (issue #1610): inner ACK blob may be
|
||||
// 5 or 6 bytes (payload_len = 1 + ack_len) instead of always 4.
|
||||
ackLen := len(buf) - 1
|
||||
if ackLen > 6 {
|
||||
ackLen = 6
|
||||
}
|
||||
p.InnerAckLen = &ackLen
|
||||
if len(buf) >= 6 {
|
||||
attempt := int(buf[5])
|
||||
p.InnerAckAttempt = &attempt
|
||||
}
|
||||
if len(buf) >= 7 {
|
||||
rnd := int(buf[6])
|
||||
p.InnerAckRand = &rnd
|
||||
}
|
||||
} else if len(buf) > 1 {
|
||||
p.InnerPayload = hex.EncodeToString(buf[1:])
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// decodeControl decodes PAYLOAD_TYPE_CONTROL (0x0B).
|
||||
//
|
||||
// byte0 high nibble is the control subtype (per firmware/src/Mesh.cpp:69
|
||||
// and firmware/examples/simple_repeater/MyMesh.cpp:773-820):
|
||||
// 0x80 = CTL_TYPE_NODE_DISCOVER_REQ — body: filter:u8 | tag:u32 LE | since:u32 LE (optional)
|
||||
// 0x90 = CTL_TYPE_NODE_DISCOVER_RESP — body: snr:i8 | tag:u32 LE | pubkey (32B full, or 8B prefix)
|
||||
// low nibble of byte0 is node_type
|
||||
//
|
||||
// The legacy CtrlZeroHop bool (bit7 of byte0) is retained for backwards
|
||||
// compatibility — it is misleading (bit7 is also set for DISCOVER_RESP)
|
||||
// and is flagged for follow-up renaming. Length checks gate every field
|
||||
// extraction; short/truncated bodies emit the subtype only and never panic.
|
||||
// decodeControl decodes PAYLOAD_TYPE_CONTROL (0x0B) byte0 flags per
|
||||
// firmware/src/Mesh.cpp:69 (high-bit set ⇒ zero-hop direct subset).
|
||||
func decodeControl(buf []byte) Payload {
|
||||
if len(buf) < 1 {
|
||||
return Payload{Type: "CONTROL", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
zeroHop := buf[0]&0x80 != 0
|
||||
length := len(buf)
|
||||
p := Payload{
|
||||
return Payload{
|
||||
Type: "CONTROL",
|
||||
CtrlFlags: fmt.Sprintf("%02x", buf[0]),
|
||||
CtrlZeroHop: &zeroHop,
|
||||
CtrlLength: &length,
|
||||
RawHex: hex.EncodeToString(buf),
|
||||
}
|
||||
|
||||
switch buf[0] & 0xF0 {
|
||||
case 0x80:
|
||||
p.CtrlSubtype = "DISCOVER_REQ"
|
||||
// REQ body: filter:u8 | tag:u32 LE | since:u32 LE (optional).
|
||||
// Total body (incl. byte0) >= 6 = 1 + 1 + 4.
|
||||
if len(buf) >= 6 {
|
||||
filter := int(buf[1])
|
||||
p.CtrlFilter = &filter
|
||||
tag := binary.LittleEndian.Uint32(buf[2:6])
|
||||
p.CtrlTag = &tag
|
||||
if len(buf) >= 10 {
|
||||
since := binary.LittleEndian.Uint32(buf[6:10])
|
||||
p.CtrlSince = &since
|
||||
}
|
||||
}
|
||||
case 0x90:
|
||||
p.CtrlSubtype = "DISCOVER_RESP"
|
||||
nodeType := int(buf[0] & 0x0F)
|
||||
p.CtrlNodeType = &nodeType
|
||||
// RESP body: snr:i8 | tag:u32 LE | pubkey. Header is 6 bytes
|
||||
// (byte0 + snr + 4B tag). Pubkey is 32B full or 8B prefix.
|
||||
if len(buf) >= 6 {
|
||||
snr := int(int8(buf[1]))
|
||||
p.CtrlSNR = &snr
|
||||
tag := binary.LittleEndian.Uint32(buf[2:6])
|
||||
p.CtrlTag = &tag
|
||||
remaining := len(buf) - 6
|
||||
if remaining >= 32 {
|
||||
p.CtrlPubKey = hex.EncodeToString(buf[6:38])
|
||||
} else if remaining >= 8 && remaining < 32 {
|
||||
p.CtrlPubKey = hex.EncodeToString(buf[6:14])
|
||||
}
|
||||
// Other lengths (e.g. 4B): omit pubkey rather than emit a
|
||||
// partial/ambiguous value.
|
||||
}
|
||||
default:
|
||||
p.CtrlSubtype = "UNKNOWN"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// decodeRawCustom decodes PAYLOAD_TYPE_RAW_CUSTOM (0x0F). Application-defined
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IngestBuffer decouples MQTT message receipt from DB writes (#1608).
|
||||
//
|
||||
// On boot the ingestor must subscribe to MQTT immediately, but the single
|
||||
// SQLite writer (#1283) can be held for minutes by a startup migration
|
||||
// (e.g. a large CREATE INDEX) or prune. Without buffering, every QoS-0 packet
|
||||
// received in that window is lost. IngestBuffer holds received work in a
|
||||
// bounded FIFO and a single consumer goroutine drains it once Ready() is
|
||||
// called — i.e. once the write path is free.
|
||||
//
|
||||
// A single consumer preserves the single-writer invariant: jobs run one at a
|
||||
// time, exactly as paho's in-order handler did before. Submit never blocks the
|
||||
// MQTT delivery goroutine; if the buffer is full it drops and counts (bounded
|
||||
// memory). Buffering replays the original messages, so it introduces NO
|
||||
// duplicates (contrast: a QoS-1 broker-queue would).
|
||||
type IngestBuffer struct {
|
||||
jobs chan func()
|
||||
ready chan struct{}
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
dropped atomic.Int64
|
||||
startOnce sync.Once
|
||||
readyOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
|
||||
// dropLogMu guards the time-based drop-log throttle (PR #1623
|
||||
// round-1 fix to #1609 M1). Per-drop logging under sustained
|
||||
// stalls could flood the log at MQTT inbound rate; instead we
|
||||
// always log the FIRST drop of a stall and then summarize at
|
||||
// most once per second until the stall ends.
|
||||
dropLogMu sync.Mutex
|
||||
stallActive bool // true between first drop and first successful Submit
|
||||
stallStart time.Time // when the current stall began
|
||||
stallStartDrop int64 // dropped() value when stall began
|
||||
lastSummaryAt time.Time // last time we wrote a summary line
|
||||
}
|
||||
|
||||
// dropLogSummaryInterval is the minimum interval between summary lines
|
||||
// during a sustained stall. Exposed as a var so tests can shrink it.
|
||||
var dropLogSummaryInterval = time.Second
|
||||
|
||||
// NewIngestBuffer returns a buffer holding up to capacity pending jobs.
|
||||
// Non-positive capacity is clamped to 1 and a WARN is logged so the
|
||||
// misconfiguration is visible (PR #1609 m2 — silent clamp hid bad
|
||||
// ingestBufferSize values).
|
||||
func NewIngestBuffer(capacity int) *IngestBuffer {
|
||||
if capacity < 1 {
|
||||
log.Printf("[ingest-buffer] WARN: requested capacity %d < 1, clamping to 1 — check ingestBufferSize config; default is 50000", capacity)
|
||||
capacity = 1
|
||||
}
|
||||
return &IngestBuffer{
|
||||
jobs: make(chan func(), capacity),
|
||||
ready: make(chan struct{}),
|
||||
stop: make(chan struct{}),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Submit enqueues a job without blocking. If the buffer is full the job is
|
||||
// dropped and the dropped counter is incremented. Safe for concurrent callers.
|
||||
//
|
||||
// Ordering invariant: callers MUST call Start() before the first Submit().
|
||||
// Submit only enqueues — without a running consumer, jobs sit in the channel
|
||||
// and (once cap is reached) are silently dropped until Start()+Ready() run.
|
||||
//
|
||||
// Drop logging (PR #1623 round-1 fix to #1609 M1) uses a time-based
|
||||
// throttle to stay loud-on-stall-start without flooding under sustained
|
||||
// stalls:
|
||||
// - the FIRST drop of a stall logs immediately
|
||||
// - subsequent drops are summarized at most once per second
|
||||
// - when the next Submit succeeds, a "drained" recovery line is
|
||||
// emitted so operators can quantify the burst
|
||||
//
|
||||
// All log lines include the buffer capacity for operator triage.
|
||||
func (b *IngestBuffer) Submit(job func()) {
|
||||
select {
|
||||
case b.jobs <- job:
|
||||
b.maybeLogRecovery()
|
||||
default:
|
||||
n := b.dropped.Add(1)
|
||||
b.logDrop(n)
|
||||
}
|
||||
}
|
||||
|
||||
// logDrop emits a drop log line under the time-based throttle. The first
|
||||
// drop of a stall always logs; subsequent drops summarize at most once
|
||||
// per dropLogSummaryInterval.
|
||||
func (b *IngestBuffer) logDrop(n int64) {
|
||||
b.dropLogMu.Lock()
|
||||
defer b.dropLogMu.Unlock()
|
||||
now := time.Now()
|
||||
if !b.stallActive {
|
||||
b.stallActive = true
|
||||
b.stallStart = now
|
||||
b.stallStartDrop = n - 1 // last successful Submit -> this is the 1st drop of the stall
|
||||
b.lastSummaryAt = now
|
||||
log.Printf("[ingest-buffer] WARNING: buffer full (cap %d), dropped %d message(s) total — write path stalled, raise ingestBufferSize or investigate slow writer", cap(b.jobs), n)
|
||||
return
|
||||
}
|
||||
if now.Sub(b.lastSummaryAt) >= dropLogSummaryInterval {
|
||||
b.lastSummaryAt = now
|
||||
stallDrops := n - b.stallStartDrop
|
||||
log.Printf("[ingest-buffer] WARNING: buffer full (cap %d), %d drop(s) in current stall, %d total — write path still stalled", cap(b.jobs), stallDrops, n)
|
||||
}
|
||||
}
|
||||
|
||||
// maybeLogRecovery is called from the success branch of Submit. If a
|
||||
// stall was active, it logs a recovery line summarizing the burst and
|
||||
// clears the stall state.
|
||||
func (b *IngestBuffer) maybeLogRecovery() {
|
||||
b.dropLogMu.Lock()
|
||||
defer b.dropLogMu.Unlock()
|
||||
if !b.stallActive {
|
||||
return
|
||||
}
|
||||
stallDrops := b.dropped.Load() - b.stallStartDrop
|
||||
dur := time.Since(b.stallStart)
|
||||
log.Printf("[ingest-buffer] INFO: buffer drained, %d drop(s) over %s (cap %d) — write path recovered", stallDrops, dur.Round(time.Millisecond), cap(b.jobs))
|
||||
b.stallActive = false
|
||||
}
|
||||
|
||||
// Start launches the consumer goroutine. It blocks until Ready() is called
|
||||
// (or Stop() fires, whichever comes first), then drains buffered jobs and
|
||||
// runs newly-submitted ones serially, in FIFO order. Idempotent.
|
||||
//
|
||||
// Lifecycle: Stop() closes b.stop, which causes the consumer to exit via
|
||||
// the stop-select arm (after draining any queued jobs if Ready() had
|
||||
// already fired). The b.jobs channel is never closed — closing it would
|
||||
// race with concurrent Submit() callers and panic; instead jobs is
|
||||
// garbage-collected with the buffer once all references drop. Done() is
|
||||
// closed when the consumer goroutine returns.
|
||||
func (b *IngestBuffer) Start() {
|
||||
b.startOnce.Do(func() {
|
||||
go func() {
|
||||
defer close(b.done)
|
||||
select {
|
||||
case <-b.ready:
|
||||
case <-b.stop:
|
||||
// Stopped before Ready — exit immediately. Pending jobs
|
||||
// are discarded; the buffer was never authorized to drain.
|
||||
return
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case job := <-b.jobs:
|
||||
job()
|
||||
case <-b.stop:
|
||||
// Stop after Ready — drain whatever is queued so
|
||||
// shutdown is graceful, then exit. b.jobs is never
|
||||
// closed (see Start godoc), so a default-case
|
||||
// non-blocking receive is the correct drain idiom.
|
||||
for {
|
||||
select {
|
||||
case job := <-b.jobs:
|
||||
job()
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// Ready signals that the write path is available; the consumer begins
|
||||
// draining. Idempotent.
|
||||
//
|
||||
// Ordering invariant: Start() MUST have been called before Ready() takes
|
||||
// effect. Calling Ready() without a prior Start() simply closes the ready
|
||||
// channel — nothing drains until a later Start() runs its consumer goroutine.
|
||||
func (b *IngestBuffer) Ready() {
|
||||
b.readyOnce.Do(func() { close(b.ready) })
|
||||
}
|
||||
|
||||
// Dropped returns the number of jobs dropped due to a full buffer.
|
||||
func (b *IngestBuffer) Dropped() int64 { return b.dropped.Load() }
|
||||
|
||||
// Pending returns the current queue depth (best-effort; for observability).
|
||||
func (b *IngestBuffer) Pending() int { return len(b.jobs) }
|
||||
|
||||
// Stop signals the consumer goroutine to exit. Test-hygiene helper so unit
|
||||
// tests don't leak the goroutine that Start() spawns. Idempotent / safe to
|
||||
// call without a prior Start(). After Stop() the consumer exits and Done()
|
||||
// is closed.
|
||||
func (b *IngestBuffer) Stop() {
|
||||
b.stopOnce.Do(func() { close(b.stop) })
|
||||
}
|
||||
|
||||
// Done returns a channel that is closed after the consumer goroutine has
|
||||
// exited. If Start() was never called, Done() never closes.
|
||||
func (b *IngestBuffer) Done() <-chan struct{} {
|
||||
return b.done
|
||||
}
|
||||
@@ -1,274 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIngestBuffer_BuffersUntilReady(t *testing.T) {
|
||||
b := NewIngestBuffer(10)
|
||||
t.Cleanup(b.Stop)
|
||||
var ran atomic.Int64
|
||||
b.Start()
|
||||
for i := 0; i < 3; i++ {
|
||||
b.Submit(func() { ran.Add(1) })
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
if ran.Load() != 0 {
|
||||
t.Fatalf("jobs ran before Ready(): %d", ran.Load())
|
||||
}
|
||||
b.Ready()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for ran.Load() < 3 && time.Now().Before(deadline) {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if ran.Load() != 3 {
|
||||
t.Fatalf("want 3 ran after Ready, got %d", ran.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBuffer_FIFOOrder(t *testing.T) {
|
||||
b := NewIngestBuffer(10)
|
||||
t.Cleanup(b.Stop)
|
||||
out := make(chan int, 5)
|
||||
b.Start()
|
||||
for i := 0; i < 5; i++ {
|
||||
i := i
|
||||
b.Submit(func() { out <- i })
|
||||
}
|
||||
b.Ready()
|
||||
for want := 0; want < 5; want++ {
|
||||
select {
|
||||
case got := <-out:
|
||||
if got != want {
|
||||
t.Fatalf("order: want %d got %d", want, got)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timeout waiting for job %d", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBuffer_DropsWhenFull(t *testing.T) {
|
||||
b := NewIngestBuffer(2)
|
||||
t.Cleanup(b.Stop) // never Ready()'d -> nothing drains
|
||||
for i := 0; i < 5; i++ {
|
||||
b.Submit(func() {})
|
||||
}
|
||||
if got := b.Dropped(); got != 3 {
|
||||
t.Fatalf("want 3 dropped (cap 2, 5 submitted), got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBuffer_ProcessesAfterReady(t *testing.T) {
|
||||
b := NewIngestBuffer(10)
|
||||
t.Cleanup(b.Stop)
|
||||
b.Start()
|
||||
b.Ready()
|
||||
done := make(chan struct{})
|
||||
b.Submit(func() { close(done) })
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("job submitted after Ready was not processed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBuffer_SerialExecution(t *testing.T) {
|
||||
b := NewIngestBuffer(50)
|
||||
t.Cleanup(b.Stop)
|
||||
var inFlight atomic.Int32
|
||||
var overlap atomic.Bool
|
||||
var wg sync.WaitGroup
|
||||
b.Start()
|
||||
const n = 20
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
b.Submit(func() {
|
||||
if inFlight.Add(1) > 1 {
|
||||
overlap.Store(true)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
wg.Done()
|
||||
})
|
||||
}
|
||||
b.Ready()
|
||||
wg.Wait()
|
||||
if overlap.Load() {
|
||||
t.Fatal("jobs overlapped — consumer is not serial (violates single-writer)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngestBuffer_ConcurrentSubmitSafe(t *testing.T) {
|
||||
b := NewIngestBuffer(20000)
|
||||
t.Cleanup(b.Stop)
|
||||
b.Start()
|
||||
var wg sync.WaitGroup
|
||||
for g := 0; g < 8; g++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 1000; i++ {
|
||||
b.Submit(func() {})
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
b.Ready()
|
||||
// Assertion is the absence of a race/panic; run under -race in CI.
|
||||
}
|
||||
|
||||
// TestIngestBuffer_StopUnblocksConsumer guards the consumer-goroutine leak
|
||||
// described in PR #1609 review m1: Start() blocks on <-b.ready forever if
|
||||
// Ready() is never called, leaking the goroutine in test runs. Stop() must
|
||||
// signal the consumer to exit cleanly without requiring Ready().
|
||||
func TestIngestBuffer_StopUnblocksConsumer(t *testing.T) {
|
||||
b := NewIngestBuffer(10)
|
||||
t.Cleanup(b.Stop)
|
||||
b.Start()
|
||||
// Do NOT call Ready(). The consumer must exit purely because of Stop().
|
||||
b.Stop()
|
||||
select {
|
||||
case <-b.Done():
|
||||
// good — consumer goroutine returned
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Stop() did not unblock the consumer goroutine within 1s (Done() never closed)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewIngestBuffer_WarnsOnSubOneClamp asserts that constructing the
|
||||
// buffer with a non-positive capacity emits a WARN log line. Silent
|
||||
// clamping (PR #1609 review m2) hid misconfigurations like
|
||||
// ingestBufferSize=-1 or 0-from-default-not-applied paths.
|
||||
func TestNewIngestBuffer_WarnsOnSubOneClamp(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
oldOut := log.Writer()
|
||||
oldFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(oldOut)
|
||||
log.SetFlags(oldFlags)
|
||||
})
|
||||
|
||||
b := NewIngestBuffer(0)
|
||||
t.Cleanup(b.Stop)
|
||||
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "WARN") || !strings.Contains(got, "ingest-buffer") {
|
||||
t.Fatalf("expected WARN log on sub-one clamp, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestBuffer_DropLogThrottle asserts the time-based throttle (PR
|
||||
// #1623 round-1 fix to #1609 M1): the FIRST drop of a stall logs
|
||||
// immediately (loud), then subsequent drops within the same stall are
|
||||
// rate-limited to at most one summary line per second, and a recovery
|
||||
// line is emitted when Submit succeeds again. This prevents log-flood
|
||||
// under sustained stalls (potentially hundreds of MB/min) while
|
||||
// preserving "loud the instant the stall starts".
|
||||
func TestIngestBuffer_DropLogThrottle(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
oldOut := log.Writer()
|
||||
oldFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(oldOut)
|
||||
log.SetFlags(oldFlags)
|
||||
})
|
||||
|
||||
b := NewIngestBuffer(2)
|
||||
t.Cleanup(b.Stop)
|
||||
// Fill to capacity (no Ready() — nothing drains).
|
||||
for i := 0; i < 2; i++ {
|
||||
b.Submit(func() {})
|
||||
}
|
||||
// 100 drops in tight loop (well under 1s).
|
||||
for i := 0; i < 100; i++ {
|
||||
b.Submit(func() {})
|
||||
}
|
||||
|
||||
got := buf.String()
|
||||
lines := strings.Count(got, "buffer full")
|
||||
if lines < 1 {
|
||||
t.Fatalf("expected the FIRST drop to log immediately; got 0 'buffer full' lines:\n%s", got)
|
||||
}
|
||||
if lines > 2 {
|
||||
t.Fatalf("expected at most 2 'buffer full' lines for 100 drops in <1s (first + at-most-one summary), got %d:\n%s", lines, got)
|
||||
}
|
||||
// Every line must include the capacity for operator triage.
|
||||
if !strings.Contains(got, "cap 2") {
|
||||
t.Fatalf("expected every drop log line to include 'cap 2', got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestBuffer_DropLogFirstAlwaysImmediate guards the "loud the
|
||||
// instant the stall starts" half of the throttle contract from PR
|
||||
// #1623: even a single drop must log immediately, not be silently
|
||||
// absorbed by the per-second summary window.
|
||||
func TestIngestBuffer_DropLogFirstAlwaysImmediate(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
oldOut := log.Writer()
|
||||
oldFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(oldOut)
|
||||
log.SetFlags(oldFlags)
|
||||
})
|
||||
|
||||
b := NewIngestBuffer(1)
|
||||
t.Cleanup(b.Stop)
|
||||
b.Submit(func() {}) // fills cap=1
|
||||
b.Submit(func() {}) // first drop
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "buffer full") {
|
||||
t.Fatalf("expected FIRST drop to log immediately; got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIngestBuffer_DropLogRecoveryAfterDrain guards the recovery-line
|
||||
// half of the throttle contract: once Submit succeeds again after one
|
||||
// or more drops, a "recovered" / "drained" line must be emitted so
|
||||
// operators can quantify the burst (PR #1623).
|
||||
func TestIngestBuffer_DropLogRecoveryAfterDrain(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
oldOut := log.Writer()
|
||||
oldFlags := log.Flags()
|
||||
log.SetOutput(&buf)
|
||||
log.SetFlags(0)
|
||||
t.Cleanup(func() {
|
||||
log.SetOutput(oldOut)
|
||||
log.SetFlags(oldFlags)
|
||||
})
|
||||
|
||||
b := NewIngestBuffer(1)
|
||||
t.Cleanup(b.Stop)
|
||||
b.Submit(func() {}) // fills cap=1
|
||||
for i := 0; i < 3; i++ {
|
||||
b.Submit(func() {}) // drops
|
||||
}
|
||||
// Drain: start consumer and Ready(), wait for queue to empty.
|
||||
b.Start()
|
||||
b.Ready()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for b.Pending() > 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
// Now a successful Submit should trigger the recovery line.
|
||||
b.Submit(func() {})
|
||||
// Give the goroutine + log a moment.
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "drained") && !strings.Contains(got, "recovered") {
|
||||
t.Fatalf("expected a 'drained'/'recovered' log line after stall ended; got:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for issue #1610: firmware 1.16.0 extended ACK support.
|
||||
//
|
||||
// Wire vectors are synthetic, derived by hand from the firmware spec:
|
||||
// - Variable-length ACK on the wire:
|
||||
// firmware/src/Mesh.cpp:545-575 createAck/createMultiAck (commit f6e6fdaa)
|
||||
// - 5-byte ACK = 4-byte truncated sha256 CRC + 1-byte attempt counter:
|
||||
// firmware/src/helpers/BaseChatMesh.cpp:218-232 (commit f6e6fdaa)
|
||||
// - 6-byte ACK = 5-byte + 1-byte RNG (so identical attempts get unique hash):
|
||||
// firmware/src/helpers/BaseChatMesh.cpp:219-234 (commit a130a95a)
|
||||
// - Multipart ACK inner blob: firmware/src/Mesh.cpp:292-307 — byte0 then
|
||||
// ack bytes, payload_len = 1 + ack_len.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- top-level ACK (decodeAck) ---
|
||||
|
||||
func TestDecodeAckLegacy4Byte(t *testing.T) {
|
||||
// Backwards-compat: 4-byte ACK leaves the new optional fields nil.
|
||||
buf := []byte{0xAA, 0xBB, 0xCC, 0xDD}
|
||||
p := decodeAck(buf)
|
||||
if p.ExtraHash != "ddccbbaa" {
|
||||
t.Errorf("extraHash=%q want ddccbbaa", p.ExtraHash)
|
||||
}
|
||||
if p.AckLen == nil || *p.AckLen != 4 {
|
||||
t.Errorf("ackLen=%v want 4", p.AckLen)
|
||||
}
|
||||
if p.AckAttempt != nil {
|
||||
t.Errorf("ackAttempt=%v want nil for legacy 4-byte ACK", *p.AckAttempt)
|
||||
}
|
||||
if p.AckRand != nil {
|
||||
t.Errorf("ackRand=%v want nil for legacy 4-byte ACK", *p.AckRand)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeAck5ByteExtended(t *testing.T) {
|
||||
// v1.16 sender (commit f6e6fdaa): 4-byte CRC + 1-byte attempt.
|
||||
buf := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0x07}
|
||||
p := decodeAck(buf)
|
||||
if p.ExtraHash != "ddccbbaa" {
|
||||
t.Errorf("extraHash=%q want ddccbbaa", p.ExtraHash)
|
||||
}
|
||||
if p.AckLen == nil || *p.AckLen != 5 {
|
||||
t.Errorf("ackLen=%v want 5", p.AckLen)
|
||||
}
|
||||
if p.AckAttempt == nil || *p.AckAttempt != 7 {
|
||||
t.Errorf("ackAttempt=%v want 7", p.AckAttempt)
|
||||
}
|
||||
if p.AckRand != nil {
|
||||
t.Errorf("ackRand=%v want nil for 5-byte ACK", *p.AckRand)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeAck6ByteExtended(t *testing.T) {
|
||||
// v1.16 sender (commit a130a95a): 4-byte CRC + 1-byte attempt + 1-byte RNG.
|
||||
buf := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0x02, 0x5A}
|
||||
p := decodeAck(buf)
|
||||
if p.ExtraHash != "ddccbbaa" {
|
||||
t.Errorf("extraHash=%q want ddccbbaa", p.ExtraHash)
|
||||
}
|
||||
if p.AckLen == nil || *p.AckLen != 6 {
|
||||
t.Errorf("ackLen=%v want 6", p.AckLen)
|
||||
}
|
||||
if p.AckAttempt == nil || *p.AckAttempt != 2 {
|
||||
t.Errorf("ackAttempt=%v want 2", p.AckAttempt)
|
||||
}
|
||||
if p.AckRand == nil || *p.AckRand != 0x5A {
|
||||
t.Errorf("ackRand=%v want 90", p.AckRand)
|
||||
}
|
||||
}
|
||||
|
||||
// --- multipart-with-ACK (decodeMultipart) ---
|
||||
|
||||
// buildMultipartAckByte0: remaining<<4 | PayloadACK (0x02).
|
||||
func buildMultipartAckByte0(remaining int) byte {
|
||||
return byte((remaining<<4)&0xF0) | byte(PayloadACK&0x0F)
|
||||
}
|
||||
|
||||
func TestDecodeMultipartAck4ByteLegacy(t *testing.T) {
|
||||
// Pre-1.16 inner ACK is 4 bytes → ackLen=4, attempt/rand nil.
|
||||
buf := []byte{buildMultipartAckByte0(3), 0xAA, 0xBB, 0xCC, 0xDD}
|
||||
p := decodeMultipart(buf)
|
||||
if p.InnerAckCrc != "ddccbbaa" {
|
||||
t.Errorf("innerAckCrc=%q want ddccbbaa", p.InnerAckCrc)
|
||||
}
|
||||
if p.InnerAckLen == nil || *p.InnerAckLen != 4 {
|
||||
t.Errorf("innerAckLen=%v want 4", p.InnerAckLen)
|
||||
}
|
||||
if p.InnerAckAttempt != nil {
|
||||
t.Errorf("innerAckAttempt=%v want nil", *p.InnerAckAttempt)
|
||||
}
|
||||
if p.InnerAckRand != nil {
|
||||
t.Errorf("innerAckRand=%v want nil", *p.InnerAckRand)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeMultipartAck5Byte(t *testing.T) {
|
||||
// v1.16: byte0 + 4-byte CRC + 1-byte attempt → payload_len = 6.
|
||||
buf := []byte{buildMultipartAckByte0(1), 0xAA, 0xBB, 0xCC, 0xDD, 0x09}
|
||||
p := decodeMultipart(buf)
|
||||
if p.InnerAckCrc != "ddccbbaa" {
|
||||
t.Errorf("innerAckCrc=%q want ddccbbaa", p.InnerAckCrc)
|
||||
}
|
||||
if p.InnerAckLen == nil || *p.InnerAckLen != 5 {
|
||||
t.Errorf("innerAckLen=%v want 5", p.InnerAckLen)
|
||||
}
|
||||
if p.InnerAckAttempt == nil || *p.InnerAckAttempt != 9 {
|
||||
t.Errorf("innerAckAttempt=%v want 9", p.InnerAckAttempt)
|
||||
}
|
||||
if p.InnerAckRand != nil {
|
||||
t.Errorf("innerAckRand=%v want nil for 5-byte inner ACK", *p.InnerAckRand)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeMultipartAck6Byte(t *testing.T) {
|
||||
// v1.16: byte0 + 4-byte CRC + 1-byte attempt + 1-byte RNG → payload_len = 7.
|
||||
buf := []byte{buildMultipartAckByte0(0), 0xAA, 0xBB, 0xCC, 0xDD, 0x04, 0xC3}
|
||||
p := decodeMultipart(buf)
|
||||
if p.InnerAckCrc != "ddccbbaa" {
|
||||
t.Errorf("innerAckCrc=%q want ddccbbaa", p.InnerAckCrc)
|
||||
}
|
||||
if p.InnerAckLen == nil || *p.InnerAckLen != 6 {
|
||||
t.Errorf("innerAckLen=%v want 6", p.InnerAckLen)
|
||||
}
|
||||
if p.InnerAckAttempt == nil || *p.InnerAckAttempt != 4 {
|
||||
t.Errorf("innerAckAttempt=%v want 4", p.InnerAckAttempt)
|
||||
}
|
||||
if p.InnerAckRand == nil || *p.InnerAckRand != 0xC3 {
|
||||
t.Errorf("innerAckRand=%v want 195", p.InnerAckRand)
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
package main
|
||||
|
||||
// Test for issue #1690 — every observation insert must denormalize the
|
||||
// transmission's last_seen so cold-load can filter on effective recency.
|
||||
//
|
||||
// Setup: insert a transmission whose first/last seen are both 7 days ago.
|
||||
// Then insert a fresh observation against the same hash. Post-fix the
|
||||
// transmissions.last_seen column must reflect the new observation time.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) {
|
||||
s, err := OpenStore(tempDBPath(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
hash := "abcdef1690cafebabe"
|
||||
weekAgo := time.Now().UTC().Add(-7 * 24 * time.Hour).Format(time.RFC3339)
|
||||
snr, rssi := 5.5, -100.0
|
||||
|
||||
first := &PacketData{
|
||||
RawHex: "0A00",
|
||||
Timestamp: weekAgo,
|
||||
ObserverID: "obs1",
|
||||
Hash: hash,
|
||||
RouteType: 2,
|
||||
PayloadType: 2,
|
||||
PayloadVersion: 0,
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
SNR: &snr,
|
||||
RSSI: &rssi,
|
||||
}
|
||||
if _, err := s.InsertTransmission(first); err != nil {
|
||||
t.Fatalf("seed insert: %v", err)
|
||||
}
|
||||
|
||||
// Sanity: confirm the seed last_seen is the 7d-ago time.
|
||||
var seededLastSeen int64
|
||||
if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = ?`, hash).Scan(&seededLastSeen); err != nil {
|
||||
t.Fatalf("seed select last_seen: %v (column missing? post-fix must add it)", err)
|
||||
}
|
||||
weekAgoUnix, _ := time.Parse(time.RFC3339, weekAgo)
|
||||
if seededLastSeen != weekAgoUnix.Unix() {
|
||||
t.Logf("seed last_seen=%d expected %d (allowed for fresh column)", seededLastSeen, weekAgoUnix.Unix())
|
||||
}
|
||||
|
||||
// New observation: nowSec timestamp.
|
||||
nowSec := time.Now().UTC().Unix()
|
||||
nowStr := time.Unix(nowSec, 0).UTC().Format(time.RFC3339)
|
||||
second := &PacketData{
|
||||
RawHex: "0A00",
|
||||
Timestamp: nowStr,
|
||||
ObserverID: "obs2", // different observer → new observation row
|
||||
Hash: hash,
|
||||
RouteType: 2,
|
||||
PayloadType: 2,
|
||||
PayloadVersion: 0,
|
||||
PathJSON: "[]",
|
||||
DecodedJSON: `{"type":"TXT_MSG"}`,
|
||||
SNR: &snr,
|
||||
RSSI: &rssi,
|
||||
}
|
||||
if _, err := s.InsertTransmission(second); err != nil {
|
||||
t.Fatalf("second insert: %v", err)
|
||||
}
|
||||
|
||||
var ls int64
|
||||
if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = ?`, hash).Scan(&ls); err != nil {
|
||||
t.Fatalf("post-insert select last_seen: %v", err)
|
||||
}
|
||||
// The post-fix writer must bump last_seen to at least the new observation's
|
||||
// epoch second. We allow ±2s slack for the unix-second round trip.
|
||||
if ls < nowSec-2 {
|
||||
t.Errorf("transmissions.last_seen=%d after fresh observation; expected ≥ %d (a recent unix-second). "+
|
||||
"Pre-fix the column is never updated on re-observation — the original cold-load bug (#1690).",
|
||||
ls, nowSec)
|
||||
}
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for #1802 — CONTROL DISCOVER_REQ / DISCOVER_RESP decode.
|
||||
// Wire format references:
|
||||
// firmware/src/Mesh.cpp:69 (CTL_TYPE constants)
|
||||
// firmware/examples/simple_repeater/MyMesh.cpp:773-820
|
||||
//
|
||||
// Subtype = byte0 & 0xF0. 0x80 = DISCOVER_REQ, 0x90 = DISCOVER_RESP.
|
||||
// REQ body (>=6B after byte0): filter:u8 | tag:u32 LE | since:u32 LE (optional)
|
||||
// RESP body (>=6+pubkey): snr:i8 | tag:u32 LE | pubkey (32B full, or 8B prefix)
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDecodeControl_DiscoverReq_FullBody(t *testing.T) {
|
||||
// byte0 = 0x80 (DISCOVER_REQ), filter=0x02, tag=0xDEADBEEF, since=0x11223344.
|
||||
buf := []byte{0x80, 0x02, 0xEF, 0xBE, 0xAD, 0xDE, 0x44, 0x33, 0x22, 0x11}
|
||||
p := decodeControl(buf)
|
||||
if p.Type != "CONTROL" {
|
||||
t.Fatalf("type=%q want CONTROL", p.Type)
|
||||
}
|
||||
if p.CtrlSubtype != "DISCOVER_REQ" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_REQ", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlFilter == nil || *p.CtrlFilter != 0x02 {
|
||||
t.Errorf("ctrlFilter=%v want 2", p.CtrlFilter)
|
||||
}
|
||||
if p.CtrlTag == nil || *p.CtrlTag != 0xDEADBEEF {
|
||||
t.Errorf("ctrlTag=%v want 0xDEADBEEF", p.CtrlTag)
|
||||
}
|
||||
if p.CtrlSince == nil || *p.CtrlSince != 0x11223344 {
|
||||
t.Errorf("ctrlSince=%v want 0x11223344", p.CtrlSince)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_DiscoverReq_NoSince(t *testing.T) {
|
||||
// 6-byte body (no optional since): byte0=0x80, filter, tag.
|
||||
buf := []byte{0x80, 0x04, 0x01, 0x02, 0x03, 0x04}
|
||||
p := decodeControl(buf)
|
||||
if p.CtrlSubtype != "DISCOVER_REQ" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_REQ", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlFilter == nil || *p.CtrlFilter != 0x04 {
|
||||
t.Errorf("ctrlFilter=%v want 4", p.CtrlFilter)
|
||||
}
|
||||
wantTag := binary.LittleEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04})
|
||||
if p.CtrlTag == nil || *p.CtrlTag != wantTag {
|
||||
t.Errorf("ctrlTag=%v want %#x", p.CtrlTag, wantTag)
|
||||
}
|
||||
if p.CtrlSince != nil {
|
||||
t.Errorf("ctrlSince should be nil for 6B REQ body, got %v", p.CtrlSince)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_DiscoverResp_FullPubKey(t *testing.T) {
|
||||
// byte0 = 0x90 | 0x02 (node_type=2, REPEATER), snr=0x10, tag, 32B pubkey.
|
||||
body := []byte{0x92, 0x10, 0x44, 0x33, 0x22, 0x11}
|
||||
for i := 0; i < 32; i++ {
|
||||
body = append(body, byte(i))
|
||||
}
|
||||
p := decodeControl(body)
|
||||
if p.CtrlSubtype != "DISCOVER_RESP" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_RESP", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlNodeType == nil || *p.CtrlNodeType != 2 {
|
||||
t.Errorf("ctrlNodeType=%v want 2", p.CtrlNodeType)
|
||||
}
|
||||
if p.CtrlSNR == nil || *p.CtrlSNR != 0x10 {
|
||||
t.Errorf("ctrlSNR=%v want 16", p.CtrlSNR)
|
||||
}
|
||||
wantTag := binary.LittleEndian.Uint32([]byte{0x44, 0x33, 0x22, 0x11})
|
||||
if p.CtrlTag == nil || *p.CtrlTag != wantTag {
|
||||
t.Errorf("ctrlTag=%v want %#x", p.CtrlTag, wantTag)
|
||||
}
|
||||
if len(p.CtrlPubKey) != 64 {
|
||||
t.Fatalf("ctrlPubKey hex len=%d want 64 (32B)", len(p.CtrlPubKey))
|
||||
}
|
||||
if p.CtrlPubKey[:4] != "0001" {
|
||||
t.Errorf("ctrlPubKey prefix=%q want 0001…", p.CtrlPubKey[:4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_DiscoverResp_PrefixPubKey(t *testing.T) {
|
||||
// 6 header bytes + 8 pubkey bytes only (prefix_only path).
|
||||
body := []byte{0x91, 0xF0, 0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88}
|
||||
p := decodeControl(body)
|
||||
if p.CtrlSubtype != "DISCOVER_RESP" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_RESP", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlNodeType == nil || *p.CtrlNodeType != 1 {
|
||||
t.Errorf("ctrlNodeType=%v want 1", p.CtrlNodeType)
|
||||
}
|
||||
// snr = signed 0xF0 = -16
|
||||
if p.CtrlSNR == nil || *p.CtrlSNR != -16 {
|
||||
t.Errorf("ctrlSNR=%v want -16", p.CtrlSNR)
|
||||
}
|
||||
if len(p.CtrlPubKey) != 16 {
|
||||
t.Errorf("ctrlPubKey hex len=%d want 16 (8B)", len(p.CtrlPubKey))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_DiscoverResp_TruncatedPubKey(t *testing.T) {
|
||||
// 6 header bytes + 4 pubkey bytes — neither 8 nor 32. Must NOT panic;
|
||||
// subtype emitted, pubkey omitted.
|
||||
body := []byte{0x91, 0x05, 0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44}
|
||||
p := decodeControl(body)
|
||||
if p.CtrlSubtype != "DISCOVER_RESP" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_RESP", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlPubKey != "" {
|
||||
t.Errorf("ctrlPubKey=%q want empty for 4B pubkey blob", p.CtrlPubKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_ShortBody_NoSubtypeFields(t *testing.T) {
|
||||
// byte0=0x80 only (no body). Must emit subtype, no panic, no body fields.
|
||||
buf := []byte{0x80}
|
||||
p := decodeControl(buf)
|
||||
if p.CtrlSubtype != "DISCOVER_REQ" {
|
||||
t.Errorf("ctrlSubtype=%q want DISCOVER_REQ", p.CtrlSubtype)
|
||||
}
|
||||
if p.CtrlFilter != nil {
|
||||
t.Errorf("ctrlFilter should be nil, got %v", p.CtrlFilter)
|
||||
}
|
||||
if p.CtrlTag != nil {
|
||||
t.Errorf("ctrlTag should be nil, got %v", p.CtrlTag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeControl_UnknownSubtype(t *testing.T) {
|
||||
// byte0 = 0xA0 — not DISCOVER_REQ/RESP. Subtype = "UNKNOWN".
|
||||
buf := []byte{0xA0, 0x11, 0x22}
|
||||
p := decodeControl(buf)
|
||||
if p.CtrlSubtype != "UNKNOWN" {
|
||||
t.Errorf("ctrlSubtype=%q want UNKNOWN", p.CtrlSubtype)
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
// formatStatusLog formats the "status: name (iata)" log line emitted on
|
||||
// MQTT status messages. name + iata are MQTT-controlled and routed
|
||||
// through sanitizeLogString so CR/LF/control bytes cannot inject forged
|
||||
// log lines.
|
||||
//
|
||||
// See audit-input-vulns-20260603 follow-up to #1540 — call site
|
||||
// cmd/ingestor/main.go:531.
|
||||
func formatStatusLog(tag, name, iata string) string {
|
||||
return fmt.Sprintf("MQTT [%s] status: %s (%s)", tag, sanitizeLogString(name), sanitizeLogString(iata))
|
||||
}
|
||||
|
||||
// formatChannelMessageLog formats the "channel message: chN from S" log line
|
||||
// emitted on MQTT channel messages. channelIdx + sender are MQTT-controlled.
|
||||
//
|
||||
// Call site cmd/ingestor/main.go:854.
|
||||
func formatChannelMessageLog(tag, channelIdx, sender string) string {
|
||||
return fmt.Sprintf("MQTT [%s] channel message: ch%s from %s", tag, sanitizeLogString(channelIdx), sanitizeLogString(sender))
|
||||
}
|
||||
|
||||
// formatDirectMessageLog formats the "direct message from S" log line
|
||||
// emitted on MQTT DM messages. sender is MQTT-controlled.
|
||||
//
|
||||
// Call site cmd/ingestor/main.go:940.
|
||||
func formatDirectMessageLog(tag, sender string) string {
|
||||
return fmt.Sprintf("MQTT [%s] direct message from %s", tag, sanitizeLogString(sender))
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFormatStatusLog_SanitizesMQTTFields pins the status log line at
|
||||
// cmd/ingestor/main.go:531 — MQTT-derived name + iata must not be able to
|
||||
// inject CR/LF/control bytes into the log stream.
|
||||
func TestFormatStatusLog_SanitizesMQTTFields(t *testing.T) {
|
||||
got := formatStatusLog("ds1", "evil\r\n[FAKE LOG LINE]", "X\nY")
|
||||
if strings.ContainsAny(got, "\r\n") {
|
||||
t.Fatalf("formatStatusLog leaked CR/LF: %q", got)
|
||||
}
|
||||
if strings.Contains(got, "[FAKE LOG LINE]") && !strings.Contains(got, "?[FAKE LOG LINE]") {
|
||||
t.Fatalf("formatStatusLog passed injection payload through unmodified: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatChannelMessageLog_SanitizesMQTTFields pins
|
||||
// cmd/ingestor/main.go:854 — channelIdx + sender are MQTT-controlled.
|
||||
func TestFormatChannelMessageLog_SanitizesMQTTFields(t *testing.T) {
|
||||
got := formatChannelMessageLog("ds1", "0\r\n[FAKE]", "evil\nguy")
|
||||
if strings.ContainsAny(got, "\r\n") {
|
||||
t.Fatalf("formatChannelMessageLog leaked CR/LF: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatDirectMessageLog_SanitizesMQTTFields pins
|
||||
// cmd/ingestor/main.go:940 — sender is MQTT-controlled.
|
||||
func TestFormatDirectMessageLog_SanitizesMQTTFields(t *testing.T) {
|
||||
got := formatDirectMessageLog("ds1", "evil\r\n[FAKE LOG LINE] something")
|
||||
if strings.ContainsAny(got, "\r\n") {
|
||||
t.Fatalf("formatDirectMessageLog leaked CR/LF: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, "??[FAKE LOG LINE]") {
|
||||
t.Fatalf("formatDirectMessageLog did not sanitize injection payload: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity: legitimate input passes through untouched apart from tag framing.
|
||||
func TestFormatLogs_LegitInputUnchanged(t *testing.T) {
|
||||
if got := formatStatusLog("ds1", "alpha-node", "BG"); got != "MQTT [ds1] status: alpha-node (BG)" {
|
||||
t.Fatalf("unexpected status line: %q", got)
|
||||
}
|
||||
if got := formatChannelMessageLog("ds1", "3", "bob"); got != "MQTT [ds1] channel message: ch3 from bob" {
|
||||
t.Fatalf("unexpected channel line: %q", got)
|
||||
}
|
||||
if got := formatDirectMessageLog("ds1", "bob"); got != "MQTT [ds1] direct message from bob" {
|
||||
t.Fatalf("unexpected DM line: %q", got)
|
||||
}
|
||||
}
|
||||
+183
-400
@@ -51,25 +51,6 @@ func main() {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
|
||||
// Apply Go runtime soft memory limit (GOMEMLIMIT). See #1010.
|
||||
// Precedence: GOMEMLIMIT env > runtime.maxMemoryMB > unset (default).
|
||||
{
|
||||
_, envSet := os.LookupEnv("GOMEMLIMIT")
|
||||
runtimeMaxMB := 0
|
||||
if cfg.Runtime != nil {
|
||||
runtimeMaxMB = cfg.Runtime.MaxMemoryMB
|
||||
}
|
||||
limit, source := applyMemoryLimit(runtimeMaxMB, envSet)
|
||||
switch source {
|
||||
case "env":
|
||||
log.Printf("[memlimit] using GOMEMLIMIT from environment (%s)", os.Getenv("GOMEMLIMIT"))
|
||||
case "config":
|
||||
log.Printf("[memlimit] runtime.maxMemoryMB=%d → SetMemoryLimit(%d MiB)", runtimeMaxMB, limit/(1024*1024))
|
||||
default:
|
||||
log.Printf("[memlimit] unset → default (no soft memory limit; recommend setting GOMEMLIMIT or runtime.maxMemoryMB to ≥1.5× working set to avoid OOM-kill)")
|
||||
}
|
||||
}
|
||||
|
||||
sources := cfg.ResolvedSources()
|
||||
|
||||
store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval())
|
||||
@@ -94,6 +75,154 @@ func main() {
|
||||
// Check auto_vacuum mode and optionally migrate (#919)
|
||||
store.CheckAutoVacuum(cfg)
|
||||
|
||||
// Node retention: move stale nodes to inactive_nodes on startup
|
||||
nodeDays := cfg.NodeDaysOrDefault()
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
|
||||
// Observer retention: remove stale observers on startup
|
||||
observerDays := cfg.ObserverDaysOrDefault()
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
|
||||
// Metrics retention: prune old metrics on startup
|
||||
metricsDays := cfg.MetricsRetentionDays()
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
|
||||
// Packet (transmissions) retention: previously lived in cmd/server,
|
||||
// moved to ingestor in #1283 to eliminate cross-process write
|
||||
// contention (SQLITE_BUSY). 0 = disabled.
|
||||
packetDays := cfg.PacketDaysOrZero()
|
||||
if packetDays > 0 {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[prune] startup pruned %d transmissions older than %d days", n, packetDays)
|
||||
}
|
||||
}
|
||||
|
||||
vacuumPages := cfg.IncrementalVacuumPages()
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
|
||||
// Daily ticker for node retention
|
||||
retentionTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
for range retentionTicker.C {
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for observer retention (every 24h, staggered 90s after startup)
|
||||
observerRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(90 * time.Second) // stagger after metrics prune
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
for range observerRetentionTicker.C {
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for metrics retention (every 24h)
|
||||
metricsRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range metricsRetentionTicker.C {
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for transmission retention (#1283).
|
||||
var packetRetentionTicker *time.Ticker
|
||||
if packetDays > 0 {
|
||||
packetRetentionTicker = time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range packetRetentionTicker.C {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays)
|
||||
}
|
||||
|
||||
// Daily neighbor_edges retention (#1287 — moved from cmd/server).
|
||||
{
|
||||
nDays := cfg.NeighborEdgesDaysOrDefault()
|
||||
neighborPruneTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(4 * time.Minute) // stagger
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] startup pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
for range neighborPruneTicker.C {
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[neighbor-prune] auto-prune enabled: edges older than %d days", nDays)
|
||||
}
|
||||
|
||||
// Periodic stats logging (every 5 minutes)
|
||||
statsTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
for range statsTicker.C {
|
||||
store.LogStats()
|
||||
}
|
||||
}()
|
||||
|
||||
// Prune-request queue (#669 M4 / #738): the read-only server enqueues
|
||||
// geo-prune requests as marker files; the ingestor (which holds the
|
||||
// write handle) executes the DELETEs. Process on startup, then every
|
||||
// 15 seconds — short enough for a one-click UX, long enough to avoid
|
||||
// useless wake-ups.
|
||||
store.RunPendingPruneRequests()
|
||||
pruneQueueTicker := time.NewTicker(15 * time.Second)
|
||||
go func() {
|
||||
for range pruneQueueTicker.C {
|
||||
store.RunPendingPruneRequests()
|
||||
}
|
||||
}()
|
||||
|
||||
// Per-second stats file writer for the server's /api/perf/write-sources
|
||||
// endpoint (#1120). Best-effort; never fatal.
|
||||
StartStatsFileWriter(store, time.Second)
|
||||
|
||||
// Multi-byte capability persister (#1324 follow-up): the server's
|
||||
// analytics cycle publishes a snapshot file via internal/mbcapqueue
|
||||
// (it cannot UPDATE itself, mode=ro since #1289). The ingestor
|
||||
// applies the snapshot here every 5 minutes — derived/cached
|
||||
// columns, ingestor owns the write.
|
||||
multibytePersistTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
time.Sleep(2 * time.Minute) // stagger after analytics warmup
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
for range multibytePersistTicker.C {
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[multibyte-persist] enabled (interval=5m)")
|
||||
|
||||
// Neighbor-edges builder (#1287 — Option 4): ingestor owns
|
||||
// neighbor_edges writes. Runs every 60s. Server reads the snapshot
|
||||
// via cmd/server/neighbor_recomputer.go on the same cadence.
|
||||
stopNeighborBuilder := store.StartNeighborEdgesBuilder(NeighborEdgesBuilderInterval)
|
||||
defer stopNeighborBuilder()
|
||||
log.Printf("[neighbor-build] enabled (interval=%s)", NeighborEdgesBuilderInterval)
|
||||
|
||||
channelKeys := loadChannelKeys(cfg, *configPath)
|
||||
if len(channelKeys) > 0 {
|
||||
log.Printf("Loaded %d channel keys for GRP_TXT decryption", len(channelKeys))
|
||||
@@ -104,13 +233,6 @@ func main() {
|
||||
regionKeys := loadRegionKeys(cfg)
|
||||
store.BackfillDefaultScopeAsync(regionKeys)
|
||||
|
||||
// Subscribe-early + buffer (#1608): the MQTT subscription is brought up
|
||||
// before startup maintenance so no packets are missed while the single
|
||||
// SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received
|
||||
// messages are buffered here and drained once Ready() is called below.
|
||||
ingestBuffer := NewIngestBuffer(cfg.IngestBufferSizeOrDefault())
|
||||
ingestBuffer.Start()
|
||||
|
||||
// Connect to each MQTT source
|
||||
var clients []mqtt.Client
|
||||
connectedCount := 0
|
||||
@@ -132,14 +254,8 @@ func main() {
|
||||
Broker: source.Broker,
|
||||
}
|
||||
|
||||
// #1043: per-source status registry. Idempotent — repeated
|
||||
// registration across reconnects returns the same state so
|
||||
// counters accumulate across the process lifetime.
|
||||
status := RegisterSourceStatus(tag, source.Broker)
|
||||
|
||||
opts.SetOnConnectHandler(func(c mqtt.Client) {
|
||||
log.Printf("MQTT [%s] connected to %s", tag, source.Broker)
|
||||
status.MarkConnect(time.Now())
|
||||
// PR #1216 r1 item 2: clear the stale LastMessageUnix from
|
||||
// before the outage so the watchdog doesn't immediately scream
|
||||
// "stalled for 2h". Also restarts the cold-start grace window
|
||||
@@ -162,7 +278,6 @@ func main() {
|
||||
|
||||
opts.SetConnectionLostHandler(func(c mqtt.Client, err error) {
|
||||
log.Printf("MQTT [%s] disconnected from %s: %v", tag, source.Broker, err)
|
||||
status.MarkDisconnect(time.Now(), err)
|
||||
})
|
||||
|
||||
opts.SetReconnectingHandler(func(c mqtt.Client, options *mqtt.ClientOptions) {
|
||||
@@ -172,16 +287,7 @@ func main() {
|
||||
// Capture source for closure
|
||||
src := source
|
||||
opts.SetDefaultPublishHandler(func(c mqtt.Client, m mqtt.Message) {
|
||||
// PR #1609 M1: stamp the RECEIPT clock here (broker liveness)
|
||||
// independently of the post-write clock that handleMessage
|
||||
// stamps. Without separation the watchdog/healthz could
|
||||
// report "fresh" while the writer was stalled and the
|
||||
// buffer was filling.
|
||||
markReceiptForTag(tag, time.Now())
|
||||
status.MarkPacket(time.Now())
|
||||
ingestBuffer.Submit(func() {
|
||||
handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg)
|
||||
})
|
||||
handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg)
|
||||
})
|
||||
|
||||
client := mqtt.NewClient(opts)
|
||||
@@ -248,211 +354,6 @@ func main() {
|
||||
log.Printf("Running — %d MQTT source(s) connected", connectedCount)
|
||||
}
|
||||
|
||||
// Node retention: move stale nodes to inactive_nodes on startup
|
||||
nodeDays := cfg.NodeDaysOrDefault()
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
|
||||
// Observer retention: remove stale observers on startup
|
||||
observerDays := cfg.ObserverDaysOrDefault()
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
|
||||
// Metrics retention: prune old metrics on startup
|
||||
metricsDays := cfg.MetricsRetentionDays()
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
|
||||
// Packet (transmissions) retention: previously lived in cmd/server,
|
||||
// moved to ingestor in #1283 to eliminate cross-process write
|
||||
// contention (SQLITE_BUSY). 0 = disabled.
|
||||
packetDays := cfg.PacketDaysOrZero()
|
||||
if packetDays > 0 {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[prune] startup pruned %d transmissions older than %d days", n, packetDays)
|
||||
}
|
||||
}
|
||||
|
||||
// Client-RX coverage retention: bound the opt-in coverage tables (#1727).
|
||||
// Independent of the feature flag, so data persists are reaped even after
|
||||
// the feature is turned off. 0 = disabled.
|
||||
clientRxDays := cfg.ClientRxDaysOrZero()
|
||||
if clientRxDays > 0 {
|
||||
if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[prune] startup pruned %d client_receptions older than %d days", n, clientRxDays)
|
||||
}
|
||||
}
|
||||
|
||||
vacuumPages := cfg.IncrementalVacuumPages()
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
|
||||
// Gate open: the synchronous startup writes above cannot return until the
|
||||
// single SQLite writer is free, which means any blocking async migration
|
||||
// (e.g. the CREATE INDEX) has finished. WaitForAsyncMigrations() makes that
|
||||
// explicit. Now drain everything the subscription buffered during startup.
|
||||
store.WaitForAsyncMigrations()
|
||||
ingestBuffer.Ready()
|
||||
if d := ingestBuffer.Dropped(); d > 0 {
|
||||
log.Printf("[ingest-buffer] write path ready; draining backlog (dropped %d during startup — consider raising ingestBufferSize)", d)
|
||||
} else {
|
||||
log.Printf("[ingest-buffer] write path ready; draining backlog (0 dropped)")
|
||||
}
|
||||
|
||||
// Daily ticker for node retention
|
||||
retentionTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
for range retentionTicker.C {
|
||||
store.MoveStaleNodes(nodeDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for observer retention (every 24h, staggered 90s after startup)
|
||||
observerRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(90 * time.Second) // stagger after metrics prune
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
for range observerRetentionTicker.C {
|
||||
store.RemoveStaleObservers(observerDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for metrics retention (every 24h)
|
||||
metricsRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range metricsRetentionTicker.C {
|
||||
store.PruneOldMetrics(metricsDays)
|
||||
store.PruneDroppedPackets(metricsDays)
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}()
|
||||
|
||||
// Daily ticker for transmission retention (#1283).
|
||||
var packetRetentionTicker *time.Ticker
|
||||
if packetDays > 0 {
|
||||
packetRetentionTicker = time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range packetRetentionTicker.C {
|
||||
if n, err := store.PruneOldPackets(packetDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays)
|
||||
}
|
||||
|
||||
// Daily ticker for client-RX coverage retention (#1727).
|
||||
if clientRxDays > 0 {
|
||||
clientRxRetentionTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for range clientRxRetentionTicker.C {
|
||||
if n, err := store.PruneOldClientReceptions(clientRxDays); err != nil {
|
||||
log.Printf("[prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
store.RunIncrementalVacuum(vacuumPages)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[prune] auto-prune enabled: client_receptions older than %d days will be removed daily", clientRxDays)
|
||||
}
|
||||
|
||||
// Hourly WAL checkpoint to prevent unbounded WAL growth.
|
||||
// TRUNCATE resets the WAL file to zero bytes when all frames are flushed;
|
||||
// if the server's read connection holds frames, remaining pages stay in the
|
||||
// WAL until the next tick. Staggered 30s after startup to avoid competing
|
||||
// with the initial burst of ingest writes.
|
||||
walCheckpointTicker := time.NewTicker(1 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(30 * time.Second)
|
||||
store.Checkpoint()
|
||||
for range walCheckpointTicker.C {
|
||||
store.Checkpoint()
|
||||
}
|
||||
}()
|
||||
log.Printf("[db] WAL checkpoint scheduled every 1h")
|
||||
|
||||
// Daily neighbor_edges retention (#1287 — moved from cmd/server).
|
||||
{
|
||||
nDays := cfg.NeighborEdgesDaysOrDefault()
|
||||
neighborPruneTicker := time.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
time.Sleep(4 * time.Minute) // stagger
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] startup pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
for range neighborPruneTicker.C {
|
||||
if n, err := store.PruneNeighborEdges(nDays); err != nil {
|
||||
log.Printf("[neighbor-prune] error: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[neighbor-prune] pruned %d edges older than %d days", n, nDays)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[neighbor-prune] auto-prune enabled: edges older than %d days", nDays)
|
||||
}
|
||||
|
||||
// Periodic stats logging (every 5 minutes)
|
||||
statsTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
for range statsTicker.C {
|
||||
store.LogStats()
|
||||
if d := ingestBuffer.Dropped(); d > 0 || ingestBuffer.Pending() > 0 {
|
||||
log.Printf("[ingest-buffer] pending=%d dropped_total=%d", ingestBuffer.Pending(), d)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Prune-request queue (#669 M4 / #738): the read-only server enqueues
|
||||
// geo-prune requests as marker files; the ingestor (which holds the
|
||||
// write handle) executes the DELETEs. Process on startup, then every
|
||||
// 15 seconds — short enough for a one-click UX, long enough to avoid
|
||||
// useless wake-ups.
|
||||
store.RunPendingPruneRequests()
|
||||
pruneQueueTicker := time.NewTicker(15 * time.Second)
|
||||
go func() {
|
||||
for range pruneQueueTicker.C {
|
||||
store.RunPendingPruneRequests()
|
||||
}
|
||||
}()
|
||||
|
||||
// Per-second stats file writer for the server's /api/perf/write-sources
|
||||
// endpoint (#1120). Best-effort; never fatal.
|
||||
StartStatsFileWriter(store, time.Second)
|
||||
|
||||
// Multi-byte capability persister (#1324 follow-up): the server's
|
||||
// analytics cycle publishes a snapshot file via internal/mbcapqueue
|
||||
// (it cannot UPDATE itself, mode=ro since #1289). The ingestor
|
||||
// applies the snapshot here every 5 minutes — derived/cached
|
||||
// columns, ingestor owns the write.
|
||||
multibytePersistTicker := time.NewTicker(5 * time.Minute)
|
||||
go func() {
|
||||
time.Sleep(2 * time.Minute) // stagger after analytics warmup
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
for range multibytePersistTicker.C {
|
||||
if _, err := store.RunMultibyteCapPersist(); err != nil {
|
||||
log.Printf("[multibyte-persist] error: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[multibyte-persist] enabled (interval=5m)")
|
||||
|
||||
// Neighbor-edges builder (#1287 — Option 4): ingestor owns
|
||||
// neighbor_edges writes. Runs every 60s. Server reads the snapshot
|
||||
// via cmd/server/neighbor_recomputer.go on the same cadence.
|
||||
stopNeighborBuilder := store.StartNeighborEdgesBuilder(NeighborEdgesBuilderInterval)
|
||||
defer stopNeighborBuilder()
|
||||
log.Printf("[neighbor-build] enabled (interval=%s)", NeighborEdgesBuilderInterval)
|
||||
|
||||
// #1212: per-source stall watchdog. Detects "silently dead" sources
|
||||
// where the client reports connected but no messages have flowed. Logs
|
||||
// a WARN line every minute for any source silent for >5m. Scan every
|
||||
@@ -472,7 +373,6 @@ func main() {
|
||||
}
|
||||
statsTicker.Stop()
|
||||
pruneQueueTicker.Stop()
|
||||
walCheckpointTicker.Stop()
|
||||
stopWatchdog()
|
||||
store.LogStats() // final stats on shutdown
|
||||
for _, c := range clients {
|
||||
@@ -536,9 +436,7 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions {
|
||||
}
|
||||
if source.RejectUnauthorized != nil && !*source.RejectUnauthorized {
|
||||
opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") || strings.HasPrefix(source.Broker, "wss://") {
|
||||
// TLS with system CA pool — valid for ssl:// MQTT brokers and
|
||||
// wss:// WebSocket brokers behind a publicly-trusted certificate.
|
||||
} else if strings.HasPrefix(source.Broker, "ssl://") {
|
||||
opts.SetTLSConfig(&tls.Config{})
|
||||
}
|
||||
return opts
|
||||
@@ -562,21 +460,6 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
return
|
||||
}
|
||||
|
||||
// Mobile client RX coverage: dedicated topic meshcore/client/{PUBLIC_KEY}/packets.
|
||||
// A roaming companion reports where it directly heard a node; handled in isolation
|
||||
// from the observer/observations path. EMQX ACL binds parts[2] to the client's own key.
|
||||
if cfg.ClientRxCoverageEnabled() && len(parts) >= 4 && parts[1] == "client" && parts[3] == "packets" {
|
||||
// The observer blacklist (checked below) only runs on the observer path,
|
||||
// so a blacklisted operator could otherwise skirt it via the client topic
|
||||
// (#1). Enforce it here before any coverage write.
|
||||
if cfg.IsObserverBlacklisted(parts[2]) {
|
||||
log.Printf("MQTT [%s] client %.8s blacklisted, dropping", tag, parts[2])
|
||||
return
|
||||
}
|
||||
handleClientPacket(store, tag, parts[2], msg, channelKeys)
|
||||
return
|
||||
}
|
||||
|
||||
// Skip status/connection topics
|
||||
if topic == "meshcore/status" || topic == "meshcore/events/connection" {
|
||||
return
|
||||
@@ -604,11 +487,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
name, _ := msg["origin"].(string)
|
||||
iata := parts[1]
|
||||
meta := extractObserverMeta(msg)
|
||||
// observer.last_seen is "when did the analyzer last hear from this
|
||||
// observer" — fundamentally an ingest-time question. Passing "" makes
|
||||
// UpsertObserverAt use time.Now(), independent of the envelope timestamp
|
||||
// (which can be stale/skewed even when well-formed). See #1465.
|
||||
if err := store.UpsertObserverAt(observerID, name, iata, meta, ""); err != nil {
|
||||
if err := store.UpsertObserverAt(observerID, name, iata, meta, resolveRxTime(msg, tag)); err != nil {
|
||||
log.Printf("MQTT [%s] observer status error: %v", tag, err)
|
||||
}
|
||||
// Insert metrics sample from status message
|
||||
@@ -627,7 +506,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] metrics insert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
log.Print(formatStatusLog(tag, firstNonEmpty(name, observerID), iata))
|
||||
log.Printf("MQTT [%s] status: %s (%s)", tag, firstNonEmpty(name, observerID), iata)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -692,14 +571,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
}
|
||||
|
||||
mqttMsg := &MQTTPacketMessage{Raw: rawHex}
|
||||
var naiveSkewSec int64
|
||||
mqttMsg.Timestamp, naiveSkewSec = resolveRxTime(msg, tag)
|
||||
if naiveSkewSec != 0 && observerID != "" {
|
||||
// Issue #1478: record so /api/observers can surface ⚠️ chip.
|
||||
if err := store.RecordNaiveSkew(observerID, naiveSkewSec, time.Now()); err != nil {
|
||||
log.Printf("MQTT [%s] RecordNaiveSkew(%s): %v", tag, observerID, err)
|
||||
}
|
||||
}
|
||||
mqttMsg.Timestamp = resolveRxTime(msg, tag)
|
||||
// Parse optional region from JSON payload (#788)
|
||||
if v, ok := msg["region"].(string); ok && v != "" {
|
||||
mqttMsg.Region = v
|
||||
@@ -756,7 +628,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
truncPK = truncPK[:16]
|
||||
}
|
||||
log.Printf("MQTT [%s] DROPPED invalid signature: hash=%s name=%s observer=%s pubkey=%s",
|
||||
tag, hash, sanitizeLogString(decoded.Payload.Name), sanitizeLogString(firstNonEmpty(mqttMsg.Origin, observerID)), truncPK)
|
||||
tag, hash, decoded.Payload.Name, firstNonEmpty(mqttMsg.Origin, observerID), truncPK)
|
||||
store.InsertDroppedPacket(&DroppedPacket{
|
||||
Hash: hash,
|
||||
RawHex: rawHex,
|
||||
@@ -786,7 +658,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
truncPK = truncPK[:16]
|
||||
}
|
||||
log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s",
|
||||
tag, truncPK, sanitizeLogString(decoded.Payload.Name), lat, lon, sanitizeLogString(firstNonEmpty(mqttMsg.Origin, observerID)))
|
||||
tag, truncPK, decoded.Payload.Name, lat, lon, firstNonEmpty(mqttMsg.Origin, observerID))
|
||||
}
|
||||
pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys)
|
||||
pktData.Foreign = foreign
|
||||
@@ -814,8 +686,8 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] node telemetry update error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
// Update default_scope when advert carries a matched transport scope (#899, #1534)
|
||||
if shouldUpdateDefaultScope(pktData) {
|
||||
// Update default_scope when advert carries a matched transport scope (#899)
|
||||
if pktData.IsTransportScoped {
|
||||
if err := store.UpdateNodeDefaultScope(decoded.Payload.PubKey, pktData.ScopeName); err != nil {
|
||||
log.Printf("MQTT [%s] node default_scope update error: %v", tag, err)
|
||||
}
|
||||
@@ -837,10 +709,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
if mqttMsg.Region != "" {
|
||||
effectiveRegion = mqttMsg.Region
|
||||
}
|
||||
// Same as the status-path call above: observer.last_seen is ingest
|
||||
// time, not envelope time. Per-packet rxTime (stored in observations
|
||||
// via InsertTransmission) still uses envelope time. See #1465.
|
||||
if err := store.UpsertObserverAt(observerID, origin, effectiveRegion, nil, ""); err != nil {
|
||||
if err := store.UpsertObserverAt(observerID, origin, effectiveRegion, nil, mqttMsg.Timestamp); err != nil {
|
||||
log.Printf("MQTT [%s] observer upsert error: %v", tag, err)
|
||||
}
|
||||
}
|
||||
@@ -950,7 +819,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
// used for claiming/health lookups. The node will get a proper entry when it
|
||||
// sends an advert. See issue #665.
|
||||
|
||||
log.Print(formatChannelMessageLog(tag, channelIdx, firstNonEmpty(sender, "unknown")))
|
||||
log.Printf("MQTT [%s] channel message: ch%s from %s", tag, channelIdx, firstNonEmpty(sender, "unknown"))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1036,7 +905,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message,
|
||||
log.Printf("MQTT [%s] DM insert error: %v", tag, err)
|
||||
}
|
||||
|
||||
log.Print(formatDirectMessageLog(tag, firstNonEmpty(sender, "unknown")))
|
||||
log.Printf("MQTT [%s] direct message from %s", tag, firstNonEmpty(sender, "unknown"))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1174,37 +1043,6 @@ func extractObserverMeta(msg map[string]interface{}) *ObserverMeta {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue #1290: firmware 1.16 publishes a `repeat` flag at the top
|
||||
// level of the /status JSON (MQTTMessageBuilder.cpp:58 — see
|
||||
// agessaman/MeshCore mqtt-bridge-implementation-flex). Accept
|
||||
// either a boolean or a case-insensitive `on|off|true|false|1|0`
|
||||
// string. Missing field → leave CanRelay nil; the writer preserves
|
||||
// the prior column value (default 1, back-compat).
|
||||
if v, ok := msg["repeat"]; ok && v != nil {
|
||||
switch t := v.(type) {
|
||||
case bool:
|
||||
b := t
|
||||
meta.CanRelay = &b
|
||||
hasData = true
|
||||
case string:
|
||||
s := strings.ToLower(strings.TrimSpace(t))
|
||||
switch s {
|
||||
case "on", "true", "1", "yes":
|
||||
b := true
|
||||
meta.CanRelay = &b
|
||||
hasData = true
|
||||
case "off", "false", "0", "no":
|
||||
b := false
|
||||
meta.CanRelay = &b
|
||||
hasData = true
|
||||
}
|
||||
case float64:
|
||||
b := t != 0
|
||||
meta.CanRelay = &b
|
||||
hasData = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasData {
|
||||
return nil
|
||||
}
|
||||
@@ -1242,28 +1080,22 @@ func firstNonEmpty(vals ...string) string {
|
||||
// the frame, not when the MQTT message is published — so a buffered packet
|
||||
// uploaded hours late still carries its true receive time. Using ingest time
|
||||
// (time.Now()) here mis-dated such packets by the upload delay.
|
||||
//
|
||||
// The returned naiveSkewSec is 0 unless a naive (zone-less) timestamp had to
|
||||
// be clamped because it was off from server-now by >15min — in which case it
|
||||
// is the signed offset in seconds (negative = observer behind UTC, positive =
|
||||
// ahead). Caller records this via Store.RecordNaiveSkew so the UI can flag
|
||||
// the observer (#1478).
|
||||
func resolveRxTime(msg map[string]interface{}, tag string) (string, int64) {
|
||||
func resolveRxTime(msg map[string]interface{}, tag string) string {
|
||||
now := time.Now().UTC()
|
||||
raw, _ := msg["timestamp"].(string)
|
||||
if raw == "" {
|
||||
return now.Format(time.RFC3339), 0
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
t, naive, err := parseEnvelopeTime(raw)
|
||||
t, err := parseEnvelopeTime(raw)
|
||||
if err != nil {
|
||||
log.Printf("MQTT [%s] unparseable timestamp %q, using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339), 0
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Hard reject: > 14h ahead is a genuine clock error (UTC+14 is the maximum
|
||||
// standard offset, so nothing valid should be further ahead than that).
|
||||
if t.After(now.Add(14 * time.Hour)) {
|
||||
log.Printf("MQTT [%s] future timestamp %q, using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339), 0
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Hard reject: > 30 days in the past is an RTC-reset node reporting a
|
||||
// factory date (e.g. 2020-01-01). Such a value would permanently drag
|
||||
@@ -1271,61 +1103,37 @@ func resolveRxTime(msg map[string]interface{}, tag string) (string, int64) {
|
||||
// InsertTransmission. No legitimate buffered upload is that stale.
|
||||
if t.Before(now.Add(-30 * 24 * time.Hour)) {
|
||||
log.Printf("MQTT [%s] stale timestamp %q (>30d old), using ingest time", tag, raw)
|
||||
return now.Format(time.RFC3339), 0
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
// Symmetric naive-timestamp clamp (issue #1463). Naive (zone-less) ISO
|
||||
// values from observers in non-UTC zones are parsed as-if UTC, leaving a
|
||||
// residual offset equal to the observer's UTC offset:
|
||||
// - UTC+N observer → value appears N hours in the future
|
||||
// - UTC-N observer → value appears N hours in the past
|
||||
// The past case was silently stored verbatim, poisoning last_seen and
|
||||
// rendering UTC-N observers perpetually "Stale" in the UI. Collapse any
|
||||
// naive value more than 15 min off server-now to now() — well-behaved
|
||||
// observers (Z-suffixed or explicit offset) are untouched regardless of
|
||||
// skew so legitimate buffered uploads remain accurate.
|
||||
const naiveTolerance = 15 * time.Minute
|
||||
if naive {
|
||||
signed := t.Sub(now) // signed: positive = ahead, negative = behind
|
||||
abs := signed
|
||||
if abs < 0 {
|
||||
abs = -abs
|
||||
}
|
||||
if abs > naiveTolerance {
|
||||
// Issue #1478: surface to UI via RecordNaiveSkew (called by handler).
|
||||
// Per-message log was silenced in #1479 — chip + banner in the UI
|
||||
// replace it.
|
||||
deltaSec := int64(signed / time.Second)
|
||||
return now.Format(time.RFC3339), deltaSec
|
||||
}
|
||||
}
|
||||
// Legacy soft clamp for zone-aware near-future values: any value ahead of
|
||||
// now is from a slightly skewed observer clock — collapse to now so we
|
||||
// don't render ⚠️ in the UI for live packets from those nodes.
|
||||
// Soft clamp: naive local-clock timestamps from UTC+N observers are parsed
|
||||
// as-if UTC, making them appear N hours in the future. A UTC+2 observer's
|
||||
// live packet looks 2h ahead, but it is NOT a buffered packet — the whole
|
||||
// point of using rxTime is to preserve the past timestamp for packets that
|
||||
// were buffered offline. If rxTime is ahead of now, the packet is live and
|
||||
// ingest time is the correct value. This also prevents storing future
|
||||
// timestamps that would show ⚠️ in the UI for every packet from UTC+N nodes.
|
||||
if t.After(now) {
|
||||
return now.Format(time.RFC3339), 0
|
||||
return now.Format(time.RFC3339)
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339), 0
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// parseEnvelopeTime parses the MQTT envelope timestamp. Two on-wire forms
|
||||
// occur: zone-aware ISO8601 (RFC3339), and a naive local-clock ISO string
|
||||
// with no zone (python datetime.isoformat()). Zone-aware layouts are tried
|
||||
// first; naive layouts are assumed UTC but the caller is informed via the
|
||||
// returned `naive` flag so it can apply a symmetric clamp (see issue #1463).
|
||||
func parseEnvelopeTime(s string) (time.Time, bool, error) {
|
||||
// Zone-aware first — RFC3339 demands Z or ±HH:MM.
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return t, false, nil
|
||||
}
|
||||
// first; naive layouts are assumed UTC, leaving a bounded residual offset
|
||||
// equal to the observer's UTC offset for naive-timestamp uploaders.
|
||||
func parseEnvelopeTime(s string) (time.Time, error) {
|
||||
for _, layout := range []string{
|
||||
time.RFC3339, // 2026-05-16T10:00:00Z / +02:00
|
||||
"2006-01-02T15:04:05.999999", // python isoformat w/ microseconds
|
||||
"2006-01-02T15:04:05", // naive ISO
|
||||
} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, true, nil
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, false, fmt.Errorf("unrecognized timestamp layout: %q", s)
|
||||
return time.Time{}, fmt.Errorf("unrecognized timestamp layout: %q", s)
|
||||
}
|
||||
|
||||
// deriveHashtagChannelKey derives an AES-128 key from a channel name.
|
||||
@@ -1335,29 +1143,12 @@ func deriveHashtagChannelKey(channelName string) string {
|
||||
return hex.EncodeToString(h[:16])
|
||||
}
|
||||
|
||||
// builtinChannelKeys returns channel keys that are part of the MeshCore firmware
|
||||
// defaults and should always be available, regardless of the rainbow file or config.
|
||||
// Adding new entries here is the right move when a key is part of the protocol spec
|
||||
// (not a community-named hashtag channel).
|
||||
func builtinChannelKeys() map[string]string {
|
||||
return map[string]string{
|
||||
// Default Public channel — well-known PSK from the MeshCore companion
|
||||
// protocol spec. Channel-hash byte = 0x11.
|
||||
"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72",
|
||||
}
|
||||
}
|
||||
|
||||
// loadChannelKeys loads channel decryption keys from config and/or a JSON file.
|
||||
// Merge priority: builtin (lowest) → rainbow → derived from hashChannels → explicit config (highest).
|
||||
// Merge priority: rainbow (lowest) → derived from hashChannels → explicit config (highest).
|
||||
func loadChannelKeys(cfg *Config, configPath string) map[string]string {
|
||||
keys := make(map[string]string)
|
||||
|
||||
// 0. Built-in firmware-default keys (lowest priority — overridable by everything else)
|
||||
for k, v := range builtinChannelKeys() {
|
||||
keys[k] = v
|
||||
}
|
||||
|
||||
// 1. Rainbow table keys
|
||||
// 1. Rainbow table keys (lowest priority)
|
||||
keysPath := os.Getenv("CHANNEL_KEYS_PATH")
|
||||
if keysPath == "" {
|
||||
keysPath = cfg.ChannelKeysPath
|
||||
@@ -1486,11 +1277,3 @@ func init() {
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
// shouldUpdateDefaultScope returns true when the packet carries a transport
|
||||
// scope whose region key matched (#1534). Without the ScopeName non-empty
|
||||
// guard, transport-scoped adverts from non-matching regions would overwrite
|
||||
// previously-correct default_scope values with the empty string.
|
||||
func shouldUpdateDefaultScope(pktData *PacketData) bool {
|
||||
return pktData.IsTransportScoped && pktData.ScopeName != ""
|
||||
}
|
||||
|
||||
+2
-167
@@ -2,10 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -616,41 +614,8 @@ func TestLoadChannelKeysHashChannelsNormalization(t *testing.T) {
|
||||
if _, ok := keys["#Spaced"]; !ok {
|
||||
t.Error("should derive key for #Spaced (trimmed)")
|
||||
}
|
||||
// 3 derived + builtins (Public)
|
||||
expected := 3 + len(builtinChannelKeys())
|
||||
if len(keys) != expected {
|
||||
t.Errorf("expected %d keys, got %d", expected, len(keys))
|
||||
}
|
||||
}
|
||||
|
||||
// Default Public channel must always be present from the built-in floor,
|
||||
// regardless of whether a rainbow file is provided.
|
||||
func TestLoadChannelKeysBuiltinPublic(t *testing.T) {
|
||||
t.Setenv("CHANNEL_KEYS_PATH", "")
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
cfg := &Config{}
|
||||
|
||||
keys := loadChannelKeys(cfg, cfgPath)
|
||||
|
||||
if got := keys["Public"]; got != "8b3387e9c5cdea6ac9e5edbaa115cd72" {
|
||||
t.Errorf("Public key = %q, want firmware-default 8b3387e9c5cdea6ac9e5edbaa115cd72", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Explicit config and rainbow entries must still override the built-in floor.
|
||||
func TestLoadChannelKeysBuiltinOverridable(t *testing.T) {
|
||||
t.Setenv("CHANNEL_KEYS_PATH", "")
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
cfg := &Config{
|
||||
ChannelKeys: map[string]string{"Public": "deadbeefdeadbeefdeadbeefdeadbeef"},
|
||||
}
|
||||
|
||||
keys := loadChannelKeys(cfg, cfgPath)
|
||||
|
||||
if got := keys["Public"]; got != "deadbeefdeadbeefdeadbeefdeadbeef" {
|
||||
t.Errorf("Public key = %q, want explicit override deadbeef...", got)
|
||||
if len(keys) != 3 {
|
||||
t.Errorf("expected 3 keys, got %d", len(keys))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1055,133 +1020,3 @@ func TestHandleMessageObserverIATAWhitelist(t *testing.T) {
|
||||
t.Errorf("observer from whitelisted IATA ARN should be accepted, got count=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPacketDataScopeMatchingNoMatch covers the #1534 regression: a
|
||||
// transport-scoped advert from a non-matching region carries
|
||||
// IsTransportScoped=true and ScopeName="". The default_scope update guard
|
||||
// must skip these packets so previously-correct scopes aren't overwritten
|
||||
// with the empty string.
|
||||
func TestBuildPacketDataScopeMatchingNoMatch(t *testing.T) {
|
||||
// Code1=2AB5 is the precomputed code for region "#test" (payload="hello",
|
||||
// payloadType=5). Build a region-key map for a DIFFERENT region so
|
||||
// matchScope() finds no match and returns "".
|
||||
const rawHex = "142AB500000068656C6C6F"
|
||||
otherKey, _ := hex.DecodeString("aabbccddeeff00112233445566778899")
|
||||
regionKeys := map[string][]byte{"#other": otherKey}
|
||||
|
||||
decoded, err := DecodePacket(rawHex, nil, false)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodePacket: %v", err)
|
||||
}
|
||||
msg := &MQTTPacketMessage{Raw: rawHex}
|
||||
pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys)
|
||||
|
||||
if !pktData.IsTransportScoped {
|
||||
t.Fatalf("precondition: IsTransportScoped should be true (Code1 != 0000)")
|
||||
}
|
||||
if pktData.ScopeName != "" {
|
||||
t.Fatalf("precondition: ScopeName should be empty (no region match), got %q", pktData.ScopeName)
|
||||
}
|
||||
|
||||
// Regression assertion: when ScopeName is empty, the guard must skip the
|
||||
// UpdateNodeDefaultScope call so an empty value never overwrites a
|
||||
// previously-correct default_scope (#1534).
|
||||
if shouldUpdateDefaultScope(pktData) {
|
||||
t.Errorf("shouldUpdateDefaultScope = true for empty ScopeName; want false (would overwrite default_scope with \"\")")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate is the call-site
|
||||
// regression test for #1534. It drives a transport-scoped ADVERT whose
|
||||
// region key does NOT match any configured region (so ScopeName=="") through
|
||||
// handleMessage end-to-end and asserts that a pre-existing default_scope on
|
||||
// the node is NOT overwritten with the empty string. This anchors the
|
||||
// call-site guard at main.go:720 — a future refactor that drops the
|
||||
// `if shouldUpdateDefaultScope(...)` wrapper and calls
|
||||
// `store.UpdateNodeDefaultScope(pubkey, pktData.ScopeName)` unconditionally
|
||||
// would re-introduce the #1534 bug and fail this test.
|
||||
func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
// A transport-scoped ADVERT: header byte 0x10 = route_type 0
|
||||
// (TRANSPORT_FLOOD) + payload_type 4 (ADVERT). Code1=AABB (non-zero, so
|
||||
// IsTransportScoped becomes true), Code2=0000, path_byte=00, then a
|
||||
// 100-byte ADVERT payload (32-byte pubkey starting 46D62D… + 4-byte ts
|
||||
// + 64-byte signature) reused from TestHandleMessageAdvertWithTelemetry.
|
||||
const rawHex = "10AABB00000046D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
const pubkey = "46d62de27d4c5194d7821fc5a34a45565dcc2537b300b9ab6275255cefb65d84"
|
||||
|
||||
// Pre-seed the node with a non-empty default_scope so we can detect an
|
||||
// erroneous overwrite with "".
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#belgium')`, pubkey); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
|
||||
// Empty regionKeys → matchScope() returns "" for any Code1 → ScopeName "".
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, map[string][]byte{}, &Config{})
|
||||
|
||||
var got sql.NullString
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil {
|
||||
t.Fatalf("read default_scope: %v", err)
|
||||
}
|
||||
if !got.Valid || got.String != "#belgium" {
|
||||
t.Errorf("default_scope after empty-scope advert = %q (valid=%v), want #belgium — call-site guard at main.go:720 is missing or broken (#1534)", got.String, got.Valid)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope is the positive
|
||||
// counterpart: a transport-scoped ADVERT whose Code1 matches a configured
|
||||
// region key MUST cause default_scope to be updated to the matched region
|
||||
// name. Together with the empty-scope test above this proves the call-site
|
||||
// branch routes correctly for both ScopeName states.
|
||||
func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
// Same ADVERT bytes; this time we compute the matching region key for
|
||||
// the (payloadType=4, payload=<advert bytes>) tuple so matchScope() will
|
||||
// return "#de".
|
||||
const advertBytes = "46D62DE27D4C5194D7821FC5A34A45565DCC2537B300B9AB6275255CEFB65D840CE5C169C94C9AED39E8BCB6CB6EB0335497A198B33A1A610CD3B03D8DCFC160900E5244280323EE0B44CACAB8F02B5B38B91CFA18BD067B0B5E63E94CFC85F758A8530B9240933402E0E6B8F84D5252322D52"
|
||||
const pubkey = "46d62de27d4c5194d7821fc5a34a45565dcc2537b300b9ab6275255cefb65d84"
|
||||
|
||||
advertRaw, _ := hex.DecodeString(advertBytes)
|
||||
// Derive the region key whose HMAC produces Code1 we can plant in the
|
||||
// header. Choose key = first 16 bytes of HMAC-SHA256(zeros, advertBytes)
|
||||
// is non-deterministic to find; instead pick an arbitrary key and
|
||||
// compute Code1 from it, then build the packet around that Code1.
|
||||
regionKey, _ := hex.DecodeString("0123456789abcdef0123456789abcdef")
|
||||
mac := hmacSHA256(regionKey, append([]byte{4}, advertRaw...))
|
||||
// Per firmware (#1534 helper logic): Code1 is the first 2 bytes of the
|
||||
// HMAC, sentinel-shifted so 0x0000 → 0x0001 and 0xFFFF → 0xFFFE.
|
||||
code := uint16(mac[0]) | (uint16(mac[1]) << 8)
|
||||
if code == 0x0000 {
|
||||
code = 0x0001
|
||||
} else if code == 0xFFFF {
|
||||
code = 0xFFFE
|
||||
}
|
||||
code1 := fmt.Sprintf("%02X%02X", byte(code&0xFF), byte(code>>8))
|
||||
rawHex := "10" + code1 + "000000" + advertBytes
|
||||
|
||||
if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#old')`, pubkey); err != nil {
|
||||
t.Fatalf("seed node: %v", err)
|
||||
}
|
||||
|
||||
msg := &mockMessage{
|
||||
topic: "meshcore/SJC/obs1/packets",
|
||||
payload: []byte(`{"raw":"` + rawHex + `"}`),
|
||||
}
|
||||
handleMessage(store, "test", source, msg, nil, map[string][]byte{"#de": regionKey}, &Config{})
|
||||
|
||||
var got sql.NullString
|
||||
if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil {
|
||||
t.Fatalf("read default_scope: %v", err)
|
||||
}
|
||||
if !got.Valid || got.String != "#de" {
|
||||
t.Errorf("default_scope after matched-scope advert = %q (valid=%v), want #de", got.String, got.Valid)
|
||||
}
|
||||
}
|
||||
|
||||
+19
-51
@@ -22,25 +22,26 @@ func (s *Store) PruneOldPackets(days int) (int64, error) {
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)
|
||||
|
||||
// Tagged for writer-perf visibility (#1340).
|
||||
var n int64
|
||||
err := s.WriterTx("prune_packets", func(tx *sql.Tx) error {
|
||||
// Delete child observations first (no CASCADE in SQLite).
|
||||
if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN (
|
||||
SELECT id FROM transmissions WHERE first_seen < ?
|
||||
)`, cutoff); err != nil {
|
||||
return fmt.Errorf("prune observations: %w", err)
|
||||
}
|
||||
|
||||
res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune transmissions: %w", err)
|
||||
}
|
||||
n, _ = res.RowsAffected()
|
||||
return nil
|
||||
})
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return 0, fmt.Errorf("prune begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// Delete child observations first (no CASCADE in SQLite).
|
||||
if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN (
|
||||
SELECT id FROM transmissions WHERE first_seen < ?
|
||||
)`, cutoff); err != nil {
|
||||
return 0, fmt.Errorf("prune observations: %w", err)
|
||||
}
|
||||
|
||||
res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prune transmissions: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("prune commit: %w", err)
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[prune] deleted %d transmissions older than %d days", n, days)
|
||||
@@ -48,39 +49,6 @@ func (s *Store) PruneOldPackets(days int) (int64, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// PruneOldClientReceptions deletes mobile client-RX coverage rows older than
|
||||
// `days` (by rx_at), and client_observers (companion names) whose last_seen has
|
||||
// aged out. This bounds the otherwise-unbounded client_receptions table the
|
||||
// opt-in coverage feature feeds. 0 disables. Owned by the ingestor writer
|
||||
// (#1283). Returns the number of client_receptions rows deleted.
|
||||
func (s *Store) PruneOldClientReceptions(days int) (int64, error) {
|
||||
if days <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)
|
||||
|
||||
var n int64
|
||||
err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error {
|
||||
res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune client_receptions: %w", err)
|
||||
}
|
||||
n, _ = res.RowsAffected()
|
||||
// Drop companion name rows not refreshed within the window.
|
||||
if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil {
|
||||
return fmt.Errorf("prune client_observers: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[prune] deleted %d client_receptions older than %d days", n, days)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// SoftDeleteBlacklistedObservers marks observers in the blacklist as
|
||||
// inactive=1 so they are hidden from API responses. Owned by ingestor
|
||||
// per #1287. Runs once at startup.
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
import "runtime/debug"
|
||||
|
||||
// applyMemoryLimit configures Go's soft memory limit (GOMEMLIMIT) for the
|
||||
// ingestor process. See #1010.
|
||||
//
|
||||
// Precedence:
|
||||
// 1. GOMEMLIMIT env var (parsed by the runtime at startup) — we do not
|
||||
// override; report source="env" with limit=0.
|
||||
// 2. runtimeMaxMB > 0 (from config runtime.maxMemoryMB) — set limit of
|
||||
// runtimeMaxMB MiB via debug.SetMemoryLimit; source="config".
|
||||
// 3. Otherwise no limit applied; source="none" (default behavior).
|
||||
//
|
||||
// Returns the limit (bytes) we set, or 0 if we did not set one.
|
||||
func applyMemoryLimit(runtimeMaxMB int, envSet bool) (int64, string) {
|
||||
if envSet {
|
||||
return 0, "env"
|
||||
}
|
||||
if runtimeMaxMB <= 0 {
|
||||
return 0, "none"
|
||||
}
|
||||
limit := int64(runtimeMaxMB) * 1024 * 1024
|
||||
debug.SetMemoryLimit(limit)
|
||||
return limit, "config"
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestApplyMemoryLimit_FromEnv: when GOMEMLIMIT env var is set, the runtime
|
||||
// already parsed it. Our function MUST NOT override and MUST report env source.
|
||||
func TestApplyMemoryLimit_FromEnv(t *testing.T) {
|
||||
t.Setenv("GOMEMLIMIT", "850MiB")
|
||||
defer debug.SetMemoryLimit(-1)
|
||||
|
||||
limit, source := applyMemoryLimit(512, true /* envSet */)
|
||||
if source != "env" {
|
||||
t.Fatalf("expected source=env, got %q", source)
|
||||
}
|
||||
if limit != 0 {
|
||||
t.Fatalf("expected limit=0 (not set by us), got %d", limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyMemoryLimit_FromConfig: when env is unset and runtime.maxMemoryMB
|
||||
// is set, derive a limit of exactly runtimeMaxMB * 1 MiB (no headroom — the
|
||||
// ingestor's working set is bounded by MQTT batch decode, not packet store).
|
||||
func TestApplyMemoryLimit_FromConfig(t *testing.T) {
|
||||
defer debug.SetMemoryLimit(-1)
|
||||
|
||||
limit, source := applyMemoryLimit(512, false /* envSet */)
|
||||
if source != "config" {
|
||||
t.Fatalf("expected source=config, got %q", source)
|
||||
}
|
||||
want := int64(512) * 1024 * 1024
|
||||
if limit != want {
|
||||
t.Fatalf("expected limit=%d, got %d", want, limit)
|
||||
}
|
||||
cur := debug.SetMemoryLimit(-1)
|
||||
if cur != want {
|
||||
t.Fatalf("runtime memory limit not set: want=%d got=%d", want, cur)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyMemoryLimit_None: neither env nor config — no limit applied,
|
||||
// default behavior preserved.
|
||||
func TestApplyMemoryLimit_None(t *testing.T) {
|
||||
defer debug.SetMemoryLimit(-1)
|
||||
debug.SetMemoryLimit(int64(1<<63 - 1)) // math.MaxInt64 = "no limit"
|
||||
|
||||
limit, source := applyMemoryLimit(0, false)
|
||||
if source != "none" {
|
||||
t.Fatalf("expected source=none, got %q", source)
|
||||
}
|
||||
if limit != 0 {
|
||||
t.Fatalf("expected limit=0, got %d", limit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyMemoryLimit_EnvWinsOverConfig: env set AND config set → env wins,
|
||||
// our function does not override. Locks the precedence triage specified.
|
||||
func TestApplyMemoryLimit_EnvWinsOverConfig(t *testing.T) {
|
||||
t.Setenv("GOMEMLIMIT", "1GiB")
|
||||
defer debug.SetMemoryLimit(-1)
|
||||
|
||||
limit, source := applyMemoryLimit(512, true /* envSet */)
|
||||
if source != "env" {
|
||||
t.Fatalf("expected source=env when both set, got %q", source)
|
||||
}
|
||||
if limit != 0 {
|
||||
t.Fatalf("expected limit=0 when env wins, got %d", limit)
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -20,60 +18,6 @@ const livenessHeartbeatInterval = time.Hour
|
||||
// reconnects on the SAME source. See processLivenessTransition.
|
||||
const forceReconnectThrottle = 60 * time.Second
|
||||
|
||||
// disconnectedReconnectMultiplier (#1749) governs how long a source may
|
||||
// stay in LivenessDisconnected before the watchdog escalates with a
|
||||
// forced reconnect. paho's SetAutoReconnect(true) normally recovers a
|
||||
// dropped connection; in production we have observed paho's reconnect
|
||||
// machinery silently dying for a single source while another source on
|
||||
// the same binary reconnects fine (prod 2026-06-30: connectCount=1,
|
||||
// disconnectCount=1, lastError="EOF", zero retries for 18h). When the
|
||||
// source stays !IsConnected for more than `multiplier × threshold`,
|
||||
// the watchdog forces a reconnect rather than trusting paho to recover.
|
||||
const disconnectedReconnectMultiplier = 5
|
||||
|
||||
// watchdogLastTickUnix (#1749) is the wall-clock unix-seconds timestamp
|
||||
// of the most recent runLivenessWatchdogLoop tick. The watchdog
|
||||
// goroutine has itself gone silent in production (#1749: 3 sources
|
||||
// stalled simultaneously for 75 min with zero WATCHDOG log lines),
|
||||
// suggesting goroutine-level failure. Exposing this clock via
|
||||
// /api/mqtt/status lets external monitoring assert that the watchdog
|
||||
// is still ticking — a stale value (e.g. > 2× the scan interval)
|
||||
// indicates the watchdog itself is dead.
|
||||
//
|
||||
// Style note (#1810 round-1, adv #5): new package-level counters in
|
||||
// this file use atomic.Int64 (typed, method-based) while per-source
|
||||
// state on SourceLivenessState uses plain int64 + atomic.StoreInt64.
|
||||
// The struct fields stay int64 because they are accessed through
|
||||
// pointer receivers all over the codebase and atomic.Int64 inside a
|
||||
// struct breaks the "noCopy" semantics expected of value receivers
|
||||
// in a few callsites; package-level vars have no such constraint and
|
||||
// the typed form catches misuse at compile time. Kept-both is
|
||||
// intentional, not drift.
|
||||
var watchdogLastTickUnix atomic.Int64
|
||||
|
||||
// watchdogPanicCount (#1810 round-1, Taleb #3) counts recovered panics
|
||||
// inside the per-source watchdog work IIFE. A loop that panic-loops
|
||||
// every tick currently looks healthy by WatchdogLastTickUnix alone —
|
||||
// the tick stamp lands BEFORE the per-source work, so a panic on every
|
||||
// source still advances the clock. This counter, surfaced via
|
||||
// /api/mqtt/status and the stats snapshot, lets external monitoring
|
||||
// alarm on a rapidly-growing value (= the loop is alive but the work
|
||||
// is broken).
|
||||
var watchdogPanicCount atomic.Int64
|
||||
|
||||
// WatchdogLastTickUnix returns the unix-seconds timestamp of the most
|
||||
// recent watchdog tick. Returns 0 if the watchdog has never ticked.
|
||||
func WatchdogLastTickUnix() int64 {
|
||||
return watchdogLastTickUnix.Load()
|
||||
}
|
||||
|
||||
// WatchdogPanicCount returns the running total of recovered panics
|
||||
// inside the watchdog per-source work IIFE (#1810). Monotonic across
|
||||
// the process lifetime; never decreases.
|
||||
func WatchdogPanicCount() int64 {
|
||||
return watchdogPanicCount.Load()
|
||||
}
|
||||
|
||||
// LivenessKind enumerates the watchdog verdicts for a source. Edge-triggered
|
||||
// transitions use this to decide whether to emit (and what severity).
|
||||
type LivenessKind int
|
||||
@@ -111,14 +55,9 @@ const (
|
||||
// window. r1's StartedAt-as-grace-clock conflated transient-stall
|
||||
// suppression with cold-start grace; r2 separates them.
|
||||
type SourceLivenessState struct {
|
||||
Tag string
|
||||
Broker string
|
||||
LastMessageUnix int64 // atomic; unix seconds of last successfully WRITTEN MQTT message (handleMessage post-write)
|
||||
// LastReceiptUnix (PR #1609 M1) is stamped at MQTT receipt time —
|
||||
// BEFORE the message is handed to the buffer/writer. STUB: unused
|
||||
// in production until the green commit wires MarkReceipt at the
|
||||
// receipt callsite and surfaces it in stats/healthz.
|
||||
LastReceiptUnix int64 // atomic; unix seconds of last RECEIPT (broker liveness)
|
||||
Tag string
|
||||
Broker string
|
||||
LastMessageUnix int64 // atomic; unix seconds of last successfully received MQTT message
|
||||
// FirstConnectedAt (PR #1216 r2 item 2) is stamped ONCE at
|
||||
// registerLivenessState time and never reset. Cold-start grace
|
||||
// checks against this so a flapping broker (CONNECT ok, SUBSCRIBE
|
||||
@@ -144,15 +83,6 @@ type SourceLivenessState struct {
|
||||
// recent forced reconnect for this source; the watchdog reads it
|
||||
// to enforce forceReconnectThrottle. atomic.
|
||||
LastForceReconnectUnix int64
|
||||
// DisconnectedSinceUnix (#1749) is the unix-seconds timestamp of
|
||||
// the FIRST tick on which the watchdog observed this source in
|
||||
// LivenessDisconnected (paho reports !IsConnected). Cleared back
|
||||
// to 0 on any tick where the source is NOT disconnected. When the
|
||||
// gap (now - DisconnectedSinceUnix) exceeds
|
||||
// disconnectedReconnectMultiplier × threshold, the watchdog
|
||||
// escalates with a forced reconnect on the assumption that paho's
|
||||
// own auto-reconnect machinery has silently died. atomic.
|
||||
DisconnectedSinceUnix int64
|
||||
// AttemptCount is incremented on every TCP/TLS connection attempt. Used
|
||||
// by ConnectionAttemptHandler to log attempt # independent of paho's
|
||||
// internal reconnect-loop state. atomic.
|
||||
@@ -165,16 +95,6 @@ func (s *SourceLivenessState) MarkMessage(now time.Time) {
|
||||
atomic.StoreInt64(&s.LastMessageUnix, now.Unix())
|
||||
}
|
||||
|
||||
// MarkReceipt records the time of an MQTT message receipt — stamped at the
|
||||
// paho receipt callback BEFORE the message enters the ingest buffer. PR
|
||||
// #1609 M1: kept separate from LastMessageUnix so the watchdog/healthz can
|
||||
// distinguish "broker alive, write path stuck" (LastReceiptUnix fresh,
|
||||
// LastMessageUnix stale) from "everything stalled" (both stale). Cheap;
|
||||
// safe to call from the message-handling hot path.
|
||||
func (s *SourceLivenessState) MarkReceipt(now time.Time) {
|
||||
atomic.StoreInt64(&s.LastReceiptUnix, now.Unix())
|
||||
}
|
||||
|
||||
// MarkReconnected clears stale liveness state so the watchdog does not
|
||||
// false-alarm on a pre-outage timestamp after paho re-establishes the
|
||||
// connection (PR #1216 r1 item 2). Resets LastMessageUnix, re-stamps
|
||||
@@ -192,14 +112,6 @@ func (s *SourceLivenessState) MarkReconnected(now time.Time) {
|
||||
atomic.StoreInt64(&s.LastMessageUnix, 0)
|
||||
atomic.StoreInt64(&s.StartedAt, now.Unix())
|
||||
atomic.StoreInt64(&s.LastAlertUnix, 0)
|
||||
// #1810 round-1 (Taleb #5): clear DisconnectedSinceUnix so the
|
||||
// next LivenessDisconnected observation is treated as a NEW outage
|
||||
// and starts its escalation timer from scratch. Without this, a
|
||||
// post-recovery disconnect immediately satisfies (now -
|
||||
// DisconnectedSinceUnix > multiplier × threshold) and force-
|
||||
// reconnects on the first tick — making the escalation a
|
||||
// trigger-on-flap instead of trigger-on-persistent-failure.
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, 0)
|
||||
}
|
||||
|
||||
// checkSourceLiveness returns (message, kind) describing the source's
|
||||
@@ -305,8 +217,7 @@ func registerLivenessOrSkip(s *SourceLivenessState) bool {
|
||||
}
|
||||
|
||||
// markLivenessForTag is the hot-path entry point: O(1) map lookup +
|
||||
// atomic store. Safe to call for unknown tags (no-op). Updates
|
||||
// LastMessageUnix (post-write clock).
|
||||
// atomic store. Safe to call for unknown tags (no-op).
|
||||
func markLivenessForTag(tag string, now time.Time) {
|
||||
livenessRegistryMu.RLock()
|
||||
s := livenessRegistry[tag]
|
||||
@@ -316,38 +227,6 @@ func markLivenessForTag(tag string, now time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
// markReceiptForTag is the hot-path entry point used at MQTT receipt
|
||||
// (BEFORE the message is buffered/written). Updates LastReceiptUnix only.
|
||||
// PR #1609 M1 — separates broker-liveness signal from write-path
|
||||
// liveness so /healthz can show a stalled writer with a live broker.
|
||||
func markReceiptForTag(tag string, now time.Time) {
|
||||
livenessRegistryMu.RLock()
|
||||
s := livenessRegistry[tag]
|
||||
livenessRegistryMu.RUnlock()
|
||||
if s != nil {
|
||||
s.MarkReceipt(now)
|
||||
}
|
||||
}
|
||||
|
||||
// SnapshotLivenessClocks returns the per-source receipt vs write-path
|
||||
// liveness pair for every registered source. Read-only; safe to call
|
||||
// from the stats-file writer. PR #1609 M1.
|
||||
func SnapshotLivenessClocks() map[string]SourceLivenessSnapshot {
|
||||
livenessRegistryMu.RLock()
|
||||
defer livenessRegistryMu.RUnlock()
|
||||
if len(livenessRegistry) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]SourceLivenessSnapshot, len(livenessRegistry))
|
||||
for tag, s := range livenessRegistry {
|
||||
out[tag] = SourceLivenessSnapshot{
|
||||
LastReceiptUnix: atomic.LoadInt64(&s.LastReceiptUnix),
|
||||
LastMessageUnix: atomic.LoadInt64(&s.LastMessageUnix),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLivenessWatchdog starts a goroutine that scans the registry every
|
||||
// `interval` and logs a warning for any source that has been silent while
|
||||
// connected for more than `threshold`. Returns a stop function that halts
|
||||
@@ -388,13 +267,6 @@ func runLivenessWatchdogLoop(tick <-chan time.Time, done <-chan struct{}, thresh
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// #1749: stamp the watchdog clock BEFORE per-source work so
|
||||
// a panic or hang inside processLivenessTransition does
|
||||
// not freeze the heartbeat that /api/mqtt/status exposes.
|
||||
// External monitoring on WatchdogLastTickUnix detects a
|
||||
// wedged loop only if this clock is fresh-while-ticking
|
||||
// and stale-when-dead.
|
||||
watchdogLastTickUnix.Store(now.Unix())
|
||||
livenessRegistryMu.RLock()
|
||||
states := make([]*SourceLivenessState, 0, len(livenessRegistry))
|
||||
for _, s := range livenessRegistry {
|
||||
@@ -402,128 +274,13 @@ func runLivenessWatchdogLoop(tick <-chan time.Time, done <-chan struct{}, thresh
|
||||
}
|
||||
livenessRegistryMu.RUnlock()
|
||||
for _, s := range states {
|
||||
// #1749: handle disconnect-escalation bookkeeping
|
||||
// BEFORE the transition dispatch. checkSourceLiveness
|
||||
// returns LivenessDisconnected when paho reports
|
||||
// !IsConnected; we track how long that has persisted
|
||||
// and escalate with a forced reconnect when paho's
|
||||
// own auto-reconnect machinery has clearly failed
|
||||
// (multiplier × threshold without recovery).
|
||||
msg, kind := checkSourceLiveness(s, threshold, now)
|
||||
// #1749: a panic in emit (blocked log pipe, full
|
||||
// Docker JSON-file driver, etc.) MUST NOT kill the
|
||||
// watchdog goroutine. Recover per-source so one bad
|
||||
// source — or one bad log call — does not silence
|
||||
// all monitoring across all sources. Both
|
||||
// maybeEscalateDisconnected and processLivenessTransition
|
||||
// call emit, so both must be inside the recover scope.
|
||||
func(state *SourceLivenessState, k LivenessKind, m string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// #1810 round-1 (Taleb #2 + #3):
|
||||
// (a) Write to os.Stderr DIRECTLY rather than via
|
||||
// log.Printf. The original log sink is the prime
|
||||
// suspect for a panic in emit (blocked stderr pipe,
|
||||
// full Docker JSON-file driver) — using the same
|
||||
// sink to report the recovery risks a second
|
||||
// panic-on-recover that kills the goroutine the
|
||||
// recover was meant to save. os.Stderr.Write is a
|
||||
// raw syscall and bypasses log's own mutex.
|
||||
// (b) Increment watchdogPanicCount so a panic-per-
|
||||
// tick loop is visible to external monitoring even
|
||||
// though WatchdogLastTickUnix continues to advance.
|
||||
watchdogPanicCount.Add(1)
|
||||
fmt.Fprintf(os.Stderr, "[ingestor] WATCHDOG RECOVERED panic processing source %q: %v\n", state.Tag, r)
|
||||
}
|
||||
}()
|
||||
maybeEscalateDisconnected(state, k, threshold, now, emit)
|
||||
processLivenessTransition(state, k, m, now, emit)
|
||||
}(s, kind, msg)
|
||||
processLivenessTransition(s, kind, msg, now, emit)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maybeEscalateDisconnected (#1749) tracks how long a source has
|
||||
// continuously been observed in LivenessDisconnected and triggers a
|
||||
// forced reconnect (subject to forceReconnectThrottle) once the gap
|
||||
// exceeds disconnectedReconnectMultiplier × threshold.
|
||||
//
|
||||
// Background: paho's SetAutoReconnect(true) is supposed to recover
|
||||
// dropped connections on its own. In production we have observed paho
|
||||
// silently giving up on one source — connectCount=1, disconnectCount=1,
|
||||
// lastError="EOF", zero retries for 18h — while another source on the
|
||||
// same binary reconnects fine. The existing watchdog's
|
||||
// processLivenessTransition stays silent on LivenessDisconnected to
|
||||
// avoid double-logging paho's own disconnect line; this escalation
|
||||
// path is the recovery hook for the case where paho never tries again.
|
||||
//
|
||||
// Behavior:
|
||||
// - kind != LivenessDisconnected: clear DisconnectedSinceUnix (reset
|
||||
// the timer; recovery or other state).
|
||||
// - kind == LivenessDisconnected, first observation: stamp
|
||||
// DisconnectedSinceUnix = now.
|
||||
// - kind == LivenessDisconnected, gap >= multiplier × threshold:
|
||||
// emit a WARN and call maybeForceReconnect (throttled). The
|
||||
// timestamp is NOT advanced beyond the original observation so the
|
||||
// emit fires on every tick past the boundary; the throttle inside
|
||||
// maybeForceReconnect handles the broker-hammering concern.
|
||||
//
|
||||
// emit is wrapped in defer/recover by the caller — a panic here does
|
||||
// not kill the loop.
|
||||
// disconnectedEscalationGap returns the gap (now - DisconnectedSinceUnix)
|
||||
// at which the watchdog escalates a persistent LivenessDisconnected
|
||||
// observation into a forced reconnect. Hoisted out of
|
||||
// maybeEscalateDisconnected (#1810 round-1, adv #6) so it is computed
|
||||
// once per call site rather than per-source-per-tick. The result
|
||||
// includes a per-source jitter offset (#1810 round-1, Taleb #4) so
|
||||
// that a shared-broker outage across N sources does NOT cause a
|
||||
// synchronized thundering-herd reconnect at the multiplier × threshold
|
||||
// boundary. Jitter range: 0..forceReconnectThrottle, deterministic per
|
||||
// tag (hash-based, no RNG state) so retries do not phase-walk.
|
||||
func disconnectedEscalationGap(tag string, threshold time.Duration) time.Duration {
|
||||
base := disconnectedReconnectMultiplier * threshold
|
||||
if forceReconnectThrottle <= 0 {
|
||||
return base
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(tag))
|
||||
jitter := time.Duration(h.Sum32()%uint32(forceReconnectThrottle/time.Millisecond)) * time.Millisecond
|
||||
return base + jitter
|
||||
}
|
||||
|
||||
func maybeEscalateDisconnected(s *SourceLivenessState, kind LivenessKind, threshold time.Duration, now time.Time, emit func(...any)) {
|
||||
if kind != LivenessDisconnected {
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, 0)
|
||||
return
|
||||
}
|
||||
disconnectedSince := atomic.LoadInt64(&s.DisconnectedSinceUnix)
|
||||
if disconnectedSince == 0 {
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, now.Unix())
|
||||
return
|
||||
}
|
||||
gap := now.Sub(time.Unix(disconnectedSince, 0))
|
||||
escalateAt := disconnectedEscalationGap(s.Tag, threshold)
|
||||
if gap < escalateAt {
|
||||
return
|
||||
}
|
||||
// #1810 round-1 (adv #2 + Taleb #1): only emit the ESCALATION WARN
|
||||
// when we are actually going to issue a force-reconnect. Without
|
||||
// this throttle, every tick past the boundary re-emits the WARN
|
||||
// — for a 1m scan interval and a 1h outage, that's 55+ duplicate
|
||||
// alert lines. maybeForceReconnect already enforces
|
||||
// forceReconnectThrottle and writes its own "forcing reconnect"
|
||||
// telemetry, so the operator-visible log surface is still
|
||||
// complete; we just no longer drown them in pre-amble.
|
||||
lastForce := atomic.LoadInt64(&s.LastForceReconnectUnix)
|
||||
if lastForce != 0 && now.Sub(time.Unix(lastForce, 0)) < forceReconnectThrottle {
|
||||
return
|
||||
}
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG ESCALATION: paho disconnected for %s (>%d×threshold=%s) with no auto-reconnect — forcing reconnect (#1749)",
|
||||
s.Tag, gap.Round(time.Second), disconnectedReconnectMultiplier, escalateAt))
|
||||
maybeForceReconnect(s, now, emit)
|
||||
}
|
||||
|
||||
// processLivenessTransition applies the edge-trigger rules and updates
|
||||
// LastAlertUnix accordingly. Separated for testability and to keep the
|
||||
// loop body small.
|
||||
@@ -602,3 +359,4 @@ func maybeForceReconnect(s *SourceLivenessState, now time.Time, emit func(...any
|
||||
emit(fmt.Sprintf("MQTT [%s] WATCHDOG reconnect attempt issued", s.Tag))
|
||||
}()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,409 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue #1749 — production CoreScope v3.9.1 experienced a complete MQTT
|
||||
// ingest stall lasting 75+ minutes during which the watchdog never
|
||||
// fired: no LivenessStalled, no LivenessNeverReceived, no
|
||||
// LivenessDisconnected log lines, no force-reconnect attempts. Two
|
||||
// distinct failure modes are addressed here:
|
||||
//
|
||||
// 1. Per-source paho machinery dies silently (recurrence with prod
|
||||
// wcmesh source 2026-06-30: connectCount=1, disconnectCount=1,
|
||||
// lastError="EOF", zero retries for ~18h while another source on
|
||||
// the same binary reconnected fine). The original watchdog returns
|
||||
// silently on LivenessDisconnected, trusting SetAutoReconnect(true)
|
||||
// to recover — when that trust is misplaced, there is no escalation
|
||||
// path.
|
||||
//
|
||||
// 2. The watchdog goroutine itself dies (3 sources going silent within
|
||||
// ~60s of each other strongly suggests a single shared dependency
|
||||
// failed, not 3 independent paho clients failing simultaneously).
|
||||
// A panic inside the emit callback (log pipe issues observed in
|
||||
// prior incidents) would kill the loop without leaving a trace.
|
||||
//
|
||||
// Fixes asserted here:
|
||||
// - Persistent LivenessDisconnected past disconnectedReconnectMultiplier
|
||||
// × threshold MUST trigger a forced reconnect with WARN telemetry.
|
||||
// - A panic inside emit MUST be recovered and the loop MUST continue
|
||||
// ticking.
|
||||
// - WatchdogLastTickUnix MUST advance with every tick so external
|
||||
// monitoring can detect a wedged watchdog goroutine.
|
||||
|
||||
// TestMQTTStallWatchdog_EscalateOnPersistentDisconnect_1749 (RED on
|
||||
// master): a source that stays !IsConnected for longer than
|
||||
// disconnectedReconnectMultiplier × threshold MUST be force-reconnected
|
||||
// at least once. On master, processLivenessTransition returns silently
|
||||
// on LivenessDisconnected — no escalation — so ForceReconnectFn is
|
||||
// never invoked.
|
||||
func TestMQTTStallWatchdog_EscalateOnPersistentDisconnect_1749(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 60 * time.Second
|
||||
scanInterval := 5 * time.Millisecond
|
||||
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "silent-paho",
|
||||
Broker: "ssl://mqtt2.example.com:8883",
|
||||
IsConnectedFn: func() bool { return false }, // paho stuck disconnected
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {})
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
// Feed ticks spanning > (multiplier × threshold) of wall clock so
|
||||
// the escalation path fires. We control the `now` parameter by
|
||||
// sending fabricated timestamps down the tick channel.
|
||||
base := time.Now()
|
||||
totalSpan := time.Duration(disconnectedReconnectMultiplier+2) * threshold
|
||||
for elapsed := time.Duration(0); elapsed <= totalSpan; elapsed += scanInterval * 200 {
|
||||
select {
|
||||
case tick <- base.Add(elapsed):
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("watchdog loop did not consume tick within 1s")
|
||||
}
|
||||
}
|
||||
|
||||
// ForceReconnectFn runs in a goroutine in production; poll for the
|
||||
// counter to land.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) && reconnectCount.Load() < 1 {
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
if got := reconnectCount.Load(); got < 1 {
|
||||
t.Fatalf("persistent LivenessDisconnected past %d×threshold MUST force-reconnect at least once (#1749); got %d invocations",
|
||||
disconnectedReconnectMultiplier, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749: once the
|
||||
// watchdog has escalated, repeat escalations on the same source MUST be
|
||||
// throttled by forceReconnectThrottle (no broker hammering during a
|
||||
// prolonged outage).
|
||||
func TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 60 * time.Second
|
||||
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "throttle-escalate",
|
||||
Broker: "ssl://example.com:8883",
|
||||
IsConnectedFn: func() bool { return false },
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {})
|
||||
|
||||
base := time.Now()
|
||||
// Pre-stamp DisconnectedSinceUnix so that the first tick is
|
||||
// already past the multiplier×threshold escalation boundary.
|
||||
// Without this we'd just observe the FIRST tick stamping the
|
||||
// timestamp and subsequent ticks would be only seconds past it.
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, base.Add(-time.Duration(disconnectedReconnectMultiplier+1)*threshold).Unix())
|
||||
// Cross the escalation boundary multiple times within a single
|
||||
// throttle window — expect ONE reconnect, not many.
|
||||
for i := 0; i < 10; i++ {
|
||||
select {
|
||||
case tick <- base.Add(time.Duration(i) * time.Second):
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("tick blocked")
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
if got := reconnectCount.Load(); got != 1 {
|
||||
t.Fatalf("escalation must be throttled within %s; got %d invocations", forceReconnectThrottle, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTStallWatchdog_LoopRecoversFromPanicInEmit_1749 (RED on
|
||||
// master): a panic inside the emit callback MUST NOT kill the watchdog
|
||||
// loop. On master there is no defer/recover around the per-source
|
||||
// processLivenessTransition call, so the first panic kills the
|
||||
// goroutine and no further ticks are processed.
|
||||
func TestMQTTStallWatchdog_LoopRecoversFromPanicInEmit_1749(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 1 * time.Minute
|
||||
|
||||
s := &SourceLivenessState{
|
||||
Tag: "panic-emit",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, time.Now().Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var calls int
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
panic("synthetic emit panic — simulates blocked log pipe (#1749 hypothesis 2)")
|
||||
}
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
|
||||
// Wrap the loop spawn with our own recover so that an unrecovered
|
||||
// panic in the loop (the bug on master) does NOT crash the test
|
||||
// process. The bug-under-test is whether the LOOP recovers; if it
|
||||
// does not, the panic propagates up to OUR recover, this goroutine
|
||||
// exits, `exited` is closed, and the loop is dead — at which point
|
||||
// the second tick will block and we assert failure with a clear
|
||||
// message rather than tearing down the whole test binary.
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
defer func() {
|
||||
_ = recover() // RED-mode safety net; production loop is what we are asserting on
|
||||
close(exited)
|
||||
}()
|
||||
runLivenessWatchdogLoop(tick, done, threshold, emit)
|
||||
}()
|
||||
|
||||
base := time.Now()
|
||||
// Tick 1: should hit the WARN edge, panic in emit, be recovered.
|
||||
select {
|
||||
case tick <- base:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first tick blocked")
|
||||
}
|
||||
// Give the goroutine a moment to recover & re-loop (or to die from
|
||||
// an unrecovered panic, which is the bug we are gating on).
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
select {
|
||||
case <-exited:
|
||||
t.Fatal("watchdog loop died from panic in emit (#1749) — production loop lacks defer/recover")
|
||||
default:
|
||||
}
|
||||
|
||||
// Reset the stalled state's alert so the second tick triggers
|
||||
// another emit (heartbeat suppression would otherwise mask the
|
||||
// second call). Use MarkReconnected then re-arm staleness.
|
||||
s.MarkReconnected(base.Add(50 * time.Millisecond))
|
||||
atomic.StoreInt64(&s.LastMessageUnix, base.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, base.Add(-20*time.Minute).Unix())
|
||||
|
||||
// Tick 2: if the loop survived, we should get a second emit call.
|
||||
select {
|
||||
case tick <- base.Add(time.Second):
|
||||
case <-exited:
|
||||
t.Fatal("watchdog loop died from panic in emit (#1749); second tick cannot be delivered because the goroutine exited")
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("watchdog loop did not survive panic in emit (#1749); second tick blocked because the goroutine died")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
got := calls
|
||||
mu.Unlock()
|
||||
if got < 2 {
|
||||
t.Fatalf("watchdog loop must survive a panic in emit and continue ticking (#1749); emit calls=%d (expected ≥2)", got)
|
||||
}
|
||||
|
||||
// Loop must still exit cleanly when signalled.
|
||||
close(done)
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("loop did not exit after done signal")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMQTTStallWatchdog_LastTickUnixExposed_1749 (RED on master):
|
||||
// WatchdogLastTickUnix MUST advance with each tick so external monitoring
|
||||
// can detect a wedged watchdog goroutine. On master no such clock is
|
||||
// exposed.
|
||||
func TestMQTTStallWatchdog_LastTickUnixExposed_1749(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
// Baseline: clock should be 0 (or stale) BEFORE the first tick of
|
||||
// this test. We can't assert exactly 0 because prior tests in the
|
||||
// same package may have ticked the loop, so just record the value
|
||||
// and assert it ADVANCES.
|
||||
before := WatchdogLastTickUnix()
|
||||
// #1810 round-1 (adv #7): this test stamps a 48h-future value into
|
||||
// the package-level watchdogLastTickUnix. Restore the prior value
|
||||
// so a downstream test that asserts "tick advanced past 'before'"
|
||||
// is not fooled by our leak.
|
||||
t.Cleanup(func() {
|
||||
watchdogLastTickUnix.Store(before)
|
||||
})
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go runLivenessWatchdogLoop(tick, done, time.Minute, func(args ...any) {})
|
||||
|
||||
stamp := time.Now().Add(48 * time.Hour) // guaranteed > before
|
||||
select {
|
||||
case tick <- stamp:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("tick blocked")
|
||||
}
|
||||
// The loop publishes the clock; poll for it to land.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var got int64
|
||||
for time.Now().Before(deadline) {
|
||||
got = WatchdogLastTickUnix()
|
||||
if got >= stamp.Unix() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("WatchdogLastTickUnix() did not advance to tick timestamp (#1749); before=%d after=%d want≥%d",
|
||||
before, got, stamp.Unix())
|
||||
}
|
||||
|
||||
// TestMQTTStallWatchdog_LoopRecoversFromPanicInEmit_EscalationPath_1749
|
||||
// (RED on the prior commit): a panic inside emit on the ESCALATION
|
||||
// path (maybeEscalateDisconnected → emit) must NOT kill the watchdog
|
||||
// loop. The fix in b3c75ca3 moved maybeEscalateDisconnected INSIDE the
|
||||
// per-source IIFE that has defer/recover, so a panic on this path is
|
||||
// recovered alongside panics on the processLivenessTransition path.
|
||||
func TestMQTTStallWatchdog_LoopRecoversFromPanicInEmit_EscalationPath_1749(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 1 * time.Minute
|
||||
|
||||
var reconnectCalls int32
|
||||
|
||||
s := &SourceLivenessState{
|
||||
Tag: "panic-escalation",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return false }, // disconnected
|
||||
ForceReconnectFn: func() {
|
||||
atomic.AddInt32(&reconnectCalls, 1)
|
||||
},
|
||||
}
|
||||
// Pre-stamp DisconnectedSinceUnix past the escalation threshold
|
||||
// (disconnectedReconnectMultiplier × threshold = 5 × 1min = 5min).
|
||||
// Set it 10 minutes in the past so escalation fires immediately.
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, time.Now().Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
// Track which tick we're on to control panic timing.
|
||||
var tickNum int32
|
||||
var mu sync.Mutex
|
||||
var emitCalls int
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
emitCalls++
|
||||
mu.Unlock()
|
||||
// Tick 1 (tickNum==1): all emits succeed → ForceReconnectFn fires.
|
||||
// Tick 2 (tickNum==2): panic on first emit in escalation path.
|
||||
// Tick 3: proves loop survived.
|
||||
if atomic.LoadInt32(&tickNum) == 2 {
|
||||
panic("synthetic emit panic on ESCALATION path (#1749 round-1 finding)")
|
||||
}
|
||||
}
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
defer func() {
|
||||
_ = recover()
|
||||
close(exited)
|
||||
}()
|
||||
runLivenessWatchdogLoop(tick, done, threshold, emit)
|
||||
}()
|
||||
|
||||
base := time.Now()
|
||||
|
||||
// Tick 1: escalation fires, ForceReconnectFn is invoked, no panic.
|
||||
atomic.StoreInt32(&tickNum, 1)
|
||||
select {
|
||||
case tick <- base:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("tick 1 blocked")
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond) // let goroutine with ForceReconnectFn run
|
||||
|
||||
if rc := atomic.LoadInt32(&reconnectCalls); rc < 1 {
|
||||
t.Fatalf("ForceReconnectFn must be invoked ≥1 time after tick 1 (got %d)", rc)
|
||||
}
|
||||
|
||||
// Reset state so tick 2 also triggers escalation.
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, base.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.LastForceReconnectUnix, 0)
|
||||
|
||||
// Tick 2: emit panics on escalation path. On unfixed code this
|
||||
// kills the loop because maybeEscalateDisconnected is outside IIFE.
|
||||
atomic.StoreInt32(&tickNum, 2)
|
||||
select {
|
||||
case tick <- base.Add(time.Second):
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("tick 2 blocked")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
select {
|
||||
case <-exited:
|
||||
t.Fatal("watchdog loop died from panic in emit on ESCALATION path (#1749 round-1) — maybeEscalateDisconnected is not panic-protected")
|
||||
default:
|
||||
}
|
||||
|
||||
// Tick 3: proves loop survived the panic on tick 2.
|
||||
atomic.StoreInt32(&tickNum, 3)
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, base.Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.LastForceReconnectUnix, 0)
|
||||
select {
|
||||
case tick <- base.Add(2 * time.Second):
|
||||
case <-exited:
|
||||
t.Fatal("watchdog loop died; tick 3 cannot be delivered")
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("tick 3 blocked — loop dead")
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
got := emitCalls
|
||||
mu.Unlock()
|
||||
// Tick 1: 2 emits (escalation + "forcing reconnect") + 1 from goroutine = 3
|
||||
// Tick 2: 1 emit (panic) = partial
|
||||
// Tick 3: ≥1 emit
|
||||
// Total should be ≥4 if loop survived
|
||||
if got < 4 {
|
||||
t.Fatalf("emit must be called ≥4 times across 3 ticks (got %d) — loop did not survive escalation panic", got)
|
||||
}
|
||||
|
||||
close(done)
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("loop did not exit after done signal")
|
||||
}
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PR #1810 round-1 follow-ups. Tests for the must-fix findings from
|
||||
// adversarial / Taleb / Kent Beck reviews on the #1749 fix:
|
||||
//
|
||||
// - C5: panic-recover writes to os.Stderr, NOT log.Printf (Taleb #2).
|
||||
// - C6: WatchdogPanicCount increments on recovered panic and is
|
||||
// surfaced via the snapshot path (Taleb #3).
|
||||
// - C7: per-source jitter on escalation prevents thundering-herd
|
||||
// (Taleb #4) — two sources escalating on the same tick yield
|
||||
// different escalation gaps.
|
||||
// - B5/C3: MarkReconnected clears DisconnectedSinceUnix so the next
|
||||
// disconnect starts its escalation timer from scratch (Taleb #5).
|
||||
// - C4 rewrite: throttled escalation WARN — the escalation log line
|
||||
// is bounded by forceReconnectThrottle even across many ticks
|
||||
// past the boundary (adv #2 + Taleb #1).
|
||||
|
||||
// captureStderr runs fn while redirecting os.Stderr to a pipe; returns
|
||||
// whatever fn wrote to stderr. log.Printf's default writer is also
|
||||
// pointed at the pipe so the test can prove a string did NOT go through
|
||||
// log.Printf vs DID go to os.Stderr.
|
||||
func captureStderrAndLog(t *testing.T, fn func()) (stderrBytes string, logBytes string) {
|
||||
t.Helper()
|
||||
rStd, wStd, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("pipe: %v", err)
|
||||
}
|
||||
rLog, wLog, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("pipe: %v", err)
|
||||
}
|
||||
origStderr := os.Stderr
|
||||
origLogOut := log.Writer()
|
||||
os.Stderr = wStd
|
||||
log.SetOutput(wLog)
|
||||
t.Cleanup(func() {
|
||||
os.Stderr = origStderr
|
||||
log.SetOutput(origLogOut)
|
||||
})
|
||||
|
||||
var stdBuf, logBuf bytes.Buffer
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); _, _ = io.Copy(&stdBuf, rStd) }()
|
||||
go func() { defer wg.Done(); _, _ = io.Copy(&logBuf, rLog) }()
|
||||
|
||||
fn()
|
||||
|
||||
_ = wStd.Close()
|
||||
_ = wLog.Close()
|
||||
wg.Wait()
|
||||
return stdBuf.String(), logBuf.String()
|
||||
}
|
||||
|
||||
// C5: panic recovery message MUST go to os.Stderr directly, NOT through
|
||||
// log.Printf — log's writer is the suspected cause of the panic
|
||||
// (blocked log pipe / full JSON-file driver) and using it for the
|
||||
// recovery message risks re-panicking on the same broken sink.
|
||||
func TestWatchdog_PanicRecoverWritesToStderrNotLog_1810(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 1 * time.Minute
|
||||
s := &SourceLivenessState{
|
||||
Tag: "panic-sink-1810",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, time.Now().Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
emit := func(args ...any) {
|
||||
panic("synthetic panic to drive recover")
|
||||
}
|
||||
|
||||
stderrOut, logOut := captureStderrAndLog(t, func() {
|
||||
tick, done, exited := setupWatchdogTestLoop(t, threshold, emit)
|
||||
sendTickOrFail(t, tick, time.Now(), time.Second, "panic-sink tick")
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
close(done)
|
||||
<-exited
|
||||
})
|
||||
|
||||
if !strings.Contains(stderrOut, "WATCHDOG RECOVERED") {
|
||||
t.Fatalf("expected WATCHDOG RECOVERED line on os.Stderr; stderr=%q log=%q", stderrOut, logOut)
|
||||
}
|
||||
if strings.Contains(logOut, "WATCHDOG RECOVERED") {
|
||||
t.Fatalf("recovery message MUST NOT go through log.Printf (#1810 Taleb #2); log=%q", logOut)
|
||||
}
|
||||
}
|
||||
|
||||
// C6: WatchdogPanicCount must increment on a recovered panic and the
|
||||
// value must be reachable via the package-level accessor (which the
|
||||
// stats snapshot reads).
|
||||
func TestWatchdog_PanicCountIncrementsOnRecover_1810(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
before := WatchdogPanicCount()
|
||||
|
||||
threshold := 1 * time.Minute
|
||||
s := &SourceLivenessState{
|
||||
Tag: "panic-count-1810",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return true },
|
||||
}
|
||||
atomic.StoreInt64(&s.LastMessageUnix, time.Now().Add(-10*time.Minute).Unix())
|
||||
atomic.StoreInt64(&s.StartedAt, time.Now().Add(-20*time.Minute).Unix())
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
emit := func(args ...any) { panic("boom") }
|
||||
|
||||
// Suppress stderr output during the recover so test output stays clean.
|
||||
_, _ = captureStderrAndLog(t, func() {
|
||||
tick, done, exited := setupWatchdogTestLoop(t, threshold, emit)
|
||||
sendTickOrFail(t, tick, time.Now(), time.Second, "panic-count tick")
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
close(done)
|
||||
<-exited
|
||||
})
|
||||
|
||||
after := WatchdogPanicCount()
|
||||
if after <= before {
|
||||
t.Fatalf("WatchdogPanicCount must advance on recovered panic (#1810 Taleb #3); before=%d after=%d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// C7: per-source jitter — two sources crossing the escalation boundary
|
||||
// on the same tick must NOT escalate at the same threshold gap value.
|
||||
// This proves the hash-based jitter offset spreads escalations and
|
||||
// avoids a thundering-herd reconnect when N sources share an upstream
|
||||
// broker outage.
|
||||
func TestWatchdog_EscalationJitterPerSource_1810(t *testing.T) {
|
||||
threshold := 60 * time.Second
|
||||
gapA := disconnectedEscalationGap("source-a", threshold)
|
||||
gapB := disconnectedEscalationGap("source-b", threshold)
|
||||
if gapA == gapB {
|
||||
t.Fatalf("expected per-source jitter on escalation gap (#1810 Taleb #4); gapA=%s gapB=%s", gapA, gapB)
|
||||
}
|
||||
// Both must be at least the unjittered base.
|
||||
base := time.Duration(disconnectedReconnectMultiplier) * threshold
|
||||
if gapA < base || gapB < base {
|
||||
t.Fatalf("escalation gap must be ≥ base (%s); gapA=%s gapB=%s", base, gapA, gapB)
|
||||
}
|
||||
// Jitter must be bounded (≤ base + 30s sanity).
|
||||
maxJitter := base + 30*time.Second
|
||||
if gapA > maxJitter || gapB > maxJitter {
|
||||
t.Fatalf("escalation jitter exceeded bound; gapA=%s gapB=%s max=%s", gapA, gapB, maxJitter)
|
||||
}
|
||||
}
|
||||
|
||||
// C3 / B5: MarkReconnected must clear DisconnectedSinceUnix. On unfixed
|
||||
// code, a post-recovery LivenessDisconnected immediately satisfies
|
||||
// (now - DisconnectedSinceUnix > multiplier×threshold) and force-
|
||||
// reconnects on the FIRST tick — making escalation a flap-trigger
|
||||
// instead of a persistent-failure-trigger.
|
||||
func TestMarkReconnected_ClearsDisconnectedSinceUnix_1810(t *testing.T) {
|
||||
s := &SourceLivenessState{Tag: "recovery-clear-1810"}
|
||||
atomic.StoreInt64(&s.DisconnectedSinceUnix, time.Now().Add(-1*time.Hour).Unix())
|
||||
s.MarkReconnected(time.Now())
|
||||
if got := atomic.LoadInt64(&s.DisconnectedSinceUnix); got != 0 {
|
||||
t.Fatalf("MarkReconnected must clear DisconnectedSinceUnix (#1810 Taleb #5); got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// C4 (rewrite): the ESCALATION WARN log line must be throttled. On
|
||||
// unfixed code (round 0) every tick past the boundary emitted the WARN;
|
||||
// for a 1m scan interval and a 1h outage that's 55+ duplicates. This
|
||||
// test does NOT pre-stamp DisconnectedSinceUnix — it starts from the
|
||||
// first-observation branch and drives ticks past the boundary,
|
||||
// asserting both that reconnects are throttled AND that the escalation
|
||||
// WARN log count stays bounded.
|
||||
func TestWatchdog_EscalationWarnThrottled_1810(t *testing.T) {
|
||||
defer snapshotAndResetRegistry(t)()
|
||||
|
||||
threshold := 1 * time.Second
|
||||
var reconnectCount atomic.Int32
|
||||
s := &SourceLivenessState{
|
||||
Tag: "warn-throttle-1810",
|
||||
Broker: "tcp://x:1883",
|
||||
IsConnectedFn: func() bool { return false },
|
||||
ForceReconnectFn: func() { reconnectCount.Add(1) },
|
||||
}
|
||||
if err := registerLivenessState(s); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var warnCount int
|
||||
emit := func(args ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, a := range args {
|
||||
if str, ok := a.(string); ok && strings.Contains(str, "WATCHDOG ESCALATION") {
|
||||
warnCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tick := make(chan time.Time)
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go runLivenessWatchdogLoop(tick, done, threshold, emit)
|
||||
|
||||
base := time.Now()
|
||||
// First tick: stamps DisconnectedSinceUnix (no escalation yet).
|
||||
tick <- base
|
||||
// Subsequent ticks: drive 60 ticks at 1s each, far past the
|
||||
// escalation boundary (5×threshold = 5s + ≤30s jitter). On
|
||||
// unfixed code this would emit 50+ WARN lines.
|
||||
for i := 1; i < 60; i++ {
|
||||
tick <- base.Add(time.Duration(i) * time.Second)
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
got := warnCount
|
||||
mu.Unlock()
|
||||
// One escalation per throttle window. With throttle=60s and the
|
||||
// test spanning ~59s of fabricated wall clock, at most ~2 WARNs
|
||||
// should fire (boundary cross + possibly one more if jitter is
|
||||
// minimal). 5 is a generous upper bound; the original bug
|
||||
// produced 50+.
|
||||
if got > 5 {
|
||||
t.Fatalf("escalation WARN must be throttled (#1810 adv #2 / Taleb #1); got %d emits across 60 ticks past the boundary", got)
|
||||
}
|
||||
if got < 1 {
|
||||
t.Fatalf("escalation WARN must fire at least once when paho stays disconnected past boundary; got 0")
|
||||
}
|
||||
if rc := reconnectCount.Load(); rc < 1 {
|
||||
t.Fatalf("ForceReconnectFn must be invoked at least once; got %d", rc)
|
||||
}
|
||||
}
|
||||
|
||||
// C2: round-trip — IngestorStatsSnapshot.WatchdogLastTickUnix and
|
||||
// WatchdogPanicCount must serialize through JSON and deserialize via
|
||||
// the server's envelope shape.
|
||||
func TestIngestorStatsSnapshot_WatchdogFieldsRoundTrip_1810(t *testing.T) {
|
||||
snap := IngestorStatsSnapshot{
|
||||
SampledAt: time.Now().UTC().Format(time.RFC3339),
|
||||
WatchdogLastTickUnix: 1700000000,
|
||||
WatchdogPanicCount: 42,
|
||||
}
|
||||
b, err := json.Marshal(snap)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
if !bytes.Contains(b, []byte(`"watchdogLastTickUnix":1700000000`)) {
|
||||
t.Fatalf("watchdogLastTickUnix missing from JSON: %s", string(b))
|
||||
}
|
||||
if !bytes.Contains(b, []byte(`"watchdogPanicCount":42`)) {
|
||||
t.Fatalf("watchdogPanicCount missing from JSON: %s", string(b))
|
||||
}
|
||||
var back IngestorStatsSnapshot
|
||||
if err := json.Unmarshal(b, &back); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if back.WatchdogLastTickUnix != 1700000000 || back.WatchdogPanicCount != 42 {
|
||||
t.Fatalf("round-trip mismatch: %+v", back)
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSourceLivenessState_ReceiptVsWriteSeparate asserts that the receipt-
|
||||
// time and post-write liveness clocks are independent (PR #1609 review
|
||||
// MAJOR M1): stamping at receipt must NOT advance the post-write clock so
|
||||
// the watchdog/healthz can distinguish "broker alive, write path stuck"
|
||||
// from "everything fine". Without separation, /healthz reports "fresh"
|
||||
// while the writer is stalled and the ingest buffer is filling.
|
||||
func TestSourceLivenessState_ReceiptVsWriteSeparate(t *testing.T) {
|
||||
s := &SourceLivenessState{Tag: "t"}
|
||||
now := time.Now()
|
||||
|
||||
// Receipt at T0; post-write never happens (writer stalled).
|
||||
s.MarkReceipt(now)
|
||||
|
||||
gotReceipt := atomic.LoadInt64(&s.LastReceiptUnix)
|
||||
gotWrite := atomic.LoadInt64(&s.LastMessageUnix)
|
||||
if gotReceipt != now.Unix() {
|
||||
t.Fatalf("LastReceiptUnix: want %d, got %d", now.Unix(), gotReceipt)
|
||||
}
|
||||
if gotWrite != 0 {
|
||||
t.Fatalf("LastMessageUnix MUST stay 0 while writer stalled (only MarkReceipt called); got %d — receipt is double-stamping the write clock and /healthz will lie about ingestion freshness", gotWrite)
|
||||
}
|
||||
|
||||
// Write completes later: only MarkMessage advances LastMessageUnix.
|
||||
later := now.Add(5 * time.Second)
|
||||
s.MarkMessage(later)
|
||||
|
||||
gotReceipt2 := atomic.LoadInt64(&s.LastReceiptUnix)
|
||||
gotWrite2 := atomic.LoadInt64(&s.LastMessageUnix)
|
||||
if gotReceipt2 != now.Unix() {
|
||||
t.Fatalf("MarkMessage must not move LastReceiptUnix backwards or forwards; want %d, got %d", now.Unix(), gotReceipt2)
|
||||
}
|
||||
if gotWrite2 != later.Unix() {
|
||||
t.Fatalf("LastMessageUnix after MarkMessage: want %d, got %d", later.Unix(), gotWrite2)
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// setupWatchdogTestLoop spawns runLivenessWatchdogLoop with a panic
|
||||
// safety net so an unrecovered panic in the loop (the historical bug
|
||||
// shape #1749 / #1810 round-1 gates against) does NOT crash the test
|
||||
// binary. Returns:
|
||||
// - tick: send fabricated timestamps to drive the loop
|
||||
// - done: caller closes to ask the loop to exit cleanly
|
||||
// - exited: closed by the helper after the loop returns (whether by
|
||||
// done-signal, normal channel close, or panic propagation)
|
||||
//
|
||||
// Extracted as part of #1810 round-1 (adv #4) — four tests in this
|
||||
// package previously open-coded the same scaffolding with subtle
|
||||
// variations.
|
||||
func setupWatchdogTestLoop(t *testing.T, threshold time.Duration, emit func(...any)) (tick chan time.Time, done chan struct{}, exited chan struct{}) {
|
||||
t.Helper()
|
||||
tick = make(chan time.Time)
|
||||
done = make(chan struct{})
|
||||
exited = make(chan struct{})
|
||||
go func() {
|
||||
defer func() {
|
||||
_ = recover() // safety net; the production loop is what we are asserting on
|
||||
close(exited)
|
||||
}()
|
||||
runLivenessWatchdogLoop(tick, done, threshold, emit)
|
||||
}()
|
||||
return tick, done, exited
|
||||
}
|
||||
|
||||
// sendTickOrFail pushes one fabricated timestamp into the loop's tick
|
||||
// channel and fails the test if the loop does not consume it within
|
||||
// timeout. Use this everywhere a test wants to step the watchdog by
|
||||
// exactly one tick — open-coded select-default-Fatal patterns are
|
||||
// repetitive and easy to get wrong (off-by-one timeouts).
|
||||
func sendTickOrFail(t *testing.T, tick chan<- time.Time, stamp time.Time, timeout time.Duration, label string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case tick <- stamp:
|
||||
case <-time.After(timeout):
|
||||
t.Fatalf("%s: tick blocked after %s — loop dead?", label, timeout)
|
||||
}
|
||||
}
|
||||
@@ -63,16 +63,6 @@ func (s *Store) StartNeighborEdgesBuilder(interval time.Duration) func() {
|
||||
// returning — first server load needs a fully-populated table.
|
||||
wuStart := time.Now()
|
||||
var wuTotal int
|
||||
// Prime the prefix index (#1547) so the very first
|
||||
// InsertTransmission after startup can resolve hop prefixes.
|
||||
if err := s.RefreshPrefixIndex(); err != nil {
|
||||
log.Printf("[neighbor-build] initial prefix-index refresh error: %v", err)
|
||||
}
|
||||
// Prime the neighbor graph (#1560) so the context-aware resolver
|
||||
// has adjacency data on the very first InsertTransmission.
|
||||
if err := s.RefreshNeighborGraph(); err != nil {
|
||||
log.Printf("[neighbor-build] initial neighbor-graph refresh error: %v", err)
|
||||
}
|
||||
for {
|
||||
n, err := s.buildAndPersistNeighborEdges()
|
||||
if err != nil {
|
||||
@@ -95,18 +85,7 @@ func (s *Store) StartNeighborEdgesBuilder(interval time.Duration) func() {
|
||||
select {
|
||||
case <-t.C:
|
||||
start := time.Now()
|
||||
// Refresh the prefix index alongside the edges build
|
||||
// (#1547) so new nodes become resolvable within a tick.
|
||||
if err := s.RefreshPrefixIndex(); err != nil {
|
||||
log.Printf("[neighbor-build] prefix-index refresh error: %v", err)
|
||||
}
|
||||
n, err := s.buildAndPersistNeighborEdges()
|
||||
// Refresh the neighbor-graph snapshot after the edges
|
||||
// build (#1560) so the context-aware resolver picks up
|
||||
// newly persisted adjacencies on the next ingest.
|
||||
if grErr := s.RefreshNeighborGraph(); grErr != nil {
|
||||
log.Printf("[neighbor-build] neighbor-graph refresh error: %v", grErr)
|
||||
}
|
||||
dur := time.Since(start)
|
||||
if err != nil {
|
||||
log.Printf("[neighbor-build] tick error after %s: %v", dur, err)
|
||||
@@ -234,36 +213,33 @@ func (s *Store) buildAndPersistNeighborEdges() (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Wrap the whole edge-persist tx under writer-perf instrumentation
|
||||
// (#1340). Slow neighbor-builder ticks (the #1339 root cause) now
|
||||
// show up on /api/perf under component=neighbor_builder.
|
||||
var inserted int
|
||||
err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error {
|
||||
stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(node_a, node_b) DO UPDATE SET
|
||||
count = count + 1,
|
||||
last_seen = MAX(last_seen, excluded.last_seen)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
var firstErr error
|
||||
for _, e := range edges {
|
||||
if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
return fmt.Errorf("upsert: %w", firstErr)
|
||||
}
|
||||
inserted = len(edges)
|
||||
return nil
|
||||
})
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return 0, fmt.Errorf("begin: %w", err)
|
||||
}
|
||||
return inserted, nil
|
||||
defer tx.Rollback()
|
||||
stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen)
|
||||
VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(node_a, node_b) DO UPDATE SET
|
||||
count = count + 1,
|
||||
last_seen = MAX(last_seen, excluded.last_seen)`)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
var firstErr error
|
||||
for _, e := range edges {
|
||||
if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
return 0, fmt.Errorf("upsert: %w", firstErr)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("commit: %w", err)
|
||||
}
|
||||
return len(edges), nil
|
||||
}
|
||||
|
||||
// canonEdge orders the pair so node_a <= node_b (matches the existing
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression tests for issue #1465 — observer.last_seen MUST always reflect
|
||||
// ingest time (server wall clock), never the MQTT envelope timestamp. Observers
|
||||
// with broken clocks (wrong TZ, RTC drift, replayed retained messages) must
|
||||
// NOT be able to drag the analyzer's "last heard from" field into the past
|
||||
// or future.
|
||||
//
|
||||
// Per-packet rxTime semantics (envelope time with naive-clamp from #1464)
|
||||
// are out of scope here — those continue to use envelope time. This file
|
||||
// asserts only the observer.last_seen path.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Status path: envelope timestamp is a well-formed RFC3339 value 3h in the
|
||||
// past. observer.last_seen must be server wall clock, NOT the envelope value.
|
||||
func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-3 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"status":"online","origin":"obs-past","timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-past/status", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-past").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope reported well-formed stale %q (3h ago) — must NOT drag last_seen into the past. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
|
||||
// Status path: envelope timestamp 5 min in the future. observer.last_seen
|
||||
// must still be server wall clock.
|
||||
func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_FutureEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
future := time.Now().UTC().Add(5 * time.Minute).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
payload := []byte(`{"status":"online","origin":"obs-future","timestamp":"` + future + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-future/status", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-future").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope reported well-formed future %q (5 min ahead) — must NOT drag last_seen into the future. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, future)
|
||||
}
|
||||
}
|
||||
|
||||
// Packet path: a transmission whose envelope timestamp is 3h in the past
|
||||
// MUST still bump observer.last_seen to server wall clock — observer is
|
||||
// clearly alive (we just ingested a packet from it), regardless of what
|
||||
// its clock claims.
|
||||
func TestPacketMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *testing.T) {
|
||||
store := newTestStore(t)
|
||||
source := MQTTSource{Name: "test"}
|
||||
|
||||
stale := time.Now().UTC().Add(-3 * time.Hour).Format(time.RFC3339)
|
||||
before := time.Now().Unix()
|
||||
|
||||
rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976"
|
||||
payload := []byte(`{"raw":"` + rawHex + `","SNR":5.5,"RSSI":-100.0,"origin":"obs-pkt","timestamp":"` + stale + `"}`)
|
||||
msg := &mockMessage{topic: "meshcore/SJC/obs-pkt/packets", payload: payload}
|
||||
|
||||
handleMessage(store, "test", source, msg, nil, nil, &Config{})
|
||||
after := time.Now().Unix()
|
||||
|
||||
var lastSeen string
|
||||
if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-pkt").Scan(&lastSeen); err != nil {
|
||||
t.Fatalf("scan last_seen: %v", err)
|
||||
}
|
||||
ls, err := time.Parse(time.RFC3339, lastSeen)
|
||||
if err != nil {
|
||||
t.Fatalf("last_seen %q not RFC3339: %v", lastSeen, err)
|
||||
}
|
||||
if ls.Unix() < before-5 || ls.Unix() > after+5 {
|
||||
t.Errorf("packet-path observer.last_seen = %q (epoch %d); want in [%d, %d] (server wall clock). "+
|
||||
"Envelope stale = %q. Observer just delivered a packet; last_seen must be NOW. Issue #1465.",
|
||||
lastSeen, ls.Unix(), before, after, stale)
|
||||
}
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Context-aware hop resolver — full restore of pre-#1289 hop
|
||||
// disambiguation semantics, ported into the ingestor (where the
|
||||
// neighbor graph + node directory now live, per #1283).
|
||||
//
|
||||
// Why this exists (issues #1547 / #1560):
|
||||
// The naive `resolvePath` only resolves hops whose prefix is unique
|
||||
// in the node table. On a >2K-node mesh the dominant case is 1-byte
|
||||
// prefix collisions (multiple candidates per prefix). Without
|
||||
// adjacency disambiguation those hops always serialize as `nil`
|
||||
// and the resolved_path remains effectively empty for the largest
|
||||
// meshes — the very deployments that need it most.
|
||||
//
|
||||
// Algorithm (ported from cmd/server/store.go @ commit 450236d5
|
||||
// `pm.resolveWithContext`, intersected with the disambiguation gating
|
||||
// from PR #1144 / #1352):
|
||||
//
|
||||
// For each hop:
|
||||
// 1. Collect candidate pubkeys by prefix-match (existing prefixIndex).
|
||||
// 2. len==0 → nil.
|
||||
// 3. len==1 → that pubkey.
|
||||
// 4. len>1 → filter by NeighborGraph adjacency to the anchor:
|
||||
// - hop 0 anchor = fromPubkey (ADVERT originator) if known;
|
||||
// - hop i (i>0) anchor = previous resolved hop's pubkey;
|
||||
// if the previous hop did not resolve, the chain breaks
|
||||
// and subsequent >1-candidate hops fall to nil.
|
||||
// Surviving candidates after filter:
|
||||
// - exactly 1 → use it
|
||||
// - 0 or >1 → nil (cannot disambiguate further)
|
||||
//
|
||||
// This is the conservative tier-1 variant. Pre-#1289 also carried
|
||||
// tier-2 (geo proximity), tier-3 (GPS preference), tier-4 (obs-count
|
||||
// fallback) — those were noisy in practice and are intentionally NOT
|
||||
// ported here; this PR is a regression restore, not an enhancement.
|
||||
|
||||
// NeighborGraph is the in-memory adjacency snapshot used by the
|
||||
// context-aware resolver. Internally lowercased.
|
||||
type NeighborGraph struct {
|
||||
adj map[string]map[string]struct{}
|
||||
}
|
||||
|
||||
// NewNeighborGraph returns an empty graph.
|
||||
func NewNeighborGraph() *NeighborGraph {
|
||||
return &NeighborGraph{adj: make(map[string]map[string]struct{})}
|
||||
}
|
||||
|
||||
// AddEdge adds an undirected adjacency a↔b. Self-loops and empty
|
||||
// endpoints are ignored.
|
||||
func (g *NeighborGraph) AddEdge(a, b string) {
|
||||
a = strings.ToLower(a)
|
||||
b = strings.ToLower(b)
|
||||
if a == "" || b == "" || a == b {
|
||||
return
|
||||
}
|
||||
if g.adj[a] == nil {
|
||||
g.adj[a] = make(map[string]struct{})
|
||||
}
|
||||
if g.adj[b] == nil {
|
||||
g.adj[b] = make(map[string]struct{})
|
||||
}
|
||||
g.adj[a][b] = struct{}{}
|
||||
g.adj[b][a] = struct{}{}
|
||||
}
|
||||
|
||||
// IsAdjacent reports whether a and b appear together in any neighbor edge.
|
||||
func (g *NeighborGraph) IsAdjacent(a, b string) bool {
|
||||
if g == nil {
|
||||
return false
|
||||
}
|
||||
a = strings.ToLower(a)
|
||||
b = strings.ToLower(b)
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
nbrs, ok := g.adj[a]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, present := nbrs[b]
|
||||
return present
|
||||
}
|
||||
|
||||
// neighborGraphHolder caches the graph for the InsertTransmission hot
|
||||
// path. atomic.Value lets the 60s rebuild publish without a read-side
|
||||
// lock.
|
||||
type neighborGraphHolder struct {
|
||||
v atomic.Value // holds *NeighborGraph
|
||||
}
|
||||
|
||||
func (h *neighborGraphHolder) load() *NeighborGraph {
|
||||
if v := h.v.Load(); v != nil {
|
||||
return v.(*NeighborGraph)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *neighborGraphHolder) store(g *NeighborGraph) {
|
||||
h.v.Store(g)
|
||||
}
|
||||
|
||||
// loadNeighborGraph reads neighbor_edges and returns an in-memory
|
||||
// adjacency snapshot. Safe to call against a fresh DB (returns an
|
||||
// empty graph).
|
||||
func loadNeighborGraph(db *sql.DB) (*NeighborGraph, error) {
|
||||
rows, err := db.Query(`SELECT node_a, node_b FROM neighbor_edges`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
g := NewNeighborGraph()
|
||||
for rows.Next() {
|
||||
var a, b string
|
||||
if err := rows.Scan(&a, &b); err != nil {
|
||||
continue
|
||||
}
|
||||
g.AddEdge(a, b)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// resolveHopWithContext resolves a single hop using NeighborGraph
|
||||
// adjacency to the anchor. Returns nil when the hop cannot be
|
||||
// disambiguated.
|
||||
//
|
||||
// exclude is a set of pubkeys to discard from the candidate pool
|
||||
// (typically the prior hops already resolved on the path — a packet
|
||||
// does not revisit a node).
|
||||
//
|
||||
// Behavior matrix:
|
||||
// len(candidates) | anchor | graph | result
|
||||
// 0 | — | — | nil
|
||||
// 1 | — | — | candidates[0]
|
||||
// >1 | "" or no graph|— | nil
|
||||
// >1 | non-empty | set | unique adjacent candidate
|
||||
// (or nil if 0 or >1 survive)
|
||||
func resolveHopWithContext(hop string, anchor string, graph *NeighborGraph, idx prefixIndex, exclude map[string]struct{}) *string {
|
||||
if idx == nil {
|
||||
return nil
|
||||
}
|
||||
h := strings.ToLower(hop)
|
||||
candidates := idx[h]
|
||||
switch len(candidates) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
pk := candidates[0]
|
||||
if _, skip := exclude[pk]; skip {
|
||||
return nil
|
||||
}
|
||||
return &pk
|
||||
}
|
||||
if graph == nil || anchor == "" {
|
||||
return nil
|
||||
}
|
||||
var match string
|
||||
survivors := 0
|
||||
for _, cand := range candidates {
|
||||
if _, skip := exclude[cand]; skip {
|
||||
continue
|
||||
}
|
||||
if graph.IsAdjacent(anchor, cand) {
|
||||
survivors++
|
||||
if survivors > 1 {
|
||||
return nil
|
||||
}
|
||||
match = cand
|
||||
}
|
||||
}
|
||||
if survivors == 1 {
|
||||
return &match
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolvePathWithContext walks the hop list, anchoring hop 0 on
|
||||
// fromPubkey (for ADVERTs) and each subsequent hop on the previous
|
||||
// resolved hop. Previously-resolved pubkeys (plus the originator) are
|
||||
// excluded from later candidate pools so the walk doesn't revisit a
|
||||
// node. Returns a `[]*string` shape compatible with
|
||||
// marshalResolvedPath (and the all-nil clobber-guard from PR #1548).
|
||||
func resolvePathWithContext(hops []string, fromPubkey string, graph *NeighborGraph, idx prefixIndex) []*string {
|
||||
if len(hops) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]*string, len(hops))
|
||||
if idx == nil {
|
||||
return out
|
||||
}
|
||||
prevAnchor := strings.ToLower(fromPubkey)
|
||||
seen := make(map[string]struct{}, len(hops)+1)
|
||||
if prevAnchor != "" {
|
||||
seen[prevAnchor] = struct{}{}
|
||||
}
|
||||
for i, hop := range hops {
|
||||
r := resolveHopWithContext(hop, prevAnchor, graph, idx, seen)
|
||||
out[i] = r
|
||||
if r != nil {
|
||||
lc := strings.ToLower(*r)
|
||||
seen[lc] = struct{}{}
|
||||
prevAnchor = lc
|
||||
} else {
|
||||
prevAnchor = ""
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RefreshNeighborGraph loads the latest neighbor_edges snapshot and
|
||||
// publishes it atomically. Called on startup and once per neighbor-
|
||||
// edges builder tick (60s) alongside RefreshPrefixIndex.
|
||||
func (s *Store) RefreshNeighborGraph() error {
|
||||
g, err := loadNeighborGraph(s.db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.neighborGraph.store(g)
|
||||
return nil
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// #1483: server's GetNodeLocationsByKeys lookup relies on stored
|
||||
// public_key being lowercase (LOWER(public_key) was dropped for perf).
|
||||
// The ingestor must normalize any legacy uppercase rows on boot so
|
||||
// the lookup remains correct.
|
||||
func TestPublicKeyLowercaseNormalizationMigration(t *testing.T) {
|
||||
dbPath := tempDBPath(t)
|
||||
s, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("first OpenStore: %v", err)
|
||||
}
|
||||
// Seed an uppercase row directly, bypassing UpsertNode's lowercase.
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT INTO nodes (public_key, name, role, last_seen, first_seen)
|
||||
VALUES ('AABBCCDDEEFF11223344', 'mixed-case-node', 'companion', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed uppercase row: %v", err)
|
||||
}
|
||||
// Sanity: verify the uppercase row is there pre-normalization.
|
||||
var pk string
|
||||
if err := s.db.QueryRow(`SELECT public_key FROM nodes WHERE public_key = 'AABBCCDDEEFF11223344'`).Scan(&pk); err != nil {
|
||||
t.Fatalf("pre-check select: %v", err)
|
||||
}
|
||||
if pk != "AABBCCDDEEFF11223344" {
|
||||
t.Fatalf("pre-check: expected uppercase, got %s", pk)
|
||||
}
|
||||
s.Close()
|
||||
|
||||
// Reopen — the boot-time migration should normalize the row.
|
||||
s2, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
|
||||
// The uppercase row should be gone.
|
||||
var still int
|
||||
if err := s2.db.QueryRow(`SELECT COUNT(*) FROM nodes WHERE public_key = 'AABBCCDDEEFF11223344'`).Scan(&still); err != nil {
|
||||
t.Fatalf("post-check uppercase count: %v", err)
|
||||
}
|
||||
if still != 0 {
|
||||
t.Fatalf("expected 0 uppercase rows after migration, got %d", still)
|
||||
}
|
||||
// The lowercase form should match.
|
||||
var lower string
|
||||
err = s2.db.QueryRow(`SELECT public_key FROM nodes WHERE public_key = 'aabbccddeeff11223344'`).Scan(&lower)
|
||||
if err == sql.ErrNoRows {
|
||||
t.Fatalf("expected lowercase row to exist after migration")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("post-check lowercase select: %v", err)
|
||||
}
|
||||
if lower != strings.ToLower("AABBCCDDEEFF11223344") {
|
||||
t.Fatalf("got %s, want lowercase form", lower)
|
||||
}
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Issue #1547 — resolved_path writer (ingestor-owned).
|
||||
//
|
||||
// Per the #1283 refactor (server is read-only; ingestor owns the
|
||||
// neighbor graph + node directory), the writer that populated
|
||||
// `observations.resolved_path` must live here in the ingestor. PR #1289
|
||||
// removed the server-side writer without porting it — this restores it.
|
||||
//
|
||||
// Approach:
|
||||
// - `resolvePath` is a pure function: hop prefixes → full pubkeys
|
||||
// using the in-memory prefix index built from `nodes.public_key`.
|
||||
// - Unique-prefix hops resolve to the full pubkey; ambiguous or
|
||||
// unknown hops resolve to `nil`. The output shape is `[]*string`
|
||||
// (with nulls for unresolved positions) — the JSON serialization
|
||||
// matches what the server's `unmarshalResolvedPath` /
|
||||
// frontend `getResolvedPath` already consume.
|
||||
// - The prefix index is rebuilt on startup and once per neighbor-
|
||||
// builder tick (60s) so new nodes start resolving within a minute
|
||||
// without blocking the MQTT ingest path.
|
||||
|
||||
// resolvePath maps each hop prefix to a full pubkey when the index
|
||||
// has exactly one candidate; returns nil at that position otherwise.
|
||||
// Returns nil for empty/no hops.
|
||||
func resolvePath(hops []string, idx prefixIndex) []*string {
|
||||
if len(hops) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]*string, len(hops))
|
||||
if idx == nil {
|
||||
return out
|
||||
}
|
||||
for i, hop := range hops {
|
||||
h := strings.ToLower(hop)
|
||||
candidates := idx[h]
|
||||
if len(candidates) == 1 {
|
||||
pk := candidates[0]
|
||||
out[i] = &pk
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// marshalResolvedPath JSON-encodes a resolved path. Returns "" when
|
||||
// the input is empty OR when every element is nil (writer treats "" as
|
||||
// SQL NULL).
|
||||
//
|
||||
// The all-nil case matters because of the UPSERT in InsertTransmission:
|
||||
//
|
||||
// resolved_path = COALESCE(excluded.resolved_path, resolved_path)
|
||||
//
|
||||
// If we emitted "[null,null]" here, nilIfEmpty() would let it through
|
||||
// as a non-NULL string and the COALESCE would OVERWRITE a previously
|
||||
// stored good resolved_path on re-ingest. Returning "" lets nilIfEmpty
|
||||
// produce SQL NULL so the COALESCE falls through to the existing value.
|
||||
// See issue #1547 / PR #1548 reviewer findings.
|
||||
func marshalResolvedPath(rp []*string) string {
|
||||
if len(rp) == 0 {
|
||||
return ""
|
||||
}
|
||||
allNil := true
|
||||
for _, p := range rp {
|
||||
if p != nil {
|
||||
allNil = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allNil {
|
||||
return ""
|
||||
}
|
||||
b, err := json.Marshal(rp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// prefixIdxHolder caches the prefix index for the InsertTransmission
|
||||
// hot path. atomic.Value lets the 60s rebuild happen without a lock on
|
||||
// the read side.
|
||||
type prefixIdxHolder struct {
|
||||
v atomic.Value // holds prefixIndex
|
||||
}
|
||||
|
||||
func (h *prefixIdxHolder) load() prefixIndex {
|
||||
if v := h.v.Load(); v != nil {
|
||||
return v.(prefixIndex)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *prefixIdxHolder) store(idx prefixIndex) {
|
||||
h.v.Store(idx)
|
||||
}
|
||||
|
||||
// RefreshPrefixIndex rebuilds the in-memory prefix index from the
|
||||
// nodes table and publishes it atomically. Called on startup and from
|
||||
// the neighbor-edges builder tick (60s) so new nodes become resolvable
|
||||
// without per-insert DB scans.
|
||||
func (s *Store) RefreshPrefixIndex() error {
|
||||
idx, err := buildPrefixIndex(s.db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.prefixIdx.store(idx)
|
||||
return nil
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func unmarshalResolvedPathLocal(s string) []*string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
var out []*string
|
||||
if json.Unmarshal([]byte(s), &out) != nil {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestResolvePathPureFunction is a unit test for the pure resolvePath
|
||||
// helper. Asserts:
|
||||
// - unique-prefix hops resolve to the full pubkey
|
||||
// - ambiguous-prefix hops resolve to nil
|
||||
// - unknown-prefix hops resolve to nil
|
||||
// - return slice length equals input hop count
|
||||
//
|
||||
// Regression gate for #1547 (resolved_path stopped being written).
|
||||
func TestResolvePathPureFunction(t *testing.T) {
|
||||
idx := prefixIndex{
|
||||
// "aa" → exactly one pubkey
|
||||
"aa": {"aaaaaaaaaa"},
|
||||
"aaaaaaaaaa": {"aaaaaaaaaa"},
|
||||
// "bb" → exactly one pubkey
|
||||
"bb": {"bbbbbbbbbb"},
|
||||
"bbbbbbbbbb": {"bbbbbbbbbb"},
|
||||
// "cc" → ambiguous (2 candidates)
|
||||
"cc": {"cccccccccc", "ccdddddddd"},
|
||||
"cccccccccc": {"cccccccccc"},
|
||||
}
|
||||
|
||||
got := resolvePath([]string{"aa", "cc", "ff", "bb"}, idx)
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("expected len 4, got %d", len(got))
|
||||
}
|
||||
if got[0] == nil || *got[0] != "aaaaaaaaaa" {
|
||||
t.Errorf("hop[0] aa: want aaaaaaaaaa, got %v", deref(got[0]))
|
||||
}
|
||||
if got[1] != nil {
|
||||
t.Errorf("hop[1] cc: want nil (ambiguous), got %v", deref(got[1]))
|
||||
}
|
||||
if got[2] != nil {
|
||||
t.Errorf("hop[2] ff: want nil (unknown), got %v", deref(got[2]))
|
||||
}
|
||||
if got[3] == nil || *got[3] != "bbbbbbbbbb" {
|
||||
t.Errorf("hop[3] bb: want bbbbbbbbbb, got %v", deref(got[3]))
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvePathEmptyHops asserts empty/no-path produces nil.
|
||||
func TestResolvePathEmptyHops(t *testing.T) {
|
||||
if got := resolvePath(nil, prefixIndex{}); got != nil {
|
||||
t.Errorf("nil hops: want nil, got %v", got)
|
||||
}
|
||||
if got := resolvePath([]string{}, prefixIndex{}); got != nil {
|
||||
t.Errorf("empty hops: want nil, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarshalResolvedPathRoundtrip asserts the JSON shape matches the
|
||||
// server's marshal/unmarshal contract: `[]*string` with nulls for
|
||||
// unresolved hops.
|
||||
func TestMarshalResolvedPathRoundtrip(t *testing.T) {
|
||||
a := "aaaaaaaaaa"
|
||||
b := "bbbbbbbbbb"
|
||||
in := []*string{&a, nil, &b}
|
||||
s := marshalResolvedPath(in)
|
||||
want := `["aaaaaaaaaa",null,"bbbbbbbbbb"]`
|
||||
if s != want {
|
||||
t.Errorf("marshal: want %s, got %s", want, s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInsertTransmissionWritesResolvedPath is the integration test that
|
||||
// gates the regression introduced by PR #1289 (issue #1547).
|
||||
//
|
||||
// Setup: seed two nodes + one observer + invoke InsertTransmission with
|
||||
// a PacketData whose PathJSON references one of the seeded nodes by
|
||||
// unique 1-byte (2-hex) prefix.
|
||||
//
|
||||
// Assert: the inserted observations row has a non-NULL resolved_path
|
||||
// whose JSON-decoded length equals the hop count, and the resolved
|
||||
// element matches the seeded node's full pubkey.
|
||||
func TestInsertTransmissionWritesResolvedPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "ingest.db")
|
||||
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
// Seed nodes with unique 1-byte prefixes.
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`,
|
||||
"aaaaaaaaaa", "from-node",
|
||||
"bbbbbbbbbb", "first-hop",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Seed one observer (needed so InsertTransmission resolves observer_idx).
|
||||
if err := store.UpsertObserver("obs-1", "observer-1", "", nil); err != nil {
|
||||
t.Fatalf("UpsertObserver: %v", err)
|
||||
}
|
||||
|
||||
// Force the prefix index to be (re)built from the seeded nodes so
|
||||
// the InsertTransmission path has something to resolve against.
|
||||
if err := store.RefreshPrefixIndex(); err != nil {
|
||||
t.Fatalf("RefreshPrefixIndex: %v", err)
|
||||
}
|
||||
|
||||
pkt := &PacketData{
|
||||
RawHex: "deadbeef",
|
||||
Timestamp: "2026-06-01T00:00:00Z",
|
||||
ObserverID: "obs-1",
|
||||
Hash: "h-1547",
|
||||
RouteType: 0,
|
||||
PayloadType: int(payloadADVERT),
|
||||
PathJSON: `["bb"]`,
|
||||
DecodedJSON: "{}",
|
||||
FromPubkey: "aaaaaaaaaa",
|
||||
}
|
||||
if _, err := store.InsertTransmission(pkt); err != nil {
|
||||
t.Fatalf("InsertTransmission: %v", err)
|
||||
}
|
||||
|
||||
var rp sql.NullString
|
||||
if err := store.db.QueryRow(
|
||||
`SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`,
|
||||
"h-1547",
|
||||
).Scan(&rp); err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if !rp.Valid || rp.String == "" {
|
||||
t.Fatalf("expected non-nil resolved_path, got NULL/empty (regression: #1547)")
|
||||
}
|
||||
got := unmarshalResolvedPathLocal(rp.String)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("resolved_path length: want 1, got %d (value=%s)", len(got), rp.String)
|
||||
}
|
||||
if got[0] == nil || *got[0] != "bbbbbbbbbb" {
|
||||
t.Errorf("resolved_path[0]: want bbbbbbbbbb, got %v (raw=%s)", deref(got[0]), rp.String)
|
||||
}
|
||||
}
|
||||
|
||||
func deref(p *string) string {
|
||||
if p == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// ─── #1560: context-aware resolution tests ─────────────────────────────────
|
||||
//
|
||||
// These exercise the post-fix behavior of resolveHopWithContext +
|
||||
// resolvePathWithContext. Until the green commit lands they MUST fail
|
||||
// on assertions (the stub falls back to naive `len==1` and returns nil
|
||||
// on every >1-candidate prefix), proving the gate is real.
|
||||
|
||||
// build5NodeAmbiguousIndex returns a prefixIndex where 3 of 5 nodes
|
||||
// share the 1-byte prefix 0x5c. Pubkeys are the "fingerprints":
|
||||
//
|
||||
// A = "5c000000000000000000000000000000aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
// B = "5c000000000000000000000000000000bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
// C = "5c000000000000000000000000000000cccccccccccccccccccccccccccccccc"
|
||||
// D = "dd000000000000000000000000000000dddddddddddddddddddddddddddddddd"
|
||||
// E = "ee000000000000000000000000000000eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
|
||||
func build5NodeAmbiguousIndex() (idx prefixIndex, A, B, C, D, E string) {
|
||||
A = "5c000000000000000000000000000000aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
B = "5c000000000000000000000000000000bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
C = "5c000000000000000000000000000000cccccccccccccccccccccccccccccccc"
|
||||
D = "dd000000000000000000000000000000dddddddddddddddddddddddddddddddd"
|
||||
E = "ee000000000000000000000000000000eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
|
||||
idx = prefixIndex{
|
||||
// 1-byte: 5c → A,B,C (collision); dd → D; ee → E
|
||||
"5c": {A, B, C},
|
||||
"dd": {D},
|
||||
"ee": {E},
|
||||
// full-key entries (so exact-match lookups still resolve)
|
||||
A: {A}, B: {B}, C: {C}, D: {D}, E: {E},
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// TestResolveHopWithContext_OneByteCollision_AdjacencyResolves
|
||||
// asserts the dominant production case (#1560): three nodes share the
|
||||
// 1-byte prefix 0x5c, but NeighborGraph adjacency narrows to exactly
|
||||
// one. The naive resolver returns nil; the context-aware resolver
|
||||
// MUST return the right pubkey.
|
||||
func TestResolveHopWithContext_OneByteCollision_AdjacencyResolves(t *testing.T) {
|
||||
idx, A, B, C, D, E := build5NodeAmbiguousIndex()
|
||||
g := NewNeighborGraph()
|
||||
// chain: A↔B, B↔C, C↔D, D↔E
|
||||
g.AddEdge(A, B)
|
||||
g.AddEdge(B, C)
|
||||
g.AddEdge(C, D)
|
||||
g.AddEdge(D, E)
|
||||
|
||||
// Anchored on A, the only 5c neighbor of A is B.
|
||||
got := resolveHopWithContext("5c", A, g, idx, nil)
|
||||
if got == nil {
|
||||
t.Fatalf("anchor=A, hop=5c: want B (%s), got <nil>", B)
|
||||
}
|
||||
if *got != B {
|
||||
t.Errorf("anchor=A, hop=5c: want %s, got %s", B, *got)
|
||||
}
|
||||
|
||||
// Anchored on B, the only 5c neighbors of B are A and C — but A is
|
||||
// the originator anchor in a path-walk; here we just assert that
|
||||
// 2 surviving candidates → nil (cannot disambiguate further).
|
||||
got = resolveHopWithContext("5c", B, g, idx, nil)
|
||||
if got != nil {
|
||||
t.Errorf("anchor=B, hop=5c: ambiguous (A and C both adjacent); want <nil>, got %s", *got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvePathWithContext_TwoHopChainAnchoredOnFromNode covers the
|
||||
// canonical 1-byte collision case end-to-end: path = [5c, 5c],
|
||||
// from_node = A → expect [B, C].
|
||||
func TestResolvePathWithContext_TwoHopChainAnchoredOnFromNode(t *testing.T) {
|
||||
idx, A, B, C, _, _ := build5NodeAmbiguousIndex()
|
||||
g := NewNeighborGraph()
|
||||
g.AddEdge(A, B)
|
||||
g.AddEdge(B, C)
|
||||
|
||||
got := resolvePathWithContext([]string{"5c", "5c"}, A, g, idx)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len(got)=%d, want 2 (raw=%v)", len(got), got)
|
||||
}
|
||||
if got[0] == nil || *got[0] != B {
|
||||
t.Errorf("hop[0]: want %s, got %v", B, deref(got[0]))
|
||||
}
|
||||
if got[1] == nil || *got[1] != C {
|
||||
t.Errorf("hop[1]: want %s, got %v", C, deref(got[1]))
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolveHopWithContext_NoAdjacencyContext_ReturnsNil asserts the
|
||||
// negative gate: 3 nodes with shared prefix, no edges between them in
|
||||
// the graph, hop=[5c] with no usable anchor → nil. Guards against an
|
||||
// over-eager resolver that just picks the first candidate.
|
||||
func TestResolveHopWithContext_NoAdjacencyContext_ReturnsNil(t *testing.T) {
|
||||
idx, _, _, _, _, _ := build5NodeAmbiguousIndex()
|
||||
g := NewNeighborGraph() // empty: no edges
|
||||
got := resolveHopWithContext("5c", "", g, idx, nil)
|
||||
if got != nil {
|
||||
t.Errorf("no anchor + empty graph: want <nil>, got %s", *got)
|
||||
}
|
||||
|
||||
// With an anchor that's not adjacent to any candidate, also nil.
|
||||
got = resolveHopWithContext("5c", "deadbeefdeadbeef", g, idx, nil)
|
||||
if got != nil {
|
||||
t.Errorf("non-adjacent anchor: want <nil>, got %s", *got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvePathWithContext_AdvertAnchoring asserts ADVERT-style
|
||||
// anchoring: from_pubkey is the originator, hop[0] is one of its
|
||||
// 1-byte-prefix neighbors → resolved.
|
||||
func TestResolvePathWithContext_AdvertAnchoring(t *testing.T) {
|
||||
idx, A, B, _, _, _ := build5NodeAmbiguousIndex()
|
||||
g := NewNeighborGraph()
|
||||
g.AddEdge(A, B) // only B is adjacent to A among the 5c candidates
|
||||
|
||||
got := resolvePathWithContext([]string{"5c"}, A, g, idx)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("len(got)=%d, want 1", len(got))
|
||||
}
|
||||
if got[0] == nil || *got[0] != B {
|
||||
t.Errorf("ADVERT anchored on A, hop=5c: want %s, got %v", B, deref(got[0]))
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvePathWithContext_RegressionMultiByteStillWorks asserts no
|
||||
// regression in the 2/3/4-byte prefix path that PR #1548 already
|
||||
// handled — unique prefixes resolve regardless of graph context.
|
||||
func TestResolvePathWithContext_RegressionMultiByteStillWorks(t *testing.T) {
|
||||
idx, _, _, _, D, E := build5NodeAmbiguousIndex()
|
||||
// dd and ee are unique 1-byte prefixes — naive path still works.
|
||||
got := resolvePathWithContext([]string{"dd", "ee"}, "", nil, idx)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len(got)=%d, want 2", len(got))
|
||||
}
|
||||
if got[0] == nil || *got[0] != D {
|
||||
t.Errorf("hop[0] dd: want %s, got %v", D, deref(got[0]))
|
||||
}
|
||||
if got[1] == nil || *got[1] != E {
|
||||
t.Errorf("hop[1] ee: want %s, got %v", E, deref(got[1]))
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvePathWithContext_AllNilContractPreserved asserts the
|
||||
// all-nil → empty-string clobber-guard contract from PR #1548 still
|
||||
// holds: an unresolvable path through the context resolver, when fed
|
||||
// to marshalResolvedPath, MUST yield "" (so nilIfEmpty → SQL NULL
|
||||
// → COALESCE preserves existing).
|
||||
func TestResolvePathWithContext_AllNilContractPreserved(t *testing.T) {
|
||||
// Empty index → every hop nil.
|
||||
got := resolvePathWithContext([]string{"5c", "dd"}, "", nil, prefixIndex{})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("len(got)=%d, want 2", len(got))
|
||||
}
|
||||
for i, p := range got {
|
||||
if p != nil {
|
||||
t.Errorf("hop[%d]: want <nil>, got %s", i, *p)
|
||||
}
|
||||
}
|
||||
if s := marshalResolvedPath(got); s != "" {
|
||||
t.Errorf("all-nil marshal: want \"\", got %q (clobber-guard regression)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarshalResolvedPathAllNilReturnsEmpty is a regression gate for
|
||||
// the data-loss clobber bug surfaced in PR #1548 review.
|
||||
//
|
||||
// When resolvePath fails to resolve ANY hop (every element nil),
|
||||
// marshalResolvedPath previously emitted "[null,null,...]" — a
|
||||
// non-empty string that bypassed nilIfEmpty and then OVERWROTE the
|
||||
// existing resolved_path via the COALESCE(excluded, current) UPSERT
|
||||
// on re-ingest. The fix returns "" so nilIfEmpty produces SQL NULL and
|
||||
// the COALESCE preserves the existing good value.
|
||||
func TestMarshalResolvedPathAllNilReturnsEmpty(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in []*string
|
||||
}{
|
||||
{"one-nil", []*string{nil}},
|
||||
{"two-nils", []*string{nil, nil}},
|
||||
{"three-nils", []*string{nil, nil, nil}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := marshalResolvedPath(tc.in)
|
||||
if got != "" {
|
||||
t.Errorf("all-nil input must return \"\" (so nilIfEmpty → SQL NULL → COALESCE preserves existing); got %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Mixed (at least one non-nil) MUST still marshal normally so we
|
||||
// don't lose partial resolutions.
|
||||
a := "aaaaaaaaaa"
|
||||
mixed := marshalResolvedPath([]*string{&a, nil})
|
||||
if mixed != `["aaaaaaaaaa",null]` {
|
||||
t.Errorf("partial resolution must still serialize; got %q", mixed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil is the
|
||||
// integration-level regression test for the data-loss bug.
|
||||
//
|
||||
// Setup: insert a transmission whose first ingest resolves cleanly to
|
||||
// a known pubkey. Then re-ingest the SAME transmission after the
|
||||
// prefix index has been cleared (simulating an empty NeighborGraph /
|
||||
// all-nil resolution path) and assert the previously stored
|
||||
// resolved_path is PRESERVED (NOT overwritten to "[null]" or NULL).
|
||||
//
|
||||
// Pre-fix behavior: marshalResolvedPath emitted "[null]", nilIfEmpty
|
||||
// kept it non-NULL, and COALESCE(excluded.resolved_path, resolved_path)
|
||||
// clobbered the original "bbbbbbbbbb".
|
||||
func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "ingest.db")
|
||||
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenStore: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.db.Exec(
|
||||
`INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`,
|
||||
"aaaaaaaaaa", "from-node",
|
||||
"bbbbbbbbbb", "first-hop",
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.UpsertObserver("obs-1", "observer-1", "", nil); err != nil {
|
||||
t.Fatalf("UpsertObserver: %v", err)
|
||||
}
|
||||
if err := store.RefreshPrefixIndex(); err != nil {
|
||||
t.Fatalf("RefreshPrefixIndex: %v", err)
|
||||
}
|
||||
|
||||
pkt := &PacketData{
|
||||
RawHex: "deadbeef",
|
||||
Timestamp: "2026-06-01T00:00:00Z",
|
||||
ObserverID: "obs-1",
|
||||
Hash: "h-clobber",
|
||||
RouteType: 0,
|
||||
PayloadType: int(payloadADVERT),
|
||||
PathJSON: `["bb"]`,
|
||||
DecodedJSON: "{}",
|
||||
FromPubkey: "aaaaaaaaaa",
|
||||
}
|
||||
if _, err := store.InsertTransmission(pkt); err != nil {
|
||||
t.Fatalf("first InsertTransmission: %v", err)
|
||||
}
|
||||
|
||||
// Sanity: first write populated resolved_path.
|
||||
var first sql.NullString
|
||||
if err := store.db.QueryRow(
|
||||
`SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`,
|
||||
"h-clobber",
|
||||
).Scan(&first); err != nil {
|
||||
t.Fatalf("first query: %v", err)
|
||||
}
|
||||
if !first.Valid || first.String == "" {
|
||||
t.Fatalf("precondition failed: first ingest left resolved_path NULL/empty; cannot test clobber")
|
||||
}
|
||||
wantPreserved := first.String
|
||||
|
||||
// Now wipe the prefix index so re-ingest produces an all-nil
|
||||
// resolution — exactly the scenario where the bug clobbers data.
|
||||
store.prefixIdx.store(prefixIndex{})
|
||||
|
||||
if _, err := store.InsertTransmission(pkt); err != nil {
|
||||
t.Fatalf("re-ingest InsertTransmission: %v", err)
|
||||
}
|
||||
|
||||
var after sql.NullString
|
||||
if err := store.db.QueryRow(
|
||||
`SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`,
|
||||
"h-clobber",
|
||||
).Scan(&after); err != nil {
|
||||
t.Fatalf("post-reingest query: %v", err)
|
||||
}
|
||||
if !after.Valid {
|
||||
t.Fatalf("data loss: resolved_path was NULL'd by re-ingest (was %q)", wantPreserved)
|
||||
}
|
||||
if after.String != wantPreserved {
|
||||
t.Errorf("data loss: resolved_path was clobbered by all-nil re-ingest\n before: %s\n after: %s", wantPreserved, after.String)
|
||||
}
|
||||
}
|
||||
+17
-93
@@ -7,27 +7,23 @@ import (
|
||||
|
||||
func TestParseEnvelopeTime(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
ok bool
|
||||
wantNaive bool
|
||||
name string
|
||||
in string
|
||||
ok bool
|
||||
}{
|
||||
{"rfc3339 utc", "2026-05-16T10:00:00Z", true, false},
|
||||
{"rfc3339 offset", "2026-05-16T12:00:00+02:00", true, false},
|
||||
{"naive iso", "2026-05-16T10:00:00", true, true},
|
||||
{"naive iso micros", "2026-05-16T10:00:00.123456", true, true},
|
||||
{"garbage", "not-a-time", false, false},
|
||||
{"empty", "", false, false},
|
||||
{"rfc3339 utc", "2026-05-16T10:00:00Z", true},
|
||||
{"rfc3339 offset", "2026-05-16T12:00:00+02:00", true},
|
||||
{"naive iso", "2026-05-16T10:00:00", true},
|
||||
{"naive iso micros", "2026-05-16T10:00:00.123456", true},
|
||||
{"garbage", "not-a-time", false},
|
||||
{"empty", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, naive, err := parseEnvelopeTime(c.in)
|
||||
_, err := parseEnvelopeTime(c.in)
|
||||
if (err == nil) != c.ok {
|
||||
t.Fatalf("parseEnvelopeTime(%q): want ok=%v, got err=%v", c.in, c.ok, err)
|
||||
}
|
||||
if err == nil && naive != c.wantNaive {
|
||||
t.Fatalf("parseEnvelopeTime(%q): want naive=%v, got %v", c.in, c.wantNaive, naive)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -52,105 +48,33 @@ func TestResolveRxTime(t *testing.T) {
|
||||
}
|
||||
|
||||
rx := now.Add(-5 * time.Hour).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": rx}, "test"); got != rx {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": rx}, "test"); got != rx {
|
||||
t.Errorf("plausible past timestamp: got %q want %q", got, rx)
|
||||
}
|
||||
if got, _ := resolveRxTime(map[string]interface{}{}, "test"); !nearNow(got) {
|
||||
if got := resolveRxTime(map[string]interface{}{}, "test"); !nearNow(got) {
|
||||
t.Errorf("missing timestamp: got %q, expected ~now", got)
|
||||
}
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": "garbage"}, "test"); !nearNow(got) {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": "garbage"}, "test"); !nearNow(got) {
|
||||
t.Errorf("garbage timestamp: got %q, expected ~now", got)
|
||||
}
|
||||
future := now.Add(48 * time.Hour).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": future}, "test"); !nearNow(got) {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": future}, "test"); !nearNow(got) {
|
||||
t.Errorf("future timestamp: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
|
||||
// RTC-reset node reporting a factory date — must not drag first_seen back.
|
||||
factory := "2020-01-01T00:00:00Z"
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": factory}, "test"); !nearNow(got) {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": factory}, "test"); !nearNow(got) {
|
||||
t.Errorf("stale factory timestamp: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
// Just past the 30-day floor → rejected.
|
||||
stale := now.Add(-31 * 24 * time.Hour).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": stale}, "test"); !nearNow(got) {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": stale}, "test"); !nearNow(got) {
|
||||
t.Errorf("stale timestamp >30d: got %q, expected ~now (rejected)", got)
|
||||
}
|
||||
// Just inside the 30-day floor → used verbatim.
|
||||
recent := now.Add(-29 * 24 * time.Hour).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": recent}, "test"); got != recent {
|
||||
if got := resolveRxTime(map[string]interface{}{"timestamp": recent}, "test"); got != recent {
|
||||
t.Errorf("recent timestamp <30d: got %q want %q", got, recent)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression: issue #1463 — naive (zone-less) ISO timestamps from observers
|
||||
// in negative-UTC-offset zones (e.g. California PDT, UTC−7) were interpreted
|
||||
// as UTC, producing rxTime values 7h in the past that poisoned `last_seen`
|
||||
// and rendered the observer perpetually "Stale" in the UI. The symmetric
|
||||
// clamp now collapses any naive timestamp more than 15 min off server-now to
|
||||
// `now()`, while zone-aware timestamps (RFC3339 with Z or offset) are still
|
||||
// honored verbatim regardless of skew (those are well-behaved observers).
|
||||
func TestResolveRxTimeNaiveTimestampClamp(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
|
||||
mustParse := func(s string) time.Time {
|
||||
t.Helper()
|
||||
parsed, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("result %q is not RFC3339: %v", s, err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
nearNow := func(s string) bool {
|
||||
d := mustParse(s).Sub(now)
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
return d <= time.Minute
|
||||
}
|
||||
|
||||
// California observer (UTC-7) emitting a naive local-clock timestamp:
|
||||
// must NOT be stored verbatim 7h in the past — clamp to ~now.
|
||||
naivePast := now.Add(-7 * time.Hour).Format("2006-01-02T15:04:05")
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": naivePast}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive past timestamp (UTC-7 observer): got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Naive future just minutes ahead (UTC+N observer, existing soft-clamp
|
||||
// behavior): still clamped to now.
|
||||
naiveFuture := now.Add(5 * time.Minute).Format("2006-01-02T15:04:05")
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": naiveFuture}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive future timestamp: got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Naive microsecond layout (python isoformat without tz) — same clamp.
|
||||
naivePastMicros := now.Add(-7 * time.Hour).Format("2006-01-02T15:04:05.000000")
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": naivePastMicros}, "test"); !nearNow(got) {
|
||||
t.Errorf("naive past timestamp w/ micros: got %q, expected ~now (clamped)", got)
|
||||
}
|
||||
|
||||
// Well-behaved observer: Z-suffixed past timestamp passes through verbatim
|
||||
// even if it's hours old (legitimate buffered uploads must be preserved).
|
||||
zPast := now.Add(-7 * time.Hour).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": zPast}, "test"); got != zPast {
|
||||
t.Errorf("Z-suffixed past timestamp must pass through: got %q want %q", got, zPast)
|
||||
}
|
||||
|
||||
// Well-behaved observer with explicit offset (UTC-7) — canonicalize to UTC
|
||||
// but preserve the moment in time. Must equal the same moment in UTC.
|
||||
offsetLoc := time.FixedZone("PDT", -7*3600)
|
||||
offsetMoment := now.Add(-7 * time.Hour).In(offsetLoc)
|
||||
offsetStr := offsetMoment.Format(time.RFC3339)
|
||||
wantUTC := offsetMoment.UTC().Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": offsetStr}, "test"); got != wantUTC {
|
||||
t.Errorf("offset-suffixed timestamp: got %q want %q", got, wantUTC)
|
||||
}
|
||||
|
||||
// Naive timestamp within tolerance window (2 min in past, observer that
|
||||
// happens to be in UTC) — within tolerance, passes through verbatim.
|
||||
naiveCloseStr := now.Add(-2 * time.Minute).Format("2006-01-02T15:04:05")
|
||||
naiveCloseWant := now.Add(-2 * time.Minute).Format(time.RFC3339)
|
||||
if got, _ := resolveRxTime(map[string]interface{}{"timestamp": naiveCloseStr}, "test"); got != naiveCloseWant {
|
||||
t.Errorf("naive timestamp within tolerance: got %q, expected %q (verbatim)", got, naiveCloseWant)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package main
|
||||
|
||||
import "strings"
|
||||
|
||||
// sanitizeLogString strips ASCII control bytes that would otherwise let a
|
||||
// node-controlled string (advert name, observer origin, channel name) inject
|
||||
// fake lines into the log stream. CR (\r), LF (\n), TAB (\t), NUL (\x00),
|
||||
// any other byte < 0x20, and 0x7F (DEL) are replaced with '?'.
|
||||
//
|
||||
// This is intentionally narrower than sanitizeName: sanitizeName preserves
|
||||
// \t and \n because they may appear in legitimately-stored display names.
|
||||
// Log sinks want neither.
|
||||
//
|
||||
// See audit-input-vulns-20260603 (LOW — log injection via newline in advert
|
||||
// name) and references at cmd/ingestor/main.go:659,689.
|
||||
func sanitizeLogString(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
// Iterate over runes so multibyte UTF-8 (Cyrillic, emoji) is preserved.
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, r := range s {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
b.WriteByte('?')
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSanitizeLogString covers the log-injection defense added to fix
|
||||
// audit-input-vulns-20260603 (LOW — log injection via newline in advert name).
|
||||
func TestSanitizeLogString(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"plain ascii preserved", "alpha-node", "alpha-node"},
|
||||
{"unicode preserved", "Иван привет 🦊", "Иван привет 🦊"},
|
||||
{"lf stripped", "evil\n[security] forged-line", "evil?[security] forged-line"},
|
||||
{"cr stripped", "evil\rfake-log", "evil?fake-log"},
|
||||
{"crlf stripped", "a\r\nb", "a??b"},
|
||||
{"tab stripped", "a\tb", "a?b"},
|
||||
{"nul stripped", "a\x00b", "a?b"},
|
||||
{"del stripped", "a\x7fb", "a?b"},
|
||||
{"bell stripped", "a\x07b", "a?b"},
|
||||
{"empty unchanged", "", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := sanitizeLogString(tc.in)
|
||||
if got != tc.want {
|
||||
t.Fatalf("sanitizeLogString(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SourceStatusSnapshot is the per-MQTT-source connection state and counter
|
||||
// view written to the ingestor stats file (under "source_statuses") and
|
||||
// consumed by cmd/server's /api/mqtt/status handler (#1043).
|
||||
//
|
||||
// All fields are unix seconds (0 = "never"). PacketsLast5m is a sliding
|
||||
// 5-minute count derived from a per-second ring buffer.
|
||||
type SourceStatusSnapshot struct {
|
||||
Name string `json:"name"`
|
||||
Broker string `json:"broker"`
|
||||
Connected bool `json:"connected"`
|
||||
LastConnectUnix int64 `json:"lastConnectUnix"`
|
||||
LastDisconnectUnix int64 `json:"lastDisconnectUnix"`
|
||||
LastPacketUnix int64 `json:"lastPacketUnix"`
|
||||
ConnectCount int64 `json:"connectCount"`
|
||||
DisconnectCount int64 `json:"disconnectCount"`
|
||||
PacketsTotal int64 `json:"packetsTotal"`
|
||||
PacketsLast5m int64 `json:"packetsLast5m"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
}
|
||||
|
||||
// sourceStatusState is the in-memory per-source counter set. All scalar
|
||||
// fields are accessed via sync/atomic so the hot-path MarkPacket /
|
||||
// MarkConnect / MarkDisconnect callsites stay lock-free. The 5-minute
|
||||
// sliding window uses a 300-element per-second ring (one slot per
|
||||
// second), guarded by ringMu only when we slide the cursor — the common
|
||||
// path increments the current second with a single atomic.AddInt64.
|
||||
//
|
||||
// Memory: one state per source (typically 1-5 in production). 300 int64
|
||||
// slots = 2.4KB/source — fine.
|
||||
type sourceStatusState struct {
|
||||
name string
|
||||
broker string // raw broker URL — server-side handler masks the password
|
||||
|
||||
connected atomic.Bool
|
||||
lastConnectUnix atomic.Int64
|
||||
lastDisconnectUnix atomic.Int64
|
||||
lastPacketUnix atomic.Int64
|
||||
connectCount atomic.Int64
|
||||
disconnectCount atomic.Int64
|
||||
packetsTotal atomic.Int64
|
||||
|
||||
// 5-minute sliding window: per-second buckets keyed by unix second.
|
||||
// Stored as parallel arrays so we can both zero-out a stale slot AND
|
||||
// know whether a slot's contents are still inside the window.
|
||||
ringMu sync.Mutex
|
||||
ringSec [300]int64 // unix second this slot represents (0 = unused)
|
||||
ringCount [300]int64 // packets received in that second
|
||||
|
||||
// lastError is rare-write/rare-read so a plain mutex is fine.
|
||||
errMu sync.RWMutex
|
||||
lastError string
|
||||
}
|
||||
|
||||
// MarkConnect records a successful (re)connection to the broker.
|
||||
// Clears any stale lastError from a prior disconnect — otherwise the UI
|
||||
// shows "connected=true, lastError='connection refused'" after a successful
|
||||
// reconnect, which is a lie (#1682 munger review r1).
|
||||
func (s *sourceStatusState) MarkConnect(now time.Time) {
|
||||
s.connected.Store(true)
|
||||
s.lastConnectUnix.Store(now.Unix())
|
||||
s.connectCount.Add(1)
|
||||
s.errMu.Lock()
|
||||
s.lastError = ""
|
||||
s.errMu.Unlock()
|
||||
}
|
||||
|
||||
// MarkDisconnect records the broker dropping the connection.
|
||||
func (s *sourceStatusState) MarkDisconnect(now time.Time, err error) {
|
||||
s.connected.Store(false)
|
||||
s.lastDisconnectUnix.Store(now.Unix())
|
||||
s.disconnectCount.Add(1)
|
||||
if err != nil {
|
||||
s.errMu.Lock()
|
||||
s.lastError = err.Error()
|
||||
s.errMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// MarkPacket records receipt of an MQTT message. Hot path.
|
||||
func (s *sourceStatusState) MarkPacket(now time.Time) {
|
||||
nowSec := now.Unix()
|
||||
s.lastPacketUnix.Store(nowSec)
|
||||
s.packetsTotal.Add(1)
|
||||
|
||||
slot := nowSec % int64(len(s.ringSec))
|
||||
s.ringMu.Lock()
|
||||
if s.ringSec[slot] != nowSec {
|
||||
s.ringSec[slot] = nowSec
|
||||
s.ringCount[slot] = 0
|
||||
}
|
||||
s.ringCount[slot]++
|
||||
s.ringMu.Unlock()
|
||||
}
|
||||
|
||||
// sumLast5m returns the count of MarkPacket calls in the last 300s. Slots
|
||||
// whose stored second falls outside the window are ignored (no stale leak).
|
||||
func (s *sourceStatusState) sumLast5m(now time.Time) int64 {
|
||||
nowSec := now.Unix()
|
||||
cutoff := nowSec - int64(len(s.ringSec)) + 1
|
||||
var total int64
|
||||
s.ringMu.Lock()
|
||||
for i := 0; i < len(s.ringSec); i++ {
|
||||
if s.ringSec[i] >= cutoff && s.ringSec[i] <= nowSec {
|
||||
total += s.ringCount[i]
|
||||
}
|
||||
}
|
||||
s.ringMu.Unlock()
|
||||
return total
|
||||
}
|
||||
|
||||
// snapshot copies the state into a serializable view.
|
||||
func (s *sourceStatusState) snapshot(now time.Time) SourceStatusSnapshot {
|
||||
s.errMu.RLock()
|
||||
errStr := s.lastError
|
||||
s.errMu.RUnlock()
|
||||
return SourceStatusSnapshot{
|
||||
Name: s.name,
|
||||
Broker: s.broker,
|
||||
Connected: s.connected.Load(),
|
||||
LastConnectUnix: s.lastConnectUnix.Load(),
|
||||
LastDisconnectUnix: s.lastDisconnectUnix.Load(),
|
||||
LastPacketUnix: s.lastPacketUnix.Load(),
|
||||
ConnectCount: s.connectCount.Load(),
|
||||
DisconnectCount: s.disconnectCount.Load(),
|
||||
PacketsTotal: s.packetsTotal.Load(),
|
||||
PacketsLast5m: s.sumLast5m(now),
|
||||
LastError: errStr,
|
||||
}
|
||||
}
|
||||
|
||||
// sourceStatusRegistry holds one sourceStatusState per source. Keyed by
|
||||
// tag (which is the source Name, or the Broker URL if the operator left
|
||||
// the name blank).
|
||||
var (
|
||||
sourceStatusRegistryMu sync.RWMutex
|
||||
sourceStatusRegistry = map[string]*sourceStatusState{}
|
||||
)
|
||||
|
||||
// RegisterSourceStatus creates (or returns the existing) state for the
|
||||
// given source. Safe for cold-start use; idempotent — re-registering the
|
||||
// same tag returns the existing state so counters aren't reset across
|
||||
// reconnects.
|
||||
func RegisterSourceStatus(tag, broker string) *sourceStatusState {
|
||||
sourceStatusRegistryMu.Lock()
|
||||
defer sourceStatusRegistryMu.Unlock()
|
||||
if s, ok := sourceStatusRegistry[tag]; ok {
|
||||
return s
|
||||
}
|
||||
s := &sourceStatusState{name: tag, broker: broker}
|
||||
sourceStatusRegistry[tag] = s
|
||||
return s
|
||||
}
|
||||
|
||||
// lookupSourceStatus returns the state for tag, or nil if unregistered.
|
||||
func lookupSourceStatus(tag string) *sourceStatusState {
|
||||
sourceStatusRegistryMu.RLock()
|
||||
defer sourceStatusRegistryMu.RUnlock()
|
||||
return sourceStatusRegistry[tag]
|
||||
}
|
||||
|
||||
// SnapshotSourceStatuses returns a slice of every registered source's
|
||||
// current snapshot. Surfaced via the ingestor stats file under
|
||||
// "source_statuses" so /api/mqtt/status can serve it (#1043).
|
||||
func SnapshotSourceStatuses(now time.Time) []SourceStatusSnapshot {
|
||||
sourceStatusRegistryMu.RLock()
|
||||
defer sourceStatusRegistryMu.RUnlock()
|
||||
out := make([]SourceStatusSnapshot, 0, len(sourceStatusRegistry))
|
||||
for _, s := range sourceStatusRegistry {
|
||||
out = append(out, s.snapshot(now))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// resetSourceStatusRegistry clears the registry. Test-only helper.
|
||||
func resetSourceStatusRegistry() {
|
||||
sourceStatusRegistryMu.Lock()
|
||||
defer sourceStatusRegistryMu.Unlock()
|
||||
sourceStatusRegistry = map[string]*sourceStatusState{}
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSourceStatus_BasicLifecycle exercises the counter wiring used by
|
||||
// the /api/mqtt/status server-side endpoint (#1043).
|
||||
func TestSourceStatus_BasicLifecycle(t *testing.T) {
|
||||
resetSourceStatusRegistry()
|
||||
defer resetSourceStatusRegistry()
|
||||
|
||||
s := RegisterSourceStatus("local", "mqtt://broker.example.com:1883")
|
||||
if s == nil {
|
||||
t.Fatal("RegisterSourceStatus returned nil")
|
||||
}
|
||||
// Re-registration is idempotent.
|
||||
if s2 := RegisterSourceStatus("local", "mqtt://other"); s2 != s {
|
||||
t.Fatal("RegisterSourceStatus not idempotent")
|
||||
}
|
||||
|
||||
now := time.Unix(1_700_000_000, 0)
|
||||
s.MarkConnect(now)
|
||||
s.MarkPacket(now)
|
||||
s.MarkPacket(now.Add(1 * time.Second))
|
||||
s.MarkPacket(now.Add(2 * time.Second))
|
||||
|
||||
snap := s.snapshot(now.Add(3 * time.Second))
|
||||
if !snap.Connected {
|
||||
t.Error("snapshot.Connected = false, want true after MarkConnect")
|
||||
}
|
||||
if snap.PacketsTotal != 3 {
|
||||
t.Errorf("PacketsTotal = %d, want 3", snap.PacketsTotal)
|
||||
}
|
||||
if snap.PacketsLast5m != 3 {
|
||||
t.Errorf("PacketsLast5m = %d, want 3", snap.PacketsLast5m)
|
||||
}
|
||||
if snap.ConnectCount != 1 {
|
||||
t.Errorf("ConnectCount = %d, want 1", snap.ConnectCount)
|
||||
}
|
||||
if snap.LastConnectUnix != now.Unix() {
|
||||
t.Errorf("LastConnectUnix = %d, want %d", snap.LastConnectUnix, now.Unix())
|
||||
}
|
||||
if snap.Broker != "mqtt://broker.example.com:1883" {
|
||||
t.Errorf("Broker = %q, want raw URL passthrough (server masks)", snap.Broker)
|
||||
}
|
||||
|
||||
// After 5 minutes idle, sliding window must be empty.
|
||||
snap2 := s.snapshot(now.Add(6 * time.Minute))
|
||||
if snap2.PacketsLast5m != 0 {
|
||||
t.Errorf("PacketsLast5m after 6m idle = %d, want 0", snap2.PacketsLast5m)
|
||||
}
|
||||
if snap2.PacketsTotal != 3 {
|
||||
t.Errorf("PacketsTotal must be lifetime-cumulative, got %d", snap2.PacketsTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceStatus_Disconnect(t *testing.T) {
|
||||
resetSourceStatusRegistry()
|
||||
defer resetSourceStatusRegistry()
|
||||
|
||||
s := RegisterSourceStatus("disco", "mqtt://x:1883")
|
||||
now := time.Unix(1_700_000_100, 0)
|
||||
s.MarkConnect(now)
|
||||
s.MarkDisconnect(now.Add(time.Minute), nil)
|
||||
|
||||
snap := s.snapshot(now.Add(2 * time.Minute))
|
||||
if snap.Connected {
|
||||
t.Error("snapshot.Connected = true after MarkDisconnect, want false")
|
||||
}
|
||||
if snap.DisconnectCount != 1 {
|
||||
t.Errorf("DisconnectCount = %d, want 1", snap.DisconnectCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotSourceStatuses_ReturnsAll(t *testing.T) {
|
||||
resetSourceStatusRegistry()
|
||||
defer resetSourceStatusRegistry()
|
||||
|
||||
RegisterSourceStatus("a", "mqtt://a")
|
||||
RegisterSourceStatus("b", "mqtt://b")
|
||||
snaps := SnapshotSourceStatuses(time.Now())
|
||||
if len(snaps) != 2 {
|
||||
t.Errorf("len(snaps) = %d, want 2", len(snaps))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSourceStatus_MarkConnectClearsLastError asserts MarkConnect wipes
|
||||
// any prior sticky error (#1682 munger r1 review). Otherwise the UI sees
|
||||
// connected=true alongside a stale "connection refused" string.
|
||||
func TestSourceStatus_MarkConnectClearsLastError(t *testing.T) {
|
||||
resetSourceStatusRegistry()
|
||||
defer resetSourceStatusRegistry()
|
||||
|
||||
s := RegisterSourceStatus("sticky", "mqtt://x:1883")
|
||||
now := time.Unix(1_700_000_200, 0)
|
||||
s.MarkConnect(now)
|
||||
s.MarkDisconnect(now.Add(time.Second), errors.New("connection refused"))
|
||||
|
||||
snap := s.snapshot(now.Add(2 * time.Second))
|
||||
if snap.LastError == "" {
|
||||
t.Fatalf("precondition: expected lastError after MarkDisconnect, got empty")
|
||||
}
|
||||
|
||||
// Reconnect — lastError must clear.
|
||||
s.MarkConnect(now.Add(3 * time.Second))
|
||||
snap = s.snapshot(now.Add(4 * time.Second))
|
||||
if snap.LastError != "" {
|
||||
t.Errorf("snapshot.LastError = %q after MarkConnect, want empty (sticky-error regression)", snap.LastError)
|
||||
}
|
||||
if !snap.Connected {
|
||||
t.Errorf("snapshot.Connected = false after MarkConnect, want true")
|
||||
}
|
||||
}
|
||||
+14
-83
@@ -43,51 +43,6 @@ type IngestorStatsSnapshot struct {
|
||||
// the server's /api/perf/io endpoint under .ingestor (#1120 — "Both
|
||||
// ingestor and server"). Optional; absent on non-Linux hosts.
|
||||
ProcIO *PerfIOSample `json:"procIO,omitempty"`
|
||||
// WriterPerf is the per-component SQLite writer-lock latency
|
||||
// snapshot (#1340) — wait_ms / hold_ms / contention_total tagged
|
||||
// by component (neighbor_builder, mqtt_handler, prune_packets,
|
||||
// prune_observers, prune_metrics, vacuum). Surfaced by the server
|
||||
// via /api/perf/write-sources under .writer_perf. Optional —
|
||||
// older ingestor builds don't publish this field.
|
||||
WriterPerf map[string]WriterStatsSnapshot `json:"writer_perf,omitempty"`
|
||||
// SourceLiveness (PR #1609 M1) is the per-MQTT-source receipt vs
|
||||
// write-path liveness snapshot. Keyed by source Tag. Surfaced by
|
||||
// the server via /api/healthz under .ingest_liveness so operators
|
||||
// can see "broker alive, write path stuck" (lastReceiptUnix recent,
|
||||
// lastMessageUnix stale) distinct from "everything stalled" (both
|
||||
// stale). Additive: omitempty so older server builds ignore it
|
||||
// gracefully.
|
||||
SourceLiveness map[string]SourceLivenessSnapshot `json:"source_liveness,omitempty"`
|
||||
// SourceStatuses (#1043) is the per-MQTT-source connection state and
|
||||
// counter view consumed by cmd/server's /api/mqtt/status handler.
|
||||
// Additive; omitempty so older server builds ignore it.
|
||||
SourceStatuses []SourceStatusSnapshot `json:"source_statuses,omitempty"`
|
||||
// WatchdogLastTickUnix (#1749) is the unix-seconds timestamp of the
|
||||
// most recent runLivenessWatchdogLoop tick. Surfaced via
|
||||
// /api/mqtt/status so external monitoring can assert the watchdog
|
||||
// goroutine is still alive — a value older than ~2× the watchdog
|
||||
// scan interval (typically 60s) indicates the watchdog itself has
|
||||
// wedged or panicked. 0 means the watchdog has never ticked
|
||||
// (e.g. cold start before the first scan). Additive — omitempty
|
||||
// so older server builds ignore it.
|
||||
WatchdogLastTickUnix int64 `json:"watchdogLastTickUnix,omitempty"`
|
||||
// WatchdogPanicCount (#1810 round-1) is the running total of
|
||||
// recovered panics inside the watchdog per-source work IIFE.
|
||||
// Exposed alongside WatchdogLastTickUnix because the tick clock is
|
||||
// stamped BEFORE the per-source work — a loop that panics on every
|
||||
// source still advances WatchdogLastTickUnix and looks healthy by
|
||||
// that signal alone. A rapidly-growing WatchdogPanicCount means
|
||||
// the loop is alive but per-source processing is broken (typically
|
||||
// a panic in emit / log sink). Monotonic; 0 means no recovered
|
||||
// panics yet. Additive — omitempty so older server builds ignore.
|
||||
WatchdogPanicCount int64 `json:"watchdogPanicCount,omitempty"`
|
||||
}
|
||||
|
||||
// SourceLivenessSnapshot is the per-source two-clock view exposed for
|
||||
// /api/healthz consumers. unixSeconds for both fields; 0 means "never".
|
||||
type SourceLivenessSnapshot struct {
|
||||
LastReceiptUnix int64 `json:"lastReceiptUnix"`
|
||||
LastMessageUnix int64 `json:"lastMessageUnix"`
|
||||
}
|
||||
|
||||
// statsFilePath returns the writable path the ingestor will publish stats to.
|
||||
@@ -106,25 +61,6 @@ func statsFilePath() string {
|
||||
|
||||
// writeStatsAtomic writes b to path via a tmp-then-rename, refusing to follow
|
||||
// symlinks on the tmp file. Returns nil on success, an error otherwise.
|
||||
//
|
||||
// Symlink semantics (refs #1170):
|
||||
//
|
||||
// - tmp side (path+".tmp"): protected by O_NOFOLLOW below. If tmp is a
|
||||
// pre-planted symlink, openat fails with ELOOP instead of writing
|
||||
// through it. This is the defensive-coding path that matters when the
|
||||
// default stats path lives under world-writable /tmp.
|
||||
//
|
||||
// - rename side (path): NOT protected by O_NOFOLLOW. Instead, os.Rename's
|
||||
// semantics are relied upon — rename atomically replaces any existing
|
||||
// entry at path (including a symlink) with the new regular file. The
|
||||
// symlink's target is NEVER written through, because all writes happened
|
||||
// to the unrelated tmp file before rename. Post-rename, path is a
|
||||
// regular file (not a symlink) and any prior symlink target's contents
|
||||
// are unchanged. The regression guardrail
|
||||
// TestWriteStatsAtomic_SymlinkAtDestIsReplaced pins this behavior so a
|
||||
// future refactor that swaps os.Rename for a destination-symlink-
|
||||
// following primitive (e.g. an open(path, O_WRONLY) without O_NOFOLLOW)
|
||||
// fails loudly.
|
||||
func writeStatsAtomic(path string, b []byte) error {
|
||||
tmp := path + ".tmp"
|
||||
// O_NOFOLLOW: if tmp is a pre-existing symlink, openat fails with ELOOP
|
||||
@@ -171,12 +107,12 @@ var readProcSelfIOFn = readProcSelfIO
|
||||
// readProcSelfIO parses /proc/self/io. Returns ok=false on non-Linux hosts or
|
||||
// any read/parse failure (caller skips the procIO block in that case).
|
||||
func readProcSelfIO() procIOSnapshot {
|
||||
out := procIOSnapshot{at: time.Now()}
|
||||
f, err := os.Open("/proc/self/io")
|
||||
if err != nil {
|
||||
return procIOSnapshot{}
|
||||
return out
|
||||
}
|
||||
defer f.Close()
|
||||
out := procIOSnapshot{at: time.Now()}
|
||||
parseProcSelfIOInto(bufio.NewScanner(f), &out)
|
||||
return out
|
||||
}
|
||||
@@ -256,23 +192,18 @@ func StartStatsFileWriter(s *Store, interval time.Duration) {
|
||||
ioRate := procIORate(prevIO, curIO, stamp)
|
||||
prevIO = curIO
|
||||
snap := IngestorStatsSnapshot{
|
||||
SampledAt: stamp,
|
||||
TxInserted: s.Stats.TransmissionsInserted.Load(),
|
||||
ObsInserted: s.Stats.ObservationsInserted.Load(),
|
||||
DuplicateTx: s.Stats.DuplicateTransmissions.Load(),
|
||||
NodeUpserts: s.Stats.NodeUpserts.Load(),
|
||||
ObserverUpserts: s.Stats.ObserverUpserts.Load(),
|
||||
WriteErrors: s.Stats.WriteErrors.Load(),
|
||||
SignatureDrops: s.Stats.SignatureDrops.Load(),
|
||||
WALCommits: s.Stats.WALCommits.Load(),
|
||||
GroupCommitFlushes: 0, // group commit reverted (refs #1129)
|
||||
BackfillUpdates: s.Stats.SnapshotBackfills(),
|
||||
ProcIO: ioRate,
|
||||
WriterPerf: s.WriterStatsSnapshot(),
|
||||
SourceLiveness: SnapshotLivenessClocks(),
|
||||
SourceStatuses: SnapshotSourceStatuses(tickAt),
|
||||
WatchdogLastTickUnix: WatchdogLastTickUnix(),
|
||||
WatchdogPanicCount: WatchdogPanicCount(),
|
||||
SampledAt: stamp,
|
||||
TxInserted: s.Stats.TransmissionsInserted.Load(),
|
||||
ObsInserted: s.Stats.ObservationsInserted.Load(),
|
||||
DuplicateTx: s.Stats.DuplicateTransmissions.Load(),
|
||||
NodeUpserts: s.Stats.NodeUpserts.Load(),
|
||||
ObserverUpserts: s.Stats.ObserverUpserts.Load(),
|
||||
WriteErrors: s.Stats.WriteErrors.Load(),
|
||||
SignatureDrops: s.Stats.SignatureDrops.Load(),
|
||||
WALCommits: s.Stats.WALCommits.Load(),
|
||||
GroupCommitFlushes: 0, // group commit reverted (refs #1129)
|
||||
BackfillUpdates: s.Stats.SnapshotBackfills(),
|
||||
ProcIO: ioRate,
|
||||
}
|
||||
buf.Reset()
|
||||
if err := enc.Encode(&snap); err != nil {
|
||||
|
||||
@@ -8,37 +8,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestProcIORate_ZeroValuePrevSuppressesRate guards against the phantom-delta
|
||||
// regression from #1169: when os.Open("/proc/self/io") fails, readProcSelfIO
|
||||
// now returns a zero-value procIOSnapshot (ok=false, zero time.Time). This
|
||||
// asserts procIORate returns nil so no inflated rate spike appears for the
|
||||
// next successful read.
|
||||
func TestProcIORate_ZeroValuePrevSuppressesRate(t *testing.T) {
|
||||
prev := procIOSnapshot{} // zero-value: ok=false, at=zero
|
||||
cur := procIOSnapshot{
|
||||
at: time.Now(),
|
||||
readBytes: 1024 * 1024 * 100,
|
||||
ok: true,
|
||||
}
|
||||
if got := procIORate(prev, cur, "2026-01-01T00:00:00Z"); got != nil {
|
||||
t.Fatalf("expected nil rate when prev is zero-value (os.Open failed), got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProcIORate_NormalPath asserts two valid snapshots produce a non-nil rate.
|
||||
func TestProcIORate_NormalPath(t *testing.T) {
|
||||
base := time.Now()
|
||||
prev := procIOSnapshot{at: base, readBytes: 0, ok: true}
|
||||
cur := procIOSnapshot{at: base.Add(time.Second), readBytes: 1024, ok: true}
|
||||
got := procIORate(prev, cur, "2026-01-01T00:00:01Z")
|
||||
if got == nil {
|
||||
t.Fatal("expected non-nil rate for valid prev/cur pair")
|
||||
}
|
||||
if got.ReadBytesPerSec != 1024.0 {
|
||||
t.Errorf("ReadBytesPerSec: want 1024.0, got %v", got.ReadBytesPerSec)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsFileWriter_PublishesProcIO asserts the ingestor's published
|
||||
// stats snapshot includes a `procIO` block with the per-process I/O rate
|
||||
// fields required by issue #1120 ("Both ingestor and server").
|
||||
@@ -96,73 +65,3 @@ func TestStatsFileWriter_PublishesProcIO(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteStatsAtomic_SymlinkAtDestIsReplaced is a regression guardrail for
|
||||
// #1170. The tmp side of writeStatsAtomic uses O_NOFOLLOW so a pre-planted
|
||||
// symlink at path+".tmp" cannot redirect the write — but the rename target
|
||||
// (`path` itself) is not protected by O_NOFOLLOW. Instead, os.Rename's
|
||||
// semantics are relied upon: rename atomically replaces any existing entry
|
||||
// at the destination, including a symlink, with the new regular file. The
|
||||
// original symlink's target is never written through (because the write
|
||||
// happened to the unrelated tmp file).
|
||||
//
|
||||
// This test pre-plants a symlink at `path` pointing to an unrelated target
|
||||
// file and asserts:
|
||||
// (a) post-write, path is a regular file (not a symlink), and
|
||||
// (b) the original target's contents are unchanged.
|
||||
//
|
||||
// If a future refactor swaps os.Rename for something that follows the
|
||||
// destination symlink (e.g. ioutil.WriteFile, or an open(path, O_WRONLY)
|
||||
// without O_NOFOLLOW), this test will fail loudly.
|
||||
func TestWriteStatsAtomic_SymlinkAtDestIsReplaced(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Unrelated target file with sentinel bytes. If writeStatsAtomic ever
|
||||
// followed the symlink at `path`, it would overwrite this file.
|
||||
target := filepath.Join(dir, "unrelated-target.bin")
|
||||
sentinel := []byte("DO-NOT-OVERWRITE-ME-#1170")
|
||||
if err := os.WriteFile(target, sentinel, 0o600); err != nil {
|
||||
t.Fatalf("seed target: %v", err)
|
||||
}
|
||||
|
||||
// Pre-plant a symlink at the destination path.
|
||||
path := filepath.Join(dir, "stats.json")
|
||||
if err := os.Symlink(target, path); err != nil {
|
||||
t.Fatalf("symlink: %v", err)
|
||||
}
|
||||
|
||||
payload := []byte(`{"sampledAt":"2026-01-01T00:00:00Z"}`)
|
||||
if err := writeStatsAtomic(path, payload); err != nil {
|
||||
t.Fatalf("writeStatsAtomic: %v", err)
|
||||
}
|
||||
|
||||
// (a) post-write, path must NOT be a symlink.
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("lstat path: %v", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
t.Errorf("post-write path is still a symlink (mode=%v); os.Rename should have atomically replaced it with a regular file", info.Mode())
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
t.Errorf("post-write path is not a regular file (mode=%v)", info.Mode())
|
||||
}
|
||||
|
||||
// Path now contains the new payload.
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read path: %v", err)
|
||||
}
|
||||
if string(got) != string(payload) {
|
||||
t.Errorf("path contents: want %q, got %q", payload, got)
|
||||
}
|
||||
|
||||
// (b) the original symlink target must be unchanged.
|
||||
gotTarget, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatalf("read target: %v", err)
|
||||
}
|
||||
if string(gotTarget) != string(sentinel) {
|
||||
t.Errorf("symlink target was clobbered: want %q, got %q", sentinel, gotTarget)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
// Fixture: migration block WITHOUT an async annotation and WITHOUT being
|
||||
// wrapped in the async-migration helper. This file exists ONLY so that
|
||||
// ~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh
|
||||
// has a known-bad sample to test against (the script is invoked with
|
||||
// BASE pointing at master and FIXTURE_DIR pointing here).
|
||||
//
|
||||
// DO NOT add a PREFLIGHT annotation to this file. DO NOT wrap the
|
||||
// migration via the async helper. The check script's correctness
|
||||
// depends on this staying BAD.
|
||||
//
|
||||
// IMPORTANT: this file must NOT contain the literal identifier of the
|
||||
// async-helper function anywhere (comments, strings, identifiers). The
|
||||
// preflight gate greps a window of lines above the migration for that
|
||||
// identifier as an "OK" signal, so mentioning it here would cause the
|
||||
// gate to *pass* this fixture — defeating its purpose. Refer to the
|
||||
// helper only obliquely as "the async-migration helper" in prose.
|
||||
package fixtures
|
||||
|
||||
const _ = `
|
||||
CREATE INDEX idx_observations_bad_sync_v1 ON observations(observer_idx, timestamp);
|
||||
`
|
||||
@@ -1,9 +0,0 @@
|
||||
// Fixture: migration block WITH an async annotation. Companion to
|
||||
// bad_sync_migration.go. The preflight check script must accept this
|
||||
// because of the PREFLIGHT line directly above the migration.
|
||||
package fixtures
|
||||
|
||||
// PREFLIGHT: async=true reason="fixture-only — ALTER ADD COLUMN is O(1) in sqlite"
|
||||
const _ = `
|
||||
ALTER TABLE observations ADD COLUMN annotated_good_fixture_col INTEGER DEFAULT 0;
|
||||
`
|
||||
@@ -1,81 +0,0 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
// advertRouteTypeFlood is ROUTE_TYPE_FLOOD from the MeshCore packet header.
|
||||
// Named distinctly from the equivalent constant in the (still open) unscoped-
|
||||
// relay PR so the two changes merge independently.
|
||||
const advertRouteTypeFlood = 1
|
||||
|
||||
// floodAdvertEntry is one advert transmission originated by a node, reduced to
|
||||
// what the windowed flood-advert count needs: first-seen timestamp, route type
|
||||
// and packet hash (for dedup across re-ingests / multi-observer rows).
|
||||
type floodAdvertEntry struct {
|
||||
ts string
|
||||
rt int
|
||||
hash string
|
||||
}
|
||||
|
||||
// countFloodAdverts counts distinct flood adverts (route_type ==
|
||||
// advertRouteTypeFlood) whose first-seen lies within the past windowHours. Entries
|
||||
// with unparseable timestamps are skipped, matching relay-liveness behaviour;
|
||||
// entries without a hash fall back to their timestamp as the dedup key.
|
||||
func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours float64) int {
|
||||
cutoff := now.Add(-time.Duration(windowHours * float64(time.Hour)))
|
||||
seen := map[string]struct{}{}
|
||||
for _, e := range entries {
|
||||
if e.rt != advertRouteTypeFlood {
|
||||
continue
|
||||
}
|
||||
t, ok := parseRelayTS(e.ts)
|
||||
if !ok || !t.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
key := e.hash
|
||||
if key == "" {
|
||||
key = e.ts
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
// CountFloodAdvertsForNode returns how many distinct FLOOD adverts pubkey
|
||||
// originated in the last windowHours - the mesh-wide-airtime kind. Zero-hop
|
||||
// adverts (route_type DIRECT) are excluded, so a nearby observer hearing a
|
||||
// node's cheap local adverts does not inflate the number.
|
||||
//
|
||||
// route_type is filtered in SQL so an advert-spamming node cannot truncate
|
||||
// the flood count (review feedback on the earlier LIMIT approach). The time
|
||||
// floor is a DATE-ONLY string with one day of slack: a date prefix compares
|
||||
// lexically the same across every first_seen format parseRelayTS accepts
|
||||
// ('T' and ' ' separators alike); the exact window check stays in Go.
|
||||
//
|
||||
// The row cap is a pure safety valve on per-request allocation: it applies to
|
||||
// flood adverts inside the floor window only, and 50000 in ~8 days is ~4 per
|
||||
// minute - any node past it is unambiguously a spammer whether the count
|
||||
// saturates or not. (An exact COUNT cannot move into SQL because the precise
|
||||
// window check needs parseRelayTS over the mixed first_seen formats.)
|
||||
// floodAdvertRowCap is the production row cap; tests pass a smaller cap
|
||||
// directly, so there is no mutable package state to race on.
|
||||
const floodAdvertRowCap = 50000
|
||||
|
||||
func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCap int) (int, error) {
|
||||
floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02")
|
||||
rows, err := db.conn.Query(
|
||||
"SELECT COALESCE(first_seen, ''), COALESCE(route_type, -1), COALESCE(hash, '') FROM transmissions WHERE from_pubkey = ? AND payload_type = ? AND route_type = ? AND first_seen >= ? ORDER BY id DESC LIMIT ?",
|
||||
pubkey, payloadTypeAdvert, advertRouteTypeFlood, floor, rowCap)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var entries []floodAdvertEntry
|
||||
for rows.Next() {
|
||||
var e floodAdvertEntry
|
||||
if err := rows.Scan(&e.ts, &e.rt, &e.hash); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
return countFloodAdverts(entries, time.Now(), windowHours), nil
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// insertAdvertTx seeds one advert transmission row - the single place that
|
||||
// knows the INSERT column list, shared by every test in this file.
|
||||
func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.Time) {
|
||||
t.Helper()
|
||||
if _, err := db.conn.Exec(
|
||||
`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', ?, ?, ?, ?, ?)`,
|
||||
hash, ts.Format("2006-01-02T15:04:05.000Z"), rt, payloadTypeAdvert, pubkey); err != nil {
|
||||
t.Fatalf("insert transmission: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func advertTS(hoursAgo float64) string {
|
||||
return time.Now().UTC().Add(-time.Duration(hoursAgo * float64(time.Hour))).Format("2006-01-02T15:04:05.000Z")
|
||||
}
|
||||
|
||||
// Flood adverts inside the window count; zero-hop (DIRECT) and out-of-window
|
||||
// ones do not; duplicate hashes collapse; broken timestamps are skipped.
|
||||
func TestCountFloodAdverts(t *testing.T) {
|
||||
now := time.Now()
|
||||
entries := []floodAdvertEntry{
|
||||
{ts: advertTS(1), rt: advertRouteTypeFlood, hash: "a1"},
|
||||
{ts: advertTS(2), rt: advertRouteTypeFlood, hash: "a1"}, // dup hash: one advert, two rows
|
||||
{ts: advertTS(3), rt: advertRouteTypeFlood, hash: "a2"},
|
||||
{ts: advertTS(4), rt: 0, hash: "a3"}, // zero-hop (DIRECT): excluded
|
||||
{ts: advertTS(9 * 24), rt: advertRouteTypeFlood, hash: "a4"}, // outside 7d window
|
||||
{ts: "not-a-time", rt: advertRouteTypeFlood, hash: "a5"}, // unparseable: skipped
|
||||
{ts: advertTS(5), rt: -1, hash: "a6"}, // route type absent: excluded
|
||||
}
|
||||
if got := countFloodAdverts(entries, now, 7*24); got != 2 {
|
||||
t.Fatalf("want 2 flood adverts in window, got %d", got)
|
||||
}
|
||||
// Hash-less entries dedup by timestamp instead of collapsing into one.
|
||||
hashless := []floodAdvertEntry{
|
||||
{ts: advertTS(1), rt: advertRouteTypeFlood},
|
||||
{ts: advertTS(2), rt: advertRouteTypeFlood},
|
||||
}
|
||||
if got := countFloodAdverts(hashless, now, 7*24); got != 2 {
|
||||
t.Fatalf("want 2 hash-less flood adverts, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The wire contract: GET /api/nodes/{pubkey} carries flood_advert_count_7d,
|
||||
// counting recent flood adverts only (zero-hop and out-of-window excluded).
|
||||
func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
now := time.Now().UTC()
|
||||
ins := func(hash string, rt int, ts time.Time) {
|
||||
insertAdvertTx(t, srv.db, "aabbccdd11223344", hash, rt, ts)
|
||||
}
|
||||
// The shared fixture may already seed adverts for this node, so assert the
|
||||
// DELTA our inserts cause rather than an absolute count.
|
||||
fetch := func() float64 {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("bad JSON: %v", err)
|
||||
}
|
||||
node, ok := body["node"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("expected node object")
|
||||
}
|
||||
got, ok := node["flood_advert_count_7d"].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("flood_advert_count_7d missing or not a number: %v", node["flood_advert_count_7d"])
|
||||
}
|
||||
return got
|
||||
}
|
||||
before := fetch()
|
||||
|
||||
ins("fa1", advertRouteTypeFlood, now.Add(-2*time.Hour))
|
||||
ins("fa2", advertRouteTypeFlood, now.Add(-30*time.Hour))
|
||||
ins("za1", 0, now.Add(-1*time.Hour)) // zero-hop: excluded
|
||||
ins("fa3", advertRouteTypeFlood, now.Add(-9*24*time.Hour)) // outside the 7d window
|
||||
// Inside the SQL date floor (window + 1d slack) but outside the exact 7d
|
||||
// window - only the Go-side check rejects this one.
|
||||
ins("fa4", advertRouteTypeFlood, now.Add(-time.Duration(7.5*24)*time.Hour))
|
||||
|
||||
if got := fetch(); got != before+2 {
|
||||
t.Fatalf("want flood_advert_count_7d = %v+2, got %v", before, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The row cap saturates the count instead of failing: with a cap of 2, three
|
||||
// qualifying flood adverts count as 2 (newest rows win via ORDER BY id DESC).
|
||||
func TestCountFloodAdvertsForNode_RowCapSaturates(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
now := time.Now().UTC()
|
||||
for i, h := range []string{"cap1", "cap2", "cap3"} {
|
||||
insertAdvertTx(t, db, "capnode11223344", h, advertRouteTypeFlood, now.Add(-time.Duration(i+1)*time.Hour))
|
||||
}
|
||||
n, err := db.CountFloodAdvertsForNode("capnode11223344", 7*24, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("want saturated count 2, got %d", n)
|
||||
}
|
||||
}
|
||||
@@ -44,14 +44,6 @@ type analyticsRecomputer struct {
|
||||
// Stats (atomic).
|
||||
computeRuns atomic.Int64
|
||||
lastComputeNs atomic.Int64 // duration of last compute in nanoseconds
|
||||
|
||||
// Issue #1659 (PR #1688 r1) — warmup gate state, inlined here so
|
||||
// hot-path readers (IsWarmingUp_1659) do lock-free atomic loads
|
||||
// only (replaces the r0 package-level map + chanLock). See
|
||||
// analytics_warmup_1659.go for full design notes.
|
||||
firstPassDoneNs atomic.Int64
|
||||
warmupStartedNs atomic.Int64
|
||||
warmupReadyGate atomic.Value // *func() bool — gate must return true for markFirstPassDone to take effect
|
||||
}
|
||||
|
||||
// newAnalyticsRecomputer constructs an unstarted recomputer.
|
||||
@@ -76,11 +68,6 @@ func newAnalyticsRecomputer(name string, interval time.Duration, compute func()
|
||||
// Calling Start multiple times is a no-op after the first call.
|
||||
func (r *analyticsRecomputer) Start() {
|
||||
r.startOnce.Do(func() {
|
||||
// Issue #1659 (#1688 munger #2): record warmup-start before
|
||||
// the first compute, so IsWarmingUp_1659's fallback timeout
|
||||
// is measured from "recomputer started" — not "first pass
|
||||
// returned", which never happens if compute() hangs.
|
||||
r.noteWarmupStart_1659()
|
||||
// Initial synchronous compute — first read must NOT see empty
|
||||
// or uninitialized data (acceptance criterion #1240).
|
||||
r.runOnce()
|
||||
@@ -108,10 +95,7 @@ func (r *analyticsRecomputer) runOnce() {
|
||||
}
|
||||
defer func() {
|
||||
// Don't let a compute panic kill the background goroutine.
|
||||
// The previous snapshot remains valid. Even on panic, we
|
||||
// still want IsWarmingUp_1659's fallback timeout to be the
|
||||
// safety net (a perpetually panicking compute would never
|
||||
// reach markFirstPassDone otherwise).
|
||||
// The previous snapshot remains valid.
|
||||
_ = recover()
|
||||
}()
|
||||
t0 := time.Now()
|
||||
@@ -121,16 +105,6 @@ func (r *analyticsRecomputer) runOnce() {
|
||||
if result != nil {
|
||||
r.cache.Store(result)
|
||||
}
|
||||
// Issue #1659: mark the first-pass clock so the warmup gate
|
||||
// in GetAnalyticsRFWithWindow / Topology / Channels handlers
|
||||
// can flip from 503-Retry-After to serving the cache.
|
||||
//
|
||||
// PR #1688 r1: called on EVERY successful pass (even nil
|
||||
// result) so a compute that returns nil but doesn't panic
|
||||
// still lifts the gate — banner-stuck-forever fix (munger #2).
|
||||
// The markFirstPassDone helper is idempotent and additionally
|
||||
// consults the chunked-loader readiness gate (munger #5).
|
||||
r.markFirstPassDone_1659()
|
||||
}
|
||||
|
||||
// Load returns the most recently computed snapshot, or nil if Start
|
||||
@@ -268,19 +242,6 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o
|
||||
}
|
||||
s.analyticsRecomputerMu.Unlock()
|
||||
|
||||
// Issue #1659 (PR #1688 r1, munger #5): wire the chunked-loader
|
||||
// readiness gate on the three warmup-gated recomputers (RF,
|
||||
// Topology, Channels). markFirstPassDone_1659 will refuse to
|
||||
// flip first-pass-done until s.LoadComplete() reports true —
|
||||
// i.e. the cold-load has populated all observations. Otherwise
|
||||
// the FIRST recomputer pass runs against the post-restart in-RAM
|
||||
// slice and the gate opens on partial data (the original #1659
|
||||
// bug class).
|
||||
loadCompleteGate := s.LoadComplete
|
||||
s.recompRF.setWarmupReadyGate_1659(loadCompleteGate)
|
||||
s.recompTopology.setWarmupReadyGate_1659(loadCompleteGate)
|
||||
s.recompChannels.setWarmupReadyGate_1659(loadCompleteGate)
|
||||
|
||||
for _, rc := range all {
|
||||
rc.Start()
|
||||
}
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
// Package main: issue #1659 — analytics warmup gating.
|
||||
//
|
||||
// Problem: after server restart, recompRF (and recompTopology /
|
||||
// recompChannels) cache the FIRST computation, which immediately after
|
||||
// boot is just the small in-RAM-observations slice (background
|
||||
// chunk-loader has not yet backfilled history). The recomputer then
|
||||
// serves that small slice from GetAnalyticsRFWithWindow's default
|
||||
// shortcut for an entire recompute interval, while the client pins it
|
||||
// via CLIENT_TTL.analyticsRF. UX: cards show a tiny "post-restart"
|
||||
// window even when the user selects "All data".
|
||||
//
|
||||
// Fix (r1 — addresses #1688 review munger #5):
|
||||
//
|
||||
// The first-pass-done signal is NOT enough on its own — the FIRST
|
||||
// recomputer pass at boot can complete against the post-restart slice
|
||||
// BEFORE the chunked loader (#1008 / chunked_load.go) has populated
|
||||
// the full observation set. Marking the gate ready in that window
|
||||
// reproduces the original #1659 bug.
|
||||
//
|
||||
// Two correctness invariants:
|
||||
//
|
||||
// 1. (#1688 munger #5) Only mark first-pass-done when BOTH:
|
||||
// a. a recomputer pass has completed, AND
|
||||
// b. the chunked loader has finished (s.LoadComplete()).
|
||||
// The gate's `readyGate` callback is wired by
|
||||
// StartAnalyticsRecomputers to `store.LoadComplete`. Passes that
|
||||
// complete while loadComplete is still false leave the gate in
|
||||
// the warming-up state; the NEXT pass after loadComplete flips
|
||||
// true is the one that opens the gate.
|
||||
//
|
||||
// 2. (#1688 munger #2 + kent-beck #2) The gate MUST lift in bounded
|
||||
// time. If compute() panics on every pass, hangs indefinitely,
|
||||
// or returns nil forever, an unguarded gate would leave the
|
||||
// 503 banner permanent. Two safeguards:
|
||||
// a. compute() panics are already caught by runOnce()'s
|
||||
// defer recover(); we additionally call markFirstPassDone
|
||||
// on EVERY pass (even nil-result), so a recomputer that
|
||||
// returns nil but doesn't panic still flips the gate.
|
||||
// b. A hard fallback timeout (warmupForceTimeout, 60s by
|
||||
// default) elapsed since the recomputer was constructed
|
||||
// forces IsWarmingUp_1659() to false — degraded mode
|
||||
// (serve whatever cache exists, possibly empty) is
|
||||
// strictly better than a permanent 503.
|
||||
//
|
||||
// Concurrency (#1688 munger #3):
|
||||
//
|
||||
// The previous r0 design used a package-level map keyed by recomputer
|
||||
// pointer, guarded by a global chanLock. Every default-shape analytics
|
||||
// request acquired that lock — a serialization point on a hot path.
|
||||
//
|
||||
// r1 inlines the warmup fields directly on `analyticsRecomputer`:
|
||||
// - firstPassDoneNs atomic.Int64
|
||||
// - warmupStartedNs atomic.Int64
|
||||
// - readyGate atomic.Value (holds func() bool, may be nil)
|
||||
//
|
||||
// Reads on the hot path are lock-free atomic loads. No package-level
|
||||
// state, no map lookups, no mutex.
|
||||
//
|
||||
// Tests: analytics_warmup_1659_test.go.
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// warmupForceTimeout is the deadline after which IsWarmingUp_1659()
|
||||
// flips false regardless of whether a successful first pass has run.
|
||||
// Operators get degraded analytics (possibly empty until the next
|
||||
// successful compute) instead of a permanent 503 banner.
|
||||
//
|
||||
// Var (not const) so tests can shorten it.
|
||||
var warmupForceTimeout = 60 * time.Second
|
||||
|
||||
// setWarmupReadyGate wires a callback that the recomputer consults
|
||||
// before honoring a markFirstPassDone_1659() request. When the gate
|
||||
// returns false, the warmup state is preserved across the pass —
|
||||
// equivalent to "this pass doesn't count; we need at least one pass
|
||||
// AFTER the gate flips true".
|
||||
//
|
||||
// nil callback means "no extra gating" (legacy behavior).
|
||||
//
|
||||
// Called from StartAnalyticsRecomputers; safe to call before Start().
|
||||
func (r *analyticsRecomputer) setWarmupReadyGate_1659(gate func() bool) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if gate == nil {
|
||||
r.warmupReadyGate.Store((*func() bool)(nil))
|
||||
return
|
||||
}
|
||||
r.warmupReadyGate.Store(&gate)
|
||||
}
|
||||
|
||||
func (r *analyticsRecomputer) loadWarmupReadyGate_1659() func() bool {
|
||||
v := r.warmupReadyGate.Load()
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
p, ok := v.(*func() bool)
|
||||
if !ok || p == nil {
|
||||
return nil
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// markFirstPassDone_1659 is called from analyticsRecomputer.runOnce()
|
||||
// after every compute attempt (success OR nil result; panics are
|
||||
// caught upstream and never reach here).
|
||||
//
|
||||
// The gate flip is conditional on the readyGate (when set) reporting
|
||||
// true — this implements the munger #5 fix: first-pass-done must
|
||||
// require BOTH a recomputer pass complete AND the chunked loader to
|
||||
// have finished populating the in-RAM observation set.
|
||||
//
|
||||
// Idempotent: only the FIRST successful flip wins; subsequent calls
|
||||
// observe a non-zero firstPassDoneNs and return immediately.
|
||||
func (r *analyticsRecomputer) markFirstPassDone_1659() {
|
||||
if r.firstPassDoneNs.Load() != 0 {
|
||||
return
|
||||
}
|
||||
if gate := r.loadWarmupReadyGate_1659(); gate != nil && !gate() {
|
||||
return
|
||||
}
|
||||
r.firstPassDoneNs.CompareAndSwap(0, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// FirstPassDoneAt_1659 reports the time the first full compute pass
|
||||
// completed (subject to the readyGate). Returns zero time if no
|
||||
// qualifying pass has completed yet.
|
||||
func (r *analyticsRecomputer) FirstPassDoneAt_1659() time.Time {
|
||||
if r == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
ns := r.firstPassDoneNs.Load()
|
||||
if ns == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(0, ns)
|
||||
}
|
||||
|
||||
// IsWarmingUp_1659 reports true when the recomputer has not yet
|
||||
// completed a qualifying first pass AND the fallback timeout has not
|
||||
// yet elapsed. Handlers for the default-shape request must return
|
||||
// 503 + Retry-After: 5 while this is true.
|
||||
//
|
||||
// Fallback timeout (warmupForceTimeout) prevents a permanent 503 in
|
||||
// pathological compute paths (perpetual panic, perpetual nil, hang).
|
||||
//
|
||||
// Lock-free: pure atomic loads.
|
||||
func (r *analyticsRecomputer) IsWarmingUp_1659() bool {
|
||||
if r == nil {
|
||||
// No recomputer registered → treat as ready; the handler
|
||||
// falls through to the legacy compute path.
|
||||
return false
|
||||
}
|
||||
if r.firstPassDoneNs.Load() != 0 {
|
||||
return false
|
||||
}
|
||||
startedNs := r.warmupStartedNs.Load()
|
||||
if startedNs != 0 {
|
||||
if time.Since(time.Unix(0, startedNs)) >= warmupForceTimeout {
|
||||
// Forced-ready: gate has been stuck too long. Stop
|
||||
// serving 503; let the handler serve whatever is in
|
||||
// the cache (possibly empty).
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// noteWarmupStart_1659 records the moment the recomputer was launched
|
||||
// (called once from Start). Used by IsWarmingUp_1659 to compute the
|
||||
// fallback-timeout elapsed window.
|
||||
func (r *analyticsRecomputer) noteWarmupStart_1659() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.warmupStartedNs.CompareAndSwap(0, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// writeAnalyticsWarmup503 emits the standard warmup response. The body
|
||||
// shape is documented for clients: error string + retry_after_s int.
|
||||
func writeAnalyticsWarmup503(w http.ResponseWriter) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = w.Write([]byte(`{"error":"analytics warming up","retry_after_s":5}`))
|
||||
}
|
||||
|
||||
// installWarmupBlocker_1659 is a test-only helper that registers the
|
||||
// RF / topology / channels recomputers with a compute function that
|
||||
// blocks on the supplied channel. firstPassDoneNs therefore stays
|
||||
// zero, simulating the post-restart warmup window for the warmup test.
|
||||
//
|
||||
// We bypass StartAnalyticsRecomputers entirely and wire the
|
||||
// recomputers manually so the background goroutines never fire. The
|
||||
// test only needs the *analyticsRecomputer pointers to be non-nil and
|
||||
// in the warmup state.
|
||||
func (s *PacketStore) installWarmupBlocker_1659(block <-chan struct{}) {
|
||||
blockCompute := func() interface{} {
|
||||
<-block
|
||||
return nil
|
||||
}
|
||||
s.analyticsRecomputerMu.Lock()
|
||||
defer s.analyticsRecomputerMu.Unlock()
|
||||
s.recompRF = newAnalyticsRecomputer("rf-test-block", time.Hour, blockCompute)
|
||||
s.recompTopology = newAnalyticsRecomputer("topo-test-block", time.Hour, blockCompute)
|
||||
s.recompChannels = newAnalyticsRecomputer("chan-test-block", time.Hour, blockCompute)
|
||||
// Do NOT call Start() — leaving firstPassDoneNs at zero is exactly
|
||||
// the warmup state the test wants to exercise.
|
||||
}
|
||||
@@ -1,330 +0,0 @@
|
||||
// Package main: issue #1659 — analytics warmup gating.
|
||||
//
|
||||
// After a server restart, the analytics recomputer caches the FIRST
|
||||
// computation (a small in-RAM slice) and serves it via the default
|
||||
// region="", zero-window shortcut in GetAnalyticsRFWithWindow until the
|
||||
// next periodic recompute fires. The client-side CLIENT_TTL.analyticsRF
|
||||
// then pins that small slice on the page even after the server flips
|
||||
// to steady-state.
|
||||
//
|
||||
// Fix: each recomputer carries a firstPassDoneAt timestamp set ONLY
|
||||
// after a full-range compute completes. While firstPassDoneAt is zero
|
||||
// AND the request is the default-shape (region="" && area="" &&
|
||||
// window.IsZero()), the handler returns 503 + Retry-After: 5 with a
|
||||
// JSON body the client recognizes and retries with backoff.
|
||||
//
|
||||
// These tests are the RED contract: they must FAIL on the assertion
|
||||
// (not a build error) when the warmup gate is absent, and PASS once
|
||||
// the fix lands.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestAnalyticsRF_WarmupReturns503 asserts that immediately after the
|
||||
// server starts — before any analytics recomputer has finished its
|
||||
// first full-range pass — GET /api/analytics/rf returns 503 with
|
||||
// Retry-After: 5 and a JSON body shaped as
|
||||
// {"error":"analytics warming up","retry_after_s":5}.
|
||||
//
|
||||
// This is the core acceptance criterion (c) from #1659.
|
||||
func TestAnalyticsRF_WarmupReturns503(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
// Register recomputers but DO NOT let them complete a first pass.
|
||||
// We install a compute func that blocks until we release it, so the
|
||||
// recomputer's firstPassDoneAt stays zero.
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
store.installWarmupBlocker_1659(block) // helper added in GREEN
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/rf", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected 503 during warmup, got %d (body=%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if got := w.Header().Get("Retry-After"); got != "5" {
|
||||
t.Fatalf("expected Retry-After: 5, got %q", got)
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON body: %v (raw=%s)", err, w.Body.String())
|
||||
}
|
||||
if resp["error"] != "analytics warming up" {
|
||||
t.Fatalf("expected error='analytics warming up', got %v", resp["error"])
|
||||
}
|
||||
if v, ok := resp["retry_after_s"].(float64); !ok || v != 5 {
|
||||
t.Fatalf("expected retry_after_s=5, got %v", resp["retry_after_s"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyticsRF_AfterFirstPassReturns200 asserts the post-warmup
|
||||
// happy path: once the recomputer's first full-range compute completes,
|
||||
// the handler serves the cached snapshot as 200.
|
||||
func TestAnalyticsRF_AfterFirstPassReturns200(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
// #1688 r1: the warmup gate now ALSO requires LoadComplete() to be
|
||||
// true before first-pass-done flips (munger #5). Tests that don't
|
||||
// exercise the chunked loader must flip it manually to model a
|
||||
// production server that has finished cold-loading.
|
||||
store.loadComplete.Store(true)
|
||||
|
||||
stop := store.StartAnalyticsRecomputers(50 * time.Millisecond)
|
||||
defer stop()
|
||||
|
||||
// Wait for the synchronous first-pass to complete. Start() runs
|
||||
// the initial compute synchronously, so by the time it returns
|
||||
// firstPassDoneAt should be set. We poll a brief moment to keep
|
||||
// the test robust to scheduling.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if store.recompRF != nil && !store.recompRF.FirstPassDoneAt_1659().IsZero() {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if store.recompRF == nil || store.recompRF.FirstPassDoneAt_1659().IsZero() {
|
||||
t.Fatal("recompRF.firstPassDoneAt never flipped after Start()")
|
||||
}
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/rf", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200 after first pass, got %d (body=%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if got := w.Header().Get("Retry-After"); got != "" {
|
||||
t.Fatalf("expected no Retry-After header on 200, got %q", got)
|
||||
}
|
||||
// Body should be a valid JSON object (the RF analytics map).
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("invalid JSON body: %v", err)
|
||||
}
|
||||
if len(resp) == 0 {
|
||||
t.Fatal("expected non-empty RF analytics response after first pass")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyticsRF_WindowedRequestNotGated asserts that even during
|
||||
// warmup, a request with an explicit time window (?since=/?until=) or
|
||||
// region/area filter is NOT gated by the warmup flag — those queries
|
||||
// bypass the recomputer entirely and hit the legacy compute-then-cache
|
||||
// path, which is unaffected by the first-pass bug.
|
||||
func TestAnalyticsRF_WindowedRequestNotGated(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
store.installWarmupBlocker_1659(block)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
// Explicit window — should bypass warmup gate.
|
||||
req := httptest.NewRequest("GET", "/api/analytics/rf?window=1h", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code == http.StatusServiceUnavailable {
|
||||
t.Fatalf("windowed request must NOT be gated by warmup (got 503)")
|
||||
}
|
||||
}
|
||||
|
||||
// === PR #1688 r1 — new test cases ===
|
||||
|
||||
// TestAnalyticsTopology_WarmupReturns503 — kent-beck #1: topology
|
||||
// gate is symmetric with RF; assert the same 503 contract.
|
||||
func TestAnalyticsTopology_WarmupReturns503(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
store.installWarmupBlocker_1659(block)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/topology", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("topology: expected 503 during warmup, got %d", w.Code)
|
||||
}
|
||||
if got := w.Header().Get("Retry-After"); got != "5" {
|
||||
t.Fatalf("topology: expected Retry-After: 5, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyticsChannels_WarmupReturns503 — kent-beck #1: channels
|
||||
// gate is symmetric with RF; assert the same 503 contract.
|
||||
func TestAnalyticsChannels_WarmupReturns503(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
store.installWarmupBlocker_1659(block)
|
||||
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/channels", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("channels: expected 503 during warmup, got %d", w.Code)
|
||||
}
|
||||
if got := w.Header().Get("Retry-After"); got != "5" {
|
||||
t.Fatalf("channels: expected Retry-After: 5, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarmup_GateBlockedUntilLoadComplete — munger #5 correctness:
|
||||
// the chunked loader readiness MUST gate first-pass-done. A recomputer
|
||||
// pass that completes while LoadComplete() is false must NOT lift the
|
||||
// gate; a SUBSEQUENT pass after LoadComplete() flips true must lift it.
|
||||
func TestWarmup_GateBlockedUntilLoadComplete(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
// LoadComplete starts false — chunked loader still running.
|
||||
|
||||
called := make(chan struct{}, 16)
|
||||
rc := newAnalyticsRecomputer("test-rf", time.Hour, func() interface{} {
|
||||
called <- struct{}{}
|
||||
return map[string]int{"x": 1}
|
||||
})
|
||||
rc.setWarmupReadyGate_1659(store.LoadComplete)
|
||||
rc.Start()
|
||||
defer rc.Stop()
|
||||
|
||||
// First pass already ran synchronously in Start(). Gate must still
|
||||
// be warming up because LoadComplete() is false.
|
||||
<-called
|
||||
if !rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("expected IsWarmingUp_1659=true while LoadComplete()=false (munger #5 bug)")
|
||||
}
|
||||
if !rc.FirstPassDoneAt_1659().IsZero() {
|
||||
t.Fatalf("expected FirstPassDoneAt zero while LoadComplete()=false")
|
||||
}
|
||||
|
||||
// Now flip the loader and trigger another pass.
|
||||
store.loadComplete.Store(true)
|
||||
rc.runOnce()
|
||||
if rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("expected gate to lift after LoadComplete()=true + another pass")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarmup_NilResultStillLiftsGate — munger #2 / kent-beck #2:
|
||||
// a compute that returns nil but doesn't panic must still flip the
|
||||
// gate (the cache stays empty but the banner does NOT get stuck).
|
||||
func TestWarmup_NilResultStillLiftsGate(t *testing.T) {
|
||||
rc := newAnalyticsRecomputer("test-nil", time.Hour, func() interface{} {
|
||||
return nil
|
||||
})
|
||||
rc.Start()
|
||||
defer rc.Stop()
|
||||
|
||||
if rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("nil-result compute must still lift warmup gate after first pass")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarmup_PanicEventuallyLiftsGate — munger #2 / kent-beck #2:
|
||||
// a compute that ALWAYS panics must not leave the gate stuck forever.
|
||||
// The fallback timeout (warmupForceTimeout) is the safety net.
|
||||
func TestWarmup_PanicEventuallyLiftsGate(t *testing.T) {
|
||||
prev := warmupForceTimeout
|
||||
warmupForceTimeout = 50 * time.Millisecond
|
||||
defer func() { warmupForceTimeout = prev }()
|
||||
|
||||
rc := newAnalyticsRecomputer("test-panic", time.Hour, func() interface{} {
|
||||
panic("compute boom")
|
||||
})
|
||||
rc.Start()
|
||||
defer rc.Stop()
|
||||
|
||||
// Panic was recovered inside runOnce; firstPassDoneNs is still 0.
|
||||
if rc.FirstPassDoneAt_1659().IsZero() == false {
|
||||
t.Fatalf("panicking compute should not have set firstPassDoneNs")
|
||||
}
|
||||
// But after warmupForceTimeout elapses, the gate must lift.
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("expected fallback timeout to lift gate after warmupForceTimeout (got still-warming)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarmup_TimeoutLiftsHangingCompute — munger #2 / kent-beck #2:
|
||||
// hung compute (blocks indefinitely on a channel) must not result in
|
||||
// permanent 503. Fallback timeout lifts it.
|
||||
func TestWarmup_TimeoutLiftsHangingCompute(t *testing.T) {
|
||||
prev := warmupForceTimeout
|
||||
warmupForceTimeout = 50 * time.Millisecond
|
||||
defer func() { warmupForceTimeout = prev }()
|
||||
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
rc := newAnalyticsRecomputer("test-hang", time.Hour, func() interface{} {
|
||||
<-block
|
||||
return nil
|
||||
})
|
||||
// Don't call Start (would block forever on synchronous initial
|
||||
// compute). Just simulate "we noted warmup start, compute is
|
||||
// hanging in another goroutine".
|
||||
rc.noteWarmupStart_1659()
|
||||
go rc.runOnce()
|
||||
|
||||
if !rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("expected initial state to be warming-up")
|
||||
}
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if rc.IsWarmingUp_1659() {
|
||||
t.Fatalf("expected fallback timeout to lift hung-compute warmup")
|
||||
}
|
||||
}
|
||||
@@ -170,13 +170,6 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string {
|
||||
|
||||
now := time.Now().UTC()
|
||||
id := 1
|
||||
// Single transaction for all inserts — see createTestDBAt for the rationale
|
||||
// (modernc.org/sqlite auto-commit per Exec fsyncs per row). numOld+numRecent
|
||||
// is small here today, but wrapping keeps the fixture robust if callers scale
|
||||
// it up, and is consistent with the other builders.
|
||||
if _, err := conn.Exec("BEGIN"); err != nil {
|
||||
t.Fatalf("test DB BEGIN: %v", err)
|
||||
}
|
||||
// Insert old packets (48 hours ago)
|
||||
for i := 0; i < numOld; i++ {
|
||||
oldT := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second)
|
||||
@@ -195,9 +188,6 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string {
|
||||
conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "")
|
||||
id++
|
||||
}
|
||||
if _, err := conn.Exec("COMMIT"); err != nil {
|
||||
t.Fatalf("test DB COMMIT: %v", err)
|
||||
}
|
||||
return dbPath
|
||||
}
|
||||
|
||||
@@ -352,27 +342,11 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) {
|
||||
defer obsStmt.Close()
|
||||
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
// Wrap the inserts in a single transaction. Without this, modernc.org/sqlite
|
||||
// (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs
|
||||
// for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the
|
||||
// fixture blows past the test timeout (the #1741 hang). A single
|
||||
// BEGIN/COMMIT makes the whole build one commit, finishing in well under a
|
||||
// second regardless of numTx.
|
||||
if _, err := conn.Exec("BEGIN"); err != nil {
|
||||
tb.Fatalf("test DB BEGIN: %v", err)
|
||||
}
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%04d", i)
|
||||
if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)); err != nil {
|
||||
tb.Fatalf("test DB insert transmission %d: %v", i, err)
|
||||
}
|
||||
if _, err := obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts); err != nil {
|
||||
tb.Fatalf("test DB insert observation %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := conn.Exec("COMMIT"); err != nil {
|
||||
tb.Fatalf("test DB COMMIT: %v", err)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
|
||||
obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,29 +396,16 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) {
|
||||
obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"}
|
||||
obsID := 1
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
// Single transaction for all inserts — see createTestDBAt for the rationale
|
||||
// (modernc.org/sqlite auto-commit per Exec would fsync per row; at numTx=30000
|
||||
// the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT.
|
||||
if _, err := conn.Exec("BEGIN"); err != nil {
|
||||
tb.Fatalf("test DB BEGIN: %v", err)
|
||||
}
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)
|
||||
hash := fmt.Sprintf("h%06d", i)
|
||||
if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)); err != nil {
|
||||
tb.Fatalf("test DB insert transmission %d: %v", i, err)
|
||||
}
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i))
|
||||
nObs := (i % 5) + 1 // 1–5 observations per transmission
|
||||
for j := 0; j < nObs; j++ {
|
||||
snr := -5.0 + float64(j)*2.5
|
||||
rssi := -90.0 + float64(j)*5.0
|
||||
if _, err := obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts); err != nil {
|
||||
tb.Fatalf("test DB insert observation %d: %v", obsID, err)
|
||||
}
|
||||
obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts)
|
||||
obsID++
|
||||
}
|
||||
}
|
||||
if _, err := conn.Exec("COMMIT"); err != nil {
|
||||
tb.Fatalf("test DB COMMIT: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,11 +21,8 @@
|
||||
// build time (#1230) so we don't have to re-filter here.
|
||||
//
|
||||
// For Dijkstra we need a DISTANCE (lower = better) not an affinity
|
||||
// (higher = better): cost = 1/weight. That conversion (plus the
|
||||
// epsilon/non-finite-weight filtering and self-loop/dedup handling) now lives
|
||||
// in the shared weightedDistanceAdjacency helper (graph_weighted.go), used by
|
||||
// both this axis and Coverage so they see byte-identical graph structure;
|
||||
// ComputeBridgeScores no longer builds the adjacency by hand.
|
||||
// (higher = better), so we convert: cost = 1 / max(epsilon, weight).
|
||||
// epsilon avoids divide-by-zero on a degenerate zero-weight edge.
|
||||
//
|
||||
// (1) Brandes, "A Faster Algorithm for Betweenness Centrality" (2001).
|
||||
package main
|
||||
@@ -33,6 +30,7 @@ package main
|
||||
import (
|
||||
"container/heap"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BridgeEdge is the algorithm-facing edge tuple consumed by
|
||||
@@ -67,9 +65,32 @@ const bridgeMinWeightEpsilon = 1e-9
|
||||
// Pure (no global state, no locks); safe to call concurrently.
|
||||
// Cost: O(V · (E + V log V)).
|
||||
func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 {
|
||||
// 1. Build the distance adjacency (cost = 1/weight) — shared with the
|
||||
// Coverage axis (graph_weighted.go) so both see identical structure.
|
||||
adj := weightedDistanceAdjacency(edges)
|
||||
// 1. Build adjacency list with distance = 1/weight.
|
||||
adj := make(map[string]map[string]float64)
|
||||
addOrMerge := func(a, b string, dist float64) {
|
||||
m, ok := adj[a]
|
||||
if !ok {
|
||||
m = make(map[string]float64)
|
||||
adj[a] = m
|
||||
}
|
||||
if existing, has := m[b]; !has || dist < existing {
|
||||
m[b] = dist
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
a := strings.ToLower(strings.TrimSpace(e.A))
|
||||
b := strings.ToLower(strings.TrimSpace(e.B))
|
||||
if a == "" || b == "" || a == b {
|
||||
continue
|
||||
}
|
||||
w := e.Weight
|
||||
if w < bridgeMinWeightEpsilon {
|
||||
continue
|
||||
}
|
||||
dist := 1.0 / w
|
||||
addOrMerge(a, b, dist)
|
||||
addOrMerge(b, a, dist)
|
||||
}
|
||||
if len(adj) == 0 {
|
||||
return map[string]float64{}
|
||||
}
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package main
|
||||
|
||||
// Issue #1551: /api/* responses must emit Cache-Control: no-store so
|
||||
// CDNs (Cloudflare, nginx, Varnish) do not cache JSON. Static assets
|
||||
// (app.js, /, etc.) intentionally remain CDN-cacheable.
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestAPIRoutesEmitNoStoreCacheControl asserts every covered /api/*
|
||||
// endpoint sets Cache-Control: no-store. This is a black-box test
|
||||
// against the real router, exercising whatever middleware chain is
|
||||
// wired by RegisterRoutes.
|
||||
func TestAPIRoutesEmitNoStoreCacheControl(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
|
||||
apiPaths := []string{
|
||||
"/api/stats",
|
||||
"/api/observers",
|
||||
"/api/packets?limit=10",
|
||||
"/api/nodes?limit=10",
|
||||
}
|
||||
|
||||
for _, p := range apiPaths {
|
||||
t.Run(p, func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", p, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("%s: expected 200, got %d (body: %s)", p, w.Code, w.Body.String())
|
||||
}
|
||||
cc := w.Header().Get("Cache-Control")
|
||||
if cc != "no-store" {
|
||||
t.Errorf("%s: expected Cache-Control: no-store, got %q", p, cc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStaticAssetsDoNotEmitNoStore guards against scope creep: the
|
||||
// no-store middleware must be scoped to /api/* only. Static assets
|
||||
// (HTML, JS, CSS) keep their existing browser-cache headers
|
||||
// ("no-cache, no-store, must-revalidate" today via spaHandler) and
|
||||
// must NOT be downgraded to bare "no-store" by the API middleware —
|
||||
// i.e. the API middleware must not run on these paths. If a future
|
||||
// change moves static assets behind no-store middleware, CDN caching
|
||||
// of immutable hashed assets breaks; assert the contract explicitly.
|
||||
func TestStaticAssetsDoNotEmitBareNoStore(t *testing.T) {
|
||||
// Build a temp public dir so spaHandler has real files to serve.
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "index.html"), []byte("<html>SPA</html>"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "app.js"), []byte("console.log('app')"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, router := setupTestServer(t)
|
||||
// Wire the SPA handler exactly the way main.go does for non-/api paths.
|
||||
fs := http.FileServer(http.Dir(dir))
|
||||
router.PathPrefix("/").Handler(spaHandler(dir, fs))
|
||||
|
||||
cases := []struct {
|
||||
path string
|
||||
wantCacheCC string
|
||||
}{
|
||||
// spaHandler sets this exact value for HTML/JS/CSS.
|
||||
{"/app.js", "no-cache, no-store, must-revalidate"},
|
||||
{"/", "no-cache, no-store, must-revalidate"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.path, func(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", c.path, nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
cc := w.Header().Get("Cache-Control")
|
||||
if cc == "no-store" {
|
||||
t.Errorf("%s: API no-store middleware leaked onto static asset (got bare %q, expected %q)", c.path, cc, c.wantCacheCC)
|
||||
}
|
||||
if cc != c.wantCacheCC {
|
||||
t.Errorf("%s: expected Cache-Control %q, got %q", c.path, c.wantCacheCC, cc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure mux import used (test compiles even if setupTestServer signature
|
||||
// changes).
|
||||
var _ = mux.NewRouter
|
||||
@@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
// Issue #1561: detect CDN-fronted deployments and warn ONCE.
|
||||
//
|
||||
// When operators put CoreScope behind Cloudflare/Fastly without
|
||||
// configuring a /api/* cache bypass, dashboards go stale — the origin
|
||||
// emits Cache-Control: no-store (#1551), but the CDN's zone-level
|
||||
// caching policy can still cache JSON responses for hours
|
||||
// (cf-cache-status: HIT, age > 0). We can't fix the CDN config from
|
||||
// the server side; the best we can do is detect the situation and
|
||||
// loudly tell the operator at the logs.
|
||||
//
|
||||
// Detection: presence of any CDN-specific request header
|
||||
// (CF-Connecting-IP, CF-Ray, Fastly-Client-IP, True-Client-IP).
|
||||
// We deliberately exclude X-Forwarded-For and X-Real-IP: every
|
||||
// generic reverse proxy (nginx, Caddy, Traefik, k8s ingress) sets
|
||||
// those, so including them would warn operators who aren't behind
|
||||
// a CDN at all and train them to ignore the warning entirely
|
||||
// (defeating the point of #1561).
|
||||
//
|
||||
// Side effects: a single log line per process boot — never blocks
|
||||
// the request, never modifies the response, never logs again.
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var cdnWarnOnce sync.Once
|
||||
|
||||
// cdnWarned is set true after the first CDN-fronted request has been
|
||||
// observed and logged. Subsequent requests short-circuit before the
|
||||
// per-request header scan in firstCDNHeader — a hot-path optimization
|
||||
// for the steady state (warning already emitted, every /api request
|
||||
// otherwise pays for 4 http.Header.Get lookups forever).
|
||||
var cdnWarned atomic.Bool
|
||||
|
||||
// cdnHeaders are HTTP request headers injected ONLY by CDNs
|
||||
// (Cloudflare, Fastly, Akamai) — never by a generic reverse proxy.
|
||||
// Detected case-insensitively by http.Header.Get.
|
||||
//
|
||||
// X-Forwarded-For / X-Real-IP are intentionally NOT in this list:
|
||||
// every nginx/Caddy/Traefik/k8s-ingress deployment sets them, so
|
||||
// using them as a CDN signal produces a false positive on every
|
||||
// reverse-proxied install (issue #1561 round-1 review).
|
||||
var cdnHeaders = []string{
|
||||
"CF-Connecting-IP", // Cloudflare
|
||||
"CF-Ray", // Cloudflare
|
||||
"Fastly-Client-IP", // Fastly
|
||||
"True-Client-IP", // Akamai (also set by Cloudflare Enterprise)
|
||||
}
|
||||
|
||||
// cdnDetectionMiddleware inspects each incoming request for CDN
|
||||
// headers and, on the FIRST one observed, logs a single warning
|
||||
// pointing the operator at docs/deployment-behind-cdn.md. The
|
||||
// middleware always calls next; it never blocks or rewrites.
|
||||
func cdnDetectionMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Fast path: once we've warned, skip the per-request header
|
||||
// scan entirely. Steady state for any CDN-fronted deploy is
|
||||
// ~every request hitting this branch.
|
||||
if cdnWarned.Load() {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if hdr := firstCDNHeader(r.Header); hdr != "" {
|
||||
cdnWarnOnce.Do(func() {
|
||||
log.Printf("[security] WARNING: detected request via CDN (%s header present). "+
|
||||
"Ensure /api/* is bypassed in your CDN config — see docs/deployment-behind-cdn.md. "+
|
||||
"Cached API responses cause observer-flap and incorrect dashboards.", hdr)
|
||||
cdnWarned.Store(true)
|
||||
})
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func firstCDNHeader(h http.Header) string {
|
||||
for _, name := range cdnHeaders {
|
||||
if h.Get(name) != "" {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package main
|
||||
|
||||
// Issue #1561: When the server is fronted by a CDN (Cloudflare, Fastly,
|
||||
// Akamai) we cannot guarantee /api/* responses are not cached unless
|
||||
// the operator configures a bypass rule. Detect CDN-specific request
|
||||
// headers at the first such request and log a one-shot warning
|
||||
// pointing the operator at the bypass doc.
|
||||
//
|
||||
// Contract:
|
||||
// - Warning logs ONLY when a CDN-specific header is present
|
||||
// (CF-Connecting-IP, CF-Ray, Fastly-Client-IP, True-Client-IP).
|
||||
// - Generic reverse-proxy headers (X-Forwarded-For, X-Real-IP) MUST
|
||||
// NOT trigger the warning — every nginx/Caddy/Traefik/k8s install
|
||||
// sets those, so warning on them defeats the entire signal.
|
||||
// - Warning logs at most ONCE per process boot (sync.Once), even
|
||||
// under concurrent first-request load.
|
||||
// - Middleware NEVER blocks the request — it always calls
|
||||
// next.ServeHTTP.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// resetCDNDetectionOnce restores a fresh sync.Once so each test starts
|
||||
// from a clean "have not warned yet" state.
|
||||
func resetCDNDetectionOnce() {
|
||||
cdnWarnOnce = sync.Once{}
|
||||
cdnWarned.Store(false)
|
||||
}
|
||||
|
||||
// runWithCDNMiddleware fires the request through the middleware and
|
||||
// returns (log output, whether next was called). The sentinel proves
|
||||
// the middleware did not silently drop the request.
|
||||
func runWithCDNMiddleware(t *testing.T, req *http.Request) (string, bool) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&buf)
|
||||
defer log.SetOutput(prev)
|
||||
|
||||
nextCalled := false
|
||||
h := cdnDetectionMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
nextCalled = true
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("middleware must not block request; got status %d", w.Code)
|
||||
}
|
||||
return buf.String(), nextCalled
|
||||
}
|
||||
|
||||
func TestCDNDetection_LogsOnCFRayHeader(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set("CF-Ray", "abc123-LAX")
|
||||
|
||||
out, nextCalled := runWithCDNMiddleware(t, req)
|
||||
|
||||
if !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if !strings.Contains(out, "detected request via CDN") {
|
||||
t.Errorf("expected log to contain 'detected request via CDN', got: %q", out)
|
||||
}
|
||||
if !strings.Contains(out, "deployment-behind-cdn") {
|
||||
t.Errorf("expected log to reference deployment-behind-cdn doc, got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDNDetection_SilentWithoutCDNHeader(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
// No CDN-typical headers set.
|
||||
|
||||
out, nextCalled := runWithCDNMiddleware(t, req)
|
||||
|
||||
if !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if strings.Contains(out, "detected request via CDN") {
|
||||
t.Errorf("expected no CDN warning without CDN headers, got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for round-1 adversarial finding: generic reverse-proxy
|
||||
// headers must NOT trigger the warning. Every nginx/Caddy/Traefik/
|
||||
// k8s-ingress reverse proxy sets X-Forwarded-For and X-Real-IP, so
|
||||
// flagging them produces a false positive on every reverse-proxied
|
||||
// install and trains operators to ignore the warning.
|
||||
func TestCDNDetection_SilentOnReverseProxyHeadersAlone(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
header string
|
||||
}{
|
||||
{"x-forwarded-for-alone", "X-Forwarded-For"},
|
||||
{"x-real-ip-alone", "X-Real-IP"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set(tc.header, "10.0.0.1")
|
||||
// No CDN-specific headers — just the generic reverse-proxy one.
|
||||
|
||||
out, nextCalled := runWithCDNMiddleware(t, req)
|
||||
|
||||
if !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if strings.Contains(out, "detected request via CDN") {
|
||||
t.Errorf("header %s alone must NOT trigger CDN warning (would false-positive every nginx/k8s deploy); got: %q", tc.header, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// When a CDN-specific header is present alongside generic proxy
|
||||
// headers (common: Cloudflare → nginx → app), the warning still fires.
|
||||
func TestCDNDetection_LogsWhenCDNHeaderAccompaniesProxyHeaders(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set("X-Forwarded-For", "10.0.0.1")
|
||||
req.Header.Set("X-Real-IP", "10.0.0.1")
|
||||
req.Header.Set("CF-Connecting-IP", "1.2.3.4")
|
||||
|
||||
out, nextCalled := runWithCDNMiddleware(t, req)
|
||||
|
||||
if !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if !strings.Contains(out, "detected request via CDN") {
|
||||
t.Errorf("expected CDN warning when CF-Connecting-IP present alongside proxy headers; got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDNDetection_LogsOnlyOnce(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
|
||||
var buf bytes.Buffer
|
||||
prev := log.Writer()
|
||||
log.SetOutput(&buf)
|
||||
defer log.SetOutput(prev)
|
||||
|
||||
nextCalled := 0
|
||||
h := cdnDetectionMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
nextCalled++
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set("CF-Ray", "abc123")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
if nextCalled != 3 {
|
||||
t.Fatalf("middleware must call next on every request; got %d calls, want 3", nextCalled)
|
||||
}
|
||||
got := strings.Count(buf.String(), "detected request via CDN")
|
||||
if got != 1 {
|
||||
t.Errorf("expected CDN warning exactly once across multiple requests; got %d in output: %q", got, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Each genuinely CDN-specific header should trip the detector on its
|
||||
// own. X-Forwarded-For / X-Real-IP are NOT in this set — see the
|
||||
// negative test TestCDNDetection_SilentOnReverseProxyHeadersAlone.
|
||||
func TestCDNDetection_RecognizesAllCommonCDNHeaders(t *testing.T) {
|
||||
headers := []string{
|
||||
"CF-Connecting-IP",
|
||||
"CF-Ray",
|
||||
"Fastly-Client-IP",
|
||||
"True-Client-IP",
|
||||
}
|
||||
for _, h := range headers {
|
||||
t.Run(h, func(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set(h, "1.2.3.4")
|
||||
out, nextCalled := runWithCDNMiddleware(t, req)
|
||||
if !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if !strings.Contains(out, "detected request via CDN") {
|
||||
t.Errorf("header %s should trip CDN detection; log was: %q", h, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Round-1 KB finding #2: sync.Once is what keeps the log from
|
||||
// spamming — verify it holds under concurrent first-request load.
|
||||
// CI runs `go test -race`, so this also stresses the underlying
|
||||
// primitive for data races. Without -race, the assertion still
|
||||
// catches a plain bool / non-atomic implementation.
|
||||
func TestCDNDetectionMiddlewareConcurrentFirstRequestLogsOnce(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
|
||||
var buf bytes.Buffer
|
||||
var bufMu sync.Mutex
|
||||
prev := log.Writer()
|
||||
// log.Printf can be called concurrently; serialize writes to buf
|
||||
// so we never race the test's own assertion read.
|
||||
log.SetOutput(writerFunc(func(p []byte) (int, error) {
|
||||
bufMu.Lock()
|
||||
defer bufMu.Unlock()
|
||||
return buf.Write(p)
|
||||
}))
|
||||
defer log.SetOutput(prev)
|
||||
|
||||
var nextCalls int64
|
||||
h := cdnDetectionMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt64(&nextCalls, 1)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
const n = 50
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
req := httptest.NewRequest("GET", "/api/observers", nil)
|
||||
req.Header.Set("CF-Ray", "abc123-LAX")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := atomic.LoadInt64(&nextCalls); got != n {
|
||||
t.Fatalf("middleware must call next on every concurrent request; got %d, want %d", got, n)
|
||||
}
|
||||
|
||||
bufMu.Lock()
|
||||
out := buf.String()
|
||||
bufMu.Unlock()
|
||||
got := strings.Count(out, "detected request via CDN")
|
||||
if got != 1 {
|
||||
t.Errorf("expected sync.Once to admit exactly ONE warning under %d concurrent first-requests; got %d. Output:\n%s", n, got, out)
|
||||
}
|
||||
}
|
||||
|
||||
// writerFunc adapts a function to io.Writer.
|
||||
type writerFunc func(p []byte) (int, error)
|
||||
|
||||
func (f writerFunc) Write(p []byte) (int, error) { return f(p) }
|
||||
|
||||
// Round-2 MAJOR finding: sync.Once only short-circuits the log.Printf,
|
||||
// not the per-request header scan. firstCDNHeader still iterates 4
|
||||
// http.Header.Get lookups on every /api request after warning fires.
|
||||
// The fix is an atomic.Bool fast-path checked BEFORE firstCDNHeader.
|
||||
// This test gates that the flag is actually set on the first CDN
|
||||
// request — without it, the middleware would have no signal to
|
||||
// short-circuit on, and the optimization would be a dead store.
|
||||
func TestCDNDetection_CdnWarnedFlagSet(t *testing.T) {
|
||||
resetCDNDetectionOnce()
|
||||
req := httptest.NewRequest("GET", "/api/x", nil)
|
||||
req.Header.Set("CF-Ray", "x")
|
||||
if _, nextCalled := runWithCDNMiddleware(t, req); !nextCalled {
|
||||
t.Fatal("middleware did not call next handler")
|
||||
}
|
||||
if !cdnWarned.Load() {
|
||||
t.Fatal("cdnWarned must be true after first CDN request (fast-path flag not set)")
|
||||
}
|
||||
}
|
||||
@@ -38,14 +38,6 @@ func newChannelTestStore(packets []*StoreTx) *PacketStore {
|
||||
}
|
||||
|
||||
func makeGrpTx(channelHash int, channel, text, sender string) *StoreTx {
|
||||
return makeGrpTxWithStatus(channelHash, channel, text, sender, "")
|
||||
}
|
||||
|
||||
// makeGrpTxWithStatus is like makeGrpTx but lets the caller set the ingestor's
|
||||
// decryptionStatus ("decrypted" / "no_key" / "decryption_failed" / ""), which
|
||||
// computeAnalyticsChannels consults to decide whether a channel name is
|
||||
// trustworthy (see #1729).
|
||||
func makeGrpTxWithStatus(channelHash int, channel, text, sender, decryptionStatus string) *StoreTx {
|
||||
decoded := map[string]interface{}{
|
||||
"type": "CHAN",
|
||||
"channelHash": float64(channelHash),
|
||||
@@ -53,9 +45,6 @@ func makeGrpTxWithStatus(channelHash int, channel, text, sender, decryptionStatu
|
||||
"text": text,
|
||||
"sender": sender,
|
||||
}
|
||||
if decryptionStatus != "" {
|
||||
decoded["decryptionStatus"] = decryptionStatus
|
||||
}
|
||||
b, _ := json.Marshal(decoded)
|
||||
pt := 5
|
||||
return &StoreTx{
|
||||
@@ -177,63 +166,3 @@ func TestIsPlaceholderName(t *testing.T) {
|
||||
t.Error("Public should NOT be placeholder")
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_PublicChannelPreserved is the regression test for
|
||||
// #1729: the firmware-default "Public" channel (channel-hash byte 0x11 = 17,
|
||||
// key-derived via SHA256(key)[0], NOT the hashtag scheme's 186) was being
|
||||
// discarded by the #978 rainbow-table validation and rendered as
|
||||
// "Encrypted (0x11)". The ingestor decrypts Public packets with the builtin
|
||||
// well-known key and marks them decryptionStatus:"decrypted"; the server must
|
||||
// trust that name instead of re-applying the hashtag hash check.
|
||||
func TestComputeAnalyticsChannels_PublicChannelPreserved(t *testing.T) {
|
||||
packets := []*StoreTx{
|
||||
makeGrpTxWithStatus(17, "Public", "hello net", "alice", "decrypted"),
|
||||
makeGrpTxWithStatus(17, "Public", "morning", "bob", "decrypted"),
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", "", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected 1 channel bucket, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
ch := channels[0]
|
||||
if ch["name"] != "Public" {
|
||||
t.Errorf("expected name 'Public' preserved, got %q (must not be downgraded to ch17)", ch["name"])
|
||||
}
|
||||
if ch["encrypted"] != false {
|
||||
t.Errorf("expected encrypted=false for decrypted Public channel, got %v", ch["encrypted"])
|
||||
}
|
||||
if ch["hash"] != "17" {
|
||||
t.Errorf("expected hash '17', got %v", ch["hash"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeAnalyticsChannels_UndecryptedNameStillValidated ensures the #1729
|
||||
// fix does not weaken #978: when the ingestor did NOT mark the packet
|
||||
// "decrypted" (no_key / decryption_failed / absent), a channel name that fails
|
||||
// the hashtag hash check is still discarded to "chNN", even if text/sender
|
||||
// happen to be present (e.g. a stale/foreign rainbow-table hit).
|
||||
func TestComputeAnalyticsChannels_UndecryptedNameStillValidated(t *testing.T) {
|
||||
// Hash 17 is NOT the hashtag hash of #Public (that is 186). Without a
|
||||
// "decrypted" status, the name must be rejected.
|
||||
packets := []*StoreTx{
|
||||
makeGrpTxWithStatus(17, "#Public", "leaked", "eve", ""),
|
||||
}
|
||||
|
||||
store := newChannelTestStore(packets)
|
||||
result := store.computeAnalyticsChannels("", "", TimeWindow{})
|
||||
|
||||
channels := result["channels"].([]map[string]interface{})
|
||||
if len(channels) != 1 {
|
||||
t.Fatalf("expected 1 channel bucket, got %d: %+v", len(channels), channels)
|
||||
}
|
||||
ch := channels[0]
|
||||
if ch["name"] != "ch17" {
|
||||
t.Errorf("expected undecrypted mismatch downgraded to 'ch17', got %q", ch["name"])
|
||||
}
|
||||
if ch["encrypted"] != true {
|
||||
t.Errorf("expected encrypted=true for rejected rainbow-table name, got %v", ch["encrypted"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,672 +0,0 @@
|
||||
package main
|
||||
|
||||
// Chunked startup load + early HTTP readiness for issue #1009.
|
||||
//
|
||||
// Design:
|
||||
// * LoadChunked paginates transmissions in id-ordered chunks of
|
||||
// `chunkSize` (default 10000 via Config.DBLoadChunkSize). After the
|
||||
// first chunk is merged into the store, FirstChunkReady is closed.
|
||||
// main.go binds the HTTP listener on that signal and serves
|
||||
// partial data while remaining chunks stream in the background.
|
||||
// * RunStartupLoad is the orchestrator: it runs LoadChunked
|
||||
// synchronously, then on success runs loadBackgroundChunks
|
||||
// synchronously so s.oldestLoaded is guaranteed set before the
|
||||
// background loader reads it (#1809). main.go typically invokes
|
||||
// RunStartupLoad inside its own goroutine and waits on
|
||||
// FirstChunkReady() in parallel to bind the HTTP listener.
|
||||
// * loadStatusMiddleware stamps X-CoreScope-Load-Status on every
|
||||
// response: "loading; progress=<rows>" until LoadComplete()
|
||||
// reports true, then "ready". Dashboards and probes can read the
|
||||
// header without parsing JSON.
|
||||
// * OnChunkLoaded registers a per-chunk callback for progress
|
||||
// logging / tests.
|
||||
//
|
||||
// Concurrency: each chunk acquires s.mu.Lock() ONLY while merging the
|
||||
// chunk's rows into store-shared maps. SQLite reads run lock-free so
|
||||
// HTTP handlers (which take s.mu.RLock) stay responsive.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
)
|
||||
|
||||
// dbLoadConfig is the server-package alias for dbconfig.LoadConfig (#1009).
|
||||
type dbLoadConfig = dbconfig.LoadConfig
|
||||
|
||||
// invariantViolation is the production-time handler for #1809-class
|
||||
// invariant failures (see loadBackgroundChunks). In prod it logs a
|
||||
// FATAL line and calls os.Exit(1) via log.Fatalf — a clean shutdown
|
||||
// with no goroutine-stack noise. Tests override it to panic so they
|
||||
// can recover() and assert the invariant message without crashing the
|
||||
// test runner. adv #6 (PR #1811).
|
||||
var invariantViolation = func(msg string) {
|
||||
log.Fatalf("invariant violation: %s", msg)
|
||||
}
|
||||
|
||||
// DBLoadChunkSize returns the configured chunk size for chunked
|
||||
// startup load (config: db.load.chunkSize), or 10000 default (#1009).
|
||||
func (c *Config) DBLoadChunkSize() int {
|
||||
return c.DB.GetLoadChunkSize()
|
||||
}
|
||||
|
||||
// chunkedLoadState holds the runtime gates for LoadChunked. It lives
|
||||
// on PacketStore via embedded fields — see store.go additions in the
|
||||
// same commit.
|
||||
|
||||
// FirstChunkReady returns a channel closed once the first chunk has
|
||||
// been merged into the store, signalling the HTTP listener can bind.
|
||||
func (s *PacketStore) FirstChunkReady() <-chan struct{} {
|
||||
s.chunkedLoadInit()
|
||||
return s.firstChunkReady
|
||||
}
|
||||
|
||||
// LoadComplete reports whether LoadChunked has finished all chunks.
|
||||
func (s *PacketStore) LoadComplete() bool {
|
||||
return s.loadComplete.Load()
|
||||
}
|
||||
|
||||
// LoadProgress reports the number of transmission rows processed by
|
||||
// the in-flight (or completed) LoadChunked call.
|
||||
func (s *PacketStore) LoadProgress() int64 {
|
||||
return s.loadProgressRows.Load()
|
||||
}
|
||||
|
||||
// OnChunkLoaded registers a callback fired once per chunk after that
|
||||
// chunk has been merged into the store. The callback receives the
|
||||
// number of transmission rows in that chunk and the running total.
|
||||
// Multiple registrations chain.
|
||||
func (s *PacketStore) OnChunkLoaded(fn func(rowsThisChunk, totalRows int)) {
|
||||
s.chunkedLoadInit()
|
||||
s.chunkCBMu.Lock()
|
||||
defer s.chunkCBMu.Unlock()
|
||||
s.chunkCallbacks = append(s.chunkCallbacks, fn)
|
||||
}
|
||||
|
||||
// chunkedLoadInit lazily initialises the readiness channel + callback
|
||||
// list under a mutex so concurrent first callers don't race.
|
||||
func (s *PacketStore) chunkedLoadInit() {
|
||||
s.chunkInitOnce.Do(func() {
|
||||
s.firstChunkReady = make(chan struct{})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PacketStore) signalFirstChunk() {
|
||||
if s.firstChunkSignaled.CompareAndSwap(false, true) {
|
||||
close(s.firstChunkReady)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) {
|
||||
s.chunkCBMu.Lock()
|
||||
cbs := append([]func(int, int){}, s.chunkCallbacks...)
|
||||
s.chunkCBMu.Unlock()
|
||||
for _, cb := range cbs {
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("[store] OnChunkLoaded callback panic: %v", r)
|
||||
}
|
||||
}()
|
||||
cb(rowsThisChunk, totalRows)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// RunStartupLoad orchestrates the startup load sequence:
|
||||
// 1. run LoadChunked synchronously (FirstChunkReady is signaled
|
||||
// internally so callers waiting on it in parallel can bind HTTP)
|
||||
// 2. on success, run loadBackgroundChunks synchronously so
|
||||
// s.oldestLoaded is guaranteed set before the bg loader reads
|
||||
// it (#1809).
|
||||
//
|
||||
// chunkSize=0 uses the LoadChunked default. The function blocks until
|
||||
// LoadChunked AND any background loader have finished. Callers that
|
||||
// want to bind the HTTP listener at FirstChunkReady should run this
|
||||
// in a goroutine and wait on FirstChunkReady() themselves.
|
||||
//
|
||||
// SYNCHRONOUS-CALL DEADLOCK WARNING (adv #10): callers that block on
|
||||
// FirstChunkReady() from the SAME goroutine that calls RunStartupLoad
|
||||
// will deadlock — LoadChunked only closes firstChunkReady from within
|
||||
// this call, and RunStartupLoad does not return until ALL chunks
|
||||
// (including the background loader) have finished. main.go's pattern
|
||||
// is correct: spawn RunStartupLoad in a goroutine and wait on
|
||||
// FirstChunkReady() on the parent stack. Do not call RunStartupLoad
|
||||
// inline.
|
||||
//
|
||||
// SINGLE-CALL INVARIANT (adv #7): RunStartupLoad is invoked exactly
|
||||
// once per process from main.go boot. Tests open a fresh store per
|
||||
// test. Re-entry would observe a stale backgroundLoadErr from a
|
||||
// previous failed call until the new LoadChunked or
|
||||
// loadBackgroundChunks rewrote it. We clear backgroundLoadErr on
|
||||
// entry to make repeat invocations in tests safe; a panic on
|
||||
// reentrant concurrent calls is not added because the production
|
||||
// invariant (one call per boot) is already documented above
|
||||
// LoadChunked.
|
||||
//
|
||||
// Steady-state contracts post-return:
|
||||
// - LoadChunked error: backgroundLoadFailed=true, backgroundLoadDone
|
||||
// is also set true (terminal observable state — see dij #1).
|
||||
// backgroundLoadErr non-empty. Returns the error.
|
||||
// - hotStartupHours == 0: backgroundLoadDone=true, failed=false
|
||||
// (no background work was needed). Coverage is computed against
|
||||
// row count so /api/perf does not report ratio=0.0 (dij #2).
|
||||
// - hotStartupHours > 0 success: terminal state is whatever
|
||||
// loadBackgroundChunks set (done=true on full coverage,
|
||||
// failed=true on partial / chunk errors — see #1690).
|
||||
//
|
||||
// Issue #1809 root cause: previously main.go spawned loadBackgroundChunks
|
||||
// at FirstChunkReady while LoadChunked was still merging the remainder
|
||||
// of the hot window. s.oldestLoaded is only assigned at the end of
|
||||
// LoadChunked, so the bg loader read "" and bailed → coverage gate
|
||||
// trips → backgroundLoadFailed=true. Running the bg loader after
|
||||
// LoadChunked returns preserves the FirstChunkReady HTTP-bind
|
||||
// parallelism while ensuring oldestLoaded has a valid floor when the
|
||||
// bg loader starts.
|
||||
func (s *PacketStore) RunStartupLoad(chunkSize int) error {
|
||||
// Clear any stale error from a previous invocation (single-call
|
||||
// invariant — see godoc above). Production never re-enters but
|
||||
// test fixtures may construct fresh stores that share no state;
|
||||
// keeping this explicit avoids surprising "ghost error" reads
|
||||
// across test runs that share a store.
|
||||
s.bgErrMu.Lock()
|
||||
s.backgroundLoadErr = ""
|
||||
s.bgErrMu.Unlock()
|
||||
|
||||
if err := s.LoadChunked(chunkSize); err != nil {
|
||||
// Pick a terminal state on the error path so /api/healthz +
|
||||
// backgroundLoadComplete don't stay undefined forever. Both
|
||||
// done AND failed are set true: the LoadChunked read-protocol
|
||||
// contract documented in store.go says `done` is the primary
|
||||
// observable terminal signal and `failed` qualifies it (read
|
||||
// done first, then check failed). See dij #1.
|
||||
s.bgErrMu.Lock()
|
||||
s.backgroundLoadErr = fmt.Sprintf("LoadChunked failed: %v", err)
|
||||
s.bgErrMu.Unlock()
|
||||
s.backgroundLoadFailed.Store(true)
|
||||
s.backgroundLoadDone.Store(true)
|
||||
return err
|
||||
}
|
||||
if s.hotStartupHours <= 0 {
|
||||
// No bg work required → terminal steady state is done=true,
|
||||
// failed=false. Without this the healthz probe would see
|
||||
// backgroundLoadComplete=false indefinitely. dij #2: compute
|
||||
// coverage so /api/perf doesn't report ratio=0 on the
|
||||
// "no-work" path — empty DB is 1.0, otherwise everything in
|
||||
// the DB is what we already loaded into memory.
|
||||
s.mu.RLock()
|
||||
loadedCount := int64(len(s.packets))
|
||||
s.mu.RUnlock()
|
||||
var totalInDB int64
|
||||
if err := s.db.conn.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&totalInDB); err != nil {
|
||||
totalInDB = -1
|
||||
}
|
||||
var ratio float64
|
||||
switch {
|
||||
case totalInDB <= 0:
|
||||
// Empty DB (0) or count failed (-1) → treat as fully covered.
|
||||
ratio = 1.0
|
||||
case loadedCount >= totalInDB:
|
||||
ratio = 1.0
|
||||
default:
|
||||
ratio = float64(loadedCount) / float64(totalInDB)
|
||||
}
|
||||
s.bgErrMu.Lock()
|
||||
s.loadCoverageRatio = ratio
|
||||
s.bgErrMu.Unlock()
|
||||
log.Printf("[store] RunStartupLoad: hotStartupHours=0 — background fill loader SKIPPED (coverage=%.1f%%, loaded=%d, totalInDB=%d)",
|
||||
ratio*100, loadedCount, totalInDB)
|
||||
s.backgroundLoadDone.Store(true)
|
||||
s.backgroundLoadProgress.Store(100)
|
||||
return nil
|
||||
}
|
||||
// INFO signal between LoadChunked completion and the bg loader
|
||||
// kick-off. The post-mortem of #1809 needed exactly this line to
|
||||
// confirm the bg loader actually started after oldestLoaded was set.
|
||||
log.Printf("[store] LoadChunked complete (oldestLoaded=%q) — starting background fill loader (retentionHours=%gh, hotStartupHours=%gh)",
|
||||
s.oldestLoaded, s.retentionHours, s.hotStartupHours)
|
||||
s.loadBackgroundChunks()
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadChunked streams transmissions + observations from SQLite into
|
||||
// the in-memory store in id-ordered chunks of `chunkSize` rows. Pass
|
||||
// 0 to use the default (10000).
|
||||
//
|
||||
// After the first chunk is merged, FirstChunkReady is closed and the
|
||||
// HTTP listener may bind. Remaining chunks stream while handlers run
|
||||
// against partially-populated data; loadStatusMiddleware advertises
|
||||
// loading status until LoadComplete() returns true.
|
||||
//
|
||||
// Re-entrancy: LoadChunked is NOT safe to call concurrently with
|
||||
// itself on the same PacketStore — it resets loadComplete /
|
||||
// loadProgressRows and mutates store-shared maps under s.mu. In
|
||||
// production it is invoked exactly once from main.go boot. Tests that
|
||||
// open a fresh store per test are also safe. If a future caller needs
|
||||
// repeat or concurrent loads, add a top-level mutex first.
|
||||
func (s *PacketStore) LoadChunked(chunkSize int) error {
|
||||
if chunkSize <= 0 {
|
||||
chunkSize = 10000
|
||||
}
|
||||
// Startup-ordering invariant (PR #1643 R1 munger #2). Mirror the
|
||||
// guard in Load() so the production async path also fast-fails when
|
||||
// neighbor_edges has rows but the graph is missing. See Load() for
|
||||
// the full rationale.
|
||||
if neighborEdgesTableExists(s.db.conn) && s.graph.Load() == nil {
|
||||
panic("packet store LoadChunked(): neighbor_edges table has rows but s.graph is nil — graph must be loaded before packet load (see main.go #1643 invariant)")
|
||||
}
|
||||
s.chunkedLoadInit()
|
||||
// Reset state for repeat calls in tests.
|
||||
s.loadComplete.Store(false)
|
||||
s.loadProgressRows.Store(0)
|
||||
|
||||
// On any return — error OR success — unblock listeners that gate on
|
||||
// the readiness signal so an empty/failed DB does not deadlock the
|
||||
// caller. Note: loadComplete is set on the success path only (see
|
||||
// the end of this function) so probes do NOT see ready=true after a
|
||||
// failed load.
|
||||
defer s.signalFirstChunk()
|
||||
|
||||
t0 := time.Now()
|
||||
|
||||
// Build the retention/memory filter the legacy Load() uses so
|
||||
// behavior is preserved when callers migrate from Load → LoadChunked.
|
||||
// Built against the `t2` alias used inside the chunk subquery so we
|
||||
// don't need brittle post-hoc string rewrites.
|
||||
var loadConditions []string
|
||||
hotCutoffHours := s.retentionHours
|
||||
if s.hotStartupHours > 0 {
|
||||
hotCutoffHours = s.hotStartupHours
|
||||
}
|
||||
var hotCutoffStr string
|
||||
var hotCutoffUnix int64
|
||||
if hotCutoffHours > 0 {
|
||||
hotCutoffT := time.Now().UTC().Add(-time.Duration(hotCutoffHours * float64(time.Hour)))
|
||||
hotCutoffStr = hotCutoffT.Format(time.RFC3339)
|
||||
hotCutoffUnix = hotCutoffT.Unix()
|
||||
_ = hotCutoffUnix
|
||||
// #1690: filter on the denormalized last_seen (effective recency)
|
||||
// rather than first_seen, so long-lived hashes with recent traffic
|
||||
// load on cold-start. first_seen is set once and never updated, so
|
||||
// the prior `t2.first_seen >= cutoff` query loaded only hashes
|
||||
// first-inserted within the window (0.3% of DB on prod).
|
||||
//
|
||||
// Test/legacy DBs without the column (PRAGMA-detected as
|
||||
// hasLastSeen=false) fall back to the legacy first_seen axis to
|
||||
// keep existing fixtures green. Production goes through
|
||||
// dbschema.AssertReady which fail-fasts when the column is
|
||||
// missing — so the fallback is only ever hit in tests.
|
||||
if s.db.hasLastSeen {
|
||||
loadConditions = append(loadConditions, fmt.Sprintf("t2.last_seen >= %d", hotCutoffUnix))
|
||||
} else {
|
||||
loadConditions = append(loadConditions, fmt.Sprintf("t2.first_seen >= '%s'", hotCutoffStr))
|
||||
}
|
||||
}
|
||||
|
||||
// COUNT honours the same retention/hot-startup filter the chunk
|
||||
// loop applies, so the logged "DB total" matches the rows the
|
||||
// loop will actually walk. Use a `t2` alias to share the WHERE
|
||||
// builder above. If the count fails (e.g. empty DB, locked WAL),
|
||||
// fall through with -1 — it's only used for the post-load log line.
|
||||
totalInDB := -1
|
||||
countSQL := "SELECT COUNT(*) FROM transmissions t2"
|
||||
if len(loadConditions) > 0 {
|
||||
countSQL += " WHERE " + strings.Join(loadConditions, " AND ")
|
||||
}
|
||||
if err := s.db.conn.QueryRow(countSQL).Scan(&totalInDB); err != nil {
|
||||
totalInDB = -1
|
||||
}
|
||||
|
||||
// Memory cap honoured by clamping the maximum cursor walk.
|
||||
var maxPackets int64
|
||||
if s.maxMemoryMB > 0 {
|
||||
avgBytes := int64(1000)
|
||||
if sample := estimateStoreTxBytesTypical(10); sample > avgBytes {
|
||||
avgBytes = sample
|
||||
}
|
||||
maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes
|
||||
if maxPackets < 1000 {
|
||||
maxPackets = 1000
|
||||
}
|
||||
}
|
||||
|
||||
chunkIdx := 0
|
||||
totalLoaded := 0
|
||||
// Start the id cursor BELOW the minimum possible row id so the
|
||||
// first chunk's `t2.id > cursorID` predicate includes id=0. The
|
||||
// e2e fixture seed for issue #1486 inserts the grouped-packet row
|
||||
// with id=0 (so it sorts LAST in the default packets view via
|
||||
// `ORDER BY id DESC` / oldest first_seen). Seeding the cursor at
|
||||
// 0 silently excluded that row, leaving the page with no
|
||||
// tr[data-hash] and timing out the playwright wait. Legacy Load()
|
||||
// had no id cursor and loaded id=0 unconditionally — we restore
|
||||
// that semantic by starting one below SQLite's minimum rowid (-1).
|
||||
var cursorID int64 = -1
|
||||
|
||||
// Relay-hop fallback inputs, fetched ONCE before the chunk-query loop.
|
||||
// getCachedNodesAndPM issues its own DB query, so calling it while a
|
||||
// chunk cursor is open would deadlock on a single-connection SQLite
|
||||
// pool. resolved_path is never persisted post-#1287, so scanAndMergeChunk
|
||||
// re-resolves relay hops from path_json using these snapshots.
|
||||
// PR #1643 R1 munger #1: cold load uses unique_prefix-only gate, so
|
||||
// the neighbor graph is no longer consulted here (affinity-tier
|
||||
// resolution against ≤168h-old observations would silently mis-attribute).
|
||||
s.mu.RLock()
|
||||
_, relayPM := s.getCachedNodesAndPM()
|
||||
s.mu.RUnlock()
|
||||
var coldLoadAmbiguousHopsSkipped int
|
||||
|
||||
for {
|
||||
conds := append([]string{}, loadConditions...)
|
||||
conds = append(conds, fmt.Sprintf("t2.id > %d", cursorID))
|
||||
whereClause := "WHERE " + strings.Join(conds, " AND ")
|
||||
|
||||
rpCol := ""
|
||||
if s.db.hasResolvedPath {
|
||||
rpCol = ", o.resolved_path"
|
||||
}
|
||||
obsRawHexCol := ""
|
||||
if s.db.hasObsRawHex {
|
||||
obsRawHexCol = ", o.raw_hex"
|
||||
}
|
||||
// #1751: scope_name is on the transmission row, so appending it as the
|
||||
// last selected column is safe regardless of the observation fan-out.
|
||||
scopeNameCol := ""
|
||||
if s.db.hasScopeName {
|
||||
scopeNameCol = ", t.scope_name"
|
||||
}
|
||||
|
||||
var chunkSQL string
|
||||
if s.db.isV3 {
|
||||
chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
|
||||
t.payload_type, t.payload_version, t.decoded_json,
|
||||
o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction,
|
||||
o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + `
|
||||
FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t
|
||||
LEFT JOIN observations o ON o.transmission_id = t.id
|
||||
LEFT JOIN observers obs ON obs.rowid = o.observer_idx
|
||||
ORDER BY t.id ASC, o.timestamp DESC`
|
||||
} else {
|
||||
chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type,
|
||||
t.payload_type, t.payload_version, t.decoded_json,
|
||||
o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction,
|
||||
o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + `
|
||||
FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t
|
||||
LEFT JOIN observations o ON o.transmission_id = t.id
|
||||
LEFT JOIN observers obs ON obs.id = o.observer_id
|
||||
ORDER BY t.id ASC, o.timestamp DESC`
|
||||
}
|
||||
|
||||
rows, err := s.db.conn.Query(chunkSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("chunk %d: query: %w", chunkIdx, err)
|
||||
}
|
||||
|
||||
chunkTxCount, lastID, err := s.scanAndMergeChunk(rows, relayPM, &coldLoadAmbiguousHopsSkipped)
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("chunk %d: scan: %w", chunkIdx, err)
|
||||
}
|
||||
|
||||
if chunkTxCount == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cursorID = lastID
|
||||
totalLoaded += chunkTxCount
|
||||
chunkIdx++
|
||||
s.loadProgressRows.Store(int64(totalLoaded))
|
||||
s.signalFirstChunk()
|
||||
s.fireChunkCallbacks(chunkTxCount, totalLoaded)
|
||||
|
||||
if maxPackets > 0 && int64(totalLoaded) >= maxPackets {
|
||||
break
|
||||
}
|
||||
if chunkTxCount < chunkSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Post-load: pick best observation, build indexes — same shape as
|
||||
// legacy Load().
|
||||
s.mu.Lock()
|
||||
for _, tx := range s.packets {
|
||||
pickBestObservation(tx)
|
||||
s.indexByNode(tx)
|
||||
}
|
||||
// Restore the "s.packets sorted oldest-first by FirstSeen" invariant
|
||||
// that legacy Load() got for free from "ORDER BY t.first_seen ASC".
|
||||
// LoadChunked walks chunks in id-ASC order so the slice ends up
|
||||
// id-ordered, which only equals first_seen-ordered when ids and
|
||||
// timestamps are correlated. After tools/freshen-fixture.sh (or any
|
||||
// real-world out-of-order ingest) they're not, leaving
|
||||
// s.packets[0].FirstSeen pointing at the newest row — which then
|
||||
// poisons oldestLoaded below and routes legitimate in-memory queries
|
||||
// to the SQL fallback. GetTimestamps (store.go) and QueryPackets
|
||||
// both rely on this invariant. See PR #1596 / mobile e2e regression.
|
||||
sort.SliceStable(s.packets, func(i, j int) bool {
|
||||
return s.packets[i].FirstSeen < s.packets[j].FirstSeen
|
||||
})
|
||||
s.buildSubpathIndex()
|
||||
s.buildPathHopIndex()
|
||||
s.buildDistanceIndex()
|
||||
if s.hotStartupHours > 0 {
|
||||
s.oldestLoaded = hotCutoffStr
|
||||
} else if len(s.packets) > 0 {
|
||||
s.oldestLoaded = s.packets[0].FirstSeen
|
||||
}
|
||||
s.loaded = true
|
||||
s.mu.Unlock()
|
||||
|
||||
// #1009 / PR #1596: flip the subpath + pathHop ready flags now that
|
||||
// the chunk loader has built both indexes synchronously above.
|
||||
// Without this, WaitIndexesReady (used by
|
||||
// StartRepeaterEnrichmentRecomputer at boot) blocks for up to
|
||||
// repeaterEnrichmentPrewarmWait (60s), delaying HTTP listener bind
|
||||
// past CI's 30s /api/healthz deadline.
|
||||
s.markIndexesReadySync()
|
||||
|
||||
elapsed := time.Since(t0)
|
||||
log.Printf("[store] LoadChunked: %d transmissions (%d observations) across %d chunk(s) in %v (chunkSize=%d, DB total=%d)",
|
||||
totalLoaded, s.totalObs, chunkIdx, elapsed, chunkSize, totalInDB)
|
||||
if coldLoadAmbiguousHopsSkipped > 0 {
|
||||
log.Printf("[store] LoadChunked: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)",
|
||||
coldLoadAmbiguousHopsSkipped)
|
||||
}
|
||||
s.loadMultibyteCapFromDB()
|
||||
// Mark complete on the success path only — see the function-level
|
||||
// defer above for why this is NOT in a deferred call. Probes that
|
||||
// read LoadComplete()==true after a failed load would otherwise
|
||||
// see ready=true for a half-loaded store.
|
||||
s.loadComplete.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanAndMergeChunk consumes one chunk's rows under s.mu.Lock and
|
||||
// returns the number of distinct transmissions seen + the max
|
||||
// transmission id (cursor for the next chunk).
|
||||
func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, coldLoadAmbiguousHopsSkipped *int) (int, int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
hopsSeen := make(map[string]bool)
|
||||
seenTxIDs := make(map[int]bool)
|
||||
var maxID int64
|
||||
|
||||
for rows.Next() {
|
||||
var txID int
|
||||
var rawHex, hash, firstSeen, decodedJSON sql.NullString
|
||||
var routeType, payloadType, payloadVersion sql.NullInt64
|
||||
var obsID sql.NullInt64
|
||||
var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString
|
||||
var snr, rssi sql.NullFloat64
|
||||
var score sql.NullInt64
|
||||
var obsRawHex sql.NullString
|
||||
var resolvedPathStr sql.NullString
|
||||
var scopeName sql.NullString
|
||||
|
||||
scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType,
|
||||
&payloadVersion, &decodedJSON,
|
||||
&obsID, &observerID, &observerName, &observerIATA, &direction,
|
||||
&snr, &rssi, &score, &pathJSON, &obsTimestamp}
|
||||
if s.db.hasObsRawHex {
|
||||
scanArgs = append(scanArgs, &obsRawHex)
|
||||
}
|
||||
if s.db.hasResolvedPath {
|
||||
scanArgs = append(scanArgs, &resolvedPathStr)
|
||||
}
|
||||
if s.db.hasScopeName {
|
||||
scanArgs = append(scanArgs, &scopeName)
|
||||
}
|
||||
if err := rows.Scan(scanArgs...); err != nil {
|
||||
log.Printf("[store] LoadChunked scan error: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if int64(txID) > maxID {
|
||||
maxID = int64(txID)
|
||||
}
|
||||
seenTxIDs[txID] = true
|
||||
|
||||
hashStr := nullStrVal(hash)
|
||||
tx := s.byHash[hashStr]
|
||||
if tx == nil {
|
||||
tx = &StoreTx{
|
||||
ID: txID,
|
||||
RawHex: nullStrVal(rawHex),
|
||||
Hash: hashStr,
|
||||
FirstSeen: nullStrVal(firstSeen),
|
||||
LatestSeen: nullStrVal(firstSeen),
|
||||
RouteType: nullIntPtr(routeType),
|
||||
PayloadType: nullIntPtr(payloadType),
|
||||
DecodedJSON: nullStrVal(decodedJSON),
|
||||
ScopeName: nullStrVal(scopeName),
|
||||
obsKeys: make(map[string]bool),
|
||||
observerSet: make(map[string]bool),
|
||||
}
|
||||
s.byHash[hashStr] = tx
|
||||
s.packets = append(s.packets, tx)
|
||||
s.byTxID[txID] = tx
|
||||
if txID > s.maxTxID {
|
||||
s.maxTxID = txID
|
||||
}
|
||||
s.indexByNode(tx)
|
||||
if tx.PayloadType != nil {
|
||||
pt := *tx.PayloadType
|
||||
s.byPayloadType[pt] = append(s.byPayloadType[pt], tx)
|
||||
}
|
||||
s.trackAdvertPubkey(tx)
|
||||
s.trackedBytes += estimateStoreTxBytes(tx)
|
||||
}
|
||||
|
||||
if obsID.Valid {
|
||||
oid := int(obsID.Int64)
|
||||
obsIDStr := nullStrVal(observerID)
|
||||
obsPJ := nullStrVal(pathJSON)
|
||||
|
||||
dk := obsIDStr + "|" + obsPJ
|
||||
if tx.obsKeys[dk] {
|
||||
continue
|
||||
}
|
||||
|
||||
obs := &StoreObs{
|
||||
ID: oid,
|
||||
TransmissionID: txID,
|
||||
ObserverID: obsIDStr,
|
||||
ObserverName: nullStrVal(observerName),
|
||||
ObserverIATA: nullStrVal(observerIATA),
|
||||
Direction: nullStrVal(direction),
|
||||
SNR: nullFloatPtr(snr),
|
||||
RSSI: nullFloatPtr(rssi),
|
||||
Score: nullIntPtr(score),
|
||||
PathJSON: obsPJ,
|
||||
// obs.RawHex deliberately NOT stored: it duplicates the parent
|
||||
// tx.RawHex (same content hash ⇒ same frame) and enrichObs falls
|
||||
// back to tx.RawHex when obs.RawHex == "". obsRawHex is still
|
||||
// scanned to keep scanArgs aligned with the o.raw_hex column.
|
||||
Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)),
|
||||
}
|
||||
|
||||
rpStr := nullStrVal(resolvedPathStr)
|
||||
if rpStr != "" {
|
||||
rp := unmarshalResolvedPath(rpStr)
|
||||
pks := extractResolvedPubkeys(rp)
|
||||
s.indexResolvedPathHops(tx, pks, hopsSeen)
|
||||
} else if relayPM != nil && obsPJ != "" && obsPJ != "[]" {
|
||||
// resolved_path is NULL on live (since #1287 relay data is
|
||||
// persisted as neighbor_edges, not per-observation). Re-resolve
|
||||
// relay-hop attribution from path_json so relay nodes keep their
|
||||
// analytics history across a restart instead of rebuilding only
|
||||
// from post-restart live traffic. relayPM is passed in from
|
||||
// LoadChunked (fetched before any chunk cursor opened).
|
||||
// byNode ONLY — see the Load() counterpart for why the
|
||||
// resolved_path/path-hop indexes must NOT be populated here.
|
||||
// PR #1643 R1 munger #1: unique_prefix-only gate.
|
||||
rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, coldLoadAmbiguousHopsSkipped)
|
||||
for _, pk := range extractResolvedPubkeys(rp) {
|
||||
s.addToByNode(tx, pk)
|
||||
}
|
||||
}
|
||||
|
||||
tx.Observations = append(tx.Observations, obs)
|
||||
tx.obsKeys[dk] = true
|
||||
if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] {
|
||||
tx.observerSet[obs.ObserverID] = true
|
||||
tx.UniqueObserverCount++
|
||||
}
|
||||
tx.ObservationCount++
|
||||
if obs.Timestamp > tx.LatestSeen {
|
||||
tx.LatestSeen = obs.Timestamp
|
||||
}
|
||||
|
||||
s.byObsID[oid] = obs
|
||||
if oid > s.maxObsID {
|
||||
s.maxObsID = oid
|
||||
}
|
||||
if obsIDStr != "" {
|
||||
s.byObserver[obsIDStr] = append(s.byObserver[obsIDStr], obs)
|
||||
}
|
||||
s.totalObs++
|
||||
s.trackedBytes += estimateStoreObsBytes(obs)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return len(seenTxIDs), maxID, err
|
||||
}
|
||||
return len(seenTxIDs), maxID, nil
|
||||
}
|
||||
|
||||
// loadStatusMiddleware sets X-CoreScope-Load-Status on every response.
|
||||
// While LoadChunked is in flight the header reports
|
||||
// "loading; progress=<rows>"; after completion it reports "ready".
|
||||
// The header is set BEFORE calling the next handler so probes can
|
||||
// observe it on any response (including streaming bodies).
|
||||
func loadStatusMiddleware(s *PacketStore, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if s != nil && s.LoadComplete() {
|
||||
w.Header().Set("X-CoreScope-Load-Status", "ready")
|
||||
} else if s != nil {
|
||||
w.Header().Set("X-CoreScope-Load-Status",
|
||||
fmt.Sprintf("loading; progress=%d", s.LoadProgress()))
|
||||
} else {
|
||||
w.Header().Set("X-CoreScope-Load-Status", "loading")
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// --- runtime state stitched into PacketStore via store_chunked.go ---
|
||||
|
||||
// Forward declarations of the new PacketStore fields used above. The
|
||||
// actual struct fields live in store.go; placing them here as a
|
||||
// reminder keeps the chunked-load surface easy to audit.
|
||||
var _ = sync.Once{}
|
||||
var _ atomic.Bool
|
||||
@@ -1,63 +0,0 @@
|
||||
package main
|
||||
|
||||
// Issue #1009 follow-up tests for PR #1596:
|
||||
//
|
||||
// (A) LoadChunked must flip subpath + pathHop index ready flags
|
||||
// after building those indexes. Otherwise WaitIndexesReady (used
|
||||
// by StartRepeaterEnrichmentRecomputer at boot) blocks the
|
||||
// caller for up to repeaterEnrichmentPrewarmWait (60s), which is
|
||||
// why CI's "Start Go server" step times out before /api/healthz
|
||||
// can answer within its 30s deadline.
|
||||
//
|
||||
// (B) LoadChunked must NOT report LoadComplete()==true when it
|
||||
// returns an error. Today a defer unconditionally calls
|
||||
// s.loadComplete.Store(true), so a failed load appears "ready"
|
||||
// to probes and the load-status middleware.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// (A) Indexes must be marked ready by LoadChunked.
|
||||
func TestLoadChunked_MarksIndexesReady(t *testing.T) {
|
||||
store := openChunkedTestStore(t, 100)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if store.SubpathIndexReady() || store.PathHopIndexReady() {
|
||||
t.Fatal("indexes must start NOT ready")
|
||||
}
|
||||
|
||||
if err := store.LoadChunked(50); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
|
||||
if !store.SubpathIndexReady() {
|
||||
t.Fatal("SubpathIndexReady() must be true after LoadChunked builds the index")
|
||||
}
|
||||
if !store.PathHopIndexReady() {
|
||||
t.Fatal("PathHopIndexReady() must be true after LoadChunked builds the index")
|
||||
}
|
||||
}
|
||||
|
||||
// (B) LoadChunked errors must not flip LoadComplete=true.
|
||||
func TestLoadChunked_ErrorDoesNotMarkComplete(t *testing.T) {
|
||||
store := openChunkedTestStore(t, 100)
|
||||
|
||||
// Close the underlying DB so the very first chunk query fails.
|
||||
if err := store.db.conn.Close(); err != nil {
|
||||
t.Fatalf("close DB: %v", err)
|
||||
}
|
||||
|
||||
err := store.LoadChunked(50)
|
||||
if err == nil {
|
||||
t.Fatal("LoadChunked must return an error when the DB query fails")
|
||||
}
|
||||
if !errors.Is(err, err) { // satisfy linters; the assertion below is what matters
|
||||
t.Fatalf("unexpected error shape: %v", err)
|
||||
}
|
||||
|
||||
if store.LoadComplete() {
|
||||
t.Fatal("LoadComplete() must remain false after LoadChunked returns an error")
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression for PR #1596 / issue #1486 e2e: LoadChunked uses
|
||||
// `cursorID = 0` with a `t2.id > cursorID` predicate, which silently
|
||||
// excludes any transmission with id=0. The e2e seed for #1486 inserts
|
||||
// the grouped-packet row with id=0 (so it sorts LAST in the default
|
||||
// packets view), and the page deep-links to /packets?hash=<seed>.
|
||||
// With the chunked loader skipping id=0, the in-memory store never
|
||||
// learns about the row; QueryGroupedPackets returns 0; the page
|
||||
// renders no `tr[data-hash]` and the e2e times out at 12s.
|
||||
//
|
||||
// Legacy Load() walked all transmissions unconditionally (no id
|
||||
// cursor) and therefore included id=0. Restoring that semantic — by
|
||||
// using a non-existent sentinel (-1) on the first iteration, or by
|
||||
// switching the predicate to `>=` for the initial pass — fixes the
|
||||
// regression.
|
||||
//
|
||||
// This test inserts a transmission with id=0 plus a handful of
|
||||
// id>=1 transmissions and asserts that LoadChunked loads the id=0
|
||||
// row into s.byHash.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER,
|
||||
payload_version INTEGER, decoded_json TEXT
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`,
|
||||
`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`,
|
||||
`INSERT INTO schema_version (version) VALUES (1)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`,
|
||||
}
|
||||
for _, s := range stmts {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
tb.Fatalf("setup exec: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
|
||||
txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
// id=0: the #1486-style seed row, within retention window.
|
||||
txStmt.Exec(0, "1500", "fae0c9e6d357a814", now.Add(-1*time.Minute).Format(time.RFC3339), 1, 5, 0, `{"type":"CHAN"}`)
|
||||
obsStmt.Exec(0, 0, "obs1", "Obs1", "rx", 5.0, -95.0, 0, `["AA"]`, now.Add(-1*time.Minute).Unix())
|
||||
|
||||
for i := 1; i <= extraTx; i++ {
|
||||
ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339)
|
||||
unixTs := now.Add(-time.Duration(i+1) * time.Minute).Unix()
|
||||
hash := fmt.Sprintf("h%04d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
|
||||
obsStmt.Exec(i, i, "obs1", "Obs1", "rx", -10.0, -80.0, 5, `["aa","bb"]`, unixTs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadChunked_IncludesIDZero: LoadChunked must load transmissions
|
||||
// with id=0. The legacy Load() (since-replaced by LoadChunked) walked
|
||||
// transmissions unconditionally; LoadChunked uses an id-cursor that
|
||||
// starts at 0 with a strict `t2.id > cursorID` predicate, so id=0
|
||||
// rows are silently dropped. This breaks the #1486 e2e fixture seed
|
||||
// which uses id=0 to sort the grouped row last in the default view.
|
||||
func TestLoadChunked_IncludesIDZero(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "idzero.db")
|
||||
createTestDBWithIDZero(t, dbPath, 10)
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDB: %v", err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{}
|
||||
store := NewPacketStore(db, cfg)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if err := store.LoadChunked(5); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
|
||||
if _, ok := store.byHash["fae0c9e6d357a814"]; !ok {
|
||||
t.Fatalf("LoadChunked dropped the id=0 transmission: "+
|
||||
"byHash[fae0c9e6d357a814] missing; loaded %d packets total "+
|
||||
"(id-cursor starts at 0 with strict `t2.id > cursorID`, "+
|
||||
"so id=0 is excluded — this is the #1486 e2e regression)",
|
||||
len(store.packets))
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression for PR #1596 (issue #1009) chunked load: when transmission
|
||||
// ids are anti-correlated with first_seen (e.g. id=1 has the NEWEST
|
||||
// timestamp), LoadChunked walks id-ASC and the post-load
|
||||
// `s.oldestLoaded = s.packets[0].FirstSeen` line set oldestLoaded to
|
||||
// the NEWEST first_seen. QueryPackets then mis-routed any
|
||||
// `since>=oldestLoaded` query to the SQL fallback, hiding fresh
|
||||
// in-memory rows. This shows up in real life on the e2e fixture after
|
||||
// tools/freshen-fixture.sh shifts timestamps so id=1 (originally
|
||||
// loaded first) carries the most recent first_seen.
|
||||
//
|
||||
// The mobile e2e test test-observer-iata-1188-e2e.js fails as a
|
||||
// result: with the default 15-minute time window, /api/packets returns
|
||||
// 0 rows and the mobile DOM has no `tr[data-hash]` to tap.
|
||||
//
|
||||
// This test asserts the in-memory invariant: after LoadChunked,
|
||||
// oldestLoaded must equal the actual oldest FirstSeen across loaded
|
||||
// transmissions, not the FirstSeen of the first row in s.packets.
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// createTestDBReverseTime builds numTx transmissions whose ids run
|
||||
// 1..numTx ASC while first_seen runs newest..oldest (id=1 = newest).
|
||||
// This mirrors the freshen-fixture-shifted e2e DB exactly.
|
||||
func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) {
|
||||
tb.Helper()
|
||||
conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL")
|
||||
if err != nil {
|
||||
tb.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
stmts := []string{
|
||||
`CREATE TABLE IF NOT EXISTS transmissions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
raw_hex TEXT, hash TEXT, first_seen TEXT,
|
||||
route_type INTEGER, payload_type INTEGER,
|
||||
payload_version INTEGER, decoded_json TEXT
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
transmission_id INTEGER, observer_id TEXT, observer_name TEXT,
|
||||
direction TEXT, snr REAL, rssi REAL, score INTEGER,
|
||||
path_json TEXT, timestamp TEXT, raw_hex TEXT
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`,
|
||||
`CREATE TABLE IF NOT EXISTS nodes (
|
||||
pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL,
|
||||
last_seen TEXT, first_seen TEXT, frequency REAL
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`,
|
||||
`INSERT INTO schema_version (version) VALUES (1)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`,
|
||||
}
|
||||
for _, s := range stmts {
|
||||
if _, err := conn.Exec(s); err != nil {
|
||||
tb.Fatalf("setup exec: %v\nSQL: %s", err, s)
|
||||
}
|
||||
}
|
||||
|
||||
txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
defer txStmt.Close()
|
||||
defer obsStmt.Close()
|
||||
|
||||
// id=1 is the NEWEST (now); id=numTx is the OLDEST (numTx minutes ago).
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
for i := 1; i <= numTx; i++ {
|
||||
ts := now.Add(-time.Duration(i-1) * time.Minute).Format(time.RFC3339)
|
||||
unixTs := now.Add(-time.Duration(i-1) * time.Minute).Unix()
|
||||
hash := fmt.Sprintf("h%04d", i)
|
||||
txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i))
|
||||
obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, unixTs)
|
||||
}
|
||||
}
|
||||
|
||||
func openReverseTimeStore(t *testing.T, numTx int) *PacketStore {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "rev.db")
|
||||
createTestDBReverseTime(t, dbPath, numTx)
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDB: %v", err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{}
|
||||
return NewPacketStore(db, cfg)
|
||||
}
|
||||
|
||||
// TestLoadChunked_OldestLoadedIsActualOldest: when LoadChunked walks
|
||||
// transmissions in id-ASC order but timestamps are anti-correlated
|
||||
// with id (PR #1596 regression scenario), oldestLoaded MUST be the
|
||||
// minimum FirstSeen across loaded packets, not the first row's
|
||||
// FirstSeen. Otherwise QueryPackets routes "since=15min ago" to SQL
|
||||
// fallback, hiding fresh rows.
|
||||
func TestLoadChunked_OldestLoadedIsActualOldest(t *testing.T) {
|
||||
store := openReverseTimeStore(t, 50)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if err := store.LoadChunked(20); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
|
||||
// Compute the actual oldest first_seen across what got loaded.
|
||||
if len(store.packets) == 0 {
|
||||
t.Fatal("no packets loaded")
|
||||
}
|
||||
actualOldest := store.packets[0].FirstSeen
|
||||
for _, p := range store.packets {
|
||||
if p.FirstSeen < actualOldest {
|
||||
actualOldest = p.FirstSeen
|
||||
}
|
||||
}
|
||||
|
||||
if store.oldestLoaded != actualOldest {
|
||||
t.Fatalf("oldestLoaded=%q must equal actual MIN(FirstSeen)=%q "+
|
||||
"(id-ordered chunk walk with anti-correlated timestamps "+
|
||||
"left oldestLoaded pointing at the newest row, which makes "+
|
||||
"QueryPackets mis-route since-windowed queries to SQL fallback "+
|
||||
"and the mobile e2e test renders 0 rows)",
|
||||
store.oldestLoaded, actualOldest)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadChunked_PacketsSortedByFirstSeenASC: QueryPackets and
|
||||
// GetTimestamps both assume s.packets is "sorted oldest-first" (see
|
||||
// store.go:2125 comment on GetTimestamps). LoadChunked walks rows
|
||||
// id-ASC which only equals first_seen-ASC when ids and timestamps
|
||||
// are correlated — not true after fixture freshen, not true after
|
||||
// any out-of-order ingest. Assert the invariant directly.
|
||||
func TestLoadChunked_PacketsSortedByFirstSeenASC(t *testing.T) {
|
||||
store := openReverseTimeStore(t, 25)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
if err := store.LoadChunked(10); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
for i := 1; i < len(store.packets); i++ {
|
||||
if store.packets[i-1].FirstSeen > store.packets[i].FirstSeen {
|
||||
t.Fatalf("s.packets must be sorted by FirstSeen ASC; "+
|
||||
"packets[%d].FirstSeen=%q > packets[%d].FirstSeen=%q",
|
||||
i-1, store.packets[i-1].FirstSeen,
|
||||
i, store.packets[i].FirstSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package main
|
||||
|
||||
// Issue #1009: chunked Load with early HTTP readiness.
|
||||
//
|
||||
// These tests gate three behaviors:
|
||||
// (a) FirstChunkReady() unblocks BEFORE LoadChunked returns, so the
|
||||
// HTTP listener can bind after the first chunk completes while
|
||||
// remaining rows continue loading in the background.
|
||||
// (b) loadStatusMiddleware stamps an X-CoreScope-Load-Status header
|
||||
// with "loading" + progress while a load is in flight, flipping
|
||||
// to "ready" once LoadComplete() reports true.
|
||||
// (c) LoadChunked honors the configured chunkSize: the per-chunk
|
||||
// progress callback fires once per chunk, so a 2500-row DB with
|
||||
// chunkSize=1000 must yield 3 callbacks (1000 + 1000 + 500).
|
||||
//
|
||||
// Each subtest fails on an assertion (not a build error) when the
|
||||
// production code is absent — that is the red-commit contract.
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func openChunkedTestStore(t *testing.T, numTx int) *PacketStore {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
dbPath := filepath.Join(dir, "chunked.db")
|
||||
createTestDBAt(t, dbPath, numTx)
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
|
||||
db, err := OpenDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDB: %v", err)
|
||||
}
|
||||
cfg := &PacketStoreConfig{}
|
||||
return NewPacketStore(db, cfg)
|
||||
}
|
||||
|
||||
// (a) FirstChunkReady fires before LoadChunked returns.
|
||||
func TestLoadChunked_FirstChunkReadyBeforeComplete(t *testing.T) {
|
||||
store := openChunkedTestStore(t, 2500)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
doneCh := make(chan error, 1)
|
||||
go func() { doneCh <- store.LoadChunked(500) }()
|
||||
|
||||
select {
|
||||
case <-store.FirstChunkReady():
|
||||
// Good: first chunk signaled. Load may or may not have completed
|
||||
// for tiny test DBs, but the gate must have fired without
|
||||
// requiring the full load.
|
||||
case err := <-doneCh:
|
||||
// If load completed before we could observe the signal, the
|
||||
// signal still must be closed.
|
||||
if err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-store.FirstChunkReady():
|
||||
default:
|
||||
t.Fatal("FirstChunkReady channel must be closed after LoadChunked completes")
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("FirstChunkReady did not fire within 10s — listener would never bind")
|
||||
}
|
||||
|
||||
// Drain background completion.
|
||||
select {
|
||||
case err := <-doneCh:
|
||||
if err != nil {
|
||||
t.Fatalf("LoadChunked returned error: %v", err)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("LoadChunked never returned")
|
||||
}
|
||||
|
||||
if !store.LoadComplete() {
|
||||
t.Fatal("LoadComplete() must report true after LoadChunked returns")
|
||||
}
|
||||
}
|
||||
|
||||
// (b) Middleware stamps X-CoreScope-Load-Status correctly across the
|
||||
// loading→ready transition.
|
||||
func TestLoadStatusMiddleware_HeaderTransition(t *testing.T) {
|
||||
store := openChunkedTestStore(t, 100)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
handler := loadStatusMiddleware(store, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
// Pre-load: header must report "loading".
|
||||
req := httptest.NewRequest("GET", "/api/healthz", nil)
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
if got := w.Header().Get("X-CoreScope-Load-Status"); got == "" || got == "ready" {
|
||||
t.Fatalf("expected loading status header before Load, got %q", got)
|
||||
}
|
||||
|
||||
if err := store.LoadChunked(50); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
|
||||
// Post-load: header must report "ready".
|
||||
req2 := httptest.NewRequest("GET", "/api/healthz", nil)
|
||||
w2 := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w2, req2)
|
||||
if got := w2.Header().Get("X-CoreScope-Load-Status"); got != "ready" {
|
||||
t.Fatalf("expected X-CoreScope-Load-Status=ready after load, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (c) LoadChunked honors the chunkSize argument — progress callback
|
||||
// fires once per chunk.
|
||||
func TestLoadChunked_ChunkSizeHonored(t *testing.T) {
|
||||
store := openChunkedTestStore(t, 2500)
|
||||
defer store.db.conn.Close()
|
||||
|
||||
var chunks []int
|
||||
store.OnChunkLoaded(func(rowsThisChunk, totalRows int) {
|
||||
chunks = append(chunks, rowsThisChunk)
|
||||
})
|
||||
|
||||
if err := store.LoadChunked(1000); err != nil {
|
||||
t.Fatalf("LoadChunked: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("expected 3 chunks for 2500 rows @ chunkSize=1000, got %d (sizes=%v)", len(chunks), chunks)
|
||||
}
|
||||
if chunks[0] != 1000 || chunks[1] != 1000 || chunks[2] != 500 {
|
||||
t.Fatalf("expected chunk sizes [1000,1000,500], got %v", chunks)
|
||||
}
|
||||
}
|
||||
|
||||
// (d) Config plumbing: DB.Load.ChunkSize threads through.
|
||||
func TestConfig_DBLoadChunkSize(t *testing.T) {
|
||||
c := &Config{}
|
||||
if got := c.DBLoadChunkSize(); got != 10000 {
|
||||
t.Fatalf("DBLoadChunkSize() default = %d, want 10000", got)
|
||||
}
|
||||
c.DB = &DBConfig{Load: &dbLoadConfig{ChunkSize: 2500}}
|
||||
if got := c.DBLoadChunkSize(); got != 2500 {
|
||||
t.Fatalf("DBLoadChunkSize() configured = %d, want 2500", got)
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// clampLimit parses a `limit`-shaped string and clamps it into [1, max].
|
||||
// Empty / non-numeric / zero / negative inputs return def.
|
||||
// Values exceeding max are clamped to max.
|
||||
//
|
||||
// This is the uniform helper for list-endpoint `limit` parameters; prefer it
|
||||
// over inline `if limit > N { limit = N }` patterns so the absolute caps stay
|
||||
// consistent across handlers. See audit-input-vulns-20260603 (MEDIUM —
|
||||
// unbounded `limit` on list endpoints).
|
||||
func clampLimit(raw string, def, max int) int {
|
||||
if raw == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
return def
|
||||
}
|
||||
if n > max {
|
||||
return max
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// queryLimit reads the `limit` query parameter from r and clamps it through
|
||||
// clampLimit. Convenience wrapper used by HTTP handlers so existing
|
||||
// queryInt(r, "limit", def) call sites can become queryLimit(r, def, max).
|
||||
func queryLimit(r *http.Request, def, max int) int {
|
||||
return clampLimit(r.URL.Query().Get("limit"), def, max)
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestClampLimit covers the uniform list-endpoint limit-clamp helper added to
|
||||
// fix audit-input-vulns-20260603 (MEDIUM).
|
||||
func TestClampLimit(t *testing.T) {
|
||||
const def = 50
|
||||
const max = 500
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
{"empty returns default", "", def},
|
||||
{"non-numeric returns default", "abc", def},
|
||||
{"negative returns default", "-1", def},
|
||||
{"zero returns default", "0", def},
|
||||
{"mid-range value preserved", "100", 100},
|
||||
{"value at cap preserved", "500", 500},
|
||||
{"over-cap clamped to max", "999999999", max},
|
||||
{"just over cap clamped", "501", max},
|
||||
{"whitespace garbage returns default", " 100 ", def},
|
||||
{"float-shaped returns default", "10.5", def},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := clampLimit(tc.raw, def, max)
|
||||
if got != tc.want {
|
||||
t.Fatalf("clampLimit(%q, %d, %d) = %d, want %d", tc.raw, def, max, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -133,7 +133,6 @@ type NodeClockSkew struct {
|
||||
Samples []SkewSample `json:"samples,omitempty"` // time-series for sparklines
|
||||
GoodFraction float64 `json:"goodFraction"` // fraction of recent samples with |skew| <= 1h
|
||||
RecentBadSampleCount int `json:"recentBadSampleCount"` // count of recent samples with |skew| > 1h
|
||||
RecentBadSamples []BadSample `json:"recentBadSamples,omitempty"` // #1094: per-bad-sample evidence (hash + bad advertTS)
|
||||
RecentSampleCount int `json:"recentSampleCount"` // total recent samples in window
|
||||
RecentHashEvidence []HashEvidence `json:"recentHashEvidence,omitempty"`
|
||||
CalibrationSummary *CalibrationSummary `json:"calibrationSummary,omitempty"`
|
||||
@@ -147,15 +146,6 @@ type SkewSample struct {
|
||||
SkewSec float64 `json:"skew"` // corrected skew in seconds
|
||||
}
|
||||
|
||||
// BadSample is a single recent advert flagged as having a nonsense timestamp
|
||||
// (|corrected skew| in the bimodal-bad band — > 1h, <= 24h). #1094: surfaced
|
||||
// so the UI can link each offender to its packet detail page.
|
||||
type BadSample struct {
|
||||
Hash string `json:"hash"` // transmission hash for packet-detail deep-link
|
||||
AdvertTS int64 `json:"advertTS"` // the offending advert Unix timestamp
|
||||
SkewSec float64 `json:"skewSec"` // corrected skew vs observer at observation time
|
||||
}
|
||||
|
||||
// HashEvidenceObserver is one observer's contribution to a per-hash evidence entry.
|
||||
type HashEvidenceObserver struct {
|
||||
ObserverID string `json:"observerID"`
|
||||
@@ -522,7 +512,7 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
lastSkew = cs.LastSkewSec
|
||||
lastAdvTS = cs.LastAdvertTS
|
||||
}
|
||||
tsSkews = append(tsSkews, tsSkewPair{ts: cs.LastObservedTS, skew: cs.MedianSkewSec, hash: tx.Hash, advertTS: cs.LastAdvertTS})
|
||||
tsSkews = append(tsSkews, tsSkewPair{ts: cs.LastObservedTS, skew: cs.MedianSkewSec})
|
||||
}
|
||||
|
||||
if len(allSkews) == 0 {
|
||||
@@ -546,7 +536,6 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
|
||||
recentSkew := lastSkew
|
||||
var recentVals []float64
|
||||
var recentPairs []tsSkewPair
|
||||
if n := len(tsSkews); n > 0 {
|
||||
latestTS := tsSkews[n-1].ts
|
||||
// Index-based window: last K samples.
|
||||
@@ -570,7 +559,6 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
start = startByTime
|
||||
}
|
||||
recentVals = make([]float64, 0, n-start)
|
||||
recentPairs = tsSkews[start:n]
|
||||
for i := start; i < n; i++ {
|
||||
recentVals = append(recentVals, tsSkews[i].skew)
|
||||
}
|
||||
@@ -595,25 +583,13 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
// adverts had nonsense timestamps") on otherwise-healthy nodes.
|
||||
var goodSamples []float64
|
||||
var rtcResetCount int
|
||||
var recentBadSamples []BadSample // #1094: per-bad-sample evidence (hash + advertTS)
|
||||
for i, v := range recentVals {
|
||||
for _, v := range recentVals {
|
||||
absV := math.Abs(v)
|
||||
switch {
|
||||
case absV > rtcResetOutlierThresholdSec:
|
||||
rtcResetCount++ // ignored for good/bad classification
|
||||
case absV <= bimodalSkewThresholdSec:
|
||||
goodSamples = append(goodSamples, v)
|
||||
default:
|
||||
// Bimodal-bad: 1h < |skew| <= 24h. Capture hash + advertTS so
|
||||
// the UI can link each offender to its packet detail page
|
||||
// instead of showing a count without evidence (#1094).
|
||||
if i < len(recentPairs) && recentPairs[i].hash != "" {
|
||||
recentBadSamples = append(recentBadSamples, BadSample{
|
||||
Hash: recentPairs[i].hash,
|
||||
AdvertTS: recentPairs[i].advertTS,
|
||||
SkewSec: round(v, 1),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
recentSampleCount := len(recentVals) - rtcResetCount
|
||||
@@ -739,7 +715,6 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew {
|
||||
Samples: samples,
|
||||
GoodFraction: round(goodFraction, 2),
|
||||
RecentBadSampleCount: recentBadCount,
|
||||
RecentBadSamples: recentBadSamples,
|
||||
RecentSampleCount: recentSampleCount,
|
||||
RecentHashEvidence: recentEvidence,
|
||||
CalibrationSummary: &calSummary,
|
||||
@@ -900,16 +875,10 @@ func mean(vals []float64) float64 {
|
||||
return sum / float64(len(vals))
|
||||
}
|
||||
|
||||
// tsSkewPair is a (timestamp, skew) pair for drift estimation. Also carries
|
||||
// the source hash + advertTS so callers building per-sample evidence (e.g.
|
||||
// recentBadSamples for #1094) can identify the offending packet without a
|
||||
// second pass. Drift code reads only ts/skew; the extra fields are inert
|
||||
// there.
|
||||
// tsSkewPair is a (timestamp, skew) pair for drift estimation.
|
||||
type tsSkewPair struct {
|
||||
ts int64
|
||||
skew float64
|
||||
hash string
|
||||
advertTS int64
|
||||
ts int64
|
||||
skew float64
|
||||
}
|
||||
|
||||
// computeDrift estimates linear drift in seconds per day from time-ordered
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
package main
|
||||
|
||||
// Regression test for #1094: the bimodal-clock warning currently exposes only
|
||||
// RecentBadSampleCount, leaving the UI to render "⚠️ N of M adverts had
|
||||
// nonsense timestamps" without telling the operator WHICH packets were bad.
|
||||
//
|
||||
// This test pins the additive API contract: alongside the count, the response
|
||||
// must expose RecentBadSamples — a slice of (hash, advertTS, skewSec) — so the
|
||||
// frontend can render each offending hash as a clickable link with its bad
|
||||
// timestamp.
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Seeds 5 recent adverts: 3 healthy (~-20s skew) and 2 with a "nonsense"
|
||||
// bimodal-bad timestamp (|skew| in (1h, 24h]). The recent window is exactly
|
||||
// 5 samples, so all five are inside it.
|
||||
func seedIssue1094Repro(t *testing.T) (*PacketStore, []string, []int64) {
|
||||
t.Helper()
|
||||
ps := NewPacketStore(nil, nil)
|
||||
pt := 4 // ADVERT
|
||||
|
||||
const pubkey = "BADTS1094"
|
||||
baseObs := int64(1779000000)
|
||||
|
||||
var txs []*StoreTx
|
||||
var badHashes []string
|
||||
var badAdvertTSs []int64
|
||||
|
||||
// 3 healthy adverts (skew = -20s).
|
||||
for i := 0; i < 3; i++ {
|
||||
obsTS := baseObs + int64(i)*60
|
||||
advTS := obsTS - 20
|
||||
txs = append(txs, &StoreTx{
|
||||
Hash: "healthy-1094-" + formatInt64(int64(i)),
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// 2 nonsense-timestamp adverts (skew = -7200s = -2h — bimodal-bad,
|
||||
// below the 24h RTC-reset exclusion so they DO count in recentBadCount).
|
||||
for i := 0; i < 2; i++ {
|
||||
obsTS := baseObs + int64(3+i)*60
|
||||
advTS := obsTS - 7200
|
||||
hash := "bad-1094-" + formatInt64(int64(i))
|
||||
txs = append(txs, &StoreTx{
|
||||
Hash: hash,
|
||||
PayloadType: &pt,
|
||||
DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`,
|
||||
Observations: []*StoreObs{
|
||||
{ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)},
|
||||
},
|
||||
})
|
||||
badHashes = append(badHashes, hash)
|
||||
badAdvertTSs = append(badAdvertTSs, advTS)
|
||||
}
|
||||
|
||||
ps.mu.Lock()
|
||||
ps.byNode[pubkey] = txs
|
||||
for _, tx := range txs {
|
||||
ps.byPayloadType[4] = append(ps.byPayloadType[4], tx)
|
||||
}
|
||||
ps.clockSkew.computeInterval = 0
|
||||
ps.mu.Unlock()
|
||||
return ps, badHashes, badAdvertTSs
|
||||
}
|
||||
|
||||
func TestIssue1094_RecentBadSamples_ExposesHashAndTimestamp(t *testing.T) {
|
||||
ps, wantHashes, wantAdvertTSs := seedIssue1094Repro(t)
|
||||
r := ps.GetNodeClockSkew("BADTS1094")
|
||||
if r == nil {
|
||||
t.Fatal("expected clock skew result")
|
||||
}
|
||||
|
||||
// Pre-condition: count must already be 2 (gates the test against the
|
||||
// existing field — if this drops we'd be measuring the wrong thing).
|
||||
if r.RecentBadSampleCount != 2 {
|
||||
t.Fatalf("RecentBadSampleCount = %d, want 2 (seed bug, not the field-under-test)",
|
||||
r.RecentBadSampleCount)
|
||||
}
|
||||
|
||||
if len(r.RecentBadSamples) != 2 {
|
||||
t.Fatalf("RecentBadSamples len = %d, want 2 — operators need to see which "+
|
||||
"adverts had nonsense timestamps, not just the count",
|
||||
len(r.RecentBadSamples))
|
||||
}
|
||||
|
||||
gotByHash := map[string]int64{}
|
||||
for _, bs := range r.RecentBadSamples {
|
||||
gotByHash[bs.Hash] = bs.AdvertTS
|
||||
}
|
||||
for i, h := range wantHashes {
|
||||
ts, ok := gotByHash[h]
|
||||
if !ok {
|
||||
t.Errorf("RecentBadSamples missing hash %q", h)
|
||||
continue
|
||||
}
|
||||
if ts != wantAdvertTSs[i] {
|
||||
t.Errorf("RecentBadSamples[%q].AdvertTS = %d, want %d (the bad advertTS)",
|
||||
h, ts, wantAdvertTSs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
-392
@@ -8,7 +8,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/dbconfig"
|
||||
@@ -25,21 +24,11 @@ type AreaEntry struct {
|
||||
LonMax *float64 `json:"lonMax,omitempty"`
|
||||
}
|
||||
|
||||
// ListLimitsConfig defines maximum row limits for list endpoints to prevent DoS.
|
||||
type ListLimitsConfig struct {
|
||||
PacketsMax int `json:"packetsMax"`
|
||||
NodesMax int `json:"nodesMax"`
|
||||
AnalyticsMax int `json:"analyticsMax"`
|
||||
ChannelMessagesMax int `json:"channelMessagesMax"`
|
||||
BulkHealthMax int `json:"bulkHealthMax"`
|
||||
}
|
||||
|
||||
// Config mirrors the Node.js config.json structure (read-only fields).
|
||||
type Config struct {
|
||||
Port int `json:"port"`
|
||||
APIKey string `json:"apiKey"`
|
||||
DBPath string `json:"dbPath"`
|
||||
ListLimits *ListLimitsConfig `json:"listLimits"`
|
||||
Port int `json:"port"`
|
||||
APIKey string `json:"apiKey"`
|
||||
DBPath string `json:"dbPath"`
|
||||
|
||||
// NodeBlacklist is a list of public keys to exclude from all API responses.
|
||||
// Blacklisted nodes are hidden from node lists, search, detail, map, and stats.
|
||||
@@ -48,40 +37,9 @@ type Config struct {
|
||||
// operator refuses to fix.
|
||||
NodeBlacklist []string `json:"nodeBlacklist"`
|
||||
|
||||
// HiddenNamePrefixes is a list of name prefixes that mark a node as
|
||||
// hidden from API responses (issue #1181). The default `["🚫"]` mirrors
|
||||
// a convention used by other MeshCore map dashboards: operators who
|
||||
// rename their node with the prefix get hidden from the map without
|
||||
// waiting for normal retention to clear stale data. DB rows are
|
||||
// preserved — the filter is applied at the API layer only, so the
|
||||
// underlying observation history remains intact.
|
||||
HiddenNamePrefixes []string `json:"hiddenNamePrefixes"`
|
||||
|
||||
// hiddenPrefixesPtr holds the active prefix slice as an atomic pointer.
|
||||
// Read path (IsNameHidden) is a single atomic load — no mutex, no
|
||||
// sync.Once. Writers always replace the whole slice; readers see either
|
||||
// the old or the new slice as a single value, never a partial state.
|
||||
// Mirrors blacklistSetPtr.
|
||||
hiddenPrefixesPtr atomic.Pointer[[]string]
|
||||
|
||||
// hiddenPrefixesGen is a monotonic counter bumped every time the
|
||||
// hidden-prefix list mutates via SetHiddenNamePrefixes. Cache wiring
|
||||
// is left for follow-up; the counter is the prerequisite primitive
|
||||
// callers will key on (mirrors blacklistGen / #1629).
|
||||
hiddenPrefixesGen atomic.Uint64
|
||||
|
||||
// blacklistSetPtr holds the active lookup set as an atomic pointer.
|
||||
// Read path is a single atomic load — no mutex, no sync.Once. Writers
|
||||
// always replace the whole map; readers see either the old or the new
|
||||
// map as a single value, never a partially-built one.
|
||||
blacklistSetPtr atomic.Pointer[map[string]bool]
|
||||
|
||||
// blacklistGen is a monotonic generation counter bumped every time the
|
||||
// blacklist mutates via SetNodeBlacklist. Callers that cache responses
|
||||
// keyed by pubkey (e.g. /api/nodes/{pubkey}/reach, #1629) include this
|
||||
// generation in their cache key so any blacklist change naturally
|
||||
// invalidates prior entries on the next request.
|
||||
blacklistGen atomic.Uint64
|
||||
// blacklistSetCached is the lazily-built set version of NodeBlacklist.
|
||||
blacklistSetCached map[string]bool
|
||||
blacklistOnce sync.Once
|
||||
|
||||
Branding map[string]interface{} `json:"branding"`
|
||||
Theme map[string]interface{} `json:"theme"`
|
||||
@@ -90,12 +48,6 @@ type Config struct {
|
||||
TypeColors map[string]interface{} `json:"typeColors"`
|
||||
Home map[string]interface{} `json:"home"`
|
||||
|
||||
// #1488 — marker stroke (outline) settings. Operators dial color, width
|
||||
// and opacity to soften the default white outline when hundreds of
|
||||
// nodes feel overwhelming. Frontend reads these as CSS vars; see
|
||||
// public/customize-v2.js applyCSS markerStroke block.
|
||||
MarkerStroke map[string]interface{} `json:"markerStroke,omitempty"`
|
||||
|
||||
MapDefaults struct {
|
||||
Center []float64 `json:"center"`
|
||||
Zoom int `json:"zoom"`
|
||||
@@ -105,8 +57,7 @@ type Config struct {
|
||||
|
||||
Roles map[string]interface{} `json:"roles"`
|
||||
HealthThresholds *HealthThresholds `json:"healthThresholds"`
|
||||
Map map[string]interface{} `json:"map"`
|
||||
Tiles map[string]interface{} `json:"tiles"` // deprecated
|
||||
Tiles map[string]interface{} `json:"tiles"`
|
||||
SnrThresholds map[string]interface{} `json:"snrThresholds"`
|
||||
DistThresholds map[string]interface{} `json:"distThresholds"`
|
||||
MaxHopDist *float64 `json:"maxHopDist"`
|
||||
@@ -118,7 +69,6 @@ type Config struct {
|
||||
|
||||
LiveMap struct {
|
||||
PropagationBufferMs int `json:"propagationBufferMs"`
|
||||
MaxNodes int `json:"maxNodes"`
|
||||
} `json:"liveMap"`
|
||||
|
||||
CacheTTL map[string]interface{} `json:"cacheTTL"`
|
||||
@@ -129,11 +79,6 @@ type Config struct {
|
||||
|
||||
PacketStore *PacketStoreConfig `json:"packetStore,omitempty"`
|
||||
|
||||
// Runtime holds Go runtime tuning knobs (#1010).
|
||||
// Currently exposes runtime.maxMemoryMB which sets a soft memory limit
|
||||
// (GOMEMLIMIT) via runtime/debug.SetMemoryLimit at startup. The
|
||||
// GOMEMLIMIT environment variable, when set, takes precedence.
|
||||
Runtime *RuntimeConfig `json:"runtime,omitempty"`
|
||||
GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"`
|
||||
|
||||
Areas map[string]AreaEntry `json:"areas,omitempty"`
|
||||
@@ -147,10 +92,6 @@ type Config struct {
|
||||
|
||||
DebugAffinity bool `json:"debugAffinity,omitempty"`
|
||||
|
||||
// MapDarkTileProvider selects the default dark-mode basemap provider for
|
||||
// new visitors. Deprecated: use Map.Tiles.DarkDefault instead.
|
||||
MapDarkTileProvider string `json:"mapDarkTileProvider,omitempty"`
|
||||
|
||||
// ObserverBlacklist is a list of observer public keys to exclude from API
|
||||
// responses (defense in depth — ingestor drops at ingest, server filters
|
||||
// any that slipped through from a prior unblocked window).
|
||||
@@ -160,44 +101,15 @@ type Config struct {
|
||||
obsBlacklistSetCached map[string]bool
|
||||
obsBlacklistOnce sync.Once
|
||||
|
||||
Compression *CompressionConfig `json:"compression,omitempty"`
|
||||
|
||||
// ClientRxCoverage gates the opt-in mobile client-RX coverage feature
|
||||
// (corescope-rx companions publishing GPS-tagged receptions). Absent/nil
|
||||
// ⇒ off; see ClientRxCoverageEnabled.
|
||||
ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"`
|
||||
|
||||
Compression *CompressionConfig `json:"compression,omitempty"`
|
||||
ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"`
|
||||
NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"`
|
||||
|
||||
// Observers cache settings (#1481 P0-3 / #1483).
|
||||
ObserversCache *ObserversCacheConfig `json:"observersCache,omitempty"`
|
||||
|
||||
// Analytics steady-state background recompute (issue #1240).
|
||||
Analytics *AnalyticsConfig `json:"analytics,omitempty"`
|
||||
|
||||
// BatteryThresholds: voltage cutoffs for low/critical alerts (#663).
|
||||
BatteryThresholds *BatteryThresholdsConfig `json:"batteryThresholds,omitempty"`
|
||||
|
||||
// Customizer controls operator-side knobs for the in-app customizer modal
|
||||
// (theme/branding/etc.). See CustomizerConfig and issue #1508.
|
||||
Customizer *CustomizerConfig `json:"customizer,omitempty"`
|
||||
|
||||
// Known-channels catalogue integration (issue #1323).
|
||||
// URL of a JSON catalogue file (channels-by-country shape) fetched
|
||||
// periodically and exposed via /api/known-channels. Empty disables.
|
||||
KnownChannelsURL string `json:"knownChannelsUrl,omitempty"`
|
||||
// Refresh interval in milliseconds. 0/missing => default 24h.
|
||||
KnownChannelsRefreshMs int64 `json:"knownChannelsRefreshMs,omitempty"`
|
||||
}
|
||||
|
||||
// CustomizerConfig holds operator-side knobs for the in-app customizer modal.
|
||||
// Today only DisabledTabs is exposed: a list of tab ids the operator wants to
|
||||
// hide from end users (e.g. ["branding","geofilter","export"]). The frontend
|
||||
// (public/customize-v2.js _renderTabs) reads this from /api/config/client and
|
||||
// filters those tabs out before rendering. Issue #1508.
|
||||
type CustomizerConfig struct {
|
||||
DisabledTabs []string `json:"disabledTabs"`
|
||||
}
|
||||
|
||||
// weakAPIKeys is the blocklist of known default/example API keys that must be rejected.
|
||||
@@ -256,17 +168,6 @@ func (c *Config) GZipEnabled() bool {
|
||||
return c.Compression != nil && c.Compression.GZip
|
||||
}
|
||||
|
||||
// ClientRxCoverageConfig gates the opt-in mobile client-RX coverage feature.
|
||||
type ClientRxCoverageConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// ClientRxCoverageEnabled reports whether the opt-in mobile client-RX coverage
|
||||
// feature is on. Nil config or absent/nil section ⇒ off (the safe default).
|
||||
func (c *Config) ClientRxCoverageEnabled() bool {
|
||||
return c != nil && c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled
|
||||
}
|
||||
|
||||
// WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled.
|
||||
func (c *Config) WSCompressionEnabled() bool {
|
||||
return c.Compression != nil && c.Compression.Websocket
|
||||
@@ -281,21 +182,6 @@ type ResolvedPathConfig struct {
|
||||
type NeighborGraphConfig struct {
|
||||
MaxAgeDays int `json:"maxAgeDays"` // edges older than this are pruned (default 5)
|
||||
MaxEdgeKm float64 `json:"maxEdgeKm"` // geo-implausibility threshold (km); 0 = default 500; negative disables (#1228)
|
||||
|
||||
// CacheRecomputeIntervalSeconds: cadence for the background
|
||||
// recomputer that rebuilds the default-shape neighbor-graph
|
||||
// response (#1481 P0-1). 0/missing = default 300 (5 min).
|
||||
// Lower = fresher data, more CPU per minute. #1483.
|
||||
CacheRecomputeIntervalSeconds int `json:"cacheRecomputeIntervalSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// ObserversCacheConfig controls the /api/observers default-shape cache.
|
||||
// #1481 P0-3 / #1483.
|
||||
type ObserversCacheConfig struct {
|
||||
// TTLSeconds: how long the cached default-shape /api/observers
|
||||
// response is served before a singleflight-collapsed refill.
|
||||
// 0/missing = default 30. Lower = fresher data, more SQL pressure.
|
||||
TTLSeconds int `json:"ttlSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// PacketStoreConfig controls in-memory packet store limits.
|
||||
@@ -309,21 +195,11 @@ type PacketStoreConfig struct {
|
||||
// GeoFilterConfig is an alias for the shared geofilter.Config type.
|
||||
type GeoFilterConfig = geofilter.Config
|
||||
|
||||
// RuntimeConfig holds Go runtime tuning knobs (#1010).
|
||||
type RuntimeConfig struct {
|
||||
// MaxMemoryMB sets the Go soft memory limit (GOMEMLIMIT) in MiB via
|
||||
// runtime/debug.SetMemoryLimit at startup. Takes precedence over the
|
||||
// implicit limit derived from packetStore.maxMemoryMB. The GOMEMLIMIT
|
||||
// environment variable, when set, takes precedence over this value.
|
||||
// 0/unset preserves default behavior.
|
||||
MaxMemoryMB int `json:"maxMemoryMB"`
|
||||
}
|
||||
|
||||
type RetentionConfig struct {
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
NodeDays int `json:"nodeDays"`
|
||||
ObserverDays int `json:"observerDays"`
|
||||
PacketDays int `json:"packetDays"`
|
||||
MetricsDays int `json:"metricsDays"`
|
||||
}
|
||||
|
||||
// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921).
|
||||
@@ -418,10 +294,6 @@ type HealthThresholds struct {
|
||||
// repeater to be considered "actively relaying" vs only "alive
|
||||
// (advert-only)". See issue #662. Defaults to 24h.
|
||||
RelayActiveHours float64 `json:"relayActiveHours"`
|
||||
// Issue #1552 — observer health classification thresholds (minutes).
|
||||
// Defaults match prior hardcoded behavior in public/observers.js (10/60).
|
||||
ObserverOnlineMinutes int `json:"observerOnlineMinutes"`
|
||||
ObserverStaleMinutes int `json:"observerStaleMinutes"`
|
||||
}
|
||||
|
||||
// ThemeFile mirrors theme.json overlay.
|
||||
@@ -432,8 +304,6 @@ type ThemeFile struct {
|
||||
NodeColors map[string]interface{} `json:"nodeColors"`
|
||||
TypeColors map[string]interface{} `json:"typeColors"`
|
||||
Home map[string]interface{} `json:"home"`
|
||||
// #1488 — marker stroke overlay from theme.json.
|
||||
MarkerStroke map[string]interface{} `json:"markerStroke,omitempty"`
|
||||
}
|
||||
|
||||
func LoadConfig(baseDirs ...string) (*Config, error) {
|
||||
@@ -456,71 +326,12 @@ func LoadConfig(baseDirs ...string) (*Config, error) {
|
||||
continue
|
||||
}
|
||||
cfg.NormalizeTimestampConfig()
|
||||
cfg.migrateDeprecatedConfig()
|
||||
cfg.applyListLimitsDefaults()
|
||||
applyCORSEnv(cfg)
|
||||
return cfg, nil
|
||||
}
|
||||
cfg.NormalizeTimestampConfig()
|
||||
cfg.migrateDeprecatedConfig()
|
||||
cfg.applyListLimitsDefaults()
|
||||
applyCORSEnv(cfg)
|
||||
return cfg, nil // defaults
|
||||
}
|
||||
|
||||
func (c *Config) applyListLimitsDefaults() {
|
||||
if c.ListLimits == nil {
|
||||
c.ListLimits = &ListLimitsConfig{}
|
||||
}
|
||||
if c.ListLimits.PacketsMax <= 0 {
|
||||
c.ListLimits.PacketsMax = 10000
|
||||
}
|
||||
if c.ListLimits.NodesMax <= 0 {
|
||||
c.ListLimits.NodesMax = 2000
|
||||
}
|
||||
if c.ListLimits.AnalyticsMax <= 0 {
|
||||
c.ListLimits.AnalyticsMax = 200
|
||||
}
|
||||
if c.ListLimits.ChannelMessagesMax <= 0 {
|
||||
c.ListLimits.ChannelMessagesMax = 500
|
||||
}
|
||||
if c.ListLimits.BulkHealthMax <= 0 {
|
||||
c.ListLimits.BulkHealthMax = 200
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) migrateDeprecatedConfig() {
|
||||
migrated := false
|
||||
if c.Map == nil {
|
||||
c.Map = make(map[string]interface{})
|
||||
}
|
||||
if c.Map["tiles"] == nil {
|
||||
c.Map["tiles"] = make(map[string]interface{})
|
||||
}
|
||||
tilesMap, ok := c.Map["tiles"].(map[string]interface{})
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if c.MapDarkTileProvider != "" {
|
||||
if tilesMap["darkDefault"] == nil {
|
||||
tilesMap["darkDefault"] = c.MapDarkTileProvider
|
||||
}
|
||||
migrated = true
|
||||
}
|
||||
if len(c.Tiles) > 0 {
|
||||
for k, v := range c.Tiles {
|
||||
if tilesMap[k] == nil {
|
||||
tilesMap[k] = v
|
||||
}
|
||||
}
|
||||
migrated = true
|
||||
}
|
||||
if migrated {
|
||||
fmt.Fprintf(os.Stderr, "[deprecated] Top-level 'mapDarkTileProvider' and 'tiles' keys in config.json are deprecated and will be ignored in v3.5.0 (see #1165). Please move them into 'map': { 'tiles': { ... } }.\n")
|
||||
}
|
||||
}
|
||||
|
||||
func LoadTheme(baseDirs ...string) *ThemeFile {
|
||||
if len(baseDirs) == 0 {
|
||||
baseDirs = []string{"."}
|
||||
@@ -569,18 +380,6 @@ func (c *Config) GetHealthThresholds() HealthThresholds {
|
||||
if c.HealthThresholds.RelayActiveHours > 0 {
|
||||
h.RelayActiveHours = c.HealthThresholds.RelayActiveHours
|
||||
}
|
||||
if c.HealthThresholds.ObserverOnlineMinutes > 0 {
|
||||
h.ObserverOnlineMinutes = c.HealthThresholds.ObserverOnlineMinutes
|
||||
}
|
||||
if c.HealthThresholds.ObserverStaleMinutes > 0 {
|
||||
h.ObserverStaleMinutes = c.HealthThresholds.ObserverStaleMinutes
|
||||
}
|
||||
}
|
||||
if h.ObserverOnlineMinutes <= 0 {
|
||||
h.ObserverOnlineMinutes = 60
|
||||
}
|
||||
if h.ObserverStaleMinutes <= 0 {
|
||||
h.ObserverStaleMinutes = 1440
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -597,14 +396,11 @@ func (h HealthThresholds) GetHealthMs(role string) (degradedMs, silentMs int) {
|
||||
// ToClientMs returns the thresholds as ms for the frontend.
|
||||
func (h HealthThresholds) ToClientMs() map[string]int {
|
||||
const hourMs = 3600000
|
||||
const minMs = 60000
|
||||
return map[string]int{
|
||||
"infraDegradedMs": int(h.InfraDegradedHours * hourMs),
|
||||
"infraSilentMs": int(h.InfraSilentHours * hourMs),
|
||||
"nodeDegradedMs": int(h.NodeDegradedHours * hourMs),
|
||||
"nodeSilentMs": int(h.NodeSilentHours * hourMs),
|
||||
"observerOnlineMs": h.ObserverOnlineMinutes * minMs,
|
||||
"observerStaleMs": h.ObserverStaleMinutes * minMs,
|
||||
"infraDegradedMs": int(h.InfraDegradedHours * hourMs),
|
||||
"infraSilentMs": int(h.InfraSilentHours * hourMs),
|
||||
"nodeDegradedMs": int(h.NodeDegradedHours * hourMs),
|
||||
"nodeSilentMs": int(h.NodeSilentHours * hourMs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -618,6 +414,7 @@ func (c *Config) ResolveDBPath(baseDir string) string {
|
||||
return filepath.Join(baseDir, "data", "meshcore.db")
|
||||
}
|
||||
|
||||
|
||||
func (c *Config) NormalizeTimestampConfig() {
|
||||
defaults := defaultTimestampConfig()
|
||||
if c.Timestamps == nil {
|
||||
@@ -670,166 +467,31 @@ func (c *Config) PropagationBufferMs() int {
|
||||
return 5000
|
||||
}
|
||||
|
||||
// LiveMapMaxNodes returns the operator-configured cap on how many nodes
|
||||
// the live map fetches (and thus renders) in a single page. Default is
|
||||
// 2000; values are clamped to [100, 20000] to defang misconfig.
|
||||
// Negative/zero falls back to default. See #1574.
|
||||
func (c *Config) LiveMapMaxNodes() int {
|
||||
const def = 2000
|
||||
const min = 100
|
||||
const max = 20000
|
||||
if c == nil || c.LiveMap.MaxNodes <= 0 {
|
||||
return def
|
||||
}
|
||||
v := c.LiveMap.MaxNodes
|
||||
if v < min {
|
||||
return min
|
||||
}
|
||||
if v > max {
|
||||
return max
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// buildBlacklistSet recomputes the lookup set from pks and returns it.
|
||||
// Empty/whitespace-only entries are skipped. Keys are lowercased + trimmed.
|
||||
// Returns nil for an empty effective set so callers can `len(m) == 0` short-circuit.
|
||||
func buildBlacklistSet(pks []string) map[string]bool {
|
||||
if len(pks) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]bool, len(pks))
|
||||
for _, pk := range pks {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
// blacklistSet lazily builds and caches the nodeBlacklist as a set for O(1) lookups.
|
||||
// Uses sync.Once to eliminate the data race on first concurrent access.
|
||||
func (c *Config) blacklistSet() map[string]bool {
|
||||
c.blacklistOnce.Do(func() {
|
||||
if len(c.NodeBlacklist) == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// SetNodeBlacklist atomically replaces NodeBlacklist with pks, rebuilds the
|
||||
// lookup set, and bumps the generation counter so any cache keyed on the
|
||||
// generation invalidates on the next request (#1629). Safe for concurrent
|
||||
// use with IsBlacklisted / BlacklistGeneration.
|
||||
func (c *Config) SetNodeBlacklist(pks []string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
// Copy so callers can mutate their slice without affecting us.
|
||||
cp := make([]string, len(pks))
|
||||
copy(cp, pks)
|
||||
c.NodeBlacklist = cp
|
||||
m := buildBlacklistSet(cp)
|
||||
c.blacklistSetPtr.Store(&m)
|
||||
c.blacklistGen.Add(1)
|
||||
}
|
||||
|
||||
// BlacklistGeneration returns a monotonic counter that increments on every
|
||||
// SetNodeBlacklist call. Response caches keyed per-pubkey embed this value
|
||||
// in their cache key so any blacklist mutation invalidates prior entries on
|
||||
// the next request (#1629).
|
||||
func (c *Config) BlacklistGeneration() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
return c.blacklistGen.Load()
|
||||
m := make(map[string]bool, len(c.NodeBlacklist))
|
||||
for _, pk := range c.NodeBlacklist {
|
||||
trimmed := strings.ToLower(strings.TrimSpace(pk))
|
||||
if trimmed != "" {
|
||||
m[trimmed] = true
|
||||
}
|
||||
}
|
||||
c.blacklistSetCached = m
|
||||
})
|
||||
return c.blacklistSetCached
|
||||
}
|
||||
|
||||
// IsBlacklisted returns true if the given public key is in the nodeBlacklist.
|
||||
// Hot read path: a single atomic pointer load + map lookup. No locks, no
|
||||
// sync.Once. The in-memory set is populated either via SetNodeBlacklist or
|
||||
// lazily on first read from c.NodeBlacklist (covering the JSON-load path
|
||||
// where the setter was never called).
|
||||
func (c *Config) IsBlacklisted(pubkey string) bool {
|
||||
if c == nil {
|
||||
if c == nil || len(c.NodeBlacklist) == 0 {
|
||||
return false
|
||||
}
|
||||
mp := c.blacklistSetPtr.Load()
|
||||
if mp == nil {
|
||||
// Lazy first-read materialisation from the JSON-loaded slice.
|
||||
// CAS-style: if another goroutine wins the race, drop ours.
|
||||
built := buildBlacklistSet(c.NodeBlacklist)
|
||||
if c.blacklistSetPtr.CompareAndSwap(nil, &built) {
|
||||
mp = &built
|
||||
} else {
|
||||
mp = c.blacklistSetPtr.Load()
|
||||
}
|
||||
}
|
||||
if mp == nil || len(*mp) == 0 {
|
||||
return false
|
||||
}
|
||||
return (*mp)[strings.ToLower(strings.TrimSpace(pubkey))]
|
||||
}
|
||||
|
||||
// IsNameHidden returns true if the given node name starts with any of the
|
||||
// operator-configured HiddenNamePrefixes (issue #1181). Empty/whitespace
|
||||
// prefixes are ignored. Used to drop nodes from /api/nodes, /api/nodes/search
|
||||
// and /api/nodes/{pubkey} without deleting the underlying DB row, so observer
|
||||
// history stays intact even after the operator hides the node.
|
||||
//
|
||||
// Hot read path: a single atomic pointer load. No locks, no sync.Once.
|
||||
// Writers always replace the whole slice; readers see either the old or
|
||||
// the new slice as a single value, never a partially-built one. Mirrors
|
||||
// IsBlacklisted's CAS-style lazy first-read materialisation for the
|
||||
// JSON-load path where SetHiddenNamePrefixes was never called.
|
||||
func (c *Config) IsNameHidden(name string) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
pp := c.hiddenPrefixesPtr.Load()
|
||||
if pp == nil {
|
||||
// Lazy first-read materialisation from the JSON-loaded slice.
|
||||
// CAS-style: if another goroutine wins the race, drop ours.
|
||||
built := make([]string, len(c.HiddenNamePrefixes))
|
||||
copy(built, c.HiddenNamePrefixes)
|
||||
if c.hiddenPrefixesPtr.CompareAndSwap(nil, &built) {
|
||||
pp = &built
|
||||
} else {
|
||||
pp = c.hiddenPrefixesPtr.Load()
|
||||
}
|
||||
}
|
||||
if pp == nil || len(*pp) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, p := range *pp {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetHiddenNamePrefixes atomically replaces HiddenNamePrefixes with the
|
||||
// given slice and bumps the generation counter. Safe for concurrent use
|
||||
// with IsNameHidden / HiddenNamePrefixesGeneration. Mirrors
|
||||
// SetNodeBlacklist (#1629).
|
||||
func (c *Config) SetHiddenNamePrefixes(prefixes []string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
cp := make([]string, len(prefixes))
|
||||
copy(cp, prefixes)
|
||||
c.HiddenNamePrefixes = cp
|
||||
c.hiddenPrefixesPtr.Store(&cp)
|
||||
c.hiddenPrefixesGen.Add(1)
|
||||
}
|
||||
|
||||
// HiddenNamePrefixesGeneration returns a monotonic counter that increments
|
||||
// on every SetHiddenNamePrefixes call. Response caches keyed per-pubkey can
|
||||
// embed this value in their cache key so any prefix mutation invalidates
|
||||
// prior entries on the next request — same pattern as BlacklistGeneration.
|
||||
func (c *Config) HiddenNamePrefixesGeneration() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
return c.hiddenPrefixesGen.Load()
|
||||
return c.blacklistSet()[strings.ToLower(strings.TrimSpace(pubkey))]
|
||||
}
|
||||
|
||||
// SaveGeoFilter writes the geo_filter section back to config.json on disk.
|
||||
@@ -924,26 +586,10 @@ func (c *Config) IsObserverBlacklisted(id string) bool {
|
||||
// data slowly." Lower values give fresher data at higher CPU cost.
|
||||
//
|
||||
// RecomputeIntervalSeconds keys (all optional):
|
||||
//
|
||||
// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew
|
||||
// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew
|
||||
type AnalyticsConfig struct {
|
||||
DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"`
|
||||
RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"`
|
||||
// LoRaPreset is the assumed PHY preset used by the relay-airtime-share
|
||||
// metric to compute true Time-on-Air (issue #1768). Defaults to the
|
||||
// EU MeshCore deployment: 869.6 MHz / BW 62.5 kHz / SF 8 / CR 4/5.
|
||||
// freq is informational only and surfaces in the analytics caption.
|
||||
LoRaPreset *LoRaPresetConfig `json:"loraPreset,omitempty"`
|
||||
}
|
||||
|
||||
// LoRaPresetConfig is the user-facing PHY preset for ToA scoring.
|
||||
// Only the four free params live here; CRC/IH/DE are firmware-fixed
|
||||
// in internal/lora and intentionally not surfaced as config.
|
||||
type LoRaPresetConfig struct {
|
||||
FreqHz float64 `json:"freq,omitempty"` // e.g. 869.6e6
|
||||
BWkHz float64 `json:"bw,omitempty"` // e.g. 62.5
|
||||
SF int `json:"sf,omitempty"` // e.g. 8
|
||||
CR int `json:"cr,omitempty"` // 5..8 (denominator suffix of 4/5..4/8)
|
||||
DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"`
|
||||
RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// AnalyticsDefaultRecomputeInterval returns the configured default
|
||||
|
||||
@@ -387,131 +387,3 @@ func TestObserverDaysOrDefault(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Issue #1552 — observer health thresholds configurable.
|
||||
|
||||
func TestObserverThresholdsOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgData := map[string]interface{}{
|
||||
"healthThresholds": map[string]interface{}{
|
||||
"observerOnlineMinutes": 30,
|
||||
"observerStaleMinutes": 120,
|
||||
},
|
||||
}
|
||||
data, _ := json.Marshal(cfgData)
|
||||
os.WriteFile(filepath.Join(dir, "config.json"), data, 0644)
|
||||
cfg, err := LoadConfig(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := cfg.GetHealthThresholds()
|
||||
if h.ObserverOnlineMinutes != 30 {
|
||||
t.Errorf("ObserverOnlineMinutes = %d, want 30", h.ObserverOnlineMinutes)
|
||||
}
|
||||
if h.ObserverStaleMinutes != 120 {
|
||||
t.Errorf("ObserverStaleMinutes = %d, want 120", h.ObserverStaleMinutes)
|
||||
}
|
||||
m := h.ToClientMs()
|
||||
if m["observerOnlineMs"] != 30*60*1000 {
|
||||
t.Errorf("observerOnlineMs = %d, want %d", m["observerOnlineMs"], 30*60*1000)
|
||||
}
|
||||
if m["observerStaleMs"] != 120*60*1000 {
|
||||
t.Errorf("observerStaleMs = %d, want %d", m["observerStaleMs"], 120*60*1000)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserverThresholdsDefaults(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
h := cfg.GetHealthThresholds()
|
||||
if h.ObserverOnlineMinutes != 60 {
|
||||
t.Errorf("default ObserverOnlineMinutes = %d, want 60", h.ObserverOnlineMinutes)
|
||||
}
|
||||
if h.ObserverStaleMinutes != 1440 {
|
||||
t.Errorf("default ObserverStaleMinutes = %d, want 1440", h.ObserverStaleMinutes)
|
||||
}
|
||||
m := h.ToClientMs()
|
||||
if m["observerOnlineMs"] != 3600000 {
|
||||
t.Errorf("default observerOnlineMs = %d, want 3600000", m["observerOnlineMs"])
|
||||
}
|
||||
if m["observerStaleMs"] != 86400000 {
|
||||
t.Errorf("default observerStaleMs = %d, want 86400000", m["observerStaleMs"])
|
||||
}
|
||||
}
|
||||
|
||||
// Loading a config with no healthThresholds block at all must still produce
|
||||
// the new 60 / 1440 defaults (not zero, not the old 10 / 60).
|
||||
func TestObserverThresholdsDefaultsFromEmptyConfigFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"port": 3000}`), 0644)
|
||||
cfg, err := LoadConfig(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := cfg.GetHealthThresholds()
|
||||
if h.ObserverOnlineMinutes != 60 {
|
||||
t.Errorf("empty-config ObserverOnlineMinutes = %d, want 60 (new default)", h.ObserverOnlineMinutes)
|
||||
}
|
||||
if h.ObserverStaleMinutes != 1440 {
|
||||
t.Errorf("empty-config ObserverStaleMinutes = %d, want 1440 (new default)", h.ObserverStaleMinutes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyListLimitsDefaults(t *testing.T) {
|
||||
t.Run("defaults when block is absent", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"port": 3000}`), 0644)
|
||||
cfg, err := LoadConfig(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.ListLimits.PacketsMax != 10000 {
|
||||
t.Errorf("expected 10000, got %d", cfg.ListLimits.PacketsMax)
|
||||
}
|
||||
if cfg.ListLimits.NodesMax != 2000 {
|
||||
t.Errorf("expected 2000, got %d", cfg.ListLimits.NodesMax)
|
||||
}
|
||||
if cfg.ListLimits.AnalyticsMax != 200 {
|
||||
t.Errorf("expected 200, got %d", cfg.ListLimits.AnalyticsMax)
|
||||
}
|
||||
if cfg.ListLimits.ChannelMessagesMax != 500 {
|
||||
t.Errorf("expected 500, got %d", cfg.ListLimits.ChannelMessagesMax)
|
||||
}
|
||||
if cfg.ListLimits.BulkHealthMax != 200 {
|
||||
t.Errorf("expected 200, got %d", cfg.ListLimits.BulkHealthMax)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("operator overrides honored", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgData := map[string]interface{}{
|
||||
"listLimits": map[string]interface{}{
|
||||
"packetsMax": 50000,
|
||||
"nodesMax": 5000,
|
||||
"analyticsMax": 500,
|
||||
"channelMessagesMax": 1000,
|
||||
"bulkHealthMax": 300,
|
||||
},
|
||||
}
|
||||
data, _ := json.Marshal(cfgData)
|
||||
os.WriteFile(filepath.Join(dir, "config.json"), data, 0644)
|
||||
cfg, err := LoadConfig(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.ListLimits.PacketsMax != 50000 {
|
||||
t.Errorf("expected 50000, got %d", cfg.ListLimits.PacketsMax)
|
||||
}
|
||||
if cfg.ListLimits.NodesMax != 5000 {
|
||||
t.Errorf("expected 5000, got %d", cfg.ListLimits.NodesMax)
|
||||
}
|
||||
if cfg.ListLimits.AnalyticsMax != 500 {
|
||||
t.Errorf("expected 500, got %d", cfg.ListLimits.AnalyticsMax)
|
||||
}
|
||||
if cfg.ListLimits.ChannelMessagesMax != 1000 {
|
||||
t.Errorf("expected 1000, got %d", cfg.ListLimits.ChannelMessagesMax)
|
||||
}
|
||||
if cfg.ListLimits.BulkHealthMax != 300 {
|
||||
t.Errorf("expected 300, got %d", cfg.ListLimits.BulkHealthMax)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+2
-40
@@ -1,47 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// applyCORSEnv overlays cfg.CORSAllowedOrigins from the CORS_ALLOWED_ORIGINS
|
||||
// env var when it is set and non-empty. Tokens are comma-separated, trimmed,
|
||||
// and empties dropped. The env var is the ops-friendly override; it lets
|
||||
// operators add cross-domain embed origins without editing config.json
|
||||
// (issue #1369). An unset or empty env var leaves cfg untouched, so
|
||||
// per-deployment config.json values still apply.
|
||||
func applyCORSEnv(cfg *Config) {
|
||||
raw, ok := os.LookupEnv("CORS_ALLOWED_ORIGINS")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
parts := strings.Split(raw, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
s := strings.TrimSpace(p)
|
||||
if s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
// Env var present but only whitespace — treat as unset, do not clobber.
|
||||
return
|
||||
}
|
||||
cfg.CORSAllowedOrigins = out
|
||||
}
|
||||
import "net/http"
|
||||
|
||||
// corsMiddleware returns a middleware that sets CORS headers based on the
|
||||
// configured allowed origins. When CORSAllowedOrigins is empty (default),
|
||||
// no Access-Control-* headers are added, preserving browser same-origin policy.
|
||||
//
|
||||
// Embed contract (issue #1369): the cross-domain surface is read-only. The
|
||||
// middleware advertises only GET, HEAD, and OPTIONS in Access-Control-Allow-
|
||||
// Methods so iframes / server-side fetchers cannot opt into POST/PUT/DELETE
|
||||
// via CORS. Same-origin writes (admin UI, API-key holders on the canonical
|
||||
// origin) are unaffected — they never go through the preflight path.
|
||||
// Credentialed CORS is intentionally NOT enabled.
|
||||
func (s *Server) corsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
origins := s.cfg.CORSAllowedOrigins
|
||||
@@ -89,8 +52,7 @@ func (s *Server) corsMiddleware(next http.Handler) http.Handler {
|
||||
w.Header().Set("Access-Control-Allow-Origin", reqOrigin)
|
||||
w.Header().Set("Vary", "Origin")
|
||||
}
|
||||
// Read-only embed contract — see comment above.
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key")
|
||||
|
||||
// Handle preflight
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue #1369: CORS_ALLOWED_ORIGINS env override + embed support.
|
||||
//
|
||||
// Red commit: these tests fail until LoadConfig honors the env var and the
|
||||
// CORS middleware advertises GET/HEAD/OPTIONS (the embed contract is
|
||||
// read-only cross-origin access).
|
||||
|
||||
// TestCORS_EnvOverridesConfig — env var CORS_ALLOWED_ORIGINS replaces config.
|
||||
func TestCORS_EnvOverridesConfig_1369(t *testing.T) {
|
||||
t.Setenv("CORS_ALLOWED_ORIGINS", "https://blog.example.com,https://embed.example.com")
|
||||
cfg, err := LoadConfig("/nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig: %v", err)
|
||||
}
|
||||
if len(cfg.CORSAllowedOrigins) != 2 {
|
||||
t.Fatalf("expected 2 origins from env, got %v", cfg.CORSAllowedOrigins)
|
||||
}
|
||||
if cfg.CORSAllowedOrigins[0] != "https://blog.example.com" ||
|
||||
cfg.CORSAllowedOrigins[1] != "https://embed.example.com" {
|
||||
t.Fatalf("env parse wrong: %v", cfg.CORSAllowedOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCORS_EnvEmptyKeepsConfig — empty env var does not clobber file config.
|
||||
func TestCORS_EnvEmptyKeepsConfig_1369(t *testing.T) {
|
||||
os.Unsetenv("CORS_ALLOWED_ORIGINS")
|
||||
cfg := &Config{CORSAllowedOrigins: []string{"https://example.com"}}
|
||||
applyCORSEnv(cfg)
|
||||
if len(cfg.CORSAllowedOrigins) != 1 || cfg.CORSAllowedOrigins[0] != "https://example.com" {
|
||||
t.Fatalf("unset env should not clobber config; got %v", cfg.CORSAllowedOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCORS_EnvTrimsWhitespace — comma-separated env tokens are trimmed.
|
||||
func TestCORS_EnvTrimsWhitespace_1369(t *testing.T) {
|
||||
t.Setenv("CORS_ALLOWED_ORIGINS", " https://a.example , https://b.example ")
|
||||
cfg := &Config{}
|
||||
applyCORSEnv(cfg)
|
||||
if len(cfg.CORSAllowedOrigins) != 2 {
|
||||
t.Fatalf("expected 2, got %v", cfg.CORSAllowedOrigins)
|
||||
}
|
||||
if cfg.CORSAllowedOrigins[0] != "https://a.example" || cfg.CORSAllowedOrigins[1] != "https://b.example" {
|
||||
t.Fatalf("not trimmed: %v", cfg.CORSAllowedOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCORS_EmbedContractGETHEAD — embed contract is read-only; the
|
||||
// Access-Control-Allow-Methods header must advertise GET, HEAD, OPTIONS only
|
||||
// (no POST/PUT/DELETE) so iframes/server-side fetchers know writes are not
|
||||
// CORS-permitted. DJB hardening: minimum surface.
|
||||
func TestCORS_EmbedContractGETHEAD_1369(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://embed.example.com"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/health", nil)
|
||||
req.Header.Set("Origin", "https://embed.example.com")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
methods := rr.Header().Get("Access-Control-Allow-Methods")
|
||||
if methods != "GET, HEAD, OPTIONS" {
|
||||
t.Fatalf("expected read-only methods 'GET, HEAD, OPTIONS', got %q", methods)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCORS_PreflightPOSTRejected — preflight asking for POST from an allowed
|
||||
// origin must NOT echo POST in Allow-Methods. The middleware advertises only
|
||||
// the read-only set; preflight succeeds (browser then blocks the POST).
|
||||
func TestCORS_PreflightPOSTRejected_1369(t *testing.T) {
|
||||
srv := newTestServerWithCORS([]string{"https://embed.example.com"})
|
||||
handler := srv.corsMiddleware(dummyHandler)
|
||||
|
||||
req := httptest.NewRequest("OPTIONS", "/api/anything", nil)
|
||||
req.Header.Set("Origin", "https://embed.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "POST")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("preflight expected 204, got %d", rr.Code)
|
||||
}
|
||||
if got := rr.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD, OPTIONS" {
|
||||
t.Fatalf("preflight must advertise read-only methods only, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func TestCORS_AllowlistMatch(t *testing.T) {
|
||||
if v := rr.Header().Get("Access-Control-Allow-Origin"); v != "https://good.example" {
|
||||
t.Fatalf("expected origin echo, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Methods"); v != "GET, HEAD, OPTIONS" {
|
||||
if v := rr.Header().Get("Access-Control-Allow-Methods"); v != "GET, POST, OPTIONS" {
|
||||
t.Fatalf("expected methods header, got %q", v)
|
||||
}
|
||||
if v := rr.Header().Get("Access-Control-Allow-Headers"); v != "Content-Type, X-API-Key" {
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// gateTestServer builds a minimal *Server with the given client-RX coverage
|
||||
// config and registers all routes, mirroring routes_test.go's setup but
|
||||
// skipping the packet store (none of the gated routes need it for the 404 /
|
||||
// registration assertions below).
|
||||
func gateTestServer(t *testing.T, cov *ClientRxCoverageConfig) *mux.Router {
|
||||
t.Helper()
|
||||
db := setupTestDB(t)
|
||||
cfg := &Config{Port: 3000, ClientRxCoverage: cov}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
return router
|
||||
}
|
||||
|
||||
func TestCoverageRoutesGatedOff(t *testing.T) {
|
||||
router := gateTestServer(t, nil)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/rx-coverage", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
router.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 for /api/rx-coverage when disabled, got %d", rr.Code)
|
||||
}
|
||||
|
||||
creq := httptest.NewRequest("GET", "/api/config/client", nil)
|
||||
crr := httptest.NewRecorder()
|
||||
router.ServeHTTP(crr, creq)
|
||||
if crr.Code != http.StatusOK {
|
||||
t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String())
|
||||
}
|
||||
if !strings.Contains(crr.Body.String(), `"clientRxCoverage":false`) {
|
||||
t.Fatalf("expected clientRxCoverage:false in config body, got %s", crr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageRoutesGatedOn(t *testing.T) {
|
||||
router := gateTestServer(t, &ClientRxCoverageConfig{Enabled: true})
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/rx-coverage", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
router.ServeHTTP(rr, req)
|
||||
if rr.Code == http.StatusNotFound {
|
||||
t.Fatalf("expected /api/rx-coverage to be registered when enabled, got 404")
|
||||
}
|
||||
|
||||
creq := httptest.NewRequest("GET", "/api/config/client", nil)
|
||||
crr := httptest.NewRecorder()
|
||||
router.ServeHTTP(crr, creq)
|
||||
if crr.Code != http.StatusOK {
|
||||
t.Fatalf("config/client status %d body %s", crr.Code, crr.Body.String())
|
||||
}
|
||||
if !strings.Contains(crr.Body.String(), `"clientRxCoverage":true`) {
|
||||
t.Fatalf("expected clientRxCoverage:true in config body, got %s", crr.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
// Package main: coverage axis of repeater usefulness score (issue #672,
|
||||
// axis 3 of 4). The "Coverage" signal is the normalized harmonic reach
|
||||
// centrality of a node in the (undirected, weighted) neighbor graph: how
|
||||
// well a repeater can reach the rest of the mesh. A node that sits close
|
||||
// (in affinity-distance) to many other nodes covers more of the network;
|
||||
// a peripheral or weakly-connected node covers little.
|
||||
//
|
||||
// Why harmonic (Σ 1/d) and not plain closeness (1 / Σ d): harmonic reach
|
||||
// is well-defined on a DISCONNECTED graph — an unreachable node simply
|
||||
// contributes 1/∞ = 0 — whereas closeness blows up. Real meshes fragment
|
||||
// into components, so this matters. (Boldi & Vigna, "Axioms for
|
||||
// Centrality", 2014.)
|
||||
//
|
||||
// It is deliberately distinct from the other axes: Traffic is empirical
|
||||
// (observed relayed load), Bridge is betweenness (being ON shortest
|
||||
// paths), Redundancy is removal-impact (criticality). Coverage is REACH
|
||||
// breadth — a hub that can get a packet close to anyone, regardless of
|
||||
// whether it currently carries that traffic.
|
||||
//
|
||||
// Edge weight and distance follow the bridge convention (#1235): weight =
|
||||
// affinity Score(now) · observer-diversity Confidence(); Dijkstra needs a
|
||||
// DISTANCE (lower = better) so cost = 1/weight. The input is the shared
|
||||
// BridgeEdge weighted-edge primitive, and the same min-heap (bridgePQ)
|
||||
// drives the per-source Dijkstra.
|
||||
//
|
||||
// Algorithm: one Dijkstra single-source shortest-path computation per
|
||||
// vertex, accumulating Σ 1/d over reachable targets, then normalize by
|
||||
// the max observed reach so per-node scores live in [0, 1]. Complexity
|
||||
// O(V · (E + V log V)) — identical to the bridge axis, comfortably a
|
||||
// background-cadence cost.
|
||||
package main
|
||||
|
||||
// ComputeCoverageScores returns a map pubkey → coverage score in [0, 1]
|
||||
// computed as normalized harmonic reach centrality on the undirected
|
||||
// weighted graph defined by `edges`. Keys are the lowercase pubkey form
|
||||
// (matching the byPathHop / persisted-edge convention).
|
||||
//
|
||||
// The graph and per-source shortest paths come from the shared
|
||||
// weightedDistanceAdjacency / dijkstraFrom helpers (graph_weighted.go). When
|
||||
// Coverage and the Bridge axis are computed from the same edge snapshot within
|
||||
// one recomputeUsefulnessAxes call they therefore see byte-identical structure;
|
||||
// the bridge map surfaced independently by the bridge recomputer (different
|
||||
// cadence/snapshot) is NOT guaranteed to match. Self-loops and edges with
|
||||
// weight < epsilon are skipped there; nodes unable to reach anyone score 0.
|
||||
//
|
||||
// Pure (no global state, no locks); safe to call concurrently.
|
||||
func ComputeCoverageScores(edges []BridgeEdge) map[string]float64 {
|
||||
adj := weightedDistanceAdjacency(edges)
|
||||
if len(adj) == 0 {
|
||||
return map[string]float64{}
|
||||
}
|
||||
|
||||
harmonic := make(map[string]float64, len(adj))
|
||||
for s := range adj {
|
||||
// dijkstraFrom returns only reachable nodes, so every distance is
|
||||
// finite — unreachable peers simply contribute nothing (harmonic
|
||||
// reach treats them as 1/∞ = 0).
|
||||
dist := dijkstraFrom(adj, s)
|
||||
var reach float64
|
||||
for t, d := range dist {
|
||||
if t == s || d <= 0 {
|
||||
continue
|
||||
}
|
||||
reach += 1.0 / d
|
||||
}
|
||||
harmonic[s] = reach
|
||||
}
|
||||
|
||||
// Normalize by the max so the best-reaching repeater is 1.0. If max is
|
||||
// 0 (e.g. a single isolated edge with no reachable pair) leave zeros.
|
||||
maxH := 0.0
|
||||
for _, v := range harmonic {
|
||||
if v > maxH {
|
||||
maxH = v
|
||||
}
|
||||
}
|
||||
if maxH > 0 {
|
||||
for k, v := range harmonic {
|
||||
harmonic[k] = v / maxH
|
||||
}
|
||||
}
|
||||
return harmonic
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestComputeCoverageScores_Empty: empty edge list yields a non-nil empty
|
||||
// map (the recomputer swaps this in before the first graph lands).
|
||||
func TestComputeCoverageScores_Empty(t *testing.T) {
|
||||
scores := ComputeCoverageScores(nil)
|
||||
if scores == nil {
|
||||
t.Fatal("want non-nil empty map, got nil")
|
||||
}
|
||||
if len(scores) != 0 {
|
||||
t.Errorf("want empty map, got %d entries", len(scores))
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeCoverageScores_LineGraph: on a 4-node line A-B-C-D the two
|
||||
// middle nodes reach the rest more cheaply (harmonic reach) than the
|
||||
// leaves, so B and C tie for the top (1.0 after normalization) and the
|
||||
// leaves A, D tie below them.
|
||||
func TestComputeCoverageScores_LineGraph(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "a", B: "b", Weight: 1.0},
|
||||
{A: "b", B: "c", Weight: 1.0},
|
||||
{A: "c", B: "d", Weight: 1.0},
|
||||
}
|
||||
s := ComputeCoverageScores(edges)
|
||||
assertInUnit(t, s)
|
||||
|
||||
if math.Abs(s["b"]-s["c"]) > 1e-9 {
|
||||
t.Errorf("symmetry: b and c should tie, got b=%v c=%v", s["b"], s["c"])
|
||||
}
|
||||
if math.Abs(s["a"]-s["d"]) > 1e-9 {
|
||||
t.Errorf("symmetry: a and d should tie, got a=%v d=%v", s["a"], s["d"])
|
||||
}
|
||||
if !(s["b"] > s["a"]) {
|
||||
t.Errorf("middle b should out-cover leaf a: b=%v a=%v", s["b"], s["a"])
|
||||
}
|
||||
if math.Abs(maxScoreValue(s)-1.0) > 1e-9 {
|
||||
t.Errorf("normalization: max should be 1.0, got %v", maxScoreValue(s))
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeCoverageScores_Star: the hub of a star reaches every leaf in
|
||||
// one hop and is the unique top scorer; the leaves tie below it (each
|
||||
// reaches the hub directly and every other leaf via the hub).
|
||||
func TestComputeCoverageScores_Star(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "s", B: "l1", Weight: 1.0},
|
||||
{A: "s", B: "l2", Weight: 1.0},
|
||||
{A: "s", B: "l3", Weight: 1.0},
|
||||
}
|
||||
s := ComputeCoverageScores(edges)
|
||||
assertInUnit(t, s)
|
||||
|
||||
if math.Abs(s["s"]-1.0) > 1e-9 {
|
||||
t.Errorf("hub should score 1.0, got %v", s["s"])
|
||||
}
|
||||
for _, leaf := range []string{"l1", "l2", "l3"} {
|
||||
if !(s[leaf] < s["s"]) {
|
||||
t.Errorf("leaf %q should cover less than the hub: %v vs %v", leaf, s[leaf], s["s"])
|
||||
}
|
||||
}
|
||||
if math.Abs(s["l1"]-s["l2"]) > 1e-9 || math.Abs(s["l2"]-s["l3"]) > 1e-9 {
|
||||
t.Errorf("leaves should tie: %v %v %v", s["l1"], s["l2"], s["l3"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeCoverageScores_Disconnected: harmonic reach must treat
|
||||
// unreachable nodes as 0 contribution. With two separate 2-node
|
||||
// components every node reaches exactly one peer at distance 1, so all
|
||||
// four tie (and normalize to 1.0). If unreachable nodes leaked in, the
|
||||
// symmetry would break.
|
||||
func TestComputeCoverageScores_Disconnected(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "a", B: "b", Weight: 1.0},
|
||||
{A: "c", B: "d", Weight: 1.0},
|
||||
}
|
||||
s := ComputeCoverageScores(edges)
|
||||
assertInUnit(t, s)
|
||||
if len(s) != 4 {
|
||||
t.Fatalf("want 4 nodes, got %d", len(s))
|
||||
}
|
||||
for _, n := range []string{"a", "b", "c", "d"} {
|
||||
if math.Abs(s[n]-1.0) > 1e-9 {
|
||||
t.Errorf("node %q: want 1.0 (each reaches one peer), got %v", n, s[n])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeCoverageScores_WeightSensitive: a stronger edge is a shorter
|
||||
// distance, so the node bridging both a strong and a weak edge reaches the
|
||||
// most. A-B weight 1.0 (near), A-C weight 0.1 (far) ⇒ A out-covers B
|
||||
// out-covers C. Flip the 1/w distance convention and this inverts.
|
||||
func TestComputeCoverageScores_WeightSensitive(t *testing.T) {
|
||||
edges := []BridgeEdge{
|
||||
{A: "a", B: "b", Weight: 1.0},
|
||||
{A: "a", B: "c", Weight: 0.1},
|
||||
}
|
||||
s := ComputeCoverageScores(edges)
|
||||
assertInUnit(t, s)
|
||||
if !(s["a"] > s["b"] && s["b"] > s["c"]) {
|
||||
t.Errorf("want a > b > c, got a=%v b=%v c=%v", s["a"], s["b"], s["c"])
|
||||
}
|
||||
}
|
||||
|
||||
// --- shared test helpers; assertInUnit and maxScoreValue are both used by
|
||||
// redundancy_score_test.go as well as the coverage tests above. ---
|
||||
|
||||
func assertInUnit(t *testing.T, m map[string]float64) {
|
||||
t.Helper()
|
||||
for k, v := range m {
|
||||
if v < 0 || v > 1 || math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
t.Errorf("score %q=%v out of [0,1]", k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxScoreValue avoids shadowing the Go 1.21 `max` builtin (#1762 nit).
|
||||
func maxScoreValue(m map[string]float64) float64 {
|
||||
largest := 0.0
|
||||
for _, v := range m {
|
||||
if v > largest {
|
||||
largest = v
|
||||
}
|
||||
}
|
||||
return largest
|
||||
}
|
||||
@@ -30,7 +30,7 @@ func setupTestDBv2(t *testing.T) *DB {
|
||||
CREATE TABLE nodes (
|
||||
public_key TEXT PRIMARY KEY, name TEXT, role TEXT,
|
||||
lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0,
|
||||
battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0
|
||||
battery_mv INTEGER, temperature_c REAL
|
||||
);
|
||||
CREATE TABLE observers (
|
||||
id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT,
|
||||
@@ -2289,10 +2289,6 @@ func TestSubpathPrecomputedIndex(t *testing.T) {
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
// #1008: indexes built in background goroutine; wait before reading.
|
||||
if !store.WaitIndexesReady(5 * time.Second) {
|
||||
t.Fatal("indexes never became ready")
|
||||
}
|
||||
|
||||
// After Load(), the precomputed index must be populated.
|
||||
if len(store.spIndex) == 0 {
|
||||
@@ -2347,10 +2343,6 @@ func TestSubpathTxIndexPopulated(t *testing.T) {
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
// #1008: indexes built in background goroutine; wait before reading.
|
||||
if !store.WaitIndexesReady(5 * time.Second) {
|
||||
t.Fatal("indexes never became ready")
|
||||
}
|
||||
|
||||
// spTxIndex must be populated alongside spIndex
|
||||
if len(store.spTxIndex) == 0 {
|
||||
@@ -2395,10 +2387,6 @@ func TestSubpathDetailMixedCaseHops(t *testing.T) {
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
store.Load()
|
||||
// #1008: indexes built in background goroutine; wait before reading.
|
||||
if !store.WaitIndexesReady(5 * time.Second) {
|
||||
t.Fatal("indexes never became ready")
|
||||
}
|
||||
|
||||
// Query with lowercase hops to establish baseline
|
||||
lower := store.GetSubpathDetail([]string{"eeff", "0011"})
|
||||
@@ -2713,17 +2701,6 @@ func TestHandleAnalyticsDistanceWithStore(t *testing.T) {
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
// #1011: lazy distance index — first request returns 202; trigger
|
||||
// the build and wait for it before asserting the 200 shape.
|
||||
store.TriggerDistanceIndexBuild()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for !store.DistanceIndexBuilt() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("distance index did not finish building within 5s")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/distance", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestConfigClientExposesCustomizerDisabledTabs verifies that the
|
||||
// /api/config/client endpoint surfaces the operator-set list of customizer
|
||||
// tabs to hide, so the customize-v2 frontend can filter them out of
|
||||
// _renderTabs(). Issue #1508.
|
||||
func TestConfigClientExposesCustomizerDisabledTabs(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
seedTestData(t, db)
|
||||
cfg := &Config{
|
||||
Port: 3000,
|
||||
Customizer: &CustomizerConfig{
|
||||
DisabledTabs: []string{"branding", "geofilter", "export"},
|
||||
},
|
||||
}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("store.Load failed: %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/config/client", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d (body=%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
custRaw, ok := body["customizer"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected body.customizer object, got %T (body=%s)", body["customizer"], w.Body.String())
|
||||
}
|
||||
tabsRaw, ok := custRaw["disabledTabs"].([]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected body.customizer.disabledTabs array, got %T", custRaw["disabledTabs"])
|
||||
}
|
||||
got := make([]string, 0, len(tabsRaw))
|
||||
for _, v := range tabsRaw {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
t.Fatalf("disabledTabs element not a string: %T", v)
|
||||
}
|
||||
got = append(got, s)
|
||||
}
|
||||
want := []string{"branding", "export", "geofilter"}
|
||||
sort.Strings(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("disabledTabs: got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigClientDefaultsCustomizerDisabledTabsEmpty verifies the backward-
|
||||
// compat default: when no customizer block is configured, the field is still
|
||||
// present and is an empty array (so the frontend can blindly call .includes()).
|
||||
func TestConfigClientDefaultsCustomizerDisabledTabsEmpty(t *testing.T) {
|
||||
_, router := setupTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/api/config/client", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
custRaw, ok := body["customizer"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected body.customizer object, got %T", body["customizer"])
|
||||
}
|
||||
tabsRaw, ok := custRaw["disabledTabs"].([]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected body.customizer.disabledTabs array, got %T", custRaw["disabledTabs"])
|
||||
}
|
||||
if len(tabsRaw) != 0 {
|
||||
t.Errorf("default disabledTabs should be empty, got %v", tabsRaw)
|
||||
}
|
||||
}
|
||||
+9
-171
@@ -12,24 +12,14 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/meshcore-analyzer/dbschema"
|
||||
"github.com/meshcore-analyzer/geofilter"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// routeTypeTransport covers TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3) —
|
||||
// the only route types that carry transport_code_1 (transport-level scope).
|
||||
// Per firmware/docs/packet_format.md § Route Types:
|
||||
// 0 = TRANSPORT_FLOOD, 1 = FLOOD, 2 = DIRECT, 3 = TRANSPORT_DIRECT.
|
||||
// Routes 1 (FLOOD) and 2 (DIRECT) never carry a scope by protocol — they are
|
||||
// inherently unscoped and are counted separately in GetScopeStats (#1838).
|
||||
// routeTypeTransport covers FLOOD (0) and DIRECT (3) route types — packets
|
||||
// that carry transport-level scoping via Code1.
|
||||
const routeTypeTransportSQL = "route_type IN (0, 3)"
|
||||
|
||||
// routeTypeNonTransportSQL matches FLOOD (1) and DIRECT (2) — non-transport
|
||||
// routes that carry no transport_code_1 and are therefore inherently unscoped
|
||||
// per MeshCore protocol (#1838).
|
||||
const routeTypeNonTransportSQL = "route_type IN (1, 2)"
|
||||
|
||||
// DB wraps a read-only connection to the MeshCore SQLite database.
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
@@ -40,7 +30,6 @@ type DB struct {
|
||||
hasScopeName bool // transmissions.scope_name column exists (#899)
|
||||
hasDefaultScope bool // nodes.default_scope column exists (#899)
|
||||
hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903)
|
||||
hasLastSeen bool // transmissions.last_seen column exists (#1690)
|
||||
|
||||
// Channel list cache (60s TTL) — avoids repeated GROUP BY scans (#762)
|
||||
channelsCacheMu sync.Mutex
|
||||
@@ -118,9 +107,6 @@ func (db *DB) detectSchema() {
|
||||
if colName == "scope_name" {
|
||||
db.hasScopeName = true
|
||||
}
|
||||
if colName == "last_seen" {
|
||||
db.hasLastSeen = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,21 +243,6 @@ type Observer struct {
|
||||
UptimeSecs *int64 `json:"uptime_secs"`
|
||||
NoiseFloor *float64 `json:"noise_floor"`
|
||||
LastPacketAt *string `json:"last_packet_at"`
|
||||
// Issue #1478: per-observer naive-clock skew tracking.
|
||||
// Written by the ingestor in cmd/ingestor/db.go RecordNaiveSkew whenever
|
||||
// resolveRxTime clamps a naive envelope timestamp >15 min off UTC. The
|
||||
// server reads these as-is; the handler derives the bool `clock_naive`
|
||||
// from clock_last_naive_at being within the last 24h.
|
||||
ClockSkewSeconds *int64 `json:"clock_skew_seconds"`
|
||||
ClockSkewCount24h int `json:"clock_skew_count_24h"`
|
||||
ClockLastNaiveAt *string `json:"clock_last_naive_at"`
|
||||
// Issue #1290: firmware 1.16 `repeat: on|off` flag persisted by the
|
||||
// ingestor. true = relay-capable, false = listener-only, nil =
|
||||
// unknown (legacy observer that never sent the field — drives the
|
||||
// tri-state UI badge so legacy rows don't masquerade as confirmed
|
||||
// repeaters). The ingestor sets can_relay_seen=1 only when it has
|
||||
// an explicit value; the read layer returns nil when seen=0.
|
||||
CanRelay *bool `json:"can_relay,omitempty"`
|
||||
}
|
||||
|
||||
// Transmission represents a row from the transmissions table.
|
||||
@@ -500,8 +471,6 @@ type PacketQuery struct {
|
||||
type PacketResult struct {
|
||||
Packets []map[string]interface{} `json:"packets"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
// QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape).
|
||||
@@ -1169,25 +1138,7 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string]
|
||||
|
||||
// GetObservers returns active observers (not soft-deleted) sorted by last_seen DESC.
|
||||
func (db *DB) GetObservers() ([]Observer, error) {
|
||||
// Issue #1290: can_relay is read via COALESCE(can_relay, 1). The
|
||||
// column is added by internal/dbschema; older test fixtures and
|
||||
// pre-migration DBs may lack it, so we probe and fall back.
|
||||
// PR #1624 MAJOR-2: can_relay_seen is the tri-state sentinel — 1
|
||||
// means the ingestor explicitly wrote a value, 0 means "unknown"
|
||||
// and the server returns CanRelay=nil so the UI shows no badge.
|
||||
canRelayClause := "COALESCE(can_relay, 1)"
|
||||
canRelaySeenClause := "0"
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||||
canRelayClause = "1"
|
||||
}
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol {
|
||||
canRelaySeenClause = "COALESCE(can_relay_seen, 0)"
|
||||
}
|
||||
rows, err := db.conn.Query(`SELECT id, name, iata, last_seen, first_seen, packet_count,
|
||||
model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at,
|
||||
clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at,
|
||||
` + canRelayClause + `, ` + canRelaySeenClause + `
|
||||
FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC`)
|
||||
rows, err := db.conn.Query("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1196,19 +1147,11 @@ func (db *DB) GetObservers() ([]Observer, error) {
|
||||
var observers []Observer
|
||||
for rows.Next() {
|
||||
var o Observer
|
||||
var batteryMv, uptimeSecs, clockSkewSec sql.NullInt64
|
||||
var clockSkewCount sql.NullInt64
|
||||
var batteryMv, uptimeSecs sql.NullInt64
|
||||
var noiseFloor sql.NullFloat64
|
||||
var canRelay, canRelaySeen int
|
||||
if err := rows.Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount,
|
||||
&o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt,
|
||||
&clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen); err != nil {
|
||||
if err := rows.Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt); err != nil {
|
||||
continue
|
||||
}
|
||||
if canRelaySeen != 0 {
|
||||
b := canRelay != 0
|
||||
o.CanRelay = &b
|
||||
}
|
||||
if batteryMv.Valid {
|
||||
v := int(batteryMv.Int64)
|
||||
o.BatteryMv = &v
|
||||
@@ -1219,103 +1162,21 @@ func (db *DB) GetObservers() ([]Observer, error) {
|
||||
if noiseFloor.Valid {
|
||||
o.NoiseFloor = &noiseFloor.Float64
|
||||
}
|
||||
if clockSkewSec.Valid {
|
||||
v := clockSkewSec.Int64
|
||||
o.ClockSkewSeconds = &v
|
||||
}
|
||||
if clockSkewCount.Valid {
|
||||
o.ClockSkewCount24h = int(clockSkewCount.Int64)
|
||||
}
|
||||
observers = append(observers, o)
|
||||
}
|
||||
return observers, nil
|
||||
}
|
||||
|
||||
// GetNonRelayObserverPubkeys returns the lowercase observer.id pubkeys
|
||||
// for observers that have advertised `repeat:off` (#1290). The server's
|
||||
// path-hop disambiguator consumes this to exclude listener-only nodes
|
||||
// from the candidate set. Inactive observers are excluded for
|
||||
// consistency with GetObservers; reactivation flips can_relay only on
|
||||
// the next status message.
|
||||
func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) {
|
||||
// Graceful no-op when can_relay column is absent (legacy DB / older
|
||||
// test fixture). Avoids noisy schema-degradation log spam.
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers
|
||||
WHERE COALESCE(can_relay, 1) = 0
|
||||
AND (inactive IS NULL OR inactive = 0)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var pk string
|
||||
if err := rows.Scan(&pk); err == nil && pk != "" {
|
||||
out = append(out, pk)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetCanRelaySeenObserverPubkeys returns the lowercase observer.id
|
||||
// pubkeys for which the ingestor has explicitly written a repeat-field
|
||||
// value (can_relay_seen=1). PR #1624 MAJOR-2: the badge surface uses
|
||||
// this to render tri-state — observers NOT in this set are "unknown"
|
||||
// and the UI shows no badge.
|
||||
func (db *DB) GetCanRelaySeenObserverPubkeys() ([]string, error) {
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); !hasCol {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers
|
||||
WHERE COALESCE(can_relay_seen, 0) = 1
|
||||
AND (inactive IS NULL OR inactive = 0)`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var pk string
|
||||
if err := rows.Scan(&pk); err == nil && pk != "" {
|
||||
out = append(out, pk)
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetObserverByID returns a single observer.
|
||||
func (db *DB) GetObserverByID(id string) (*Observer, error) {
|
||||
var o Observer
|
||||
var batteryMv, uptimeSecs, clockSkewSec sql.NullInt64
|
||||
var clockSkewCount sql.NullInt64
|
||||
var batteryMv, uptimeSecs sql.NullInt64
|
||||
var noiseFloor sql.NullFloat64
|
||||
var canRelay, canRelaySeen int
|
||||
canRelayClause := "COALESCE(can_relay, 1)"
|
||||
canRelaySeenClause := "0"
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol {
|
||||
canRelayClause = "1"
|
||||
}
|
||||
if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol {
|
||||
canRelaySeenClause = "COALESCE(can_relay_seen, 0)"
|
||||
}
|
||||
err := db.conn.QueryRow(`SELECT id, name, iata, last_seen, first_seen, packet_count,
|
||||
model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at,
|
||||
clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at,
|
||||
`+canRelayClause+`, `+canRelaySeenClause+`
|
||||
FROM observers WHERE id = ?`, id).
|
||||
Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount,
|
||||
&o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt,
|
||||
&clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen)
|
||||
err := db.conn.QueryRow("SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at FROM observers WHERE id = ?", id).
|
||||
Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if canRelaySeen != 0 {
|
||||
b := canRelay != 0
|
||||
o.CanRelay = &b
|
||||
}
|
||||
if batteryMv.Valid {
|
||||
v := int(batteryMv.Int64)
|
||||
o.BatteryMv = &v
|
||||
@@ -1326,13 +1187,6 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) {
|
||||
if noiseFloor.Valid {
|
||||
o.NoiseFloor = &noiseFloor.Float64
|
||||
}
|
||||
if clockSkewSec.Valid {
|
||||
v := clockSkewSec.Int64
|
||||
o.ClockSkewSeconds = &v
|
||||
}
|
||||
if clockSkewCount.Valid {
|
||||
o.ClockSkewCount24h = int(clockSkewCount.Int64)
|
||||
}
|
||||
return &o, nil
|
||||
}
|
||||
|
||||
@@ -2110,10 +1964,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf
|
||||
placeholders[i] = "?"
|
||||
args[i] = strings.ToLower(k)
|
||||
}
|
||||
// #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and
|
||||
// forces a full scan. Nodes are stored lowercase already; we lowercase
|
||||
// args in Go above so a plain IN matches the index on public_key.
|
||||
query := "SELECT public_key, lat, lon, role FROM nodes WHERE public_key IN (" + strings.Join(placeholders, ",") + ")"
|
||||
query := "SELECT public_key, lat, lon, role FROM nodes WHERE LOWER(public_key) IN (" + strings.Join(placeholders, ",") + ")"
|
||||
rows, err := db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return result
|
||||
@@ -2799,19 +2650,6 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) {
|
||||
return nil, fmt.Errorf("scope summary query: %w", err)
|
||||
}
|
||||
|
||||
// #1838: non-transport routes (FLOOD=1, DIRECT=2) never carry
|
||||
// transport_code_1 per MeshCore protocol, so they are inherently unscoped.
|
||||
// Fold their count into Summary.Unscoped so the analytics denominator
|
||||
// reflects total-observed-transmissions rather than only transport-eligible.
|
||||
var nonTransportUnscoped int
|
||||
if err := db.conn.QueryRow(`
|
||||
SELECT COUNT(*) FROM transmissions
|
||||
WHERE `+routeTypeNonTransportSQL+` AND first_seen >= ?
|
||||
`, since).Scan(&nonTransportUnscoped); err != nil {
|
||||
return nil, fmt.Errorf("scope non-transport count query: %w", err)
|
||||
}
|
||||
resp.Summary.Unscoped += nonTransportUnscoped
|
||||
|
||||
// Per-region counts (named regions only)
|
||||
rows, err := db.conn.Query(`
|
||||
SELECT scope_name, COUNT(*) AS cnt
|
||||
|
||||
+4
-12
@@ -51,10 +51,7 @@ func setupTestDB(t *testing.T) *DB {
|
||||
uptime_secs INTEGER,
|
||||
noise_floor REAL,
|
||||
inactive INTEGER DEFAULT 0,
|
||||
last_packet_at TEXT DEFAULT NULL,
|
||||
clock_skew_seconds INTEGER DEFAULT NULL,
|
||||
clock_skew_count_24h INTEGER DEFAULT 0,
|
||||
clock_last_naive_at TEXT DEFAULT NULL
|
||||
last_packet_at TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE transmissions (
|
||||
@@ -2254,26 +2251,21 @@ func TestGetScopeStats(t *testing.T) {
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', ?, 0, '')`, now)
|
||||
// Transport unscoped (NULL)
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', ?, 0, NULL)`, now)
|
||||
// Non-transport FLOOD (route_type=1) — inherently unscoped per MeshCore protocol (#1838)
|
||||
// Non-transport (should not count)
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', ?, 1, NULL)`, now)
|
||||
// Non-transport DIRECT (route_type=2) — inherently unscoped per MeshCore protocol (#1838)
|
||||
db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', ?, 2, NULL)`, now)
|
||||
|
||||
stats, err := db.GetScopeStats("24h")
|
||||
if err != nil {
|
||||
t.Fatalf("GetScopeStats: %v", err)
|
||||
}
|
||||
// TransportTotal still counts only routes 0+3 (transport-code-carrying).
|
||||
if stats.Summary.TransportTotal != 3 {
|
||||
t.Errorf("TransportTotal = %d, want 3", stats.Summary.TransportTotal)
|
||||
}
|
||||
if stats.Summary.Scoped != 2 {
|
||||
t.Errorf("Scoped = %d, want 2", stats.Summary.Scoped)
|
||||
}
|
||||
// Unscoped now folds in non-transport (routes 1+2) — see #1838.
|
||||
// 1 transport-NULL + 2 non-transport (routes 1 and 2) = 3.
|
||||
if stats.Summary.Unscoped != 3 {
|
||||
t.Errorf("Unscoped = %d, want 3 (1 transport-null + 2 non-transport)", stats.Summary.Unscoped)
|
||||
if stats.Summary.Unscoped != 1 {
|
||||
t.Errorf("Unscoped = %d, want 1", stats.Summary.Unscoped)
|
||||
}
|
||||
if stats.Summary.UnknownScope != 1 {
|
||||
t.Errorf("UnknownScope = %d, want 1", stats.Summary.UnknownScope)
|
||||
|
||||
+1
-44
@@ -91,11 +91,6 @@ type Payload struct {
|
||||
MAC string `json:"mac,omitempty"`
|
||||
EncryptedData string `json:"encryptedData,omitempty"`
|
||||
ExtraHash string `json:"extraHash,omitempty"`
|
||||
// Extended ACK fields per firmware 1.16.0 (issue #1610) — populated by
|
||||
// decodeAck once the server-side re-decoder is upgraded (issue #1694).
|
||||
AckLen *int `json:"ackLen,omitempty"`
|
||||
AckAttempt *int `json:"ackAttempt,omitempty"`
|
||||
AckRand *int `json:"ackRand,omitempty"`
|
||||
PubKey string `json:"pubKey,omitempty"`
|
||||
Timestamp uint32 `json:"timestamp,omitempty"`
|
||||
TimestampISO string `json:"timestampISO,omitempty"`
|
||||
@@ -129,11 +124,6 @@ type Payload struct {
|
||||
InnerType *int `json:"innerType,omitempty"`
|
||||
InnerTypeName string `json:"innerTypeName,omitempty"`
|
||||
InnerAckCrc string `json:"innerAckCrc,omitempty"`
|
||||
// Extended ACK inner fields (issue #1610 / #1694) — populated by
|
||||
// decodeMultipart once ACK parity is ported from the ingestor.
|
||||
InnerAckLen *int `json:"innerAckLen,omitempty"`
|
||||
InnerAckAttempt *int `json:"innerAckAttempt,omitempty"`
|
||||
InnerAckRand *int `json:"innerAckRand,omitempty"`
|
||||
InnerPayload string `json:"innerPayload,omitempty"`
|
||||
// CONTROL (PAYLOAD_TYPE_CONTROL=0x0B) byte0 flags, per
|
||||
// firmware/src/Mesh.cpp:69 — high-bit = zero-hop direct subset.
|
||||
@@ -251,27 +241,10 @@ func decodeAck(buf []byte) Payload {
|
||||
return Payload{Type: "ACK", Error: "too short", RawHex: hex.EncodeToString(buf)}
|
||||
}
|
||||
checksum := binary.LittleEndian.Uint32(buf[0:4])
|
||||
ackLen := len(buf)
|
||||
if ackLen > 6 {
|
||||
ackLen = 6
|
||||
}
|
||||
p := Payload{
|
||||
return Payload{
|
||||
Type: "ACK",
|
||||
ExtraHash: fmt.Sprintf("%08x", checksum),
|
||||
AckLen: &ackLen,
|
||||
}
|
||||
// Firmware 1.16.0 extended ACK (issue #1610): 5th byte is the attempt
|
||||
// counter (commit f6e6fdaa), 6th byte is a random byte added so identical
|
||||
// attempts still hash uniquely (commit a130a95a).
|
||||
if len(buf) >= 5 {
|
||||
attempt := int(buf[4])
|
||||
p.AckAttempt = &attempt
|
||||
}
|
||||
if len(buf) >= 6 {
|
||||
rnd := int(buf[5])
|
||||
p.AckRand = &rnd
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func decodeAdvert(buf []byte, validateSignatures bool) Payload {
|
||||
@@ -405,22 +378,6 @@ func decodeMultipart(buf []byte) Payload {
|
||||
if innerType == PayloadACK && len(buf) >= 5 {
|
||||
crc := binary.LittleEndian.Uint32(buf[1:5])
|
||||
p.InnerAckCrc = fmt.Sprintf("%08x", crc)
|
||||
// Firmware 1.16.0 extended ACK (issue #1610): inner ACK blob may be
|
||||
// 5 or 6 bytes (payload_len = 1 + ack_len) instead of always 4.
|
||||
// Attempt counter added in commit f6e6fdaa, RNG byte in commit a130a95a.
|
||||
ackLen := len(buf) - 1
|
||||
if ackLen > 6 {
|
||||
ackLen = 6
|
||||
}
|
||||
p.InnerAckLen = &ackLen
|
||||
if len(buf) >= 6 {
|
||||
attempt := int(buf[5])
|
||||
p.InnerAckAttempt = &attempt
|
||||
}
|
||||
if len(buf) >= 7 {
|
||||
rnd := int(buf[6])
|
||||
p.InnerAckRand = &rnd
|
||||
}
|
||||
} else if len(buf) > 1 {
|
||||
p.InnerPayload = hex.EncodeToString(buf[1:])
|
||||
}
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
package main
|
||||
|
||||
// Tests for issue #1694 — server-side decoder parity with the ingestor's
|
||||
// firmware-1.16.0 extended ACK support (issue #1610). Wire vectors mirror
|
||||
// the ingestor's tests so both decoders agree byte-for-byte.
|
||||
//
|
||||
// - decodeAck: firmware/src/helpers/BaseChatMesh.cpp:218-234
|
||||
// - decodeMultipart: firmware/src/Mesh.cpp:287-310
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDecodeAckExtended(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
buf []byte
|
||||
wantLen int
|
||||
wantAttPtr bool
|
||||
wantAtt int
|
||||
wantRndPtr bool
|
||||
wantRnd int
|
||||
}{
|
||||
{
|
||||
name: "legacy 4-byte ACK (CRC only)",
|
||||
buf: []byte{0xEF, 0xBE, 0xAD, 0xDE},
|
||||
wantLen: 4,
|
||||
},
|
||||
{
|
||||
name: "5-byte ACK (CRC + attempt)",
|
||||
buf: []byte{0xEF, 0xBE, 0xAD, 0xDE, 0x07},
|
||||
wantLen: 5,
|
||||
wantAttPtr: true,
|
||||
wantAtt: 7,
|
||||
},
|
||||
{
|
||||
name: "6-byte ACK (CRC + attempt + rand)",
|
||||
buf: []byte{0xEF, 0xBE, 0xAD, 0xDE, 0x07, 0x42},
|
||||
wantLen: 6,
|
||||
wantAttPtr: true,
|
||||
wantAtt: 7,
|
||||
wantRndPtr: true,
|
||||
wantRnd: 0x42,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
p := decodeAck(tc.buf)
|
||||
if p.Type != "ACK" {
|
||||
t.Fatalf("type=%q want ACK", p.Type)
|
||||
}
|
||||
if p.AckLen == nil {
|
||||
t.Fatalf("AckLen=nil want %d", tc.wantLen)
|
||||
}
|
||||
if *p.AckLen != tc.wantLen {
|
||||
t.Errorf("AckLen=%d want %d", *p.AckLen, tc.wantLen)
|
||||
}
|
||||
if tc.wantAttPtr {
|
||||
if p.AckAttempt == nil {
|
||||
t.Errorf("AckAttempt=nil want %d", tc.wantAtt)
|
||||
} else if *p.AckAttempt != tc.wantAtt {
|
||||
t.Errorf("AckAttempt=%d want %d", *p.AckAttempt, tc.wantAtt)
|
||||
}
|
||||
} else if p.AckAttempt != nil {
|
||||
t.Errorf("AckAttempt=%d want nil", *p.AckAttempt)
|
||||
}
|
||||
if tc.wantRndPtr {
|
||||
if p.AckRand == nil {
|
||||
t.Errorf("AckRand=nil want %d", tc.wantRnd)
|
||||
} else if *p.AckRand != tc.wantRnd {
|
||||
t.Errorf("AckRand=%d want %d", *p.AckRand, tc.wantRnd)
|
||||
}
|
||||
} else if p.AckRand != nil {
|
||||
t.Errorf("AckRand=%d want nil", *p.AckRand)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeMultipartAckExtendedInner(t *testing.T) {
|
||||
// byte0 = (remaining<<4)|inner_type = (3<<4)|0x03 = 0x33
|
||||
// inner ACK = CRC(deadbeef LE) + attempt(0x07) + rand(0x42) = 6 bytes
|
||||
// total buf = 1 + 6 = 7 bytes.
|
||||
buf := []byte{0x33, 0xEF, 0xBE, 0xAD, 0xDE, 0x07, 0x42}
|
||||
p := decodeMultipart(buf)
|
||||
if p.InnerAckCrc != "deadbeef" {
|
||||
t.Fatalf("InnerAckCrc=%q want deadbeef", p.InnerAckCrc)
|
||||
}
|
||||
if p.InnerAckLen == nil || *p.InnerAckLen != 6 {
|
||||
t.Errorf("InnerAckLen=%v want 6", p.InnerAckLen)
|
||||
}
|
||||
if p.InnerAckAttempt == nil || *p.InnerAckAttempt != 7 {
|
||||
t.Errorf("InnerAckAttempt=%v want 7", p.InnerAckAttempt)
|
||||
}
|
||||
if p.InnerAckRand == nil || *p.InnerAckRand != 0x42 {
|
||||
t.Errorf("InnerAckRand=%v want 0x42", p.InnerAckRand)
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Issue #1011: distance index must NOT be built eagerly at startup.
|
||||
// It is constructed lazily on first /api/analytics/distance request,
|
||||
// the first request returns 202 + Retry-After while the build runs,
|
||||
// and concurrent requests during the build also get 202 (one build
|
||||
// only, not N parallel builds).
|
||||
//
|
||||
// These three assertions encode the acceptance criteria from the
|
||||
// triage Fix path (sync.Once-style first-request trigger, 202+Retry-After).
|
||||
|
||||
// TestDistanceIndexNotBuiltOnLoad: Load() must complete without
|
||||
// populating distHops / distPaths. Eager build is gone.
|
||||
func TestDistanceIndexNotBuiltOnLoad(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("Load(): %v", err)
|
||||
}
|
||||
store.mu.RLock()
|
||||
nHops := len(store.distHops)
|
||||
nPaths := len(store.distPaths)
|
||||
store.mu.RUnlock()
|
||||
if nHops != 0 || nPaths != 0 {
|
||||
t.Fatalf("expected distance index empty after Load() (lazy build, #1011); got %d hops, %d paths — eager build still firing in Load()", nHops, nPaths)
|
||||
}
|
||||
if store.DistanceIndexBuilt() {
|
||||
t.Fatalf("expected DistanceIndexBuilt() = false directly after Load(); got true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDistanceFirstRequestReturns202: first /api/analytics/distance call
|
||||
// must trigger async build and return 202 + Retry-After. The handler must
|
||||
// NOT block for the full build.
|
||||
func TestDistanceFirstRequestReturns202(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("Load(): %v", err)
|
||||
}
|
||||
srv.store = store
|
||||
r := mux.NewRouter()
|
||||
srv.RegisterRoutes(r)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/analytics/distance", nil)
|
||||
w := httptest.NewRecorder()
|
||||
t0 := time.Now()
|
||||
r.ServeHTTP(w, req)
|
||||
elapsed := time.Since(t0)
|
||||
|
||||
if w.Code != 202 {
|
||||
t.Fatalf("expected 202 Accepted on first request (lazy build, #1011); got %d (body=%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if ra := w.Header().Get("Retry-After"); ra == "" {
|
||||
t.Fatalf("expected non-empty Retry-After header on 202 response; got none")
|
||||
}
|
||||
// Handler must return quickly — must not block on the full build.
|
||||
if elapsed > 500*time.Millisecond {
|
||||
t.Fatalf("first-request handler took %v — must not block on build (#1011)", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDistanceConcurrentRequestsDuringBuildReturn202: 10 requests fired
|
||||
// in close succession while the build is in flight must all receive 202;
|
||||
// exactly one build runs.
|
||||
func TestDistanceConcurrentRequestsDuringBuildReturn202(t *testing.T) {
|
||||
db := setupRichTestDB(t)
|
||||
defer db.Close()
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
store := NewPacketStore(db, nil)
|
||||
if err := store.Load(); err != nil {
|
||||
t.Fatalf("Load(): %v", err)
|
||||
}
|
||||
// Hold the lazy build open until every concurrent request has been served,
|
||||
// so the "requests during the build window -> 202" guarantee is exercised
|
||||
// deterministically. Without this the build of the tiny test DB finishes
|
||||
// almost instantly, so on a fast/idle machine some requests race past it and
|
||||
// get 200 — a flake, not a regression (the code only ever promised 202 for
|
||||
// requests that arrive WHILE the build is in flight).
|
||||
release := make(chan struct{})
|
||||
store.distanceBuildHook = func() { <-release }
|
||||
srv.store = store
|
||||
r := mux.NewRouter()
|
||||
srv.RegisterRoutes(r)
|
||||
|
||||
const N = 10
|
||||
var wg sync.WaitGroup
|
||||
var got202 atomic.Int32
|
||||
wg.Add(N)
|
||||
for i := 0; i < N; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
req := httptest.NewRequest("GET", "/api/analytics/distance", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
if w.Code == 202 {
|
||||
got202.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(release) // let the gated build finish
|
||||
if got202.Load() != N {
|
||||
t.Fatalf("expected all %d concurrent first-window requests to get 202; only %d did", N, got202.Load())
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// TestFirstSeen_1166_HandleNodesSurface pins issue #1166: the /api/nodes
|
||||
// response carries a `first_seen` ISO timestamp per node so the frontend
|
||||
// can show a sortable "First Seen" column.
|
||||
func TestFirstSeen_1166_HandleNodesSurface(t *testing.T) {
|
||||
db := setupCapabilityTestDB(t)
|
||||
defer db.conn.Close()
|
||||
if _, err := db.conn.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pk := "cccc000000000000000000000000000000000000000000000000000000000000"
|
||||
first := time.Now().Add(-72 * time.Hour).UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
last := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
if _, err := db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 5)`,
|
||||
pk, last, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := NewPacketStore(db, nil)
|
||||
cfg := &Config{Port: 3000}
|
||||
hub := NewHub()
|
||||
srv := NewServer(db, cfg, hub)
|
||||
srv.store = store
|
||||
|
||||
router := mux.NewRouter()
|
||||
srv.RegisterRoutes(router)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes?limit=10", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
router.ServeHTTP(rr, req)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("/api/nodes status: want 200, got %d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Nodes []map[string]interface{} `json:"nodes"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v body=%s", err, rr.Body.String())
|
||||
}
|
||||
var got map[string]interface{}
|
||||
for _, n := range resp.Nodes {
|
||||
if k, _ := n["public_key"].(string); k == pk {
|
||||
got = n
|
||||
break
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatalf("node missing from /api/nodes response")
|
||||
}
|
||||
fs, hasFS := got["first_seen"]
|
||||
if !hasFS {
|
||||
t.Fatalf("first_seen absent from /api/nodes response (issue #1166)")
|
||||
}
|
||||
s, _ := fs.(string)
|
||||
if s == "" {
|
||||
t.Errorf("first_seen empty, want ISO timestamp, got %v", fs)
|
||||
}
|
||||
if s != first {
|
||||
t.Errorf("first_seen = %q, want %q", s, first)
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestGetStoreStats_CacheHit verifies that a second call within 30s returns
|
||||
// the cached observation counts without re-querying the database.
|
||||
func TestGetStoreStats_CacheHit(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
store := srv.store
|
||||
|
||||
store.statsCacheMu.Lock()
|
||||
store.statsCacheTime = time.Now()
|
||||
store.statsLastHour = 42
|
||||
store.statsLast24h = 777
|
||||
store.statsCacheMu.Unlock()
|
||||
|
||||
st, err := store.GetStoreStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetStoreStats: %v", err)
|
||||
}
|
||||
if st.PacketsLastHour != 42 {
|
||||
t.Errorf("cache hit: PacketsLastHour want 42 got %d", st.PacketsLastHour)
|
||||
}
|
||||
if st.PacketsLast24h != 777 {
|
||||
t.Errorf("cache hit: PacketsLast24h want 777 got %d", st.PacketsLast24h)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetStoreStats_CacheExpiry verifies that a cache older than 30s is
|
||||
// discarded and the database query re-runs to refresh the values.
|
||||
func TestGetStoreStats_CacheExpiry(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
store := srv.store
|
||||
|
||||
store.statsCacheMu.Lock()
|
||||
store.statsCacheTime = time.Now().Add(-35 * time.Second)
|
||||
store.statsLastHour = 9999
|
||||
store.statsLast24h = 9999
|
||||
store.statsCacheMu.Unlock()
|
||||
|
||||
st, err := store.GetStoreStats()
|
||||
if err != nil {
|
||||
t.Fatalf("GetStoreStats: %v", err)
|
||||
}
|
||||
if st.PacketsLastHour == 9999 || st.PacketsLast24h == 9999 {
|
||||
t.Errorf("stale cache not expired: got PacketsLastHour=%d PacketsLast24h=%d — DB values expected, not sentinel",
|
||||
st.PacketsLastHour, st.PacketsLast24h)
|
||||
}
|
||||
|
||||
store.statsCacheMu.Lock()
|
||||
age := time.Since(store.statsCacheTime)
|
||||
store.statsCacheMu.Unlock()
|
||||
if age > 5*time.Second {
|
||||
t.Errorf("cache not refreshed after expiry: statsCacheTime age=%v", age)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetStoreStats_CacheConcurrentReaders verifies that 100 concurrent
|
||||
// callers produce no data race on the stats cache fields.
|
||||
// Run with: go test -race ./... -run TestGetStoreStats_CacheConcurrentReaders
|
||||
func TestGetStoreStats_CacheConcurrentReaders(t *testing.T) {
|
||||
srv, _ := setupTestServer(t)
|
||||
store := srv.store
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 100)
|
||||
for range 100 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if _, err := store.GetStoreStats(); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Errorf("concurrent GetStoreStats: %v", err)
|
||||
}
|
||||
}
|
||||
+1
-8
@@ -30,10 +30,6 @@ require github.com/meshcore-analyzer/dbschema v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema
|
||||
|
||||
require github.com/meshcore-analyzer/lora v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/lora => ../../internal/lora
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
@@ -50,9 +46,6 @@ require github.com/meshcore-analyzer/prunequeue v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue
|
||||
|
||||
require (
|
||||
github.com/meshcore-analyzer/mbcapqueue v0.0.0
|
||||
golang.org/x/sync v0.10.0
|
||||
)
|
||||
require github.com/meshcore-analyzer/mbcapqueue v0.0.0
|
||||
|
||||
replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue
|
||||
|
||||
@@ -16,8 +16,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
// Package main: shared weighted-graph primitives for the structural
|
||||
// usefulness axes (issue #672). The Bridge (betweenness) and Coverage
|
||||
// (harmonic reach) axes operate on the same undirected, affinity-weighted
|
||||
// neighbor graph and previously each built the distance adjacency by hand —
|
||||
// a line-for-line duplicate (#1762 review). These helpers are the single
|
||||
// source of truth for that construction. Within a single recomputeUsefulnessAxes
|
||||
// call the Bridge and Coverage axes are fed from the SAME bridgeEdgesFromGraph
|
||||
// snapshot, so they see byte-identical structure there. This does NOT extend
|
||||
// across recomputers: the bridge recomputer and the usefulness-axes recomputer
|
||||
// run on independent cadences and take their own graph snapshots, so the bridge
|
||||
// map surfaced by handleNodes need not match the Coverage axis's snapshot.
|
||||
package main
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// weightedDistanceAdjacency builds the symmetric distance adjacency from a
|
||||
// weighted edge list: cost = 1/weight (lower distance = stronger affinity),
|
||||
// keeping the cheapest edge per pair. Self-loops and edges with a non-finite
|
||||
// weight (NaN/±Inf) or weight < bridgeMinWeightEpsilon are skipped — they
|
||||
// would break Dijkstra's relaxation invariant. Note `w < epsilon` is false
|
||||
// for NaN, so NaN must be rejected explicitly; otherwise 1/NaN = NaN would
|
||||
// poison every reach computation downstream (#1762 review). Keys are the
|
||||
// lowercased pubkey form.
|
||||
func weightedDistanceAdjacency(edges []BridgeEdge) map[string]map[string]float64 {
|
||||
adj := make(map[string]map[string]float64)
|
||||
addOrMerge := func(a, b string, dist float64) {
|
||||
m, ok := adj[a]
|
||||
if !ok {
|
||||
m = make(map[string]float64)
|
||||
adj[a] = m
|
||||
}
|
||||
if existing, has := m[b]; !has || dist < existing {
|
||||
m[b] = dist
|
||||
}
|
||||
}
|
||||
for _, e := range edges {
|
||||
a := strings.ToLower(strings.TrimSpace(e.A))
|
||||
b := strings.ToLower(strings.TrimSpace(e.B))
|
||||
if a == "" || b == "" || a == b {
|
||||
continue
|
||||
}
|
||||
w := e.Weight
|
||||
if math.IsNaN(w) || math.IsInf(w, 0) || w < bridgeMinWeightEpsilon {
|
||||
continue
|
||||
}
|
||||
dist := 1.0 / w
|
||||
addOrMerge(a, b, dist)
|
||||
addOrMerge(b, a, dist)
|
||||
}
|
||||
return adj
|
||||
}
|
||||
|
||||
// dijkstraFrom returns the shortest-path distance from src to every reachable
|
||||
// node over the distance adjacency (unreachable nodes are absent). Reuses the
|
||||
// bridgePQ min-heap. Used by the Coverage axis; the Bridge axis runs its own
|
||||
// Brandes-coupled SSSP that additionally tracks predecessors and path counts.
|
||||
func dijkstraFrom(adj map[string]map[string]float64, src string) map[string]float64 {
|
||||
dist := map[string]float64{src: 0}
|
||||
pq := &bridgePQ{}
|
||||
heap.Init(pq)
|
||||
heap.Push(pq, bridgePQItem{node: src, dist: 0})
|
||||
|
||||
visited := make(map[string]bool)
|
||||
for pq.Len() > 0 {
|
||||
top := heap.Pop(pq).(bridgePQItem)
|
||||
v := top.node
|
||||
if visited[v] {
|
||||
continue
|
||||
}
|
||||
visited[v] = true
|
||||
for w, edgeDist := range adj[v] {
|
||||
alt := top.dist + edgeDist
|
||||
if cur, ok := dist[w]; !ok || alt < cur {
|
||||
dist[w] = alt
|
||||
heap.Push(pq, bridgePQItem{node: w, dist: alt})
|
||||
}
|
||||
}
|
||||
}
|
||||
return dist
|
||||
}
|
||||
+2
-12
@@ -42,7 +42,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
// processed<total).
|
||||
bfTotal, bfProcessed, bfDone := fromPubkeyBackfillSnapshot()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
resp := map[string]interface{}{
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"ready": true,
|
||||
"loadedTx": loadedTx,
|
||||
"loadedObs": loadedObs,
|
||||
@@ -51,15 +51,5 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
"processed": bfProcessed,
|
||||
"done": bfDone,
|
||||
},
|
||||
}
|
||||
// PR #1609 M1: surface per-MQTT-source receipt vs write-path
|
||||
// liveness so operators can distinguish "broker alive, write
|
||||
// path stuck" (lastReceiptUnix recent, lastMessageUnix stale)
|
||||
// from "everything stalled" (both stale). Additive — older
|
||||
// ingestor builds simply produce no entry and the field is
|
||||
// omitted. Schema-compatible with prior /healthz consumers.
|
||||
if liveness := readIngestorSourceLiveness(); len(liveness) > 0 {
|
||||
resp["ingest_liveness"] = liveness
|
||||
}
|
||||
json.NewEncoder(w).Encode(resp)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the
|
||||
// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are
|
||||
// projected to Web Mercator and snapped to a pointy-top hex grid whose size
|
||||
// depends on the display resolution. Cell ids are "res:q:r" (axial coords).
|
||||
// At city/region scale this looks like H3/mapme.sh coverage without any deps.
|
||||
|
||||
const hexEarthRadius = 6378137.0 // Web Mercator sphere radius (m)
|
||||
|
||||
// hexTargetPx is the desired on-screen hex size (point-to-point height) in CSS
|
||||
// pixels. mercUPPZ0 is Web Mercator units per pixel at zoom 0 (world span / 256);
|
||||
// Leaflet halves it each zoom level, independent of latitude. Sizing the hex in
|
||||
// these units therefore renders it at a constant ~hexTargetPx at every zoom — the
|
||||
// old fixed-meter buckets looked like specks when zoomed out (issue: hexes too small).
|
||||
const hexTargetPx = 28.0
|
||||
const mercUPPZ0 = 156543.03392
|
||||
|
||||
func hexMercator(lat, lon float64) (float64, float64) {
|
||||
x := hexEarthRadius * lon * math.Pi / 180
|
||||
y := hexEarthRadius * math.Log(math.Tan(math.Pi/4+lat*math.Pi/360))
|
||||
return x, y
|
||||
}
|
||||
|
||||
func hexInvMercator(x, y float64) (lat, lon float64) {
|
||||
lon = x / hexEarthRadius * 180 / math.Pi
|
||||
lat = (2*math.Atan(math.Exp(y/hexEarthRadius)) - math.Pi/2) * 180 / math.Pi
|
||||
return lat, lon
|
||||
}
|
||||
|
||||
// hexSizeForRes is the hex circumradius (center→corner) in Web Mercator units for a
|
||||
// display resolution. Resolution equals the Leaflet zoom level (see zoomToHexRes), so
|
||||
// the size scales as 2^-zoom and the hex keeps a constant ~hexTargetPx on-screen size
|
||||
// regardless of zoom. hexCellAt (binning) and hexBoundary (drawing) both read this, so
|
||||
// they stay consistent for a given cell id.
|
||||
func hexSizeForRes(res int) float64 {
|
||||
return (hexTargetPx / 2) * mercUPPZ0 / math.Pow(2, float64(res))
|
||||
}
|
||||
|
||||
// hexMaxLat is the Web Mercator latitude limit. The projection (hexMercator)
|
||||
// diverges toward ±90° — tan(π/4 + lat·π/360) → ∞ — so points beyond this would
|
||||
// produce NaN cell rings via hexInvMercator. Coverage is therefore only defined
|
||||
// within ±hexMaxLat; polar submissions are clamped to the edge (#17).
|
||||
const hexMaxLat = 85.05112878
|
||||
|
||||
// hexCellAt returns a stable cell id ("res:q:r") for the lat/lon at res. Latitude
|
||||
// is clamped to ±hexMaxLat so near-polar points bin to the edge instead of
|
||||
// producing NaN geometry.
|
||||
func hexCellAt(lat, lon float64, res int) string {
|
||||
if lat > hexMaxLat {
|
||||
lat = hexMaxLat
|
||||
} else if lat < -hexMaxLat {
|
||||
lat = -hexMaxLat
|
||||
}
|
||||
size := hexSizeForRes(res)
|
||||
x, y := hexMercator(lat, lon)
|
||||
q := (math.Sqrt(3)/3*x - 1.0/3*y) / size
|
||||
r := (2.0 / 3 * y) / size
|
||||
qi, ri := hexRound(q, r)
|
||||
return fmt.Sprintf("%d:%d:%d", res, qi, ri)
|
||||
}
|
||||
|
||||
// hexRound rounds fractional axial coords to the nearest hex via cube rounding.
|
||||
func hexRound(q, r float64) (int, int) {
|
||||
x, z := q, r
|
||||
y := -x - z
|
||||
rx, ry, rz := math.Round(x), math.Round(y), math.Round(z)
|
||||
dx, dy, dz := math.Abs(rx-x), math.Abs(ry-y), math.Abs(rz-z)
|
||||
switch {
|
||||
case dx > dy && dx > dz:
|
||||
rx = -ry - rz
|
||||
case dy > dz:
|
||||
ry = -rx - rz
|
||||
default:
|
||||
rz = -rx - ry
|
||||
}
|
||||
return int(rx), int(rz)
|
||||
}
|
||||
|
||||
// hexBoundary returns the cell's 6 corners as a closed [lon,lat] ring (GeoJSON
|
||||
// order), or nil if the cell id is malformed.
|
||||
func hexBoundary(cellID string) [][2]float64 {
|
||||
res, q, r, ok := parseHexCell(cellID)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
size := hexSizeForRes(res)
|
||||
cx := size * (math.Sqrt(3)*float64(q) + math.Sqrt(3)/2*float64(r))
|
||||
cy := size * (1.5 * float64(r))
|
||||
ring := make([][2]float64, 0, 7)
|
||||
for i := 0; i < 6; i++ {
|
||||
ang := math.Pi / 180 * float64(60*i-30)
|
||||
lat, lon := hexInvMercator(cx+size*math.Cos(ang), cy+size*math.Sin(ang))
|
||||
ring = append(ring, [2]float64{lon, lat})
|
||||
}
|
||||
ring = append(ring, ring[0]) // close the ring
|
||||
return ring
|
||||
}
|
||||
|
||||
func parseHexCell(id string) (res, q, r int, ok bool) {
|
||||
p := strings.Split(id, ":")
|
||||
if len(p) != 3 {
|
||||
return 0, 0, 0, false
|
||||
}
|
||||
a, e1 := strconv.Atoi(p[0])
|
||||
b, e2 := strconv.Atoi(p[1])
|
||||
c, e3 := strconv.Atoi(p[2])
|
||||
if e1 != nil || e2 != nil || e3 != nil {
|
||||
return 0, 0, 0, false
|
||||
}
|
||||
return a, b, c, true
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestHexCellAtClampsPolarLatitude verifies #17: latitudes past the Web Mercator
|
||||
// limit are clamped, so near-polar submissions bin to the edge cell and produce
|
||||
// finite geometry instead of NaN rings.
|
||||
func TestHexCellAtClampsPolarLatitude(t *testing.T) {
|
||||
for _, lat := range []float64{89.9, 90.0, -89.9, -90.0} {
|
||||
cell := hexCellAt(lat, 3.72, 9)
|
||||
clamped := math.Copysign(hexMaxLat, lat)
|
||||
if want := hexCellAt(clamped, 3.72, 9); cell != want {
|
||||
t.Fatalf("lat %.1f should clamp to %q, got %q", lat, want, cell)
|
||||
}
|
||||
ring := hexBoundary(cell)
|
||||
if ring == nil {
|
||||
t.Fatalf("lat %.1f produced no ring", lat)
|
||||
}
|
||||
for _, pt := range ring {
|
||||
if math.IsNaN(pt[0]) || math.IsNaN(pt[1]) || math.IsInf(pt[0], 0) || math.IsInf(pt[1], 0) {
|
||||
t.Fatalf("lat %.1f produced non-finite ring point %v", lat, pt)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHexCellAtStableAndDistinct(t *testing.T) {
|
||||
a := hexCellAt(51.0500, 3.7200, 9)
|
||||
b := hexCellAt(51.0500, 3.7200, 9)
|
||||
if a == "" || a != b {
|
||||
t.Fatalf("stable cell expected, got %q %q", a, b)
|
||||
}
|
||||
c := hexCellAt(51.2000, 3.7200, 9) // ~17 km away
|
||||
if c == a {
|
||||
t.Fatalf("distant point should differ, both %q", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHexBoundaryClosedRing(t *testing.T) {
|
||||
cell := hexCellAt(51.05, 3.72, 9)
|
||||
ring := hexBoundary(cell)
|
||||
if len(ring) != 7 {
|
||||
t.Fatalf("expected 7 points (closed hex), got %d", len(ring))
|
||||
}
|
||||
if ring[0] != ring[6] {
|
||||
t.Fatalf("ring not closed: %v vs %v", ring[0], ring[6])
|
||||
}
|
||||
if hexBoundary("garbage") != nil {
|
||||
t.Fatalf("malformed cell should return nil")
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestHiddenNamePrefix_1181_NodeHealth asserts that /api/nodes/{pk}/health
|
||||
// returns 404 for a node whose name starts with a hidden prefix — mirroring
|
||||
// the existing blacklist guard at the top of handleNodeHealth.
|
||||
//
|
||||
// Anti-tautology: this test FAILS if the IsNameHidden guard is removed from
|
||||
// handleNodeHealth (the handler would 200 with health data instead of 404).
|
||||
func TestHiddenNamePrefix_1181_NodeHealth(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
pk := "deadbeef00001184"
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`,
|
||||
pk, "🚫 health me", "companion"); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
get := func() *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest("GET", "/api/nodes/"+pk+"/health", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
return w
|
||||
}
|
||||
|
||||
srv.cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
w := get()
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("hidden: expected 404 from /api/nodes/%s/health, got %d body=%s", pk, w.Code, w.Body.String())
|
||||
}
|
||||
if strings.Contains(w.Body.String(), "health me") {
|
||||
t.Fatalf("hidden: name leaked in /health 404 body: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHiddenNamePrefix_1181_BulkHealth asserts /api/nodes/bulk-health filters
|
||||
// out nodes whose name starts with a hidden prefix — same shape as the
|
||||
// existing blacklist filter inside handleBulkHealth.
|
||||
//
|
||||
// Anti-tautology: remove the IsNameHidden branch from handleBulkHealth and
|
||||
// the hidden node leaks back into the response array; this assertion fails.
|
||||
func TestHiddenNamePrefix_1181_BulkHealth(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
pk := "deadbeef00001185"
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`,
|
||||
pk, "🚫 bulk me", "companion"); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
srv.cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
srv.cfg.NodeBlacklist = []string{"force-filter-branch"} // force the existing blacklist branch on so results-array path is taken
|
||||
srv.cfg.SetNodeBlacklist(srv.cfg.NodeBlacklist)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/nodes/bulk-health?limit=2000", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var arr []map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &arr); err != nil {
|
||||
t.Fatalf("unmarshal: %v body=%s", err, w.Body.String())
|
||||
}
|
||||
for _, e := range arr {
|
||||
if got, _ := e["public_key"].(string); strings.EqualFold(got, pk) {
|
||||
t.Fatalf("hidden node %s leaked through /api/nodes/bulk-health", pk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHiddenNamePrefix_1181_Paths asserts /api/nodes/{pk}/paths returns 404
|
||||
// for a hidden-prefix node, mirroring blacklist behaviour.
|
||||
func TestHiddenNamePrefix_1181_Paths(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
pk := "deadbeef00001186"
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`,
|
||||
pk, "🚫 paths me", "companion"); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
srv.cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
req := httptest.NewRequest("GET", "/api/nodes/"+pk+"/paths", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("hidden: expected 404 from /api/nodes/%s/paths, got %d body=%s", pk, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHiddenNamePrefix_1181_Analytics asserts /api/nodes/{pk}/analytics 404s
|
||||
// for hidden-prefix nodes.
|
||||
func TestHiddenNamePrefix_1181_Analytics(t *testing.T) {
|
||||
srv, router := setupTestServer(t)
|
||||
|
||||
pk := "deadbeef00001187"
|
||||
if _, err := srv.db.conn.Exec(`INSERT INTO nodes
|
||||
(public_key, name, role, lat, lon, last_seen, first_seen, advert_count)
|
||||
VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`,
|
||||
pk, "🚫 analytics me", "companion"); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
srv.cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
req := httptest.NewRequest("GET", "/api/nodes/"+pk+"/analytics", nil)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("hidden: expected 404 from /api/nodes/%s/analytics, got %d body=%s", pk, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHiddenNamePrefixesGeneration_Increments asserts the per-source
|
||||
// generation counter bumps on every Set call — mirrors
|
||||
// TestConfig_BlacklistGenerationIncrements behaviour. Cache wiring lives in
|
||||
// a follow-up; the counter is the prerequisite primitive.
|
||||
func TestHiddenNamePrefixesGeneration_Increments(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
g0 := cfg.HiddenNamePrefixesGeneration()
|
||||
cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
g1 := cfg.HiddenNamePrefixesGeneration()
|
||||
if g1 != g0+1 {
|
||||
t.Fatalf("first SetHiddenNamePrefixes: gen %d -> %d (want +1)", g0, g1)
|
||||
}
|
||||
cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
g2 := cfg.HiddenNamePrefixesGeneration()
|
||||
if g2 != g1+1 {
|
||||
t.Fatalf("second SetHiddenNamePrefixes: gen %d -> %d (want +1)", g1, g2)
|
||||
}
|
||||
cfg.SetHiddenNamePrefixes(nil)
|
||||
g3 := cfg.HiddenNamePrefixesGeneration()
|
||||
if g3 != g2+1 {
|
||||
t.Fatalf("nil SetHiddenNamePrefixes: gen %d -> %d (want +1)", g2, g3)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHiddenNamePrefixes_ConcurrentAccess hammers Set + IsNameHidden from
|
||||
// multiple goroutines. Doesn't assert anything beyond "doesn't panic" —
|
||||
// atomic.Pointer correctness is what we're verifying, race detector is not
|
||||
// in scope for this PR's CI (see PR scope).
|
||||
func TestHiddenNamePrefixes_ConcurrentAccess(t *testing.T) {
|
||||
cfg := &Config{}
|
||||
cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
|
||||
var stop atomic.Bool
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Writer
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; !stop.Load(); i++ {
|
||||
if i%2 == 0 {
|
||||
cfg.SetHiddenNamePrefixes([]string{"🚫", "test"})
|
||||
} else {
|
||||
cfg.SetHiddenNamePrefixes([]string{"🚫"})
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Readers
|
||||
for r := 0; r < 4; r++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !stop.Load() {
|
||||
_ = cfg.IsNameHidden("🚫 something")
|
||||
_ = cfg.IsNameHidden("normal name")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
stop.Store(true)
|
||||
wg.Wait()
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user