diff --git a/.eslintrc.json b/.eslintrc.json index d178d5df..a10d6429 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -209,6 +209,7 @@ "escapeHtml": "readonly", "exports": "readonly", "favStar": "readonly", + "fetchAllNodes": "readonly", "filterPacketsByRoute": "readonly", "formatAbsoluteTimestamp": "readonly", "formatChartAxisLabel": "readonly", diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e77606af..8e33da8a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -354,6 +354,7 @@ jobs: BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/public/analytics.js b/public/analytics.js index 5b0f3964..3e2bae36 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2148,7 +2148,7 @@ try { const rq = RegionFilter.regionQueryString() + AreaFilter.areaQueryString(); const [nodesResp, bulkHealth] = await Promise.all([ - api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), + fetchAllNodes('&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), api('/nodes/bulk-health?limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF }) ]); const nodes = nodesResp.nodes || nodesResp; @@ -2885,7 +2885,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = let nodesResp, hashSizesResp; try { [nodesResp, hashSizesResp] = await Promise.all([ - api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), + fetchAllNodes('&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), // #1270: fetch CONFIGURED-hash-size counts so the Network Overview // tells the operational story (matching Hash Stats "By Repeaters"), // not just a math-only count of unique pubkey slices. diff --git a/public/app.js b/public/app.js index f41f1cf8..bf6836be 100644 --- a/public/app.js +++ b/public/app.js @@ -130,6 +130,55 @@ async function api(path, { ttl = 0, bust = false } = {}) { return promise; } +// Fetch the COMPLETE /api/nodes set, transparently paging around the server's +// per-request row cap. /api/nodes clamps ?limit to `listLimits.nodesMax` +// (default 2000, operator-configurable; originally a hard 500 in PR #1540, +// raised/made configurable in PR #1589). A single ?limit=N fetch therefore +// silently truncates to the top nodesMax rows by last_seen DESC, so on a mesh +// with more nodes than that cap every node-list consumer (map, live, +// analytics, packets, area-map) loses the older-advert tail — a node that +// relays constantly but last self-advertised hours ago drops off the map even +// though it is plainly alive. #1606 fixed this for the Nodes page; this helper +// generalizes the same loop for all callers, using a fixed client page size +// well under the server cap. +// +// extraQuery: query fragment appended after the paged limit/offset, each piece +// already '&'-prefixed exactly as callers build it today +// (e.g. '&lastHeard=30d&area=x', '&sortBy=lastSeen®ion=y'); pass '' for none. +// safetyCap: hard ceiling on BOTH pages fetched and nodes returned — the result +// is sliced to it (callers like live.js pass their render ceiling here). +// Returns { nodes, counts, total }: counts is from the first page; total is the +// real deduped/capped node count (NOT the server's per-query `total`). +async function fetchAllNodes(extraQuery = '', { ttl = 0, pageSize = 500, safetyCap = 10000 } = {}) { + const accumulated = []; + let counts = {}; + for (let offset = 0; offset < safetyCap; offset += pageSize) { + const data = await api(`/nodes?limit=${pageSize}&offset=${offset}${extraQuery}`, { ttl }); + const page = data && Array.isArray(data.nodes) ? data.nodes + : (Array.isArray(data) ? data : []); + accumulated.push.apply(accumulated, page); + if (offset === 0) counts = (data && data.counts) || {}; + // Canonical stop: a short page is the end. The server's `total` is a real + // COUNT(*) for the query, but the handler overwrites it with the filtered + // length under area/geo/blacklist filtering — so we never loop on it, nor + // surface it; a short page is the reliable end-of-data signal. See #1606. + if (page.length < pageSize) break; + } + // Dedup by public_key: the sort window (last_seen DESC by default) can shift + // under concurrent ingest, repeating a row across a page boundary. Rows + // missing a public_key get a unique synthetic key so they are NOT collapsed. + const seen = new Map(); + for (let i = 0; i < accumulated.length; i++) { + const n = accumulated[i]; + seen.set((n && n.public_key) || ('__nokey' + i), n); + } + // Enforce safetyCap as a real node-count ceiling (the page loop only bounds + // it to the next pageSize multiple), so e.g. live.js's LIVE_MAP_MAX_NODES is + // honored exactly rather than overshooting by up to pageSize-1. + const nodes = Array.from(seen.values()).slice(0, safetyCap); + return { nodes, counts, total: nodes.length }; +} + function invalidateApiCache(prefix) { for (const key of _apiCache.keys()) { if (key.startsWith(prefix || '')) _apiCache.delete(key); diff --git a/public/area-map.html b/public/area-map.html index 8681c730..abde38b8 100644 --- a/public/area-map.html +++ b/public/area-map.html @@ -152,16 +152,11 @@ async function load() { document.getElementById('show-nodes').checked = false; try { - const [areasResp, nodesResp] = await Promise.all([ - fetch(`${baseUrl}/api/config/areas/polygons`), - fetch(`${baseUrl}/api/nodes?limit=9999`) - ]); + const areasResp = await fetch(`${baseUrl}/api/config/areas/polygons`); if (!areasResp.ok) throw new Error(`areas/polygons: ${areasResp.status}`); - if (!nodesResp.ok) throw new Error(`nodes: ${nodesResp.status}`); areas = await areasResp.json(); - const nodesData = await nodesResp.json(); - allNodes = nodesData.nodes || nodesData || []; + allNodes = await fetchAllNodesPaged(''); buildSidebar(); buildNodeLayer(); @@ -229,19 +224,38 @@ function escapeHtml(s) { return String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); } +// Paginate past the server's per-request node cap (listLimits.nodesMax) so the +// full node set is surfaced. This page is embeddable cross-origin via baseUrl, +// so it can't use app.js's fetchAllNodes; the loop is inlined. extra: query +// fragment '&'-prefixed (e.g. '&area=x'). Throws on a non-OK response so callers +// surface the error instead of silently rendering a truncated, complete-looking set. +async function fetchAllNodesPaged(extra) { + const PAGE = 500, CAP = 10000, out = []; + for (let off = 0; off < CAP; off += PAGE) { + const r = await fetch(`${baseUrl}/api/nodes?limit=${PAGE}&offset=${off}${extra}`); + if (!r.ok) throw new Error(`/api/nodes ${r.status} at offset ${off}`); + const d = await r.json(); + const page = Array.isArray(d) ? d : (d.nodes || []); + out.push.apply(out, page); + if (page.length < PAGE) break; + } + // Dedup by public_key; rows missing one get a unique key so they aren't collapsed. + const seen = new Map(); + out.forEach((n, i) => seen.set((n && n.public_key) || ('__nokey' + i), n)); + return Array.from(seen.values()); +} + async function loadAreaNodes(areaKey, color, group) { try { - const resp = await fetch(`${baseUrl}/api/nodes?area=${encodeURIComponent(areaKey)}&limit=9999`); - if (!resp.ok) return; - const data = await resp.json(); - (data.nodes || data || []).forEach(n => { + const list = await fetchAllNodesPaged(`&area=${encodeURIComponent(areaKey)}`); + list.forEach(n => { if (!n.lat || !n.lon) return; const m = L.circleMarker([n.lat, n.lon], { radius: 7, color, fillColor: color, fillOpacity: 0.85, weight: 2 }); m.bindPopup(`
${escapeHtml(n.name || n.public_key?.slice(0,8) || '?')}
${escapeHtml(n.public_key?.slice(0,16) || '')}…
GPS: ${n.lat.toFixed(5)}, ${n.lon.toFixed(5)}
${escapeHtml(areaKey)}
`); group.addLayer(m); }); - } catch(_) {} + } catch (e) { console.error('loadAreaNodes failed for', areaKey, e); } } function buildNodeLayer() { diff --git a/public/live.js b/public/live.js index 077acca9..61b0ca4f 100644 --- a/public/live.js +++ b/public/live.js @@ -2577,9 +2577,9 @@ // "Show all nodes" is enabled. Empty string when no region set. const rqs = (window.RegionFilter && typeof RegionFilter.nodesRegionQueryString === 'function') ? RegionFilter.nodesRegionQueryString() : ''; - const url = beforeTs - ? `/api/nodes?limit=${window.LIVE_MAP_MAX_NODES}&before=${encodeURIComponent(new Date(beforeTs).toISOString())}${aqs}${rqs}` - : `/api/nodes?limit=${window.LIVE_MAP_MAX_NODES}${aqs}${rqs}`; + const beforeQs = beforeTs + ? `&before=${encodeURIComponent(new Date(beforeTs).toISOString())}` + : ''; // Full reload (no beforeTs): clear existing markers so switching areas // removes nodes that no longer belong to the selected area. if (!beforeTs) { @@ -2587,9 +2587,13 @@ nodeMarkers = {}; nodeData = {}; } - const resp = await fetch(url); - const nodes = await resp.json(); - const list = Array.isArray(nodes) ? nodes : (nodes.nodes || []); + // Paginate past the server's per-request node cap (listLimits.nodesMax) + // so the live map isn't truncated to the top-N by advert. LIVE_MAP_MAX_NODES + // is passed as safetyCap, which fetchAllNodes enforces as a hard ceiling on + // returned nodes — so the live map never renders more than the configured max. + const { nodes: list } = await fetchAllNodes(`${beforeQs}${aqs}${rqs}`, { + safetyCap: window.LIVE_MAP_MAX_NODES || 10000, + }); var now = Date.now(); list.forEach(n => { if (n.lat != null && n.lon != null && !(n.lat === 0 && n.lon === 0)) { diff --git a/public/map.js b/public/map.js index cfcbb4bb..5f0c0b42 100644 --- a/public/map.js +++ b/public/map.js @@ -1216,7 +1216,10 @@ try { REGION_NAMES = await api('/config/regions', { ttl: 3600 }); } catch {} const aqs = AreaFilter.areaQueryString(); - const data = await api(`/nodes?limit=10000&lastHeard=${filters.lastHeard}${aqs}`, { ttl: CLIENT_TTL.nodeList }); + // Paginate past the server's per-request node cap (listLimits.nodesMax) + // so actively-relaying repeaters that last advertised hours ago still + // appear instead of being truncated by the top-N window. See fetchAllNodes. + const data = await fetchAllNodes(`&lastHeard=${filters.lastHeard}${aqs}`, { ttl: CLIENT_TTL.nodeList }); nodes = data.nodes || []; // Load observers for jump buttons + map markers diff --git a/public/packets.js b/public/packets.js index 54c63d0a..238ded68 100644 --- a/public/packets.js +++ b/public/packets.js @@ -903,7 +903,7 @@ if (!HopResolver.ready()) { try { const [nodeData, obsData, coordData] = await Promise.all([ - api('/nodes?limit=2000', { ttl: 60000 }), + fetchAllNodes('', { ttl: 60000 }), api('/observers', { ttl: 60000 }), api('/iata-coords', { ttl: 300000 }).catch(() => ({ coords: {} })), ]); @@ -911,7 +911,11 @@ observers: obsData.observers || obsData || [], iataCoords: coordData.coords || {}, }); - } catch {} + } catch (e) { + // Non-fatal: hops will render as unresolved hex prefixes until a later + // call succeeds. Log so a paginated /api/nodes failure isn't silent. + console.warn('[packets] HopResolver init failed:', e); + } } } diff --git a/test-all.sh b/test-all.sh index 004a6dd2..3bcc95cb 100755 --- a/test-all.sh +++ b/test-all.sh @@ -14,6 +14,7 @@ node test-packet-filter-ux.js node test-aging.js node test-issue-1065-gesture-hints-gates.js node test-frontend-helpers.js +node test-fetch-all-nodes-pagination.js node test-url-state.js node test-perf-go-runtime.js node test-channel-psk-ux.js diff --git a/test-fetch-all-nodes-pagination.js b/test-fetch-all-nodes-pagination.js new file mode 100644 index 00000000..b5368949 --- /dev/null +++ b/test-fetch-all-nodes-pagination.js @@ -0,0 +1,247 @@ +/* Regression test for the /api/nodes per-request row cap (#1606 class). + * + * Bug: /api/nodes clamps ?limit to listLimits.nodesMax (default 2000; + * originally a hard 500 in PR #1540, raised/made configurable in #1589) and + * orders by last_seen DESC. Every node-list consumer (map.js, live.js, + * analytics.js, packets.js, area-map.html) issued a single ?limit=N fetch and + * trusted the response as the full set, so a mesh with more nodes than the cap + * silently saw only the top rows. A repeater that relays constantly but last + * self-advertised hours ago fell outside that window and vanished from the + * map/live view even though it was plainly alive. + * + * Fix: a shared app.js `fetchAllNodes()` helper pages through /api/nodes (fixed + * client page size, well under the server cap) until a short page, deduping by + * public_key, and caps the result at safetyCap. This test drives the REAL + * api()+fetchAllNodes against a mocked fetch using a 500-per-page fixture (the + * client page size) past which a node is hidden, plus the server's unreliable + * per-page `total`. Pre-fix consumers saw one page; the helper surfaces all. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +const pending = []; +function test(name, fn) { + try { + const out = fn(); + if (out && typeof out.then === 'function') { + pending.push(out.then(() => { passed++; console.log(' ✅ ' + name); }) + .catch(e => { failed++; console.log(' ❌ ' + name + ': ' + e.message); })); + return; + } + passed++; console.log(' ✅ ' + name); + } catch (e) { + failed++; console.log(' ❌ ' + name + ': ' + e.message); + } +} + +function makeSandbox() { + const ctx = { + window: {}, + document: { + readyState: 'complete', + createElement: () => ({ id: '', textContent: '', innerHTML: '' }), + head: { appendChild: () => {} }, + getElementById: () => null, + addEventListener: () => {}, + querySelectorAll: () => [], + querySelector: () => null, + }, + console, Date, Infinity, Math, Array, Object, String, Number, JSON, RegExp, Error, TypeError, + parseInt, parseFloat, isNaN, isFinite, encodeURIComponent, decodeURIComponent, + setTimeout: () => {}, clearTimeout: () => {}, setInterval: () => {}, clearInterval: () => {}, + performance: { now: () => Date.now() }, + localStorage: (() => { const s = {}; return { getItem: k => s[k] || null, setItem: (k, v) => { s[k] = String(v); }, removeItem: k => { delete s[k]; } }; })(), + location: { hash: '' }, + CustomEvent: class CustomEvent {}, + Map, Set, Promise, URLSearchParams, + addEventListener: () => {}, dispatchEvent: () => {}, + fetch: () => Promise.resolve({ ok: true, json: () => Promise.resolve({}) }), + }; + ctx.window.addEventListener = () => {}; + ctx.window.dispatchEvent = () => {}; + vm.createContext(ctx); + return ctx; +} + +function loadInCtx(ctx, file) { + vm.runInContext(fs.readFileSync(file, 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; +} + +// A fetch mock that mirrors the real /api/nodes: clamps ?limit to `cap`, +// honors ?offset, and returns the same kind of unreliable `total` the server +// emits (clamped to the page size). `extraDup` injects one duplicate +// public_key straddling a page boundary to exercise dedup. +function makeNodesFetch(total, cap, opts = {}) { + const calls = []; + const fixture = []; + for (let i = 0; i < total; i++) fixture.push({ public_key: 'pk' + i, name: 'N' + i }); + // Optionally repeat the last row of page 1 as the first row of page 2. + if (opts.dupAtBoundary) fixture[cap] = fixture[cap - 1]; + return { + calls, + fetch: (url) => { + calls.push(url); + const qs = url.split('?')[1] || ''; + const p = new URLSearchParams(qs); + const limit = Math.min(parseInt(p.get('limit') || '50', 10), cap); + const offset = parseInt(p.get('offset') || '0', 10); + const page = fixture.slice(offset, offset + limit); + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ nodes: page, counts: { repeaters: total }, total: page.length }), + }); + }, + }; +} + +console.log('\n=== fetchAllNodes: pagination past the 500-row cap ==='); + +test('surfaces ALL nodes past the 500 server cap (1200 > 500)', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + const m = makeNodesFetch(1200, 500); + ctx.fetch = m.fetch; + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.nodes.length, 1200, 'expected all 1200 nodes, got ' + out.nodes.length); + assert.strictEqual(out.total, 1200, 'total must be the real deduped count, not the clamped per-page total'); + assert.strictEqual(m.calls.length, 3, 'expected 3 pages (500+500+200), got ' + m.calls.length); +}); + +test('stops on a short page rather than the unreliable server total', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + // Exactly 1000 → pages 500, 500, then a 0-length page stops the loop. + const m = makeNodesFetch(1000, 500); + ctx.fetch = m.fetch; + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.nodes.length, 1000); + assert.strictEqual(m.calls.length, 3, 'expected one extra empty page after two full pages'); +}); + +test('dedups a public_key repeated across a page boundary', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + const m = makeNodesFetch(1200, 500, { dupAtBoundary: true }); + ctx.fetch = m.fetch; + const out = await ctx.fetchAllNodes(''); + const keys = out.nodes.map(n => n.public_key); + assert.strictEqual(new Set(keys).size, keys.length, 'result must contain no duplicate public_key'); +}); + +test('passes counts from the first page through', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + const m = makeNodesFetch(600, 500); + ctx.fetch = m.fetch; + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.counts.repeaters, 600); +}); + +test('appends extraQuery verbatim and caps at a 500-aligned safetyCap', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + const m = makeNodesFetch(5000, 500); + ctx.fetch = m.fetch; + const out = await ctx.fetchAllNodes('&lastHeard=30d&area=BE', { safetyCap: 1000 }); + // safetyCap=1000 → offsets 0 and 500 → exactly 1000 nodes (tightened so an + // off-by-pageSize regression is caught here, not in production). + assert.strictEqual(out.nodes.length, 1000, 'expected exactly 1000 nodes, got ' + out.nodes.length); + assert.ok(m.calls[0].indexOf('&lastHeard=30d&area=BE') !== -1, 'extraQuery must be appended to the request'); + assert.ok(m.calls[0].indexOf('limit=500') !== -1 && m.calls[0].indexOf('offset=0') !== -1, 'first page must request limit/offset'); +}); + +test('safetyCap is a hard node-count ceiling (no pageSize overshoot)', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + const m = makeNodesFetch(5000, 500); + ctx.fetch = m.fetch; + // safetyCap=800 is NOT a multiple of pageSize: the loop fetches offsets 0 and + // 500 (1000 rows) but the result must be sliced back to exactly 800 — this is + // the live.js LIVE_MAP_MAX_NODES ceiling that previously overshot. + const out = await ctx.fetchAllNodes('', { safetyCap: 800 }); + assert.strictEqual(out.nodes.length, 800, 'safetyCap must cap returned nodes exactly, got ' + out.nodes.length); + assert.strictEqual(out.total, 800, 'total must equal the capped count'); +}); + +test('rows missing public_key are NOT collapsed into one', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + // Two distinct rows both lacking public_key must survive as two entries. + ctx.fetch = () => Promise.resolve({ + ok: true, + json: () => Promise.resolve({ nodes: [{ name: 'A' }, { name: 'B' }, { public_key: 'pk1', name: 'C' }] }), + }); + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.nodes.length, 3, 'falsy-key rows must not collapse, got ' + out.nodes.length); +}); + +test('empty result → exactly one request, zero nodes', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + let calls = 0; + ctx.fetch = () => { calls++; return Promise.resolve({ ok: true, json: () => Promise.resolve({ nodes: [], counts: {}, total: 0 }) }); }; + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.nodes.length, 0); + assert.strictEqual(calls, 1, 'a 0-length first page must stop after one request, got ' + calls); +}); + +test('handles a bare-array /api/nodes response shape', async () => { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/app.js'); + ctx.fetch = () => Promise.resolve({ ok: true, json: () => Promise.resolve([{ public_key: 'a' }, { public_key: 'b' }]) }); + const out = await ctx.fetchAllNodes(''); + assert.strictEqual(out.nodes.length, 2, 'bare-array body must be accepted'); +}); + +// ===== area-map.html inline fetchAllNodesPaged (separate impl, can't import app.js) ===== +// area-map.html is embeddable cross-origin, so it carries its own copy of the +// loop. Extract that function from the HTML and exercise the SAME behaviors plus +// its distinct error contract (throws on a non-OK page rather than returning a +// silent partial). Keeping this test next to the helper's prevents the two +// copies from drifting unnoticed. +function loadAreaMapHelper(fetchImpl) { + const html = fs.readFileSync('public/area-map.html', 'utf8'); + const start = html.indexOf('async function fetchAllNodesPaged'); + assert(start !== -1, 'fetchAllNodesPaged not found in area-map.html'); + let depth = 0, end = -1; + for (let i = html.indexOf('{', start); i < html.length; i++) { + if (html[i] === '{') depth++; + else if (html[i] === '}' && --depth === 0) { end = i + 1; break; } + } + const src = html.slice(start, end); + // Provide fetch + baseUrl as closure params (the function references both). + return new Function('fetch', 'baseUrl', src + '\nreturn fetchAllNodesPaged;')(fetchImpl, 'http://host'); +} + +test('area-map inline helper paginates past the cap and dedups', async () => { + const m = makeNodesFetch(1200, 500, { dupAtBoundary: true }); + const fetchAllNodesPaged = loadAreaMapHelper(m.fetch); + const list = await fetchAllNodesPaged(''); + const keys = list.map(n => n.public_key); + assert.strictEqual(new Set(keys).size, keys.length, 'no duplicate public_key'); + assert.ok(list.length >= 1199, 'expected ~1200 nodes past the cap, got ' + list.length); +}); + +test('area-map inline helper throws on a non-OK page (no silent partial)', async () => { + const fixture500 = []; + for (let i = 0; i < 500; i++) fixture500.push({ public_key: 'q' + i }); + const fetchImpl = (url) => { + const off = parseInt(new URLSearchParams(url.split('?')[1] || '').get('offset') || '0', 10); + if (off === 0) return Promise.resolve({ ok: true, json: () => Promise.resolve({ nodes: fixture500 }) }); + return Promise.resolve({ ok: false, status: 500, json: () => Promise.resolve({}) }); + }; + const fetchAllNodesPaged = loadAreaMapHelper(fetchImpl); + let threw = false; + try { await fetchAllNodesPaged(''); } catch (_) { threw = true; } + assert.ok(threw, 'a non-OK page must reject, not return a truncated complete-looking list'); +}); + +(async () => { + await Promise.all(pending); + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed > 0 ? 1 : 0); +})(); diff --git a/test-map-nodes-pagination-e2e.js b/test-map-nodes-pagination-e2e.js new file mode 100644 index 00000000..cee5db03 --- /dev/null +++ b/test-map-nodes-pagination-e2e.js @@ -0,0 +1,153 @@ +/** + * E2E: the Map view must paginate /api/nodes past the server's per-request cap. + * + * Bug (this PR): map.js issued a single `/api/nodes?limit=10000` fetch. The + * server clamps ?limit to listLimits.nodesMax (default 2000; originally 500 in + * PR #1540) and orders by last_seen DESC, so on a mesh larger than the cap the + * map silently dropped every node whose last self-advert fell outside the top + * window — even ones relaying constantly. The node still appeared in the + * (paginated, #1606) Nodes list but vanished from the map. + * + * This test mocks /api/nodes with a 500-per-page cap (the client page size) and + * a 501st node ("PAGE2 RP") reachable only on the second page. After the map + * loads, the app's node set (window.__mc_nodes, populated by map.js loadNodes() + * → fetchAllNodes()) must contain all 501 nodes including the page-2 node, and a + * marker for it must exist on the map. Pre-fix, __mc_nodes would hold 500 and + * the page-2 node would be absent. + * + * Backend-independent: every /api/* call the map makes at load is mocked via + * page.route, so it runs against any static host at BASE_URL (the CI fixture + * server, or a plain static server locally). + * + * Run: BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; +const PAGE_CAP = 500; // client page size; a node sits past it on page 2 +const PAGE2_KEY = 'page2deadbeef00000000000000000000000000000000000000000000000beef02'; +const PAGE2_NAME = 'PAGE2 RP'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +// Build the full fixture: 500 "page 1" repeaters + 1 page-2 repeater. +function buildFixture() { + const nodes = []; + for (let i = 0; i < PAGE_CAP; i++) { + nodes.push({ + public_key: 'p1' + String(i).padStart(62, '0'), + name: 'P1-' + i, + role: 'repeater', + lat: 51 + (i % 100) * 0.001, + lon: 5 + Math.floor(i / 100) * 0.001, + last_seen: '2026-06-09T07:40:00Z', + hash_size: 1, + }); + } + nodes.push({ + public_key: PAGE2_KEY, + name: PAGE2_NAME, + role: 'repeater', + lat: 50.5, + lon: 4.5, + last_seen: '2026-06-08T13:55:14Z', // older advert → would be cut by the 500 cap + hash_size: 2, + }); + return nodes; +} + +(async () => { + const launchOpts = { + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }; + const browser = await chromium.launch(launchOpts); + const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(15000); + page.on('pageerror', (e) => console.error('[pageerror]', e.message)); + + console.log(`\n=== Map /api/nodes pagination E2E against ${BASE} ===`); + + const fixture = buildFixture(); + let nodesRequests = 0; + + // Mock every /api/* call the map makes at load. /api/nodes?... is paginated + // with the same 500-row clamp + offset semantics as the real server; all + // other endpoints get harmless stubs so the test is backend-independent. + await page.route('**/api/**', (route) => { + const url = new URL(route.request().url()); + const path = url.pathname; + if (path === '/api/nodes') { + nodesRequests++; + const limit = Math.min(parseInt(url.searchParams.get('limit') || '50', 10), PAGE_CAP); + const offset = parseInt(url.searchParams.get('offset') || '0', 10); + const slice = fixture.slice(offset, offset + limit); + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + nodes: slice, + total: slice.length, // deliberately wrong per-page total; the helper must ignore it + counts: { repeaters: fixture.length, rooms: 0, companions: 0, sensors: 0 }, + }), + }); + } + if (path === '/api/observers') { + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ observers: [] }) }); + } + // Generic stub for config/regions/map/etc. — empty object is safe. + return route.fulfill({ status: 200, contentType: 'application/json', body: '{}' }); + }); + + await page.goto(BASE + '/#/map', { waitUntil: 'load', timeout: 60000 }); + await page.waitForSelector('#leaflet-map', { timeout: 15000 }); + + await step('map loads all 501 nodes by paginating past the 500-row cap', async () => { + // Wait until loadNodes() has populated the app node set. + await page.waitForFunction( + () => Array.isArray(window.__mc_nodes) && window.__mc_nodes.length >= 501, + { timeout: 15000 } + ); + const len = await page.evaluate(() => window.__mc_nodes.length); + assert(len === 501, 'expected 501 nodes in __mc_nodes, got ' + len); + assert(nodesRequests >= 2, 'expected ≥2 /api/nodes page requests, got ' + nodesRequests); + }); + + await step('the page-2 node (cut by the cap pre-fix) is present in the node set', async () => { + const found = await page.evaluate( + (key) => window.__mc_nodes.some((n) => n.public_key === key), + PAGE2_KEY + ); + assert(found, 'page-2 node ' + PAGE2_NAME + ' missing from __mc_nodes'); + }); + + await step('a marker for the page-2 node is rendered on the map', async () => { + const hasMarker = await page.evaluate((key) => { + let found = false; + const scan = (layer) => { + if (found || !layer || !layer.eachLayer) return; + layer.eachLayer((m) => { + if (found) return; + if (m._nodeKey === key) { found = true; return; } + if (m.eachLayer) scan(m); // cluster groups nest their markers + }); + }; + // markerLayer + clusterGroup are internal; reach them via the map's layers. + if (window.__mc_map && window.__mc_map.eachLayer) window.__mc_map.eachLayer(scan); + return found; + }, PAGE2_KEY); + assert(hasMarker, 'no marker with _nodeKey for the page-2 node was rendered'); + }); + + await browser.close(); + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed > 0 ? 1 : 0); +})().catch((e) => { console.error(e); process.exit(1); });