diff --git a/.github/workflows/pages-geocaching.yml b/.github/workflows/pages-geocaching.yml index e11578c0..dd7f3f8b 100644 --- a/.github/workflows/pages-geocaching.yml +++ b/.github/workflows/pages-geocaching.yml @@ -46,6 +46,7 @@ jobs: npm run build:geocaching node --test ../tests/site/geocaching-protocol.test.mjs node --test ../tests/site/geocaching-regions.test.mjs + node --test ../tests/site/geocaching-snapshot.test.mjs - name: Overlay the Geocaching page and navigation run: | cp -a site/geocaching "$RUNNER_TEMP/publish/" diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d52fbc6e..d2936501 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -102,6 +102,7 @@ jobs: npm run build:components node build-geocaching.mjs node --test ../tests/site/geocaching-protocol.test.mjs + node --test ../tests/site/geocaching-snapshot.test.mjs rm -rf node_modules - name: Upload GitHub Pages artifact diff --git a/site/geocaching/data/public/.gitignore b/site/geocaching/data/public/.gitignore new file mode 100644 index 00000000..fcab935f --- /dev/null +++ b/site/geocaching/data/public/.gitignore @@ -0,0 +1,2 @@ +!objects/ +!objects/*.bin diff --git a/site/geocaching/data/public/README.md b/site/geocaching/data/public/README.md new file mode 100644 index 00000000..eae3c455 --- /dev/null +++ b/site/geocaching/data/public/README.md @@ -0,0 +1,24 @@ +# Published public directory + +The map loads this directory even when its live Reticulum bridge is offline. +`index.json` includes the export time and current public summaries. Immutable +signed objects are fetched on demand and verified against their summaries before +displaying details or producing GPX. Archived heads are retained so archive +filters and later exports preserve the publisher's state. + +This is a dated copy, not a live global inventory. An unavailable bridge must not +be presented as an empty live result. The worker can use live directories when +available and falls back to the published copy when a live query fails. + +To refresh from an operator's directory database: + +```sh +python tools/geocaching/export_public_directory.py --database /path/to/directory.sqlite --output site/geocaching/data/public +``` + +Run `node --test tests/site/geocaching-snapshot.test.mjs` and publish the resulting +files through the normal Pages deployment. The exporter opens SQLite read-only, +verifies each signed object, excludes conflicting heads, and exports only public +records. Never copy a service state directory, identity file or SQLite database +into the website. Automatic collection and deployment still require an available +directory source and an authorized scheduled deployment. diff --git a/site/geocaching/data/public/index.json b/site/geocaching/data/public/index.json new file mode 100644 index 00000000..bf5c5bb1 --- /dev/null +++ b/site/geocaching/data/public/index.json @@ -0,0 +1 @@ +{"version":1,"updatedAt":"2026-09-28T09:25:41.666848+00:00","summaries":["m8QgPjDvQphX74IMV5HHSMCxfnI5sftPiWd2GooXGm1BvoICxCB8R9Y20Voo4TFwaXef3IGHgER4nHqyrnLe17eagCz3LwIAALFBcmNoaXZlZCBsb2NhdGlvbgICAM0BBg==","m8QgkeI2uB8xiX5BTqpHoVUqvLkZp4OyZ5a5Ki04f7mGtMwBxCDNtFrWgLOpafU+MDYGnyH5SyBvyUFPg2VECHqL3ZrGkwDOBfXhANLSs1IAsFNhbiBKb3NlIEdhbGxlb24CAgDNAW0=","m8Qg31G0+16F7BrdNsYbLAahfe4UyiQcm7hsCuoKO3QeE5gCxCAJSCAIyc/6t1oylnbiFnoATIeWWlKSnwBnpyap3L70EwLOBfXhANLSs1IAsUFyY2hpdmVkIGxvY2F0aW9uAgIAzQEO"]} diff --git a/site/geocaching/data/public/objects/09482008c9cffab75a329676e2167a004c87965a52929f0067a726a9dcbef413.bin b/site/geocaching/data/public/objects/09482008c9cffab75a329676e2167a004c87965a52929f0067a726a9dcbef413.bin new file mode 100644 index 00000000..7efeb203 Binary files /dev/null and b/site/geocaching/data/public/objects/09482008c9cffab75a329676e2167a004c87965a52929f0067a726a9dcbef413.bin differ diff --git a/site/geocaching/data/public/objects/7c47d636d15a28e1317069779fdc81878044789c7ab2ae72ded7b79a802cf72f.bin b/site/geocaching/data/public/objects/7c47d636d15a28e1317069779fdc81878044789c7ab2ae72ded7b79a802cf72f.bin new file mode 100644 index 00000000..d408c8db Binary files /dev/null and b/site/geocaching/data/public/objects/7c47d636d15a28e1317069779fdc81878044789c7ab2ae72ded7b79a802cf72f.bin differ diff --git a/site/geocaching/data/public/objects/cdb45ad680b3a969f53e3036069f21f94b206fc9414f836544087a8bdd9ac693.bin b/site/geocaching/data/public/objects/cdb45ad680b3a969f53e3036069f21f94b206fc9414f836544087a8bdd9ac693.bin new file mode 100644 index 00000000..76e5908e Binary files /dev/null and b/site/geocaching/data/public/objects/cdb45ad680b3a969f53e3036069f21f94b206fc9414f836544087a8bdd9ac693.bin differ diff --git a/site/geocaching/src/page.js b/site/geocaching/src/page.js index 6f4f9932..f54404e7 100644 --- a/site/geocaching/src/page.js +++ b/site/geocaching/src/page.js @@ -17,7 +17,7 @@ const labels = { show:tr('Show caches','显示藏宝点'),active:tr('Active','开放'),disabled:tr('Disabled','停用'),archived:tr('Archived','归档'), emptyCount:tr('Loaded 0 caches','已加载 0 个藏宝点'),start:tr('Preparing your map…','正在准备你的寻宝地图…'), more:tr('Load more','加载更多'),downloadSelected:tr('Download selected GPX','下载所选 GPX'),downloadPartial:tr('Download successful items only','仅下载成功的项目'), - scope:tr('Results cover the directories reached in this session, not a global total.','结果来自本次连接到的目录，数量仅指已加载内容，不代表全网总数。'), + scope:tr('Results cover published copies or live directories, not a global total.','结果来自已同步副本或实时目录，数量不代表全网总数。'), search:tr('Search this area','搜索此区域'),download:tr('Download GPX','下载 GPX'), region:tr('Where to explore','探索范围'),regionKind:tr('Search by','按范围查找'), mapRegion:tr('Current map area','当前地图区域'),countries:tr('Country / region','国家／地区'),seas:tr('Ocean / sea','海洋／海域'), @@ -28,6 +28,7 @@ for (const node of document.querySelectorAll('[data-label]')) node.textContent = const $ = id => document.getElementById(id); const worker = new Worker(new URL('./reticulum-worker.js', import.meta.url), {type:'module'}); let nextId = 0, ready = false, rows = [], detailId = null, detailGeneration = 0, partial = null; +let snapshotAt = null; const pending = new Map(), selected = new Set(); const map = L.map('map', {worldCopyJump:true, minZoom:0}).fitWorld(); const tiles = L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', {maxZoom:19, updateWhenIdle:true, @@ -153,7 +154,7 @@ function renderMarkers() { for (const cluster of cells.values()) { const {row,lat,lon} = cluster[0], multi = cluster.length > 1; const content = document.createElement('span'), image = document.createElement('img'); - image.src = cacheIconUrl; image.alt = ''; image.width = image.height = 32; + image.src = new URL(cacheIconUrl, import.meta.url).href; image.alt = ''; image.width = image.height = 32; content.append(image); if (multi) { const count = document.createElement('span'); @@ -176,13 +177,14 @@ async function openDetail(row) { try { const item = await rpc('get',{cacheId:row.id}); if (generation !== detailGeneration) return; - $('detail-state').textContent = item.isCurrent ? tr('AUTHOR SIGNATURE VERIFIED','作者签名已验证') : tr('VERIFIED HISTORICAL VERSION','已验证的历史版本'); + $('detail-state').textContent = item.snapshotAt ? tr('VERIFIED PUBLISHED COPY','已验证的公开副本') : item.isCurrent ? tr('AUTHOR SIGNATURE VERIFIED','作者签名已验证') : tr('VERIFIED HISTORICAL VERSION','已验证的历史版本'); const description = document.createElement('p'); description.textContent = item.record[9]; const hint = document.createElement('details'), summary = document.createElement('summary'), text = document.createElement('p'); summary.textContent = tr('Show hint','展开提示'); text.textContent = item.record[10] || tr('No hint provided.','未提供提示。'); hint.append(summary,text); const metadata = document.createElement('p'); metadata.className = 'fingerprint'; metadata.textContent = `${tr('Author','作者')}: ${item.authorHash}\n${tr('Version','版本')}: ${item.record[3]}\n` + - `${tr('Verified at','验证时间')}: ${new Date(item.checkedAt).toLocaleString()}\n${item.sourceName}`; + `${tr('Verified at','验证时间')}: ${new Date(item.checkedAt).toLocaleString()}\n${item.sourceName}` + + (item.snapshotAt ? `\n${tr('Directory synced','目录同步时间')}: ${new Date(item.snapshotAt).toLocaleString()}` : ''); $('detail-content').append(description,hint,metadata); $('download-one').disabled = false; } catch (error) { if (generation === detailGeneration) $('detail-state').textContent = error.message; } } @@ -216,7 +218,17 @@ worker.onmessage = ({data}) => { return; } const event=data.event; - if (event.type==='status') { + if (event.type==='snapshot') { + snapshotAt = event.updatedAt; ready=true; + $('search').disabled=false; $('region-apply').disabled=false; + $('connection-state').textContent=tr('Published directory','已同步目录'); + $('directory-status').textContent=tr('Directory synced: ','目录同步时间：')+new Date(snapshotAt).toLocaleString(); + initialLocation.finally(()=>{if(ready && querySerial===0)search();}); + } else if (event.type==='status') { + if (snapshotAt) { + $('connection-state').textContent=tr('Published directory','已同步目录'); + return; + } $('connection-state').textContent = event.state==='disconnected'?tr('Disconnected','连接已断开'):tr('Discovering directories…','正在发现目录…'); if (event.state==='disconnected') { ready=false; $('search').disabled=true; $('region-apply').disabled=true; $('connection-state').classList.remove('ready'); updateSelection(); } } else if (event.type==='directory') { @@ -229,11 +241,14 @@ worker.onmessage = ({data}) => { rows=event.rows; $('more').hidden=!event.more; renderRows(); notice(event.limited?tr('500-item display limit reached. Narrow the area to explore more.','已达到 500 项显示上限，请缩小区域继续探索。'): !rows.length && event.more?tr('No matches in these pages yet. Load more to continue searching this region.','当前批次暂无匹配点，可加载更多继续检索此区域。'): + event.snapshotAt ? tr('Showing the published directory. Last synced: ','正在显示已同步目录，最后同步：')+new Date(event.snapshotAt).toLocaleString() : tr('Loaded from live directory responses. Open a cache to verify its details.','已加载目录实时响应。打开藏宝点以验证完整详情。')); } else if (event.type==='source-error') notice(`${event.name}: ${event.message}`); }; worker.onerror = () => { notice(tr('The map service stopped. Refresh the page to try again.','地图服务已停止，请刷新页面重试。')); ready=false; $('search').disabled=true; $('region-apply').disabled=true; }; async function startService() { + try { await rpc('snapshot', {url:new URL('../data/public/index.json', import.meta.url).href}); } + catch { /* Live discovery can still work when no published copy is available. */ } try { const response = await fetch(new URL('../network.json', import.meta.url), {cache:'no-store'}); if (!response.ok) throw Error('Missing deployment configuration'); @@ -242,6 +257,11 @@ async function startService() { if (url.protocol !== 'wss:' && !(url.protocol === 'ws:' && ['localhost','127.0.0.1','[::1]'].includes(url.hostname))) throw Error('Invalid deployment endpoint'); await rpc('connect',{url:url.href,discoverySeeds:config.discoverySeeds}); } catch { + if (snapshotAt) { + $('connection-state').textContent=tr('Published directory','已同步目录'); + notice(tr('Live updates unavailable. Published caches, details and GPX remain accessible.','实时更新暂不可用，仍可查看已同步藏宝点、详情并下载 GPX。')); + return; + } $('connection-state').textContent = tr('Service unavailable','服务暂不可用'); $('directory-status').textContent = tr('Please try again later.','请稍后再试。'); notice(tr('The map service is temporarily unavailable. No setup is needed on your side.','寻宝地图服务暂时不可用，你无需进行任何网络设置。')); diff --git a/site/geocaching/src/reticulum-worker.js b/site/geocaching/src/reticulum-worker.js index 808d2a0d..d6059c9a 100644 --- a/site/geocaching/src/reticulum-worker.js +++ b/site/geocaching/src/reticulum-worker.js @@ -1,27 +1,48 @@ import {DirectoryClient} from './protocol-client.js'; import {makeGpx} from './gpx-download.js'; +import {SnapshotClient} from './snapshot-client.js'; -let client = null; +let client = null, snapshot = null, liveReady = false, selected = null; +const notify = event => self.postMessage({event}); self.onmessage = async ({data}) => { const {id, command, args = {}} = data; try { let result; - if (command === 'connect') { + if (command === 'snapshot') { + const candidate = new SnapshotClient(notify); + result = await candidate.load(args.url); + snapshot = candidate; + notify({type:'snapshot', updatedAt:result}); + } else if (command === 'connect') { await client?.close(); - client = new DirectoryClient(event => self.postMessage({event})); + liveReady = false; + client = new DirectoryClient(event => { + if (event.type === 'directory') liveReady = true; + if (event.type === 'status' && event.state === 'disconnected') liveReady = false; + notify(event); + }); await client.connect(args.url, null, args.discoverySeeds); } else if (command === 'disconnect') { - await client?.close(); client = null; + await client?.close(); client = null; liveReady = false; } else { - if (!client) throw Error('Connect to Reticulum first'); - if (command === 'query') await client.query(args.bounds, args.stateMask, args.region, args.queryToken); - else if (command === 'more') await client.more(); - else if (command === 'get') result = await client.get(args.cacheId); + if (!client && !snapshot) throw Error('Directory unavailable'); + if (command === 'query') { + selected = liveReady ? client : snapshot; + if (!selected) throw Error('Still discovering public directories'); + try { await selected.query(args.bounds, args.stateMask, args.region, args.queryToken); } + catch (error) { + if (!snapshot || selected === snapshot) throw error; + selected = snapshot; + await selected.query(args.bounds, args.stateMask, args.region, args.queryToken); + } + } + else if (command === 'more') { if (selected === client) await client.more(); } + else if (command === 'get') result = await selected.get(args.cacheId); else if (command === 'download') { if (!Array.isArray(args.cacheIds) || !args.cacheIds.length || args.cacheIds.length > 20) throw Error('Select 1–20 caches'); - const records = [], failures = []; + const records = [], failures = [], provider = selected; for (const cacheId of args.cacheIds) { - try { records.push((await client.get(cacheId)).signed); } + try { records.push((await provider.get(cacheId)).signed); } catch (error) { failures.push({cacheId, message: error.message}); } } result = {gpx: records.length ? await makeGpx(records) : null, succeeded: records.length, failures}; diff --git a/site/geocaching/src/snapshot-client.js b/site/geocaching/src/snapshot-client.js new file mode 100644 index 00000000..5153dabb --- /dev/null +++ b/site/geocaching/src/snapshot-client.js @@ -0,0 +1,52 @@ +import {decode, validateSummary, verifySigned, viewportBoxes} from './protocol.js'; +import {countryContains} from './country-boundaries.js'; + +const hex = bytes => Array.from(bytes, value => value.toString(16).padStart(2, '0')).join(''); + +// Public summaries load independently of WSS. Full objects remain immutable, +// are fetched on demand, and use the same author verification as live replies. +export class SnapshotClient { + constructor(notify, fetcher = (...args) => fetch(...args)) { this.notify = notify; this.fetcher = fetcher; this.rows = new Map(); } + + async load(url) { + this.base = new URL('.', url); + const response = await this.fetcher(url, {cache:'no-cache', signal:AbortSignal.timeout(15000)}); + if (!response.ok) throw Error('Published directory unavailable'); + const text = await response.text(); + if (text.length > 16 * 1024 * 1024) throw Error('Published directory too large'); + const data = JSON.parse(text); + if (data.version !== 1 || !Number.isFinite(Date.parse(data.updatedAt)) || !Array.isArray(data.summaries) || data.summaries.length > 20000) throw Error('Invalid published directory'); + const rows = new Map(); + for (const encoded of data.summaries) { + if (typeof encoded !== 'string' || encoded.length > 1024) throw Error('Invalid published summary'); + const summary = validateSummary(decode(Uint8Array.from(atob(encoded), c => c.charCodeAt(0)))); + const id = hex(summary[0]); + if (rows.has(id)) throw Error('Duplicate published cache'); + rows.set(id, {id, summary, conflict:false, sourceName:'Published directory'}); + } + this.rows = rows; this.updatedAt = data.updatedAt; + return this.updatedAt; + } + + query(bounds, stateMask, region, queryToken) { + const boxes = viewportBoxes(bounds), found = []; + for (const row of this.rows.values()) { + const s = row.summary; + if (!(stateMask & (1 << s[3])) || !boxes.some(([south,west,north,east]) => s[4] >= south && s[4] <= north && s[5] >= west && s[5] <= east)) continue; + if (region && !countryContains(region, s[4]/1e7, s[5]/1e7)) continue; + found.push(row); + if (found.length > 500) break; + } + this.notify({type:'results', queryToken, rows:found.slice(0,500), more:false, limited:found.length > 500, snapshotAt:this.updatedAt}); + } + + async get(id) { + const row = this.rows.get(id); + if (!row) throw Error('Cache not present in published directory'); + const response = await this.fetcher(new URL(`objects/${hex(row.summary[2])}.bin`, this.base), {signal:AbortSignal.timeout(15000)}); + if (!response.ok) throw Error('Published details unavailable'); + const raw = new Uint8Array(await response.arrayBuffer()); + const verified = await verifySigned(decode(raw, 8192), row.summary); + return {...verified, isCurrent:false, snapshotAt:this.updatedAt, checkedAt:Date.now(), sourceName:'Published directory'}; + } +} diff --git a/tests/site/geocaching-snapshot.test.mjs b/tests/site/geocaching-snapshot.test.mjs new file mode 100644 index 00000000..41a73540 --- /dev/null +++ b/tests/site/geocaching-snapshot.test.mjs @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import {readFile} from 'node:fs/promises'; +import {test} from 'node:test'; +import {SnapshotClient} from '../../site/geocaching/src/snapshot-client.js'; +import {decode, encode, verifySigned} from '../../site/geocaching/src/protocol.js'; +import {DirectoryClient} from '../../site/geocaching/src/protocol-client.js'; + +const bytes = await readFile(new URL('../../modules/core_geocaching/tests/fixtures/signed-cache-v1.bin', import.meta.url)); +const verified = await verifySigned(decode(bytes)); +const manifest = {version:1, updatedAt:'2026-09-28T00:00:00Z', summaries:[Buffer.from(encode(verified.summary)).toString('base64')]}; +const fetcher = async url => new Response(String(url).endsWith('index.json') ? JSON.stringify(manifest) : bytes); + +test('published directory filters locally and verifies full signed details', async () => { + let event; + const client = new SnapshotClient(value => event=value, fetcher); + await client.load('https://example.org/geocaching/data/public/index.json'); + client.query([-90,-180,90,180],1,null,7); + assert.equal(event.rows.length,1); + assert.equal(event.queryToken,7); + assert.equal(event.snapshotAt,manifest.updatedAt); + const detail = await client.get(verified.cacheId); + assert.equal(detail.record[9],verified.record[9]); + assert.equal(detail.isCurrent,false); + client.query([-90,-180,90,180],4,null,8); + assert.equal(event.rows.length,0); + client.query([-90,-180,90,180],1,{type:'Feature',geometry:{type:'Polygon',coordinates:[[[0,0],[1,0],[1,1],[0,1],[0,0]]]}},9); + assert.equal(event.rows.length,0); + const damaged=Uint8Array.from(bytes); damaged[damaged.length-1]^=1; + client.fetcher=async()=>new Response(damaged); + await assert.rejects(client.get(verified.cacheId),/signature/); +}); + +test('worker keeps query, details and GPX available when WSS cannot connect', async () => { + const events=[], originalFetch=globalThis.fetch, originalConnect=DirectoryClient.prototype.connect; + globalThis.self={postMessage:value=>events.push(value)}; + globalThis.fetch=fetcher; + DirectoryClient.prototype.connect=async()=>{throw Error('Bridge offline');}; + try { + await import('../../site/geocaching/src/reticulum-worker.js'); + await self.onmessage({data:{id:1,command:'snapshot',args:{url:'https://example.org/data/public/index.json'}}}); + await self.onmessage({data:{id:2,command:'connect',args:{url:'wss://offline.example.org'}}}); + assert.equal(events.find(value=>value.id===2).error,'Bridge offline'); + await self.onmessage({data:{id:3,command:'query',args:{bounds:[-90,-180,90,180],stateMask:1,queryToken:1}}}); + assert.equal(events.find(value=>value.event?.type==='results').event.rows.length,1); + await self.onmessage({data:{id:4,command:'get',args:{cacheId:verified.cacheId}}}); + assert.equal(events.find(value=>value.id===4).result.record[9],verified.record[9]); + await self.onmessage({data:{id:5,command:'download',args:{cacheIds:[verified.cacheId]}}}); + const download=events.find(value=>value.id===5).result; + assert.equal(download.succeeded,1); assert.equal(download.failures.length,0); + assert.match(download.gpx,/ { + const base=new URL('../../site/geocaching/data/public/',import.meta.url); + const fetchFiles=async url=>new Response(await readFile(new URL(url))); + let event; + const client=new SnapshotClient(value=>event=value,fetchFiles); + await client.load(new URL('index.json',base)); + client.query([-90,-180,90,180],1,null,1); + const row=event.rows.find(row=>row.summary[6]==='San Jose Galleon'); + assert.ok(row); + const detail=await client.get(row.id); + assert.ok(detail.record[9].length>20); + assert.equal(detail.cacheId,row.id); +}); diff --git a/tools/geocaching/export_public_directory.py b/tools/geocaching/export_public_directory.py new file mode 100644 index 00000000..bb46cade --- /dev/null +++ b/tools/geocaching/export_public_directory.py @@ -0,0 +1,42 @@ +"""Export current public signed records for GitHub Pages, without private state.""" +import argparse +import base64 +from datetime import datetime, timezone +import json +from pathlib import Path +import sqlite3 + +from directory_store import packed, verify_cache + + +def export(database, output): + output = Path(output) + objects = output / "objects" + objects.mkdir(parents=True, exist_ok=True) + summaries = [] + with sqlite3.connect(Path(database).resolve().as_uri() + "?mode=ro", uri=True) as db: + db.execute("BEGIN") + rows = db.execute("""SELECT o.cache, o.hash, o.raw, o.signature FROM heads h + JOIN objects o ON o.hash=h.hash JOIN public_refs p ON p.hash=o.hash + WHERE h.conflict=0 ORDER BY o.cache""") + for cache_id, revision_hash, raw, signature in rows: + verified = verify_cache([raw, signature]) + if verified.cache_id != cache_id or verified.revision_hash != revision_hash: + raise ValueError("Stored object identity mismatch") + (objects / f"{revision_hash.hex()}.bin").write_bytes(packed([raw, signature])) + summaries.append(base64.b64encode(packed(verified.summary)).decode("ascii")) + if len(summaries) > 20000: + raise ValueError("Published directory requires partitioning above 20000 caches") + data = {"version": 1, "updatedAt": datetime.now(timezone.utc).isoformat(), "summaries": summaries} + temporary = output / "index.json.tmp" + temporary.write_text(json.dumps(data, ensure_ascii=True, separators=(",", ":")) + "\n", encoding="utf-8") + temporary.replace(output / "index.json") + return len(summaries) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + print(f"Exported {export(args.database, args.output)} public caches")