diff --git a/.eslintrc.json b/.eslintrc.json index f6e644a5..7207011a 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -12,12 +12,14 @@ "CACHE_INVALIDATE_MS": "readonly", "CLIENT_CONFIG": "readonly", "CLIENT_TTL": "readonly", + "CSApi": "readonly", "CSAuth": "readonly", "ChannelColorPicker": "readonly", "ChannelColors": "readonly", "ChannelDecrypt": "readonly", "ChannelQR": "readonly", "Chart": "readonly", + "CSBeta": "readonly", "DIST_THRESHOLDS": "readonly", "DragManager": "readonly", "EXTERNAL_URLS": "readonly", @@ -298,6 +300,59 @@ "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" } + ], + "no-restricted-syntax": [ + "error", + { + "selector": "CallExpression[callee.name='fetch']", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "CallExpression[callee.property.name='fetch'][callee.object.name=/^(window|self|globalThis)$/]", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "NewExpression[callee.name=/^(XMLHttpRequest|EventSource)$/]", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "CallExpression[callee.property.name='sendBeacon']", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "NewExpression[callee.name='WebSocket']:not([arguments.0.callee.property.name='wsUrl'])", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "Literal[value=/(href|src|action)=[\"']\\/api(\\/|$)/]", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "TemplateElement[value.raw=/(href|src|action)=[\"']\\/api(\\/|$)/]", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "AssignmentExpression:matches([left.property.name=/^(href|src|action|location)$/], [left.name='location']):matches([right.value=/^\\/api(\\/|$)/], [right.left.value=/^\\/api(\\/|$)/], [right.left.left.value=/^\\/api(\\/|$)/], [right.quasis.0.value.raw=/^\\/api(\\/|$)/])", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "CallExpression:matches([callee.name='open'], [callee.property.name='open'], [callee.object.name='location'][callee.property.name=/^(assign|replace)$/], [callee.object.property.name='location'][callee.property.name=/^(assign|replace)$/]):matches([arguments.0.value=/^\\/api(\\/|$)/], [arguments.0.left.value=/^\\/api(\\/|$)/], [arguments.0.left.left.value=/^\\/api(\\/|$)/], [arguments.0.quasis.0.value.raw=/^\\/api(\\/|$)/])", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + }, + { + "selector": "CallExpression[callee.property.name='setAttribute'][arguments.0.value=/^(href|src|action|formaction)$/]:matches([arguments.1.value=/^\\/api(\\/|$)/], [arguments.1.left.value=/^\\/api(\\/|$)/], [arguments.1.left.left.value=/^\\/api(\\/|$)/], [arguments.1.quasis.0.value.raw=/^\\/api(\\/|$)/])", + "message": "Every request to the CoreScope server goes through CSApi (public/api-base.js) so the UI also works from a static host or CDN: CSApi.fetch, CSApi.postJSON, CSApi.url, CSApi.wsUrl, or api() and CSAuth.request, which use it. See \"API calls\" in AGENTS.md." + } ] - } -} \ No newline at end of file + }, + "overrides": [ + { + "files": [ + "public/api-base.js" + ], + "rules": { + "no-restricted-syntax": "off" + } + } + ] +} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 350c2864..a22c3425 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -553,6 +553,25 @@ jobs: sleep 1 done + - name: Start split frontend (static build + API-only server) + # The anonymous static build on :8788 against a stock server on :13591 + # that lists that origin in CORS_ALLOWED_ORIGINS, the way a static + # frontend (Cloudflare Pages) calls the containers. -public on a missing + # directory is API-only mode, so nothing is served same-origin. + if: matrix.shard == 1 + run: | + CHANNEL=dev COMMIT=e2e API_BASE=http://localhost:13591 AUTH=false CLASSIC_URL=http://localhost:13591 bash scripts/static/build.sh + fuser -k 13591/tcp 8788/tcp 2>/dev/null || true + CORS_ALLOWED_ORIGINS=http://localhost:8788 ./corescope-server -port 13591 -db test-fixtures/e2e-fixture.db -public /nonexistent & + echo $! > .server-split.pid + node scripts/dev-static.mjs --root dist/web --port 8788 & + echo $! > .static-split.pid + for i in $(seq 1 30); do + if curl -sf http://localhost:13591/api/healthz > /dev/null 2>&1 && curl -sf -o /dev/null http://localhost:8788/; then echo "split frontend ready after ${i}s"; break; fi + if [ "$i" -eq 30 ]; then echo "split frontend failed to start within 30s"; exit 1; fi + sleep 1 + done + - name: Run Playwright E2E tests (fail-fast) env: # On master pushes every suite records the frontend coverage of the @@ -587,6 +606,8 @@ jobs: # Allowlist: tests/a11y-allowlist.yaml (0 entries — hard pass policy). # Per-viewport summary printed at the end; any net>0 fails the build. suite 1 test-a11y-axe-1668.js BASE_URL=$B AXE_SCREENSHOT_DIR=/tmp/axe-1668 + # The static build on another origin than the API (scripts/static/build.sh). + suite 1 test-split-anonymous-e2e.js SPLIT_URL=http://localhost:8788 API_URL=http://localhost:13591 suite 2 test-filter-ux-e2e.js BASE_URL=$B suite 2 test-radio-setting-1869-e2e.js BASE_URL=$B suite 2 test-observer-neighbors-2270-e2e.js BASE_URL=$B @@ -719,6 +740,8 @@ jobs: suite 3 test-path-inspector-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-pr-1490-live-map-gpu-animations-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 1 test-region-quick-picks.js + # The lint rule that keeps network calls on CSApi (needs eslint). + suite 1 test-lint-network-calls.js suite 3 test-table-sort.js suite 1 test-touch-targets.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-live-dedup.js BASE_URL=$B @@ -784,6 +807,9 @@ jobs: kill $(cat .server-um.pid) 2>/dev/null || true rm -f .server-um.pid fi + for f in .server-split.pid .static-split.pid; do + if [ -f "$f" ]; then kill $(cat "$f") 2>/dev/null || true; rm -f "$f"; fi + done - name: Upload shard results if: success() @@ -944,9 +970,34 @@ jobs: - name: Validate staging compose file run: docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet + # Split containers: operators build the web image from their checkout and + # nothing publishes it, so without this a broken base image, package or + # Caddyfile would reach an operator first. + - name: Validate split compose file + run: CORESCOPE_IMAGE=corescope:check docker compose -f docker-compose.split.yml config --quiet + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build and smoke the web image (Dockerfile.web) + env: + GIT_COMMIT: ${{ steps.meta.outputs.git_commit }} + run: | + set -euo pipefail + docker buildx build . -f Dockerfile.web --load -t corescope-web:check --build-arg GIT_COMMIT="$GIT_COMMIT" + # BACKEND is a closed port: /api must reach it (502), not the file server (404). + docker run -d --name web-check -p 8080:80 -e BACKEND=127.0.0.1:9 corescope-web:check + trap 'docker logs web-check 2>&1 | tail -20; docker rm -f web-check >/dev/null' EXIT + for i in $(seq 1 30); do curl -fs -o /dev/null http://localhost:8080/ && break; sleep 1; done + page=$(curl -fsS -D headers.txt http://localhost:8080/) + echo "$page" | grep -q "app.js?v=$GIT_COMMIT" || { echo "::error::index.html is not busted with the commit"; exit 1; } + if echo "$page" | grep -q __BUST__; then echo "::error::index.html still has __BUST__"; exit 1; fi + grep -qi '^cache-control: no-cache' headers.txt || { echo "::error::index.html may be cached"; exit 1; } + curl -fsS http://localhost:8080/deployment.js | grep -q "apiBase: ''" || { echo "::error::deployment.js is not same-origin"; exit 1; } + code=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/api/healthz) + [ "$code" = 502 ] || { echo "::error::/api/healthz answered $code, not 502 from the closed BACKEND"; exit 1; } + echo "web image serves the packaged UI and forwards /api ✅" + - name: Set up QEMU (arm64 runtime stage + arm64 smoke) uses: docker/setup-qemu-action@v3 diff --git a/AGENTS.md b/AGENTS.md index fd35106a..0e3f1726 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,8 @@ Guide for AI agents working on this codebase. Read this before writing any code. ## Architecture -Go backend + static frontend. No build step. No framework. No bundler. +Go backend + static frontend. No build step: `public/` is what the browser runs. No framework. No bundler. +`scripts/static/build.sh` is packaging, not a build: it copies `public/` for a static host or the web container, writes `deployment.js` and fills in `__BUST__`, and never transforms code. **⚠️ The Node.js server (server.js) is DEPRECATED and has been removed. All backend code is in Go.** **⚠️ DO NOT create or modify any Node.js server files. All backend changes go in `cmd/server/` or `cmd/ingestor/`.** @@ -18,7 +19,9 @@ cmd/server/ — Go API server (REST + WebSocket broadcast + static file s decoder.go — MeshCore packet decoder cmd/ingestor/ — Go MQTT ingestor (separate binary, writes to shared SQLite DB) public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT DEPRECATED - app.js — SPA router, shared globals, theme loading + app.js — SPA router, shared globals, theme loading, api() (cached GETs) + deployment.js — where the API is (rewritten by scripts/static/build.sh for a static host) + api-base.js — CSApi: the ONLY code that builds API URLs or opens connections (see "API calls") roles.js — ROLE_COLORS, TYPE_COLORS, health thresholds, shared helpers nodes.js — Nodes list + side pane + full detail page map.js — Leaflet map with markers, legend, filters @@ -33,9 +36,10 @@ public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT style.css — Main styles, CSS variables for theming live.css — Live page styles home.css — Home page styles - index.html — SPA shell, script/style tags with __BUST__ placeholder (auto-replaced at server startup) + index.html — SPA shell, script/style tags with __BUST__ placeholder (filled in automatically; see rule 3) test-fixtures/ — Real data SQLite fixture from staging (used for E2E tests) -scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation) +scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation; + static/build.sh packages the UI for a static host or the web container) ``` ### Data Flow @@ -105,7 +109,7 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`, and `npx eslint public/*.js` (CI fails on any error). If you add new logic, add tests. No exceptions. The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. @@ -113,7 +117,7 @@ The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` stil After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. ### 3. Cache busters are automatic — do NOT manually edit them -Cache busters are injected automatically by the Go server at startup. The `__BUST__` placeholder in `index.html` is replaced with a Unix timestamp when the server reads the file. No manual bumping needed — every server restart picks up new asset versions. Do NOT replace `__BUST__` with hardcoded timestamps. +Cache busters are injected automatically. The Go server replaces the `__BUST__` placeholder in every top-level page (`index.html`, `area-map.html`, `geofilter-*.html`) with a Unix timestamp at startup, so every server restart picks up new asset versions. `scripts/static/build.sh` replaces it with the commit when it packages the UI. No manual bumping needed. Do NOT replace `__BUST__` with hardcoded timestamps, and give every local script or stylesheet a new page loads `?v=__BUST__`. ### 4. Verify API response shape before building UI Before writing client code that consumes an API endpoint, check what the endpoint ACTUALLY returns. Use `curl` or check the server code. Don't assume fields exist — grouped packets (`groupByHash=true`) have different fields than raw packets. This has caused multiple breakages. @@ -196,6 +200,22 @@ If you need to know how something works — a flag, a field, a timing, a behavio ## Frontend Conventions +### API calls (api-base.js) +The UI runs embedded in the server, in the web container, and from a static host or CDN on another origin (see "Deployment modes" in `docs/deployment.md`). `public/deployment.js` says where the API is, and `public/api-base.js` (`window.CSApi`) is the only code that turns a path into a URL or opens a connection. Everything else goes through it: + +| Need | Use | +|------|-----| +| GET JSON, cached | `api('/nodes?…')` (app.js) | +| Any other request | `CSApi.fetch('/api/…', opts)` | +| JSON POST | `CSApi.postJSON('/api/…', body)`: sends `text/plain`, so a cross-origin POST has no preflight (the server decodes JSON whatever the Content-Type; `cmd/server/simple_post_test.go`) | +| Logged-in request | `CSAuth.request(method, '/api/…', body)` | +| Link, download, `src` | `CSApi.url('/api/…')` | +| WebSocket | `new WebSocket(CSApi.wsUrl('/ws'))` | + +- **Never** call `fetch`, `XMLHttpRequest`, `EventSource`, `navigator.sendBeacon` or `new WebSocket(url)` directly, or put a bare `/api/…` path in an `href`, `src` or `action` (assigned, set with `setAttribute` or written into markup) or in `window.open`, `location.assign`, `location.replace` or `location =`. It works embedded and breaks on a static host. The `no-restricted-syntax` rule in `.eslintrc.json` rejects all of these in `public/*.js` (only `api-base.js` is exempt) and CI fails on it: run `npx eslint public/*.js` before pushing. That command never sees the inline scripts and markup of `public/*.html`; `tests/e2e/test-lint-network-calls.js` applies the same rule to them, and pins what the rule reports: extend it when you extend the rule. The rule sees a path at the start of a string, a concatenation or a template; a URL assembled elsewhere and passed in a variable gets past it, so it does not replace review. Do not silence it with `eslint-disable`. The one exception is `area-map.html`, whose Server field can name another server. +- **Logins and writes:** the anonymous static build (`CSApi.anonymous`) sends no cookies. A feature that needs a login or a write shows `CSBeta.classicOnlyHtml('…')` (or links with `CSApi.classicHref(hash)`) instead, as `account.js`, `admin.js` and `perf.js` do. +- **Tests:** a test that runs a frontend file outside the browser must load `deployment.js` and `api-base.js` first, or `CSApi` is undefined and its fetches fail silently inside their `try` blocks. vm sandboxes: `tests/unit/api-base-sandbox.js`. jsdom: `w.eval` both files before the file under test (`tests/e2e/test-region-quick-picks.js`). + ### Theming All colors MUST use CSS variables. Never hardcode `#hex` values outside of `:root` definitions. The customizer controls colors via `THEME_CSS_MAP` in customize.js. If you add a new color, add it as a CSS variable and map it in the customizer. @@ -378,6 +398,7 @@ One logical change per commit. Each commit is deployable. Each commit has its te | Gave up testing locally | 2 | Basic tests work on ARM — only heavy coverage scripts crash | | Copy-pasted functions for "coverage" | 1 | Test the real code, not copies in a helper file | | Subagent timed out mid-work | 4 | Give clear scope, don't try to run slow pipelines locally | +| Frontend file tested without api-base.js | 1 | Load `deployment.js` + `api-base.js` first (see "API calls") | ## File Naming - Tests: `test-{feature}.js` in `tests/unit/` (standalone, listed in `test-all.sh`) or `tests/e2e/` (browser, listed in `scripts/non-unit-tests.json`) — never in repo root @@ -397,5 +418,6 @@ Existing patterns: `#/nodes/{pubkey}?section=node-neighbors`, `#/analytics?tab=c - Don't add framework abstractions (React, Vue, etc.) - Don't hardcode colors — use CSS variables - Don't make per-packet server API calls from the frontend +- Don't call `fetch` / `WebSocket` / XHR directly from the frontend — go through `CSApi` (see "API calls") - Don't push without running tests - Don't start implementing without plan approval diff --git a/DEPLOY.md b/DEPLOY.md index 040cd098..444ed565 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -24,6 +24,13 @@ curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-c docker compose up -d ``` +### Other deployment modes + +The UI can also run in a container of its own (`docker-compose.split.yml`, with +`CORESCOPE_IMAGE` set to the server image of your checkout's version) or from a +static host or CDN such as Cloudflare Pages. See +[Deployment modes](docs/deployment.md#deployment-modes). + ## Image Tags | Tag | Description | diff --git a/Dockerfile.web b/Dockerfile.web new file mode 100644 index 00000000..e22799c9 --- /dev/null +++ b/Dockerfile.web @@ -0,0 +1,38 @@ +# CoreScope web container: the static UI on its own, served by Caddy, which +# forwards /api and /ws to a CoreScope server container. The UI and the API +# then share one origin, so the server needs no CORS settings. Logins work as +# on the classic UI once the server's userManagement.publicBaseUrl is this +# container's public origin. docker-compose.split.yml runs the two together; +# see "Deployment modes" in docs/deployment.md. +# +# Runtime environment: +# BACKEND host:port of the CoreScope server. Default corescope:3000. +# SITE_ADDRESS :80 (default) for plain HTTP, or a domain name for Caddy's +# automatic HTTPS (publish 80 and 443, and keep /data). + +FROM alpine:3.24 AS build +RUN apk add --no-cache bash +WORKDIR /src +COPY public/ public/ +COPY scripts/static/ scripts/static/ +COPY deploy/static/ deploy/static/ +# GIT_COMMIT is the cache buster on every asset (?v=); without one, the build +# time, as the server does at start. CHANNEL beta, next or dev shows a badge. +ARG GIT_COMMIT= +ARG CHANNEL=static +RUN COMMIT="$(printf '%.12s' "${GIT_COMMIT:-$(date +%s)}")" \ + API_BASE=same-origin AUTH=true CHANNEL="$CHANNEL" OUT=/srv \ + bash scripts/static/build.sh && \ + rm /srv/_headers + +FROM alpine:3.24 +RUN apk add --no-cache caddy +COPY --from=build /srv /srv +COPY docker/web/Caddyfile /etc/caddy/Caddyfile +ENV BACKEND=corescope:3000 \ + SITE_ADDRESS=:80 \ + XDG_DATA_HOME=/data \ + XDG_CONFIG_HOME=/config +EXPOSE 80 443 +VOLUME ["/data"] +CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] diff --git a/Makefile b/Makefile index 345d628f..cc633614 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ # make crossbuild # linux/amd64 + linux/arm64, static, into dist/ # make test / vet / fmt-check # across all modules # make docker-build # multi-arch image via buildx +# make web API_BASE=… AUTH=… # the UI alone, for a static host, into dist/web +# make dev-split # UI on :8788 calling the API on :3000 cross-origin GO ?= $(shell command -v go) GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || echo unknown) @@ -59,12 +61,12 @@ DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope DOCKER_TAG ?= $(GIT_VERSION) DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 -.PHONY: all build crossbuild test vet fmt-check tidy clean docker-build docker-push help +.PHONY: all build crossbuild test vet fmt-check tidy clean web dev-split docker-build docker-push help all: build help: - @echo "targets: build crossbuild test vet fmt-check tidy clean docker-build docker-push" + @echo "targets: build crossbuild test vet fmt-check tidy clean web dev-split docker-build docker-push" @echo " build-{$(shell echo $(CMDS) | tr ' ' ',')}" # -- host builds --------------------------------------------------------------- @@ -119,6 +121,35 @@ tidy: clean: rm -rf $(DIST) +# -- static frontend ------------------------------------------------------------ +# +# web packages public/ into dist/web for a static host or a CDN. It reads +# API_BASE, AUTH, CHANNEL, CLASSIC_URL and COMMIT; see scripts/static/build.sh. +# The Go binaries share dist/, which is why the UI gets a folder of its own. +web: + bash scripts/static/build.sh + +# dev-split runs the UI the way a static host serves it: the anonymous +# build on :8788 calling an API-only server on :3000 cross-origin, with no +# proxy. The server change is the one env var, exactly as in production. +# DEV_SPLIT_DB=path/to.db picks the data (default: the E2E fixture); it runs on a +# migrated copy in dist/, so the original is never touched. +DEV_SPLIT_DB ?= test-fixtures/e2e-fixture.db +DEV_SPLIT_API_PORT ?= 3000 +DEV_SPLIT_STATIC_PORT ?= 8788 + +dev-split: build-server build-migrate + cp $(DEV_SPLIT_DB) $(DIST)/dev-split.db + $(DIST)/corescope-migrate -db $(DIST)/dev-split.db + CHANNEL=dev COMMIT=dev AUTH=false \ + API_BASE=http://localhost:$(DEV_SPLIT_API_PORT) CLASSIC_URL=http://localhost:$(DEV_SPLIT_API_PORT) \ + bash scripts/static/build.sh + @echo "UI: http://localhost:$(DEV_SPLIT_STATIC_PORT) API: http://localhost:$(DEV_SPLIT_API_PORT) (Ctrl-C stops both)" + @trap 'kill 0' INT TERM EXIT; \ + CORS_ALLOWED_ORIGINS=http://localhost:$(DEV_SPLIT_STATIC_PORT) \ + $(DIST)/corescope-server -port $(DEV_SPLIT_API_PORT) -db $(DIST)/dev-split.db -public /nonexistent & \ + node scripts/dev-static.mjs --root $(DIST)/web --port $(DEV_SPLIT_STATIC_PORT) + # -- docker -------------------------------------------------------------------- docker-build: diff --git a/cmd/server/helpers_test.go b/cmd/server/helpers_test.go index b2bff47a..f146ffc5 100644 --- a/cmd/server/helpers_test.go +++ b/cmd/server/helpers_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "regexp" "strings" "testing" ) @@ -447,6 +448,37 @@ func TestSpaHandlerCacheBust(t *testing.T) { }) } +// Every top-level page gets the same cache buster as index.html, so a tool +// page (area-map.html, geofilter-builder.html) served fresh never runs a +// script cached from before the server started. +func TestSpaHandlerBustsEveryTopLevelPage(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "index.html"), []byte(``), 0644) + os.WriteFile(filepath.Join(dir, "tool.html"), []byte(``), 0644) + handler := spaHandler(dir, http.FileServer(http.Dir(dir))) + + get := func(p string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("GET", p, nil)) + return w + } + bust := regexp.MustCompile(`v=(\d+)`) + index, tool := get("/"), get("/tool.html") + if strings.Contains(tool.Body.String(), "__BUST__") { + t.Fatalf("tool.html kept its placeholder: %s", tool.Body.String()) + } + im, tm := bust.FindStringSubmatch(index.Body.String()), bust.FindStringSubmatch(tool.Body.String()) + if im == nil || tm == nil || im[1] != tm[1] { + t.Errorf("tool.html buster %v, index.html buster %v: want the same timestamp", tm, im) + } + if got := tool.Header().Get("Cache-Control"); got != "no-cache, no-store, must-revalidate" { + t.Errorf("tool.html Cache-Control = %q", got) + } + if got := tool.Header().Get("Content-Type"); !strings.HasPrefix(got, "text/html") { + t.Errorf("tool.html Content-Type = %q", got) + } +} + func TestWriteJSON(t *testing.T) { w := httptest.NewRecorder() writeJSON(w, map[string]interface{}{"key": "value"}) diff --git a/cmd/server/main.go b/cmd/server/main.go index ac880ada..4958be1d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -606,16 +606,22 @@ func main() { // It reads index.html once at creation time and replaces the __BUST__ placeholder // with a Unix timestamp so browsers fetch fresh JS/CSS after each server restart. func spaHandler(root string, fs http.Handler) http.Handler { - // Pre-process index.html: replace __BUST__ with a cache-bust timestamp - indexPath := filepath.Join(root, "index.html") - rawHTML, err := os.ReadFile(indexPath) - if err != nil { - log.Printf("[static] warning: could not read index.html for cache-bust: %v", err) - rawHTML = []byte("
index.html not found
") - } + // Pre-process every top-level page, index.html and the tool pages + // (area-map.html, geofilter-builder.html): replace __BUST__ with a + // cache-bust timestamp, so each loads the scripts of this start. bustValue := fmt.Sprintf("%d", time.Now().Unix()) - indexHTML := []byte(strings.ReplaceAll(string(rawHTML), "__BUST__", bustValue)) - log.Printf("[static] cache-bust value: %s", bustValue) + pages := bustPages(root, bustValue) + indexHTML, ok := pages["/index.html"] + if !ok { + log.Printf("[static] warning: could not read index.html for cache-bust") + indexHTML = []byte("index.html not found
") + } + log.Printf("[static] cache-bust value: %s (%d pages)", bustValue, len(pages)) + servePage := func(w http.ResponseWriter, body []byte) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") + w.Write(body) + } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Defense-in-depth: explicitly reject path-traversal attempts before @@ -628,20 +634,21 @@ func spaHandler(root string, fs http.Handler) http.Handler { http.Error(w, "bad request", http.StatusBadRequest) return } - // Serve pre-processed index.html for root and /index.html - if r.URL.Path == "/" || r.URL.Path == "/index.html" { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - w.Write(indexHTML) + // Serve the pre-processed index.html for the root, and each + // pre-processed page (/index.html included) at its own path + if r.URL.Path == "/" { + servePage(w, indexHTML) + return + } + if page, ok := pages[r.URL.Path]; ok { + servePage(w, page) return } path := filepath.Join(root, r.URL.Path) if _, err := os.Stat(path); os.IsNotExist(err) { // SPA fallback — serve pre-processed index.html - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - w.Write(indexHTML) + servePage(w, indexHTML) return } // Disable caching for JS/CSS/HTML @@ -652,6 +659,29 @@ func spaHandler(root string, fs http.Handler) http.Handler { }) } +// bustPages reads each .html file at the top of root, keyed by its URL path +// ("/index.html"), with every __BUST__ replaced by bust. +func bustPages(root, bust string) map[string][]byte { + pages := map[string][]byte{} + entries, err := os.ReadDir(root) + if err != nil { + log.Printf("[static] warning: could not list %s for cache-bust: %v", root, err) + return pages + } + for _, e := range entries { + if e.IsDir() || !strings.EqualFold(filepath.Ext(e.Name()), ".html") { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name())) + if err != nil { + log.Printf("[static] warning: could not read %s for cache-bust: %v", e.Name(), err) + continue + } + pages["/"+e.Name()] = []byte(strings.ReplaceAll(string(raw), "__BUST__", bust)) + } + return pages +} + // isSafeStaticPath rejects request paths that contain traversal sequences // or backslashes — defense-in-depth for the SPA static handler so a future // router with SkipClean(true) cannot expose the filesystem. Empty input is diff --git a/cmd/server/websocket_test.go b/cmd/server/websocket_test.go index fbdd6210..17bf337d 100644 --- a/cmd/server/websocket_test.go +++ b/cmd/server/websocket_test.go @@ -454,3 +454,31 @@ func TestHubDefaultPingInterval(t *testing.T) { t.Fatalf("pingInterval = %v, want 30s (public/app.js WS_STALE_MS depends on it)", got) } } + +// UIs cached from before the UI moved its WebSocket to /ws still connect on +// "/": wsOrStatic upgrades any path the static handler serves. Keep that +// until those clients are gone; this pins it. +func TestWebSocketUpgradeOnRootStillWorks(t *testing.T) { + hub := NewHub() + static := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("static")) + }) + srv := httptest.NewServer(wsOrStatic(hub, static)) + defer srv.Close() + + conn, resp, err := websocket.DefaultDialer.Dial("ws"+srv.URL[4:]+"/", nil) + if err != nil { + t.Fatalf("dial /: %v", err) + } + defer conn.Close() + if resp.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("dial /: status %d, want 101", resp.StatusCode) + } + deadline := time.Now().Add(2 * time.Second) + for hub.ClientCount() != 1 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if hub.ClientCount() != 1 { + t.Errorf("expected the root connection to register with the hub, got %d clients", hub.ClientCount()) + } +} diff --git a/deploy/static/README.md b/deploy/static/README.md new file mode 100644 index 00000000..a7d53f00 --- /dev/null +++ b/deploy/static/README.md @@ -0,0 +1,174 @@ +# Frontend on a static host or CDN + +The CoreScope UI is static files. This directory and `scripts/static/build.sh` +let it be served from Cloudflare Pages or any other static host, while the +existing container keeps serving the API and the classic, embedded UI. This is +the third of the deployment modes in +[docs/deployment.md](../../docs/deployment.md#deployment-modes); the other two +run the UI in a container. + +This is the **anonymous** build (`AUTH=false`): it reads everything a logged-out +visitor can read, live updates included, and links to the classic site for +logging in and admin work. On the server it needs one setting, +`CORS_ALLOWED_ORIGINS` (see [Server](#server)), on a version that includes #2362: +`edge` now, and releases cut from `master` after it. Older servers accept the +origin too, but they hide the `Retry-After` and `X-CoreScope-Load-Status` +headers from it, so the warm-up banner never shows, and they refuse wildcard +preview origins. + +## How it works + +- `public/deployment.js` says where the API is. The embedded UI ships it with + `apiBase: ''` (same origin, as today). `build.sh` rewrites it for the + static build. +- `public/api-base.js` (`window.CSApi`) is the only place that builds API and + WebSocket URLs. A lint rule keeps raw `fetch('/api…')` out of every other file. +- The anonymous build sends no cookies, sends its three JSON `POST`s as + `text/plain` (so the browser sends no preflight), and hides login, account and + admin controls in favour of an "available on classic" link. + +## Building + +```bash +make web API_BASE=https://analyzer.example.org AUTH=false +``` + +writes `dist/web`: `public/` byte for byte, except that every top-level page +(`index.html` and the tool pages) gets the cache buster and `deployment.js` says +where the API is. Serve that folder at the root of a hostname; the UI uses +absolute paths such as `/icons/…`. `_headers` holds the cache rules: the pages +and `deployment.js` always revalidate, while everything they load carries +`?v=One JSON file with everything this instance stores about your account: profile, devices, synced settings, proposals, notifications, and your account and mail history. Passwords and login tokens are not included.
' + - '' + + '' + '