mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-07-02 03:11:36 +00:00
e04c7113cb
Fixes #1323 ## Summary Adds a small in-memory cache of the community-maintained hashtag-channels catalogue (`marcelverdult/meshcore-channels`) and exposes it as `GET /api/known-channels?region=XX` plus a collapsed sidebar section on the Channels view ("Known channels (catalogue)") with a one-click "+ Add" button per row. Per triage (#1323): new `cmd/server/known_channels_cache.go`, new `GET /api/known-channels?region=…`, frontend section in `public/channels.js`. No new DB tables — cache is in-memory only. ## What changed - `cmd/server/known_channels_cache.go` — `knownChannelsCache` with an atomic snapshot pointer, 24h default refresh, 30s HTTP timeout, 4 MB body cap, custom `User-Agent`. Fail-soft: a failed refresh leaves the last-known snapshot in place. Background goroutine started from `main.go` after the neighbor-graph recomputer; never blocks startup. - `cmd/server/known_channels_route.go` — `GET /api/known-channels?region=` serves the cached snapshot off the atomic pointer (never blocks on upstream). Region filter is case-insensitive ISO 3166-1 alpha-2. Empty/missing cache returns 200 with an empty entries list (fail-soft for the UI). - `cmd/server/config.go` — `KnownChannelsURL` + `KnownChannelsRefreshMs`. - `config.example.json` — example values + `_comment_knownChannels`. - `public/channels.js` — new collapsed sidebar section "Known channels (catalogue)" that lazy-fetches `/api/known-channels` on first render and renders rows with a "+ Add" button. The button calls the existing `addUserChannel(name)` path, so adding catalogue channels reuses the full save-key + decrypt flow that user-typed hashtags already use. - `cmd/server/known_channels_cache_test.go` — failing-first tests: - `TestKnownChannelsParseFixture` asserts the parser populates `GeneratedAt`/`License` and region-stamps every entry while skipping empty countries. - `TestKnownChannelsRouteRegionFilter` asserts the route returns 200 with exactly the filtered subset for `?region=be`. - `TestKnownChannelsFailSoftOn500` asserts a failed upstream fetch leaves the prior snapshot in place and bumps `failCount`. ## Upstream pinning The default URL is pinned to the specific file `channels-by-country.json` on `main`: > https://raw.githubusercontent.com/marcelverdult/meshcore-channels/main/channels-by-country.json Shape (verified 2026-05-24): ```json { "generated_at": "...", "license": "CC0-1.0", "countries": { "be": [{"channel": "#antwerpen", "description": "..."}], ... } } ``` ## Test plan ``` cd cmd/server && go test -run 'TestKnownChannels' -count=1 . ok github.com/corescope/server 0.008s ``` Red commit:5c43cff3(all three tests fail on assertions, build clean). Green commit:54a1080e(parser + cache + route implemented, all three pass). ## TDD evidence (red → green) - **Red commit `5c43cff3427afd8aa2f3cce20c31058190aebc37`** — tests added with stub implementations that compile but return zero/empty so each test fails on an assertion (not a compile/import error). `go test -run TestKnownChannels` output captured in the commit message. - **Green commit `54a1080e45fd2e10da2caa156f376bf4d0212976`** — parser, cache, route, main-wiring, frontend section land; all three tests pass. ## Frontend verification Browser verified: http://analyzer-stg.00id.net/#/channels (with the `/api/known-channels` response stubbed in DevTools to simulate the cache being populated on staging, which is still on master and doesn't have the new endpoint yet). E2E assertion added: cmd/server/known_channels_cache_test.go:71 — asserts the route returns 200 and the response body's `entries` length matches the filtered subset. ## Limitations / follow-ups (not in scope of this PR) - The catalogue only ships PSK keys for a small subset of entries (the upstream schema makes `key` optional). For entries WITHOUT a `key`, the "+ Add" button still wires through `addUserChannel("#name")` — which derives the standard public-channel key from the name (the same path used today when a user types `#foo` into the Add Channel modal). For entries WITH a `key`, a follow-up PR can pass the key through to `addUserChannel` so the UX matches "paste-a-PSK". Today the key is shown in the JSON payload but not yet wired into the FE button. - No deduplication against the in-memory `/api/channels` list — the catalogue section is intentionally separate so the user sees which channels exist worldwide even if their server hasn't seen traffic. - No per-section region selector yet — the section shows the full catalogue regardless of the page-level region filter. Future work: add a dropdown. ## Preflight ``` ═══ Preflight clean. ═══ ``` cross-stack: justified — issue #1323 spans `cmd/server` (cache + route) and `public/channels.js` (sidebar surface); same feature, both halves required. --------- Co-authored-by: Kpa-clawbot <bot@corescope.local>
237 lines
8.2 KiB
Go
237 lines
8.2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gorilla/mux"
|
|
)
|
|
|
|
// Canned fixture mirroring the upstream channels-by-country.json shape
|
|
// (https://raw.githubusercontent.com/marcelverdult/meshcore-channels/main/channels-by-country.json
|
|
// pinned 2026-05-24). Two countries: one with entries, one empty (to test
|
|
// the "skip empty countries" branch).
|
|
const knownChannelsFixture = `{
|
|
"generated_at": "2026-05-24T22:29:02Z",
|
|
"license": "CC0-1.0",
|
|
"countries": {
|
|
"be": [
|
|
{"channel": "#antwerpen", "description": "antwerpen"},
|
|
{"channel": "#bemesh", "description": "bemesh"}
|
|
],
|
|
"us": [
|
|
{"channel": "#bayarea", "description": "Bay Area"}
|
|
],
|
|
"ad": []
|
|
}
|
|
}`
|
|
|
|
// (a) Cache parses a canned JSON fixture into a snapshot.
|
|
func TestKnownChannelsParseFixture(t *testing.T) {
|
|
snap, err := parseKnownChannelsJSON([]byte(knownChannelsFixture), "fixture://test", time.Unix(1700000000, 0))
|
|
if err != nil {
|
|
t.Fatalf("parseKnownChannelsJSON: %v", err)
|
|
}
|
|
if snap == nil {
|
|
t.Fatal("snapshot is nil")
|
|
}
|
|
if snap.GeneratedAt != "2026-05-24T22:29:02Z" {
|
|
t.Errorf("GeneratedAt = %q, want 2026-05-24T22:29:02Z", snap.GeneratedAt)
|
|
}
|
|
if snap.License != "CC0-1.0" {
|
|
t.Errorf("License = %q, want CC0-1.0", snap.License)
|
|
}
|
|
if snap.Source != "fixture://test" {
|
|
t.Errorf("Source = %q, want fixture://test", snap.Source)
|
|
}
|
|
if got, want := len(snap.Entries), 3; got != want {
|
|
t.Fatalf("len(Entries) = %d, want %d (empty country ad must be skipped)", got, want)
|
|
}
|
|
// Spot-check one entry's region stamping.
|
|
var foundAntwerpen bool
|
|
for _, e := range snap.Entries {
|
|
if e.Channel == "#antwerpen" {
|
|
foundAntwerpen = true
|
|
if e.Region != "be" {
|
|
t.Errorf("antwerpen Region = %q, want be", e.Region)
|
|
}
|
|
}
|
|
}
|
|
if !foundAntwerpen {
|
|
t.Fatal("antwerpen entry missing from snapshot")
|
|
}
|
|
}
|
|
|
|
// (b) The route returns 200 + filtered list.
|
|
func TestKnownChannelsRouteRegionFilter(t *testing.T) {
|
|
snap, err := parseKnownChannelsJSON([]byte(knownChannelsFixture), "fixture://test", time.Now())
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
srv := &Server{
|
|
knownChannels: &knownChannelsCache{},
|
|
}
|
|
srv.knownChannels.ptr.Store(snap)
|
|
|
|
r := mux.NewRouter()
|
|
r.HandleFunc("/api/known-channels", srv.handleKnownChannels).Methods("GET")
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/api/known-channels?region=be", nil)
|
|
w := httptest.NewRecorder()
|
|
r.ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp KnownChannelsSnapshot
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("unmarshal: %v; body=%s", err, w.Body.String())
|
|
}
|
|
if got := len(resp.Entries); got != 2 {
|
|
t.Fatalf("filtered entries = %d, want 2 (be has 2); got body=%s", got, w.Body.String())
|
|
}
|
|
for _, e := range resp.Entries {
|
|
if e.Region != "be" {
|
|
t.Errorf("entry %q has region %q, want be", e.Channel, e.Region)
|
|
}
|
|
if !strings.HasPrefix(e.Channel, "#") {
|
|
t.Errorf("entry channel %q missing # prefix", e.Channel)
|
|
}
|
|
}
|
|
}
|
|
|
|
// (c) Cache survives upstream 500 (fail-soft): a prior good snapshot must
|
|
// remain available after a failed refresh.
|
|
func TestKnownChannelsFailSoftOn500(t *testing.T) {
|
|
// First server: returns the fixture (success).
|
|
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(knownChannelsFixture))
|
|
}))
|
|
defer good.Close()
|
|
|
|
c := newKnownChannelsCache(good.URL, time.Hour)
|
|
if err := c.fetchOnce(context.Background()); err != nil {
|
|
t.Fatalf("initial fetchOnce: %v", err)
|
|
}
|
|
first := c.load()
|
|
if first == nil || len(first.Entries) == 0 {
|
|
t.Fatal("first snapshot must be populated")
|
|
}
|
|
|
|
// Second server: always 500.
|
|
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "boom", http.StatusInternalServerError)
|
|
}))
|
|
defer bad.Close()
|
|
|
|
// Re-point the cache to the failing upstream and fetch.
|
|
c.url = bad.URL
|
|
err := c.fetchOnce(context.Background())
|
|
if err == nil {
|
|
t.Fatal("expected fetchOnce to return error on 500")
|
|
}
|
|
after := c.load()
|
|
if after == nil {
|
|
t.Fatal("snapshot wiped after failed fetch — must be fail-soft")
|
|
}
|
|
if len(after.Entries) != len(first.Entries) {
|
|
t.Errorf("snapshot entry count changed after failed fetch: was %d, now %d", len(first.Entries), len(after.Entries))
|
|
}
|
|
if c.failCount.Load() < 1 {
|
|
t.Errorf("failCount = %d, want >=1", c.failCount.Load())
|
|
}
|
|
}
|
|
|
|
// (d) Malformed JSON returns an error AND increments failCount via
|
|
// fetchOnce (the parse path lives inside fetchOnce so the metric is
|
|
// the cache-level signal operators see, not just the parser's return).
|
|
func TestKnownChannelsParseError(t *testing.T) {
|
|
// parser-level: garbage in, error out.
|
|
if _, err := parseKnownChannelsJSON([]byte("{not json"), "fixture://bad", time.Now()); err == nil {
|
|
t.Fatal("parseKnownChannelsJSON: expected error on malformed JSON")
|
|
}
|
|
// cache-level: a 200 with malformed body must bump failCount and
|
|
// leave any prior snapshot in place.
|
|
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte("{not json"))
|
|
}))
|
|
defer bad.Close()
|
|
c := newKnownChannelsCache(bad.URL, time.Hour)
|
|
before := c.failCount.Load()
|
|
if err := c.fetchOnce(context.Background()); err == nil {
|
|
t.Fatal("fetchOnce: expected parse error to surface")
|
|
}
|
|
if c.failCount.Load() <= before {
|
|
t.Errorf("failCount did not increment: before=%d after=%d", before, c.failCount.Load())
|
|
}
|
|
if c.fetchCount.Load() != 0 {
|
|
t.Errorf("fetchCount = %d, want 0 (parse failed)", c.fetchCount.Load())
|
|
}
|
|
}
|
|
|
|
// (e) The handler tolerates a nil cache (the startup-window fail-soft
|
|
// guarantee): server still serves 200 + an empty entries snapshot
|
|
// rather than 500. Mirrors the production code path where the route
|
|
// is registered before — or independently of — knownChannels being
|
|
// instantiated (the OPT-IN gating leaves it nil entirely when disabled).
|
|
func TestKnownChannelsHandlerNilCache(t *testing.T) {
|
|
srv := &Server{} // knownChannels intentionally nil
|
|
r := mux.NewRouter()
|
|
r.HandleFunc("/api/known-channels", srv.handleKnownChannels).Methods("GET")
|
|
req := httptest.NewRequest(http.MethodGet, "/api/known-channels", nil)
|
|
w := httptest.NewRecorder()
|
|
r.ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status = %d, want 200 (nil cache must fail-soft); body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp KnownChannelsSnapshot
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("unmarshal: %v; body=%s", err, w.Body.String())
|
|
}
|
|
if resp.Entries == nil {
|
|
t.Fatal("Entries is nil, want non-nil empty slice (JSON [] not null)")
|
|
}
|
|
if len(resp.Entries) != 0 {
|
|
t.Errorf("Entries len = %d, want 0", len(resp.Entries))
|
|
}
|
|
if cc := w.Header().Get("Cache-Control"); cc == "" {
|
|
t.Errorf("Cache-Control header missing on nil-cache response")
|
|
}
|
|
}
|
|
|
|
// (f) An empty region query param ("?region=") must pass through as if
|
|
// no filter was supplied — i.e. the full snapshot is returned, NOT an
|
|
// empty list. Guards against an off-by-one in the trim+filter path.
|
|
func TestKnownChannelsRegionEmptyPassthrough(t *testing.T) {
|
|
snap, err := parseKnownChannelsJSON([]byte(knownChannelsFixture), "fixture://test", time.Now())
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
srv := &Server{knownChannels: &knownChannelsCache{}}
|
|
srv.knownChannels.ptr.Store(snap)
|
|
r := mux.NewRouter()
|
|
r.HandleFunc("/api/known-channels", srv.handleKnownChannels).Methods("GET")
|
|
req := httptest.NewRequest(http.MethodGet, "/api/known-channels?region=", nil)
|
|
w := httptest.NewRecorder()
|
|
r.ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status = %d, want 200; body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp KnownChannelsSnapshot
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("unmarshal: %v; body=%s", err, w.Body.String())
|
|
}
|
|
if got, want := len(resp.Entries), len(snap.Entries); got != want {
|
|
t.Fatalf("empty region must return unfiltered snapshot: got %d entries, want %d", got, want)
|
|
}
|
|
if cc := w.Header().Get("Cache-Control"); cc == "" {
|
|
t.Errorf("Cache-Control header missing on populated response")
|
|
}
|
|
}
|