diff --git a/cmd/server/channel_message_scope_name_test.go b/cmd/server/channel_message_scope_name_test.go new file mode 100644 index 00000000..3d2af471 --- /dev/null +++ b/cmd/server/channel_message_scope_name_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// #1851: the Channels view shows the region scope a message was sent under. +// transmissions.scope_name encodes three states (see StoreTx.ScopeName): +// SQL NULL = not transport-scoped, "" = transport-scoped but no configured +// region key matched, "#name" = matched region. /api/channels/{hash}/messages +// is served by the DB query when a DB is attached and by the in-memory store +// otherwise, and live messages arrive over the WebSocket broadcast, so all +// three paths must carry the same three states. + +const ( + chScopeHashUnscoped = "cccccccccccccc01" + chScopeHashUnknown = "cccccccccccccc02" + chScopeHashMatched = "cccccccccccccc03" +) + +var chScopeWant = map[string]interface{}{ + chScopeHashUnscoped: nil, + chScopeHashUnknown: "", + chScopeHashMatched: "#belgium", +} + +// setupChannelScopeDB seeds one channel with three messages, one per scope +// state, each heard by one observation. +func setupChannelScopeDB(t *testing.T) *DB { + t.Helper() + db := setupTestDB(t) + if _, err := db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + t.Fatalf("add scope_name column: %v", err) + } + db.hasScopeName = true + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'BRU')`); err != nil { + t.Fatalf("insert observer: %v", err) + } + now := time.Now().UTC() + rows := []struct { + hash string + routeType int + scope interface{} + text string + }{ + {chScopeHashUnscoped, 1, nil, "Alice: plain flood"}, + {chScopeHashUnknown, 0, "", "Bob: unknown region"}, + {chScopeHashMatched, 0, "#belgium", "Carol: matched region"}, + } + for i, r := range rows { + ts := now.Add(time.Duration(i-3) * time.Minute) + res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', ?, ?, ?, 5, ?, '#scopetest', ?)`, + r.hash, ts.Format(time.RFC3339), r.routeType, + `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope) + if err != nil { + t.Fatalf("insert tx %s: %v", r.hash, err) + } + txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 9.5, -90, '[]', ?)`, txID, ts.Unix()); err != nil { + t.Fatalf("insert obs %s: %v", r.hash, err) + } + } + return db +} + +func assertChannelMessageScopes(t *testing.T, messages []map[string]interface{}) { + t.Helper() + if len(messages) != len(chScopeWant) { + t.Fatalf("expected %d messages, got %d", len(chScopeWant), len(messages)) + } + for _, m := range messages { + h, _ := m["packetHash"].(string) + want, known := chScopeWant[h] + if !known { + t.Errorf("unexpected packetHash %q", h) + continue + } + got, present := m["scope_name"] + if !present { + t.Errorf("%s: scope_name key missing", h) + continue + } + if got != want { + t.Errorf("%s: scope_name = %#v, want %#v", h, got, want) + } + } +} + +func TestDBGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + messages, _, err := db.GetChannelMessages("#scopetest", 100, 0) + if err != nil { + t.Fatal(err) + } + assertChannelMessageScopes(t, messages) +} + +func TestStoreGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + messages, _ := store.GetChannelMessages("#scopetest", 100, 0) + for i := range messages { + // The store keeps the value as a *string; normalise through JSON so + // the comparison is on what the browser receives. + b, err := json.Marshal(messages[i]) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + messages[i] = decoded + } + assertChannelMessageScopes(t, messages) +} + +// A database the ingestor has not migrated yet has no scope_name column. The +// DB query must not select it (the query would fail) and every message reports +// null, the same as a non-transport-scoped message. +func TestDBGetChannelMessagesWithoutScopeNameColumn(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + messages, _, err := db.GetChannelMessages("#test", 100, 0) + if err != nil { + t.Fatalf("GetChannelMessages on a schema without scope_name: %v", err) + } + if len(messages) == 0 { + t.Fatal("expected messages for #test") + } + for _, m := range messages { + if got := m["scope_name"]; got != nil { + t.Errorf("scope_name = %#v, want nil without the column", got) + } + } +} + +func TestChannelMessagesEndpointExposesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + for _, withDB := range []bool{true, false} { + name := "db" + if !withDB { + name = "store" + } + t.Run(name, func(t *testing.T) { + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + srv.store = store + if !withDB { + srv.db = nil + } + router := mux.NewRouter() + srv.RegisterRoutes(router) + + req := httptest.NewRequest("GET", "/api/channels/%23scopetest/messages?limit=50", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + var body struct { + Messages []map[string]interface{} `json:"messages"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal: %v", err) + } + assertChannelMessageScopes(t, body.Messages) + }) + } +} + +// Live messages in the Channels view are built from the WebSocket broadcast, +// not from the REST response, so both broadcast builders must carry scope_name +// or a new message shows no region until the next REST refresh replaces it. +func TestBroadcastMapsCarryScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + + assertBroadcast := func(t *testing.T, maps []map[string]interface{}, hash string, want interface{}) { + t.Helper() + found := false + for _, bm := range maps { + if bm["hash"] != hash { + continue + } + found = true + b, err := json.Marshal(bm) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got, ok := decoded["scope_name"]; !ok || got != want { + t.Errorf("top-level scope_name = %#v (present=%v), want %#v", got, ok, want) + } + pkt, _ := decoded["packet"].(map[string]interface{}) + if got, ok := pkt["scope_name"]; !ok || got != want { + t.Errorf("packet.scope_name = %#v (present=%v), want %#v", got, ok, want) + } + } + if !found { + t.Fatalf("no broadcast map for %s", hash) + } + } + + t.Run("IngestNewFromDB", func(t *testing.T) { + txMax := store.MaxTransmissionID() + now := time.Now().UTC() + res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', 'cccccccccccccc04', ?, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium')`, + now.Format(time.RFC3339)) + if err != nil { + t.Fatalf("insert tx: %v", err) + } + txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 7.0, -95, '[]', ?)`, txID, now.Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps, _ := store.IngestNewFromDB(txMax, 100) + assertBroadcast(t, maps, "cccccccccccccc04", "#belgium") + }) + + t.Run("IngestNewObservations", func(t *testing.T) { + obsMax := db.GetMaxObservationID() + var txID int + if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = ?`, chScopeHashUnknown).Scan(&txID); err != nil { + t.Fatalf("lookup tx: %v", err) + } + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 3.0, -110, '["aa"]', ?)`, txID, time.Now().Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps := store.IngestNewObservations(obsMax, 100) + assertBroadcast(t, maps, chScopeHashUnknown, "") + }) +} diff --git a/cmd/server/db.go b/cmd/server/db.go index dac63be8..9ed1b9af 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2094,10 +2094,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders[i] = "?" obsArgs[i] = id } + // #1851: scope_name lives on the transmission row, so appending it as the + // last selected column is safe for both query shapes. + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } var obsSQL string if db.isV3 { obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - obs.id, obs.name, o.snr, o.path_json, o.timestamp + obs.id, obs.name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx @@ -2105,7 +2111,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . ORDER BY o.id ASC` } else { obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp + o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `) @@ -2130,7 +2136,12 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . var pktHash, dj, fs, obsID, obsName, pathJSON sql.NullString var snr sql.NullFloat64 var obsTs sql.NullInt64 - rows.Scan(&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs) + var scopeName sql.NullString + scanArgs := []interface{}{&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs} + if db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } + rows.Scan(scanArgs...) if !dj.Valid { continue } @@ -2181,6 +2192,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . "observers": []string{}, "hops": hops, "snr": nullFloat(snr), + "scope_name": nullStr(scopeName), }, Repeats: 1, } diff --git a/cmd/server/store.go b/cmd/server/store.go index 6b0b2546..5a0a84e0 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -2968,6 +2968,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac "path_json": strOrNil(obs.PathJSON), "direction": strOrNil(obs.Direction), "observation_count": tx.ObservationCount, + "scope_name": strPtrOrNil(tx.ScopeName), } // Use decode-window resolved path for broadcast (never from struct) if broadcastRP != nil { @@ -3245,6 +3246,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] "path_json": strOrNil(obs.PathJSON), "direction": strOrNil(obs.Direction), "observation_count": tx.ObservationCount, + "scope_name": strPtrOrNil(tx.ScopeName), } // Use decode-window resolved path for broadcast if obsRPMap != nil { @@ -5665,6 +5667,7 @@ func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int, "observers": observers, "hops": hops, "snr": snrVal, + "scope_name": strPtrOrNil(tx.ScopeName), }, Repeats: 1, Observers: observers, diff --git a/docs/api-spec.md b/docs/api-spec.md index 3d2c19cd..09d38889 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -1169,13 +1169,24 @@ Messages for a specific channel. "repeats": number, // dedup count "observers": [string], // observer names "hops": number, - "snr": number | null + "snr": number | null, + "scope_name": string | null // region scope, see below } ], "total": number // total deduplicated messages } ``` +`scope_name` is the transmission's region scope (`transmissions.scope_name`), the same field `/api/packets` returns: + +| Value | Meaning | +|-------|---------| +| `null` | No transport code: the message was not region-scoped. Also `null` when the database has no `scope_name` column yet (ingestor migration not run). | +| `""` | Transport-scoped, but the ingestor could not match it to a single region: no region key matched, or several matched with no single operator-configured key among them. | +| `"#name"` | The matched region name. | + +The same field is on the WebSocket `packet` broadcast, both top-level and inside `packet`. + --- ## GET /api/analytics/rf diff --git a/public/channels.js b/public/channels.js index b4688932..964bfa87 100644 --- a/public/channels.js +++ b/public/channels.js @@ -335,6 +335,20 @@ return str.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); } + // #1851: region scope from the packet's transport code (scope_name). + // null = no transport code, so no chip; '' = a transport code the ingestor + // could not match to a single region (no key matched, or several did with no + // operator-configured winner); otherwise the matched region name. Chip + // classes are the Scope Audit's (scope-audit.css), so a scope looks the same + // everywhere. + function messageScopeChipHtml(scopeName) { + if (scopeName == null) return ''; + if (scopeName === '') { + return 'unknown scope'; + } + return '' + escapeHtml(scopeName) + ''; + } + function truncate(str, len) { if (!str) return ''; return str.length > len ? str.slice(0, len) + '…' : str; @@ -579,7 +593,10 @@ // M5: Cache invalidation — if total candidate count changed, re-decrypt everything var totalCandidates = candidates.length; - var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt; + // #1851: a cache written before messages carried scope_name would keep + // those messages chipless on the delta path, so decrypt them again. + var cacheLacksScope = cachedMsgs.some(function (m) { return !('scope_name' in m); }); + var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt || cacheLacksScope; // M5: Delta fetch — only decrypt packets newer than lastTs if (!needFullDecrypt && cachedMsgs.length > 0 && lastTs) { @@ -663,6 +680,7 @@ packetHash: c.packet.hash, packetId: c.packet.id, hops: d.path_len || 0, snr: c.packet.snr || null, observers: c.packet.observer_name ? [c.packet.observer_name] : [], + scope_name: c.packet.scope_name ?? null, repeats: 1 }); continue; @@ -679,6 +697,7 @@ packetHash: c.packet.hash, packetId: c.packet.id, hops: 0, snr: c.packet.snr || null, observers: c.packet.observer_name ? [c.packet.observer_name] : [], + scope_name: c.packet.scope_name ?? null, repeats: 1 }); } else { @@ -1417,6 +1436,8 @@ var pktId = m.data?.id || null; var snr = m.data?.snr ?? m.data?.packet?.snr ?? payload.SNR ?? null; var observer = m.data?.packet?.observer_name || m.data?.observer || null; + // ?? not ||: '' (transport-scoped, region unmatched) must survive. + var scopeName = m.data?.scope_name ?? m.data?.packet?.scope_name ?? null; // Update channel list entry — only once per unique packet hash var isFirstObservation = pktHash && !seenHashes.has(pktHash + ':' + channelKey); @@ -1468,6 +1489,7 @@ observers: observer ? [observer] : [], hops: payload.path_len || 0, snr: snr, + scope_name: scopeName, // #1498: mark as WS-pushed so a later REST replacement // (selectChannel / refreshMessages) can merge instead of // stomp. Without this flag the REST response wipes any @@ -2258,6 +2280,8 @@ if (msg.observers?.length > 1) meta.push(`${msg.observers.length} observers`); if (msg.hops > 0) meta.push(`${msg.hops} hops`); if (msg.snr !== null && msg.snr !== undefined) meta.push(`SNR ${msg.snr}`); + const scopeChip = messageScopeChipHtml(msg.scope_name); + if (scopeChip) meta.push(scopeChip); const safeId = btoa(encodeURIComponent(sender)); // #1367: emit BOTH the new chat-app class names (.ch-message / diff --git a/test-all.sh b/test-all.sh index 9e15b728..2e5950a9 100755 --- a/test-all.sh +++ b/test-all.sh @@ -125,6 +125,7 @@ node test-issue-1836-crossnav-case-normalization.js node test-issue-1843-node-qr-quiet-zone.js node test-issue-1846-observers-width.js node test-issue-1849-trace-hashbytes.js +node test-issue-1851-channel-message-scope.js node test-issue-1868-control-decode.js node test-issue-1890-og-url.js node test-issue-1956-release-routing.js diff --git a/test-issue-1851-channel-message-scope.js b/test-issue-1851-channel-message-scope.js new file mode 100644 index 00000000..4352fa00 --- /dev/null +++ b/test-issue-1851-channel-message-scope.js @@ -0,0 +1,330 @@ +/* #1851: each channel message shows the region scope it was sent with. + * + * scope_name has three states (transmissions.scope_name): null = the packet + * carried no transport code, '' = it carried one the ingestor could not match + * to a single region, '#name' = matched region. A message reaches the Channels view by + * three routes (REST /api/channels/{hash}/messages, the WebSocket broadcast, + * and client-side decryption of /api/packets rows), so each route must keep + * the field, and the render must keep the '' state apart from null. + * + * Loads the real public/channels.js in a vm sandbox and reads the HTML it + * writes into #chMessages. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); +const { createCipheriv, createHmac, createHash } = require('crypto'); + +let passed = 0, failed = 0; +const pending = []; +function test(name, fn) { + const done = (err) => { + if (err) { failed++; console.log(` ❌ ${name}: ${err.message}`); } + else { passed++; console.log(` ✅ ${name}`); } + }; + try { + const out = fn(); + if (out && typeof out.then === 'function') { pending.push(out.then(() => done(), done)); return; } + done(); + } catch (e) { done(e); } +} + +function makeChannelsSandbox(apiImpl) { + const dom = {}; + function makeEl(id) { + if (dom[id]) return dom[id]; + dom[id] = { + id, innerHTML: '', textContent: '', value: '', + scrollTop: 0, scrollHeight: 100, clientHeight: 80, + style: {}, dataset: {}, + classList: { add() {}, remove() {}, toggle() {}, contains() { return false; } }, + addEventListener() {}, removeEventListener() {}, + querySelector() { return null; }, querySelectorAll() { return []; }, + getBoundingClientRect() { return { left: 0, bottom: 0, width: 0 }; }, + setAttribute() {}, removeAttribute() {}, focus() {}, + }; + return dom[id]; + } + const headerText = { textContent: '' }; + makeEl('chHeader').querySelector = (sel) => (sel === '.ch-header-text' ? headerText : null); + ['chMessages', 'chList', 'chScrollBtn', 'chAriaLive', 'chBackBtn', 'chRegionFilter'].forEach(makeEl); + const layout = { classList: { add() {}, remove() {}, contains() { return false; } } }; + const appEl = { + innerHTML: '', + querySelector(sel) { return sel === '.ch-layout' ? layout : makeEl(sel); }, + addEventListener() {}, + }; + const storage = {}; + const ctx = { + window: { addEventListener() {}, dispatchEvent() {} }, + document: { + readyState: 'complete', + createElement: () => ({ id: '', textContent: '', innerHTML: '' }), + head: { appendChild() {} }, + body: { appendChild() {}, removeChild() {}, contains() { return false; } }, + documentElement: { getAttribute: () => null, setAttribute() {} }, + getElementById: makeEl, + querySelector: (sel) => (sel === '.ch-layout' ? layout : null), + querySelectorAll: () => [], + addEventListener() {}, removeEventListener() {}, + }, + console, Date, Math, Array, Object, String, Number, JSON, RegExp, Error, TypeError, + Map, Set, Promise, URLSearchParams, Infinity, parseInt, parseFloat, isNaN, isFinite, + encodeURIComponent, decodeURIComponent, + setTimeout: () => 0, clearTimeout() {}, setInterval: () => 0, clearInterval() {}, + performance: { now: () => Date.now() }, + localStorage: { + getItem: (k) => (k in storage ? storage[k] : null), + setItem: (k, v) => { storage[k] = String(v); }, + removeItem: (k) => { delete storage[k]; }, + }, + location: { hash: '' }, + getHashParams: () => new URLSearchParams(''), + history: { replaceState() {} }, + matchMedia: () => ({ matches: false }), + MutationObserver: function () { this.observe = () => {}; this.disconnect = () => {}; }, + RegionFilter: { init() {}, onChange() { return () => {}; }, offChange() {}, getRegionParam() { return ''; } }, + debouncedOnWS: (fn) => fn, onWS() {}, offWS() {}, + api: apiImpl, + CLIENT_TTL: { observers: 120000, channels: 15000, channelMessages: 10000, nodeDetail: 10000 }, + ROLE_EMOJI: {}, ROLE_LABELS: {}, + timeAgo: () => '1m ago', + registerPage: (name, handlers) => { ctx._pageHandlers = handlers; }, + btoa: (s) => Buffer.from(String(s), 'utf8').toString('base64'), + atob: (s) => Buffer.from(String(s), 'base64').toString('utf8'), + crypto: { subtle: require('crypto').webcrypto.subtle }, + TextEncoder, TextDecoder, Uint8Array, + }; + ctx.window.matchMedia = ctx.matchMedia; + vm.createContext(ctx); + for (const file of ['public/vendor/aes-ecb.js', 'public/channel-decrypt.js', 'public/channels.js']) { + vm.runInContext(fs.readFileSync(file, 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + } + ctx._pageHandlers.init(appEl); + return { ctx, dom }; +} + +function listApi(extra) { + return (path) => { + if (path.indexOf('/observers') === 0) return Promise.resolve({ observers: [] }); + if (path.indexOf('/channels') === 0 && path.indexOf('/messages') === -1) { + return Promise.resolve({ channels: [{ hash: 'general', name: 'general', messageCount: 4, lastActivity: null }] }); + } + return extra(path); + }; +} + +// Splits the rendered message list into one HTML chunk per message so each +// assertion is about a specific message, not about the page as a whole. +function messageChunks(html) { + return html.split('