mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-11-11 21:18:04 +00:00
Alex BandClaude Opus 5.5
2cdb18e1dd
## Problem
`ParsedDecoded` caches each packet's decoded payload for as long as the
packet is stored, and the analytics passes fill it for every packet. It
kept fields that no server code reads:
- the ciphertext and MAC of encrypted payloads;
- advert signatures (and their validity flag);
- ISO copies of timestamps;
- raw path data;
- control-packet details.
Together those are about a third of the cached JSON.
## Change
- After decoding, the cache deletes those keys
(`decodedCacheDroppedKeys`).
- The live feed was the only place that sent the whole map.
`broadcastPayload` now passes the stored `DecodedJSON` through as a
`json.RawMessage` (after `json.Valid`), so clients still get every
field, byte for byte. Nothing in the server reads the broadcast payload
as a map; it is only serialised. This replaces the shallow copy of the
cache, so no JSON is decoded or map copied under the store's write lock
in either ingest path, and no nested map (such as `flags`) is shared
with the cache.
- `TestDecodedCacheDroppedKeysAreUnread` is a guard: it fails if any
server code starts naming a dropped key, so a future reader can't
silently get nothing.
## Measured
| | before | after |
|---|---|---|
| decoded cache's live heap (copy of a busy week) | 46 MB | 39.5 MB |
The saving is smaller than the JSON share suggests, because the
per-packet map overhead dominates. A typed struct would be the next
step, but that touches 27 call sites.
## Tests
- `TestParsedDecodedDropsUnreadKeys` and
`TestBroadcastPayloadKeepsEveryKey` (every key present in the broadcast,
and invalid JSON left out as before): red commit first.
- The guard test above.
- `go test ./cmd/server` passes. `map[string]interface{}` in
`cmd/server`: 933 on master, 932 here.
No config or UI changes.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
90 lines
3.0 KiB
Go
90 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const decodedCacheFixture = `{"type":"ADVERT","pubKey":"ab12","name":"Rpt","lat":55.9,"lon":-3.2,` +
|
|
`"flags":{"hasLocation":true,"repeater":true},"timestamp":1760000000,"timestampISO":"2025-10-09T08:53:20Z",` +
|
|
`"signature":"` + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff" + `",` +
|
|
`"signatureValid":true,"encryptedData":"deadbeef","mac":"cafe","pathData":"0102","ctrlPubKey":"ff","ctrlSubtype":1,` +
|
|
`"ctrlTag":2,"ctrlFlags":"x","ctrlZeroHop":true,"ctrlLength":3}`
|
|
|
|
// The decoded-payload cache is filled for every stored packet by the
|
|
// analytics passes and kept for the packet's lifetime: about 40MB on a busy
|
|
// week. A third of the JSON it holds is in fields no server code reads
|
|
// (ciphertext, MACs, signatures, ISO copies of timestamps, control-packet
|
|
// details), so the cache drops them.
|
|
func TestParsedDecodedDropsUnreadKeys(t *testing.T) {
|
|
tx := &StoreTx{DecodedJSON: decodedCacheFixture}
|
|
d := tx.ParsedDecoded()
|
|
for _, k := range decodedCacheDroppedKeys {
|
|
if _, ok := d[k]; ok {
|
|
t.Errorf("the cache keeps %q, which no server code reads", k)
|
|
}
|
|
}
|
|
for _, k := range []string{"type", "pubKey", "name", "lat", "lon", "flags", "timestamp"} {
|
|
if _, ok := d[k]; !ok {
|
|
t.Errorf("the cache lost %q", k)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The live feed still sends the whole payload: the stored JSON goes out as
|
|
// it is, without decoding it again under the store lock or copying the
|
|
// cache, so the frontend sees every field.
|
|
func TestBroadcastPayloadKeepsEveryKey(t *testing.T) {
|
|
tx := &StoreTx{DecodedJSON: decodedCacheFixture}
|
|
tx.ParsedDecoded()
|
|
raw := broadcastPayload(tx)
|
|
var p map[string]interface{}
|
|
if err := json.Unmarshal(raw, &p); err != nil {
|
|
t.Fatalf("broadcast payload is not JSON: %v", err)
|
|
}
|
|
for _, k := range append([]string{"type", "pubKey", "flags"}, decodedCacheDroppedKeys...) {
|
|
if _, ok := p[k]; !ok {
|
|
t.Errorf("broadcast payload is missing %q", k)
|
|
}
|
|
}
|
|
if broadcastPayload(&StoreTx{DecodedJSON: "{not json"}) != nil {
|
|
t.Error("an unreadable payload must be left out of the broadcast")
|
|
}
|
|
}
|
|
|
|
// Guard: if server code starts reading one of the dropped keys, it must come
|
|
// off the list. Any string literal naming one outside a struct tag fails.
|
|
func TestDecodedCacheDroppedKeysAreUnread(t *testing.T) {
|
|
files, _ := filepath.Glob("*.go")
|
|
for _, f := range files {
|
|
if strings.HasSuffix(f, "_test.go") {
|
|
continue
|
|
}
|
|
b, err := os.ReadFile(f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inList := false
|
|
for i, line := range strings.Split(string(b), "\n") {
|
|
if strings.HasPrefix(line, "var decodedCacheDroppedKeys") {
|
|
inList = true
|
|
}
|
|
if inList {
|
|
inList = !strings.HasPrefix(line, "}")
|
|
continue
|
|
}
|
|
if strings.Contains(line, "`json:\"") {
|
|
continue
|
|
}
|
|
for _, k := range decodedCacheDroppedKeys {
|
|
if strings.Contains(line, `"`+k+`"`) {
|
|
t.Errorf("%s:%d names %q, which ParsedDecoded drops: take it off decodedCacheDroppedKeys", f, i+1, k)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|