mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-11-11 12:37:33 +00:00
Alex BandClaude Opus 5.5
9529bf9d41
Refs #1870. ## Problem `Load`, `loadChunk` and `scanAndMergeChunk` decode each observation's `resolved_path` only to collect its non-empty pubkeys. On live data that's a JSON array of pubkeys and nulls, about 420 characters, present on most observations. Decoding into `[]*string` allocated a string and a pointer for every hop, plus two slices. ## Change - `resolvedPubkeysFromJSON` reads the usual array of escape-free strings and nulls in place. Anything else falls back to decoding, so it returns exactly `extractResolvedPubkeys(unmarshalResolvedPath(s))`. - Relay pubkeys repeat across nearly every observation, so each one goes through `internStr` (from #2183) and is kept once. A pubkey seen before is found by substring, without copying, and the indexes it feeds (`byNode` and friends) share that copy. Interned keys live for the process, evicted nodes' included; the table is bounded at 64k entries. - The three bulk-load call sites use it. - **Eviction too.** `resolvedPubkeysForEvictionBatch` runs on every eviction batch, about once a minute in steady state, and spent about 130 MB per five minutes on a busy week. It now reads each pubkey in place and de-duplicates each packet's short list by scanning it instead of building a map per packet. - Other callers of `unmarshalResolvedPath` are unchanged. ## Measured | | before | after | |---|---|---| | allocations for 3 known pubkeys | 13 | 1 | | eviction, allocations per evicted packet (3 observations × 4 hops) | 51 | 15 | | this path's allocations over startup (copy of a busy week) | 236 MB | 61 MB | | server CPU over the run | 175.6 s | 170.6 s | The startup rows need that dataset and are from a single run. The benchmarks are rerunnable (`go test -run X -bench ResolvedPubkeys -benchmem ./cmd/server`, medians of 3): | | master | this PR | |---|---|---| | `BenchmarkResolvedPubkeysFromJSON` (5 relays + 3 nulls, pool of 2,000) | n/a | 619 ns, 128 B, 1 alloc | | `BenchmarkResolvedPubkeysForEvictionBatch` (1,000 packets × 3 observations) | 14.3 ms, 3.96 MB, 65,495 allocs | 6.7 ms, 1.36 MB, 18,501 allocs | ## Tests - `TestResolvedPubkeysFromJSONAllocations`: red commit first, with the bound tightened in green. - `TestResolvedPubkeysForEvictionBatch`: keys, order, and an allocation bound. It has its own red commit. - `TestResolvedPubkeysFromJSONMatchesDecoding` compares against decode-then-extract on 26 edge cases and 20,000 random inputs. - `go test ./cmd/server` passes. No config or UI changes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
53 lines
2.0 KiB
Go
53 lines
2.0 KiB
Go
package main
|
|
|
|
import (
|
|
"math/rand"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// The bulk loaders decode each observation's resolved_path (a JSON array of
|
|
// pubkeys and nulls, ~420 chars on live data, set on most observations)
|
|
// only to collect its non-empty pubkeys. Decoding into []*string allocated
|
|
// a string and a pointer per hop plus two slices: about 210MB over one
|
|
// startup on a busy week. resolvedPubkeysFromJSON reads the usual array in
|
|
// place, and a pubkey it has seen before costs nothing: only the result is
|
|
// allocated.
|
|
func TestResolvedPubkeysFromJSONAllocations(t *testing.T) {
|
|
pk := strings.Repeat("ab", 32)
|
|
rp := `["` + pk + `",null,"` + strings.Repeat("cd", 32) + `",null,"` + strings.Repeat("ef", 32) + `"]`
|
|
_ = resolvedPubkeysFromJSON(rp) // first sight keeps one copy of each
|
|
if n := testing.AllocsPerRun(100, func() { _ = resolvedPubkeysFromJSON(rp) }); n > 1 {
|
|
t.Fatalf("resolvedPubkeysFromJSON allocates %.0f times for 3 known pubkeys, want at most 1 (the result)", n)
|
|
}
|
|
}
|
|
|
|
// It returns exactly what decoding and extracting would, odd inputs included.
|
|
func TestResolvedPubkeysFromJSONMatchesDecoding(t *testing.T) {
|
|
cases := []string{
|
|
"", "[]", "null", "[null]", `["aa"]`, `["aa",null,"bb"]`, ` [ "aa" , null ] `, `[""]`, `["",null,""]`,
|
|
`["aa",]`, `["aa"`, `["aa"]x`, `[1]`, `["aa",1]`, `[nul]`, `[nulll]`, `["a\"b"]`, `["é"]`,
|
|
`[["aa"]]`, `{"a":1}`, "[\"a\x01\"]", `["aa" "bb"]`, `[,]`, `[null,]`, "[\"\xff\"]", `[true]`,
|
|
}
|
|
rng := rand.New(rand.NewSource(2))
|
|
alphabet := []string{`[`, `]`, `"`, `,`, ` `, `a`, `1`, `\`, `null`, `nul`, `{`, `:`}
|
|
for i := 0; i < 20000; i++ {
|
|
var b strings.Builder
|
|
for j := rng.Intn(12); j >= 0; j-- {
|
|
b.WriteString(alphabet[rng.Intn(len(alphabet))])
|
|
}
|
|
cases = append(cases, b.String())
|
|
}
|
|
for _, s := range cases {
|
|
got, want := resolvedPubkeysFromJSON(s), extractResolvedPubkeys(unmarshalResolvedPath(s))
|
|
if len(got) != len(want) {
|
|
t.Fatalf("%q: got %q, want %q", s, got, want)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("%q: got %q, want %q", s, got, want)
|
|
}
|
|
}
|
|
}
|
|
}
|