mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-11-12 00:57:39 +00:00
Alex BandClaude Opus 5.5
9529bf9d41
Refs #1870. ## Problem `Load`, `loadChunk` and `scanAndMergeChunk` decode each observation's `resolved_path` only to collect its non-empty pubkeys. On live data that's a JSON array of pubkeys and nulls, about 420 characters, present on most observations. Decoding into `[]*string` allocated a string and a pointer for every hop, plus two slices. ## Change - `resolvedPubkeysFromJSON` reads the usual array of escape-free strings and nulls in place. Anything else falls back to decoding, so it returns exactly `extractResolvedPubkeys(unmarshalResolvedPath(s))`. - Relay pubkeys repeat across nearly every observation, so each one goes through `internStr` (from #2183) and is kept once. A pubkey seen before is found by substring, without copying, and the indexes it feeds (`byNode` and friends) share that copy. Interned keys live for the process, evicted nodes' included; the table is bounded at 64k entries. - The three bulk-load call sites use it. - **Eviction too.** `resolvedPubkeysForEvictionBatch` runs on every eviction batch, about once a minute in steady state, and spent about 130 MB per five minutes on a busy week. It now reads each pubkey in place and de-duplicates each packet's short list by scanning it instead of building a map per packet. - Other callers of `unmarshalResolvedPath` are unchanged. ## Measured | | before | after | |---|---|---| | allocations for 3 known pubkeys | 13 | 1 | | eviction, allocations per evicted packet (3 observations × 4 hops) | 51 | 15 | | this path's allocations over startup (copy of a busy week) | 236 MB | 61 MB | | server CPU over the run | 175.6 s | 170.6 s | The startup rows need that dataset and are from a single run. The benchmarks are rerunnable (`go test -run X -bench ResolvedPubkeys -benchmem ./cmd/server`, medians of 3): | | master | this PR | |---|---|---| | `BenchmarkResolvedPubkeysFromJSON` (5 relays + 3 nulls, pool of 2,000) | n/a | 619 ns, 128 B, 1 alloc | | `BenchmarkResolvedPubkeysForEvictionBatch` (1,000 packets × 3 observations) | 14.3 ms, 3.96 MB, 65,495 allocs | 6.7 ms, 1.36 MB, 18,501 allocs | ## Tests - `TestResolvedPubkeysFromJSONAllocations`: red commit first, with the bound tightened in green. - `TestResolvedPubkeysForEvictionBatch`: keys, order, and an allocation bound. It has its own red commit. - `TestResolvedPubkeysFromJSONMatchesDecoding` compares against decode-then-extract on 26 edge cases and 20,000 random inputs. - `go test ./cmd/server` passes. No config or UI changes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
55 lines
1.9 KiB
Go
55 lines
1.9 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"runtime"
|
|
"testing"
|
|
)
|
|
|
|
func evictionResolvedStore(t testing.TB, n int) (*PacketStore, []int) {
|
|
t.Helper()
|
|
db := setupTestDB(t)
|
|
t.Cleanup(func() { db.Close() })
|
|
pk := func(i int) string { return fmt.Sprintf("%064x", i) }
|
|
ids := make([]int, 0, n)
|
|
for i := 1; i <= n; i++ {
|
|
if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '00', ?, '2026-01-01T00:00:00Z')`, i, fmt.Sprintf("h%015d", i)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for o := 0; o < 3; o++ {
|
|
rp := `["` + pk(i%9) + `",null,"` + pk(o+20) + `","` + pk(i%9) + `"]`
|
|
if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, 1, '["aa","bb","cc","dd"]', 0, ?)`, i, rp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
ids = append(ids, i)
|
|
}
|
|
return &PacketStore{db: db}, ids
|
|
}
|
|
|
|
// Eviction looks up the resolved pubkeys of every packet it drops, once a
|
|
// minute in steady state. It decoded each resolved_path into a pointer per
|
|
// hop and built a de-dup map per packet: about 130MB of allocations per
|
|
// five minutes on a busy week.
|
|
func TestResolvedPubkeysForEvictionBatch(t *testing.T) {
|
|
s, ids := evictionResolvedStore(t, 300)
|
|
got := s.resolvedPubkeysForEvictionBatch(ids)
|
|
want := []string{fmt.Sprintf("%064x", 1), fmt.Sprintf("%064x", 20), fmt.Sprintf("%064x", 21), fmt.Sprintf("%064x", 22)}
|
|
if fmt.Sprint(got[1]) != fmt.Sprint(want) {
|
|
t.Fatalf("tx 1: %v, want %v (first-seen order, de-duplicated)", got[1], want)
|
|
}
|
|
if len(got) != 300 {
|
|
t.Fatalf("%d packets resolved, want 300", len(got))
|
|
}
|
|
var a, b runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&a)
|
|
_ = s.resolvedPubkeysForEvictionBatch(ids)
|
|
runtime.ReadMemStats(&b)
|
|
per := float64(b.Mallocs-a.Mallocs) / float64(len(ids))
|
|
t.Logf("%.1f allocations per evicted packet (3 observations, 4 hops each)", per)
|
|
if per > 20 {
|
|
t.Fatalf("%.1f allocations per evicted packet, limit 20", per)
|
|
}
|
|
}
|