mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-11-11 12:37:33 +00:00
Alex BandClaude Opus 5.5
92dd172f6e
## Problem
`Load`, `loadChunk` and `scanAndMergeChunk` declared their ~20
`sql.Null*` scan targets, and built the `Scan` argument slice, inside
the row loop. Each target's address goes into the `[]interface{}`, so
each one escaped and was heap-allocated again on every row, along with
the slice and its appends. On a week of a busy network that was the
biggest flat cost inside `loadChunk`, about 330 MB.
## Change
- The targets and the argument slice are declared once, before the loop.
- `Scan` overwrites every target on each row.
- Nothing else takes their addresses; the `null*` helpers copy their
argument.
## Measured
| | before | after |
|---|---|---|
| allocations per row, fixture (`Load` / `LoadChunked` / `loadChunk`),
on current master | 85.9 / 87.1 / 86.0 | 63.9 / 65.1 / 64.0 |
| allocated over startup (copy of a busy week) | 5.41 GB | 5.05 GB |
| server CPU over the run | 175.6 s | 172.4 s |
The per-row figures are from the test on current master. The startup
figures were measured before #2183 and #2185 landed; this change saves
the same ~22 allocations per row on top of them.
**Scale:** this is the cold-load hot path, once per observation row
(about 400k rows on that week). The change removes per-row work and adds
none; complexity is unchanged.
## Tests
- `TestBulkLoadersReuseScanTargets` sets an allocations-per-row bound
for each loader, between the before and after values. Red commit first,
then green.
- `go vet ./...` is clean and `go test ./...` passes in `cmd/server`;
`-race -count=2` passes on the loader and observer-string tests.
- Local SonarQube: quality gate passed, no issues on added lines.
- `map[string]interface{}` in `cmd/server`: 934 on master and here.
No config or UI changes.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
2.1 KiB
Go
74 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The bulk loaders declared their ~20 scan targets and built the Scan
|
|
// argument slice inside the row loop. Each target escapes (its address goes
|
|
// into the []interface{}), so every row allocated them all again, plus the
|
|
// slice and its growth: about 330MB of the allocations loadChunk made over a
|
|
// startup on a week of a busy network. Scan overwrites every target on each
|
|
// row, so one set serves the whole loop. On this fixture that took each
|
|
// loader from about 89 allocations per row to about 67; the limits sit
|
|
// between.
|
|
|
|
func rowScanStore(t *testing.T, n int) *PacketStore {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "rows.db")
|
|
createTestDBAt(t, dbPath, n)
|
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
|
db, err := OpenDB(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("OpenDB: %v", err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
return NewPacketStore(db, &PacketStoreConfig{})
|
|
}
|
|
|
|
func mallocsPerRow(t *testing.T, n int, load func(*PacketStore) error) float64 {
|
|
t.Helper()
|
|
s := rowScanStore(t, n)
|
|
var a, b runtime.MemStats
|
|
runtime.GC()
|
|
runtime.ReadMemStats(&a)
|
|
if err := load(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runtime.ReadMemStats(&b)
|
|
if len(s.packets) != n {
|
|
t.Fatalf("loaded %d packets, want %d", len(s.packets), n)
|
|
}
|
|
return float64(b.Mallocs-a.Mallocs) / float64(n)
|
|
}
|
|
|
|
func TestBulkLoadersReuseScanTargets(t *testing.T) {
|
|
const n = 2000
|
|
cases := []struct {
|
|
name string
|
|
limit float64
|
|
load func(*PacketStore) error
|
|
}{
|
|
{"Load", 78, func(s *PacketStore) error { return s.Load() }},
|
|
{"LoadChunked", 83, func(s *PacketStore) error { return s.LoadChunked(500) }},
|
|
{"loadChunk", 78, func(s *PacketStore) error {
|
|
from := time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC)
|
|
return s.loadChunk(from, from.Add(72*time.Hour))
|
|
}},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
got := mallocsPerRow(t, n, c.load)
|
|
t.Logf("%s: %.1f allocations per row", c.name, got)
|
|
if got > c.limit {
|
|
t.Errorf("%s makes %.1f allocations per row, limit %.0f: are the scan targets rebuilt per row?", c.name, got, c.limit)
|
|
}
|
|
})
|
|
}
|
|
}
|