mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-10-24 05:53:19 +00:00
perf(store): count path hops without decoding the JSON (#2187)
## Problem
`pathLen` is how `pickBestObservation` chooses each transmission's best
(longest-path) observation, so it runs for every observation on every
load and ingest, and again from the resolved-path index. It decoded the
path JSON into a `[]interface{}` only to take its length.
## Change
- `countStringArray` counts the usual array of plain hex strings in
place, with no allocation (helpers `skipJSONSpace` and
`skipPlainJSONString`).
- Anything else falls back to the decoder exactly as before: escapes,
non-string elements, `null`, or malformed input. So the result is always
what decoding gives, including 0 for invalid JSON.
## Perf justification (AGENTS.md rule 0)
- **Complexity.** O(len(path_json)), a single pass with no allocation.
The old version was also linear but went through reflection and
allocated per element.
- **Scale.** It runs once per observation on every load and ingest (400k
observations in a busy regional network's 7-day window), plus each time
the best observation is re-picked.
Throwaway benchmark (not committed), current master vs this branch:
| `path_json` | master | this PR |
|---|---|---|
| `[]` | 215 ns, 32 B, 2 allocs | 4 ns, 0 allocs |
| 6 hops, 1-byte | 1825 ns, 488 B, 18 allocs | 27 ns, 0 allocs |
| 10 hops, 3-byte | 2815 ns, 936 B, 27 allocs | 72 ns, 0 allocs |
On a copy of that 7-day window:
| | master | this PR |
|---|---|---|
| allocated over startup (the whole server) | 5.41 GB | 5.10 GB |
| server CPU over the run | 175.6 s | 169.2 s |
`pathLen` accounted for about 275 MB of the startup allocations.
## Tests
- `TestPathLenDoesNotAllocate`: zero allocations for typical paths. Red
commit first (2–11 allocations), then green.
- `TestPathLenMatchesDecoding` compares the result with `json.Unmarshal`
into `[]interface{}` on 27 edge cases and 20,000 random inputs built
from JSON punctuation, escapes, `null`, objects and numbers.
- `cd cmd/server && go test ./...` passes.
- SonarQube: quality gate passed, no issues on added lines.
Backend only: no UI, API shape or config change.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01KaR4xu8gVykQBkH8yG8m6H
---------
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a48dc32b8a
commit
38ed9d6ce6
2 files changed
+124
No files matched your search
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func pathLenReference(pathJSON string) int {
|
||||
if pathJSON == "" {
|
||||
return 0
|
||||
}
|
||||
var hops []interface{}
|
||||
if json.Unmarshal([]byte(pathJSON), &hops) != nil {
|
||||
return 0
|
||||
}
|
||||
return len(hops)
|
||||
}
|
||||
|
||||
// pathLen picks the best observation of every transmission, so it runs for
|
||||
// every observation on every load and ingest. Decoding the path into a
|
||||
// []interface{} just to count it allocated about 275MB over one startup on a
|
||||
// week of a busy network. A path is an array of hex strings: those are
|
||||
// counted without decoding.
|
||||
func TestPathLenDoesNotAllocate(t *testing.T) {
|
||||
for _, pj := range []string{`[]`, `["aa"]`, `["a1b2c3","d4e5f6","0708"]`, ` [ "aa" , "bb" ] `} {
|
||||
if n := testing.AllocsPerRun(100, func() { _ = pathLen(pj) }); n != 0 {
|
||||
t.Errorf("pathLen(%s) allocates %.0f times, want 0", pj, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pathLen counts exactly what decoding would, including the odd cases it
|
||||
// hands to the decoder.
|
||||
func TestPathLenMatchesDecoding(t *testing.T) {
|
||||
cases := []string{
|
||||
"", "[]", "[ ]", `["aa"]`, `["aa","bb"]`, ` ["aa" ,"bb" ]`, "\t[\n\"aa\"\r]\n",
|
||||
`["aa",]`, `["aa"`, `["aa"]x`, `[,]`, `["a\"b","c"]`, `["é"]`, `["a\\"]`,
|
||||
`[1,2,3]`, `["aa",null]`, `[["aa"]]`, `{"a":1}`, `"aa"`, `null`, `[""]`, `["",""]`,
|
||||
"[\"a\x01\"]", `["aa" "bb"]`, `["aa",,"bb"]`, "[\"\xff\xfe\"]", `[ "aa" , ]`,
|
||||
}
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
alphabet := []string{`[`, `]`, `"`, `,`, ` `, `a`, `1`, `\`, `n`, `{`, `}`, `:`}
|
||||
for i := 0; i < 20000; i++ {
|
||||
var b strings.Builder
|
||||
for j := rng.Intn(12); j >= 0; j-- {
|
||||
b.WriteString(alphabet[rng.Intn(len(alphabet))])
|
||||
}
|
||||
cases = append(cases, b.String())
|
||||
}
|
||||
for _, pj := range cases {
|
||||
if got, want := pathLen(pj), pathLenReference(pj); got != want {
|
||||
t.Fatalf("pathLen(%q) = %d, decoding gives %d", pj, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1777,10 +1777,17 @@ func rechargeTx(tx *StoreTx) int64 {
|
||||
return d
|
||||
}
|
||||
|
||||
// pathLen returns the number of hops in a path_json array, or 0 when it is
|
||||
// not a JSON array. It runs for every observation on every load and ingest
|
||||
// (pickBestObservation), so the usual array of plain strings is counted by
|
||||
// countStringArray without decoding; anything else is decoded.
|
||||
func pathLen(pathJSON string) int {
|
||||
if pathJSON == "" {
|
||||
return 0
|
||||
}
|
||||
if n, ok := countStringArray(pathJSON); ok {
|
||||
return n
|
||||
}
|
||||
var hops []interface{}
|
||||
if json.Unmarshal([]byte(pathJSON), &hops) != nil {
|
||||
return 0
|
||||
@@ -1788,6 +1795,66 @@ func pathLen(pathJSON string) int {
|
||||
return len(hops)
|
||||
}
|
||||
|
||||
// countStringArray counts the elements of s if it is a JSON array whose
|
||||
// elements are all strings without escapes, with JSON whitespace allowed
|
||||
// between tokens. ok is false for anything else (escapes, other element
|
||||
// types, malformed input), which the caller decodes instead.
|
||||
func countStringArray(s string) (n int, ok bool) {
|
||||
i := skipJSONSpace(s, 0)
|
||||
if i >= len(s) || s[i] != '[' {
|
||||
return 0, false
|
||||
}
|
||||
i = skipJSONSpace(s, i+1)
|
||||
if i < len(s) && s[i] == ']' {
|
||||
return 0, skipJSONSpace(s, i+1) == len(s)
|
||||
}
|
||||
for {
|
||||
if i, ok = skipPlainJSONString(s, i); !ok {
|
||||
return 0, false
|
||||
}
|
||||
n++
|
||||
i = skipJSONSpace(s, i)
|
||||
if i >= len(s) {
|
||||
return 0, false
|
||||
}
|
||||
switch s[i] {
|
||||
case ',':
|
||||
i = skipJSONSpace(s, i+1)
|
||||
case ']':
|
||||
return n, skipJSONSpace(s, i+1) == len(s)
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// skipJSONSpace returns the index of the first byte at or after i that is
|
||||
// not JSON whitespace.
|
||||
func skipJSONSpace(s string, i int) int {
|
||||
for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// skipPlainJSONString expects a JSON string starting at s[i] and returns
|
||||
// the index just past its closing quote. ok is false if there is no string
|
||||
// there, or it has an escape or a control character.
|
||||
func skipPlainJSONString(s string, i int) (int, bool) {
|
||||
if i >= len(s) || s[i] != '"' {
|
||||
return i, false
|
||||
}
|
||||
for i++; i < len(s) && s[i] != '"'; i++ {
|
||||
if s[i] == '\\' || s[i] < 0x20 {
|
||||
return i, false
|
||||
}
|
||||
}
|
||||
if i >= len(s) {
|
||||
return i, false
|
||||
}
|
||||
return i + 1, true
|
||||
}
|
||||
|
||||
// indexResolvedPathHops indexes a transmission under every relay-hop pubkey
|
||||
// extracted from an observation's resolved_path, and refreshes the dependent
|
||||
// resolved-pubkey + path-hop indexes. This is the single point of truth for
|
||||
|
||||
Reference in new issue
Block a user