diff --git a/cmd/server/path_len_test.go b/cmd/server/path_len_test.go new file mode 100644 index 00000000..13828790 --- /dev/null +++ b/cmd/server/path_len_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "encoding/json" + "math/rand" + "strings" + "testing" +) + +func pathLenReference(pathJSON string) int { + if pathJSON == "" { + return 0 + } + var hops []interface{} + if json.Unmarshal([]byte(pathJSON), &hops) != nil { + return 0 + } + return len(hops) +} + +// pathLen picks the best observation of every transmission, so it runs for +// every observation on every load and ingest. Decoding the path into a +// []interface{} just to count it allocated about 275MB over one startup on a +// week of a busy network. A path is an array of hex strings: those are +// counted without decoding. +func TestPathLenDoesNotAllocate(t *testing.T) { + for _, pj := range []string{`[]`, `["aa"]`, `["a1b2c3","d4e5f6","0708"]`, ` [ "aa" , "bb" ] `} { + if n := testing.AllocsPerRun(100, func() { _ = pathLen(pj) }); n != 0 { + t.Errorf("pathLen(%s) allocates %.0f times, want 0", pj, n) + } + } +} + +// pathLen counts exactly what decoding would, including the odd cases it +// hands to the decoder. +func TestPathLenMatchesDecoding(t *testing.T) { + cases := []string{ + "", "[]", "[ ]", `["aa"]`, `["aa","bb"]`, ` ["aa" ,"bb" ]`, "\t[\n\"aa\"\r]\n", + `["aa",]`, `["aa"`, `["aa"]x`, `[,]`, `["a\"b","c"]`, `["é"]`, `["a\\"]`, + `[1,2,3]`, `["aa",null]`, `[["aa"]]`, `{"a":1}`, `"aa"`, `null`, `[""]`, `["",""]`, + "[\"a\x01\"]", `["aa" "bb"]`, `["aa",,"bb"]`, "[\"\xff\xfe\"]", `[ "aa" , ]`, + } + rng := rand.New(rand.NewSource(1)) + alphabet := []string{`[`, `]`, `"`, `,`, ` `, `a`, `1`, `\`, `n`, `{`, `}`, `:`} + for i := 0; i < 20000; i++ { + var b strings.Builder + for j := rng.Intn(12); j >= 0; j-- { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + cases = append(cases, b.String()) + } + for _, pj := range cases { + if got, want := pathLen(pj), pathLenReference(pj); got != want { + t.Fatalf("pathLen(%q) = %d, decoding gives %d", pj, got, want) + } + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index 9f55f252..acd86c63 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -1777,10 +1777,17 @@ func rechargeTx(tx *StoreTx) int64 { return d } +// pathLen returns the number of hops in a path_json array, or 0 when it is +// not a JSON array. It runs for every observation on every load and ingest +// (pickBestObservation), so the usual array of plain strings is counted by +// countStringArray without decoding; anything else is decoded. func pathLen(pathJSON string) int { if pathJSON == "" { return 0 } + if n, ok := countStringArray(pathJSON); ok { + return n + } var hops []interface{} if json.Unmarshal([]byte(pathJSON), &hops) != nil { return 0 @@ -1788,6 +1795,66 @@ func pathLen(pathJSON string) int { return len(hops) } +// countStringArray counts the elements of s if it is a JSON array whose +// elements are all strings without escapes, with JSON whitespace allowed +// between tokens. ok is false for anything else (escapes, other element +// types, malformed input), which the caller decodes instead. +func countStringArray(s string) (n int, ok bool) { + i := skipJSONSpace(s, 0) + if i >= len(s) || s[i] != '[' { + return 0, false + } + i = skipJSONSpace(s, i+1) + if i < len(s) && s[i] == ']' { + return 0, skipJSONSpace(s, i+1) == len(s) + } + for { + if i, ok = skipPlainJSONString(s, i); !ok { + return 0, false + } + n++ + i = skipJSONSpace(s, i) + if i >= len(s) { + return 0, false + } + switch s[i] { + case ',': + i = skipJSONSpace(s, i+1) + case ']': + return n, skipJSONSpace(s, i+1) == len(s) + default: + return 0, false + } + } +} + +// skipJSONSpace returns the index of the first byte at or after i that is +// not JSON whitespace. +func skipJSONSpace(s string, i int) int { + for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { + i++ + } + return i +} + +// skipPlainJSONString expects a JSON string starting at s[i] and returns +// the index just past its closing quote. ok is false if there is no string +// there, or it has an escape or a control character. +func skipPlainJSONString(s string, i int) (int, bool) { + if i >= len(s) || s[i] != '"' { + return i, false + } + for i++; i < len(s) && s[i] != '"'; i++ { + if s[i] == '\\' || s[i] < 0x20 { + return i, false + } + } + if i >= len(s) { + return i, false + } + return i + 1, true +} + // indexResolvedPathHops indexes a transmission under every relay-hop pubkey // extracted from an observation's resolved_path, and refreshes the dependent // resolved-pubkey + path-hop indexes. This is the single point of truth for