perf: batch observation fetching to eliminate N+1 API calls on sort change (#586)

## Summary

Fixes the N+1 API call pattern when changing observation sort mode on
the packets page. Previously, switching sort to Path or Time fired
individual `/api/packets/{hash}` requests for **every**
multi-observation group without cached children — potentially 100+
concurrent requests.

## Changes

### Backend: Batch observations endpoint
- **New endpoint:** `POST /api/packets/observations` accepts `{"hashes":
["h1", "h2", ...]}` and returns all observations keyed by hash in a
single response
- Capped at 200 hashes per request to prevent abuse
- 4 test cases covering empty input, invalid JSON, too-many-hashes, and
valid requests

### Frontend: Use batch endpoint
- `packets.js` sort change handler now collects all hashes needing
observation data and sends a single POST request instead of N individual
GETs
- Same behavior, single round-trip

## Performance

- **Before:** Changing sort with 100 visible groups → 100 concurrent API
requests, browser connection queueing (6 per host), several seconds of
lag
- **After:** Single POST request regardless of group count, response
time proportional to store lookup (sub-millisecond per hash in memory)

Fixes #389

---------

Co-authored-by: you <you@example.com>
This commit is contained in:
Kpa-clawbot
2026-04-04 10:18:40 -07:00
committed by GitHub
co-authored by you
parent 7ff89d8607
commit cd470dffbe
3 changed files with 117 additions and 7 deletions
+65
View File
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
@@ -4232,3 +4233,67 @@ func TestDistanceIncrementalUpdate(t *testing.T) {
t.Logf("Distance index: %d→%d hops, %d→%d paths (incremental)",
initialHops, len(store.distHops), initialPaths, len(store.distPaths))
}
func TestHandleBatchObservations(t *testing.T) {
_, router := setupNoStoreServer(t)
t.Run("empty hashes returns empty results", func(t *testing.T) {
body := strings.NewReader(`{"hashes":[]}`)
req := httptest.NewRequest("POST", "/api/packets/observations", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
}
var resp map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &resp)
results, ok := resp["results"].(map[string]interface{})
if !ok || len(results) != 0 {
t.Fatalf("expected empty results map, got %v", resp)
}
})
t.Run("invalid JSON returns 400", func(t *testing.T) {
body := strings.NewReader(`not json`)
req := httptest.NewRequest("POST", "/api/packets/observations", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 400 {
t.Fatalf("expected 400, got %d", w.Code)
}
})
t.Run("too many hashes returns 400", func(t *testing.T) {
hashes := make([]string, 201)
for i := range hashes {
hashes[i] = fmt.Sprintf("hash%d", i)
}
data, _ := json.Marshal(map[string][]string{"hashes": hashes})
req := httptest.NewRequest("POST", "/api/packets/observations", bytes.NewReader(data))
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 400 {
t.Fatalf("expected 400, got %d", w.Code)
}
})
t.Run("valid hashes with no store returns empty results", func(t *testing.T) {
body := strings.NewReader(`{"hashes":["abc123","def456"]}`)
req := httptest.NewRequest("POST", "/api/packets/observations", body)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
}
var resp map[string]interface{}
json.Unmarshal(w.Body.Bytes(), &resp)
_, ok := resp["results"].(map[string]interface{})
if !ok {
t.Fatalf("expected results map, got %v", resp)
}
})
}