perf: combine 4 subpath API calls into single bulk endpoint (#587)

## Summary Consolidates the 4 parallel `/api/analytics/subpaths` calls in the Route Patterns tab into a single `/api/analytics/subpaths-bulk` endpoint, eliminating 3 redundant server-side scans of the subpath index on cache miss. ## Changes ### Backend (`cmd/server/routes.go`, `cmd/server/store.go`) - New `GET /api/analytics/subpaths-bulk?groups=2-2:50,3-3:30,4-4:20,5-8:15` endpoint - Groups format: `minLen-maxLen:limit` comma-separated - `GetAnalyticsSubpathsBulk()` iterates `spIndex` once, bucketing entries into per-group accumulators by hop length - Hop name resolution is done once per raw hop and shared across groups - Results are cached per-group for compatibility with existing single-key cache lookups - Region-filtered queries fall back to individual `GetAnalyticsSubpaths()` calls (region filtering requires per-transmission observer checks) ### Frontend (`public/analytics.js`) - `renderSubpaths()` now makes 1 API call instead of 4 - Response shape: `{ results: [{ subpaths, totalPaths }, ...] }` — destructured into the same `[d2, d3, d4, d5]` variables ### Tests (`cmd/server/routes_test.go`) - `TestAnalyticsSubpathsBulk`: validates 3-group response shape, missing params error, invalid format error ## Performance - **Before:** 4 API calls → 4 scans of `spIndex` + 4× hop resolution on cache miss - **After:** 1 API call → 1 scan of `spIndex` + 1× hop resolution (shared cache) - Cache miss cost reduced by ~75% for this tab - No change on cache hit (individual group caching still works) Fixes #398 Co-authored-by: you <you@example.com>
2026-05-25 12:44:01 +00:00 · 2026-04-04 10:19:18 -07:00
parent cd470dffbe
commit 790a713ba9
4 changed files with 216 additions and 6 deletions
@@ -146,6 +146,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
 	r.HandleFunc("/api/analytics/hash-sizes", s.handleAnalyticsHashSizes).Methods("GET")
 	r.HandleFunc("/api/analytics/hash-collisions", s.handleAnalyticsHashCollisions).Methods("GET")
 	r.HandleFunc("/api/analytics/subpaths", s.handleAnalyticsSubpaths).Methods("GET")
+	r.HandleFunc("/api/analytics/subpaths-bulk", s.handleAnalyticsSubpathsBulk).Methods("GET")
 	r.HandleFunc("/api/analytics/subpath-detail", s.handleAnalyticsSubpathDetail).Methods("GET")
 	r.HandleFunc("/api/analytics/neighbor-graph", s.handleNeighborGraph).Methods("GET")

@@ -1383,6 +1384,57 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request)
 	})
 }

+// handleAnalyticsSubpathsBulk returns multiple length-range buckets in a single
+// response, avoiding repeated scans of the same packet data. Query format:
+//   ?groups=2-2:50,3-3:30,4-4:20,5-8:15   (minLen-maxLen:limit per group)
+func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Request) {
+	region := r.URL.Query().Get("region")
+	groupsParam := r.URL.Query().Get("groups")
+	if groupsParam == "" {
+		writeJSON(w, ErrorResp{Error: "groups parameter required (e.g. groups=2-2:50,3-3:30)"})
+		return
+	}
+
+	var groups []subpathGroup
+	for _, g := range strings.Split(groupsParam, ",") {
+		parts := strings.SplitN(g, ":", 2)
+		if len(parts) != 2 {
+			writeJSON(w, ErrorResp{Error: "invalid group format: " + g})
+			return
+		}
+		rangeParts := strings.SplitN(parts[0], "-", 2)
+		if len(rangeParts) != 2 {
+			writeJSON(w, ErrorResp{Error: "invalid range format: " + parts[0]})
+			return
+		}
+		mn, err1 := strconv.Atoi(rangeParts[0])
+		mx, err2 := strconv.Atoi(rangeParts[1])
+		lim, err3 := strconv.Atoi(parts[1])
+		if err1 != nil || err2 != nil || err3 != nil || mn < 2 || mx < mn || lim < 1 {
+			writeJSON(w, ErrorResp{Error: "invalid group: " + g})
+			return
+		}
+		groups = append(groups, subpathGroup{mn, mx, lim})
+	}
+
+	if s.store == nil {
+		results := make([]map[string]interface{}, len(groups))
+		for i := range groups {
+			results[i] = map[string]interface{}{"subpaths": []interface{}{}, "totalPaths": 0}
+		}
+		writeJSON(w, map[string]interface{}{"results": results})
+		return
+	}
+
+	results := s.store.GetAnalyticsSubpathsBulk(region, groups)
+	writeJSON(w, map[string]interface{}{"results": results})
+}
+
+// subpathGroup defines a length-range + limit for the bulk subpaths endpoint.
+type subpathGroup struct {
+	MinLen, MaxLen, Limit int
+}
+
 func (s *Server) handleAnalyticsSubpathDetail(w http.ResponseWriter, r *http.Request) {
 	hops := r.URL.Query().Get("hops")
 	if hops == "" {
@@ -1105,6 +1105,63 @@ func TestAnalyticsSubpaths(t *testing.T) {
 	}
 }

+func TestAnalyticsSubpathsBulk(t *testing.T) {
+	_, router := setupTestServer(t)
+
+	// Valid request with multiple groups.
+	req := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk?groups=2-2:50,3-3:30,5-8:15", nil)
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+	if w.Code != 200 {
+		t.Fatalf("expected 200, got %d", w.Code)
+	}
+	var body map[string]interface{}
+	json.Unmarshal(w.Body.Bytes(), &body)
+	results, ok := body["results"].([]interface{})
+	if !ok {
+		t.Fatal("expected results array")
+	}
+	if len(results) != 3 {
+		t.Errorf("expected 3 result groups, got %d", len(results))
+	}
+	// Each result should have subpaths and totalPaths.
+	for i, r := range results {
+		rm, ok := r.(map[string]interface{})
+		if !ok {
+			t.Fatalf("result %d not a map", i)
+		}
+		if _, ok := rm["subpaths"]; !ok {
+			t.Errorf("result %d missing subpaths", i)
+		}
+		if _, ok := rm["totalPaths"]; !ok {
+			t.Errorf("result %d missing totalPaths", i)
+		}
+	}
+
+	// Missing groups param → error.
+	req2 := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk", nil)
+	w2 := httptest.NewRecorder()
+	router.ServeHTTP(w2, req2)
+	if w2.Code != 200 {
+		t.Fatalf("expected 200 with error body, got %d", w2.Code)
+	}
+	var errBody map[string]interface{}
+	json.Unmarshal(w2.Body.Bytes(), &errBody)
+	if _, ok := errBody["error"]; !ok {
+		t.Error("expected error field for missing groups param")
+	}
+
+	// Invalid group format.
+	req3 := httptest.NewRequest("GET", "/api/analytics/subpaths-bulk?groups=bad", nil)
+	w3 := httptest.NewRecorder()
+	router.ServeHTTP(w3, req3)
+	var errBody3 map[string]interface{}
+	json.Unmarshal(w3.Body.Bytes(), &errBody3)
+	if _, ok := errBody3["error"]; !ok {
+		t.Error("expected error for invalid group format")
+	}
+}
+
 func TestAnalyticsSubpathDetailWithHops(t *testing.T) {
 	_, router := setupTestServer(t)
 	req := httptest.NewRequest("GET", "/api/analytics/subpath-detail?hops=aa,bb", nil)
@@ -5919,6 +5919,111 @@ func (s *PacketStore) GetAnalyticsSubpaths(region string, minLen, maxLen, limit
 	return result
 }

+// GetAnalyticsSubpathsBulk returns multiple length-range buckets from a single
+// scan of the subpath index, avoiding repeated iterations.
+func (s *PacketStore) GetAnalyticsSubpathsBulk(region string, groups []subpathGroup) []map[string]interface{} {
+	// For region queries or when there are few groups, fall back to individual calls
+	// which benefit from per-key caching.
+	if region != "" {
+		results := make([]map[string]interface{}, len(groups))
+		for i, g := range groups {
+			results[i] = s.GetAnalyticsSubpaths(region, g.MinLen, g.MaxLen, g.Limit)
+		}
+		return results
+	}
+
+	// Check if all groups are cached.
+	allCached := true
+	cachedResults := make([]map[string]interface{}, len(groups))
+	s.cacheMu.Lock()
+	for i, g := range groups {
+		cacheKey := fmt.Sprintf("|%d|%d|%d", g.MinLen, g.MaxLen, g.Limit)
+		if cached, ok := s.subpathCache[cacheKey]; ok && time.Now().Before(cached.expiresAt) {
+			cachedResults[i] = cached.data
+		} else {
+			allCached = false
+			break
+		}
+	}
+	if allCached {
+		s.cacheHits += int64(len(groups))
+		s.cacheMu.Unlock()
+		return cachedResults
+	}
+	s.cacheMu.Unlock()
+
+	// Single scan: bucket by hop length into per-group accumulators.
+	s.mu.RLock()
+	_, pm := s.getCachedNodesAndPM()
+	hopCache := make(map[string]*nodeInfo)
+	resolveHop := func(hop string) string {
+		if cached, ok := hopCache[hop]; ok {
+			if cached != nil {
+				return cached.Name
+			}
+			return hop
+		}
+		r, _, _ := pm.resolveWithContext(hop, nil, s.graph)
+		hopCache[hop] = r
+		if r != nil {
+			return r.Name
+		}
+		return hop
+	}
+
+	perGroup := make([]map[string]*subpathAccum, len(groups))
+	for i := range groups {
+		perGroup[i] = make(map[string]*subpathAccum)
+	}
+
+	for rawKey, count := range s.spIndex {
+		hops := strings.Split(rawKey, ",")
+		hopLen := len(hops)
+
+		// Resolve hop names once, reuse across groups.
+		var named []string
+		var namedKey string
+		resolved := false
+
+		for gi, g := range groups {
+			if hopLen < g.MinLen || hopLen > g.MaxLen {
+				continue
+			}
+			if !resolved {
+				named = make([]string, hopLen)
+				for i, h := range hops {
+					named[i] = resolveHop(h)
+				}
+				namedKey = strings.Join(named, " → ")
+				resolved = true
+			}
+			entry := perGroup[gi][namedKey]
+			if entry == nil {
+				entry = &subpathAccum{raw: rawKey}
+				perGroup[gi][namedKey] = entry
+			}
+			entry.count += count
+		}
+	}
+	totalPaths := s.spTotalPaths
+	s.mu.RUnlock()
+
+	results := make([]map[string]interface{}, len(groups))
+	for i, g := range groups {
+		results[i] = s.rankSubpaths(perGroup[i], totalPaths, g.Limit)
+	}
+
+	// Cache individual results for future single-key lookups too.
+	s.cacheMu.Lock()
+	for i, g := range groups {
+		cacheKey := fmt.Sprintf("|%d|%d|%d", g.MinLen, g.MaxLen, g.Limit)
+		s.subpathCache[cacheKey] = &cachedResult{data: results[i], expiresAt: time.Now().Add(s.rfCacheTTL)}
+	}
+	s.cacheMu.Unlock()
+
+	return results
+}
+
 // subpathAccum holds a running count for a single named subpath.
 type subpathAccum struct {
 	count int
@@ -1398,12 +1398,8 @@
    el.innerHTML = '<div class="text-center text-muted" style="padding:40px">Analyzing route patterns…</div>';
    try {
      const rq = RegionFilter.regionQueryString();
-      const [d2, d3, d4, d5] = await Promise.all([
-        api('/analytics/subpaths?minLen=2&maxLen=2&limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF }),
-        api('/analytics/subpaths?minLen=3&maxLen=3&limit=30' + rq, { ttl: CLIENT_TTL.analyticsRF }),
-        api('/analytics/subpaths?minLen=4&maxLen=4&limit=20' + rq, { ttl: CLIENT_TTL.analyticsRF }),
-        api('/analytics/subpaths?minLen=5&maxLen=8&limit=15' + rq, { ttl: CLIENT_TTL.analyticsRF })
-      ]);
+      const bulk = await api('/analytics/subpaths-bulk?groups=2-2:50,3-3:30,4-4:20,5-8:15' + rq, { ttl: CLIENT_TTL.analyticsRF });
+      const [d2, d3, d4, d5] = bulk.results;

      function renderTable(data, title) {
        if (!data.subpaths.length) return `<h4>${title}</h4><div class="text-muted">No data</div>`;