test: de-flake distance-202 and anchor-bias tests (deterministic timing) (#1808)

Two server tests flaked intermittently and reddened CI on unrelated
(frontend)
PRs that merged master:

- TestDistanceConcurrentRequestsDuringBuildReturn202 asserted all 10
concurrent
requests get 202 'during the build window', but the lazy distance build
on the
tiny test DB finishes almost instantly, so on a fast machine some
requests
raced past it and got 200 (~50% flake). Add a nil-by-default
distanceBuildHook
seam on PacketStore (zero overhead in prod) that the test uses to hold
the
build open until all requests have been served — making the window
guarantee
  deterministic.

- TestHandleNodePaths_AnchorBiasInconsistency_Issue1278 queried /paths
right
  after store.Load(), racing the path-hop index that Load() builds in a
  background goroutine (#1008); the membership/canonical result was thus
  non-deterministic (rarer flake, worse under suite load). Wait for
  PathHopIndexReady() before querying.

Both run 30x green and pass -race. No production behavior change (hook
is nil).

Co-authored-by: Waydroid Builder <claude@michael.arcan.de>
This commit is contained in:
Michael J. Arcan
2026-06-29 15:53:59 -07:00
committed by GitHub
co-authored by Waydroid Builder
parent ec0ebeda2f
commit 9ae547ed7b
3 changed files with 30 additions and 0 deletions
+9
View File
@@ -88,6 +88,14 @@ func TestDistanceConcurrentRequestsDuringBuildReturn202(t *testing.T) {
if err := store.Load(); err != nil {
t.Fatalf("Load(): %v", err)
}
// Hold the lazy build open until every concurrent request has been served,
// so the "requests during the build window -> 202" guarantee is exercised
// deterministically. Without this the build of the tiny test DB finishes
// almost instantly, so on a fast/idle machine some requests race past it and
// get 200 — a flake, not a regression (the code only ever promised 202 for
// requests that arrive WHILE the build is in flight).
release := make(chan struct{})
store.distanceBuildHook = func() { <-release }
srv.store = store
r := mux.NewRouter()
srv.RegisterRoutes(r)
@@ -108,6 +116,7 @@ func TestDistanceConcurrentRequestsDuringBuildReturn202(t *testing.T) {
}()
}
wg.Wait()
close(release) // let the gated build finish
if got202.Load() != N {
t.Fatalf("expected all %d concurrent first-window requests to get 202; only %d did", N, got202.Load())
}
+12
View File
@@ -78,6 +78,18 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) {
if err := store.Load(); err != nil {
t.Fatalf("store.Load: %v", err)
}
// The path-hop index that /paths reads is built in a background goroutine
// (#1008); querying before it is ready races the build and yields a
// non-deterministic membership/canonical result (the flake). Wait for
// readiness so the test asserts against the fully-built index.
pathHopDeadline := time.After(5 * time.Second)
for !store.PathHopIndexReady() {
select {
case <-pathHopDeadline:
t.Fatal("path-hop index not ready within 5s")
case <-time.After(10 * time.Millisecond):
}
}
srv.store = store
router := mux.NewRouter()
srv.RegisterRoutes(router)
+9
View File
@@ -287,6 +287,11 @@ type PacketStore struct {
distLazyBuilding bool
distLazyLastBuilt time.Time
distLazyLastObs int // totalObs at last build, for Δobs debounce
// distanceBuildHook, if non-nil, runs at the start of the lazy build
// goroutine (after distLazyBuilding is set, before any lock is held). Tests
// use it to hold the build open so concurrent requests deterministically
// observe the "building" window; nil (and zero overhead) in production.
distanceBuildHook func()
// Cached GetNodeHashSizeInfo result — recomputed at most once every 15s
hashSizeInfoMu sync.Mutex
@@ -4278,6 +4283,10 @@ func (s *PacketStore) TriggerDistanceIndexBuild() {
s.distLazyBuilding = true
s.distLazyMu.Unlock()
if s.distanceBuildHook != nil {
s.distanceBuildHook() // test seam: hold the build window open
}
s.mu.Lock()
s.buildDistanceIndex()
obsAtBuild := s.totalObs