From e1a1be1735db21fbe5624e2fd82fad09a883f913 Mon Sep 17 00:00:00 2001
From: Kpa-clawbot <kpaclawbot@outlook.com>
Date: Fri, 1 May 2026 19:04:23 -0700
Subject: [PATCH] fix(server): add observers.inactive column at startup if
 missing (root cause of CI flake) (#961)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## The actual root cause

PR #954 added `WHERE inactive IS NULL OR inactive = 0` to the server's
observer queries, but the `inactive` column is only added by the
**ingestor** migration (`cmd/ingestor/db.go:344-354`). When the server
runs against a DB the ingestor never touched (e.g. the e2e fixture), the
column doesn't exist:

```
$ sqlite3 test-fixtures/e2e-fixture.db "SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0;"
Error: no such column: inactive
```

The server's `db.QueryRow().Scan()` swallows that error →
`totalObservers` stays 0 → `/api/observers` returns empty → map test
fails with "No map markers/overlays found".

This explains all the failing CI runs since #954 merged. PR #957
(freshen fixture) helped with the `nodes` time-rot but couldn't fix the
missing-column problem. PR #960 (freshen observers) added the right
timestamps but the column was still missing. PR #959 (data-loaded in
finally) fixed a different real bug. None of those touched the actual
mechanism.

## Fix

Mirror the existing `ensureResolvedPathColumn` pattern: add
`ensureObserverInactiveColumn` that runs at server startup, checks if
the column exists via `PRAGMA table_info`, adds it with `ALTER TABLE
observers ADD COLUMN inactive INTEGER DEFAULT 0` if missing.

Wired into `cmd/server/main.go` immediately after
`ensureResolvedPathColumn`.

## Verification

End-to-end on a freshened fixture:

```
$ sqlite3 /tmp/e2e-verify.db "PRAGMA table_info(observers);" | grep inactive
(no output — column absent)

$ ./cs-fixed -port 13702 -db /tmp/e2e-verify.db -public public &
[store] Added inactive column to observers

$ curl 'http://localhost:13702/api/observers'
returned=31    # was 0 before fix
```

`go test ./...` passes (19.8s).

## Lessons

I should have run `sqlite3 fixture "SELECT ... WHERE inactive ..."`
directly the first time the map test failed after #954 instead of
writing four "fix" PRs that didn't address the actual mechanism.
Apologies for the wild goose chase.

Co-authored-by: Kpa-clawbot <bot@example.invalid>
---
 cmd/server/main.go             |  6 ++++++
 cmd/server/neighbor_persist.go | 39 ++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/cmd/server/main.go b/cmd/server/main.go
index 0f3656c6..aea2c305 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -174,6 +174,12 @@ func main() {
 		database.hasResolvedPath = true // detectSchema ran before column was added; fix the flag
 	}
 
+	// Ensure observers.inactive column exists (PR #954 filters on it; ingestor migration
+	// adds it but server may run against DBs ingestor never touched, e.g. e2e fixture).
+	if err := ensureObserverInactiveColumn(dbPath); err != nil {
+		log.Printf("[store] warning: could not add observers.inactive column: %v", err)
+	}
+
 	// WaitGroup for background init steps that gate /api/healthz readiness.
 	var initWg sync.WaitGroup
 
diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go
index 53f83437..feb77efb 100644
--- a/cmd/server/neighbor_persist.go
+++ b/cmd/server/neighbor_persist.go
@@ -281,6 +281,45 @@ func ensureResolvedPathColumn(dbPath string) error {
 	return nil
 }
 
+// ensureObserverInactiveColumn adds the inactive column to observers if missing.
+// The column was originally added by ingestor migration (cmd/ingestor/db.go:344) to
+// support soft-delete via RemoveStaleObservers + filtered reads (PR #954). When the
+// server starts against a DB that was never touched by the ingestor (e.g. the e2e
+// fixture), the column is missing and read queries that filter on it (GetObservers,
+// GetStats) silently fail with "no such column: inactive" — leaving /api/observers
+// returning empty.
+func ensureObserverInactiveColumn(dbPath string) error {
+	rw, err := openRW(dbPath)
+	if err != nil {
+		return err
+	}
+	defer rw.Close()
+
+	rows, err := rw.Query("PRAGMA table_info(observers)")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var cid int
+		var colName string
+		var colType sql.NullString
+		var notNull, pk int
+		var dflt sql.NullString
+		if rows.Scan(&cid, &colName, &colType, &notNull, &dflt, &pk) == nil && colName == "inactive" {
+			return nil // already exists
+		}
+	}
+
+	_, err = rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0")
+	if err != nil {
+		return fmt.Errorf("add inactive column: %w", err)
+	}
+	log.Println("[store] Added inactive column to observers")
+	return nil
+}
+
 // resolvePathForObs resolves hop prefixes to full pubkeys for an observation.
 // Returns nil if path is empty.
 func resolvePathForObs(pathJSON, observerID string, tx *StoreTx, pm *prefixMap, graph *NeighborGraph) []*string {