From 79162d8bdb6acea8974b4becc17557a30f0114cf Mon Sep 17 00:00:00 2001 From: efiten Date: Tue, 14 Apr 2026 10:27:29 +0200 Subject: [PATCH] feat: one-click prune nodes outside geofilter (#669 M4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds POST /api/admin/prune-geo-filter endpoint (dry-run by default, ?confirm=true to delete). Wires a Prune section into the GeoFilter customizer tab — preview lists affected nodes, confirm deletes them. Requires write-capable apiKey (writeEnabled gate, same as PUT). Co-Authored-By: Claude Sonnet 4.6 --- cmd/server/db.go | 56 ++++++++++++ cmd/server/routes.go | 64 +++++++++++++ cmd/server/routes_test.go | 172 +++++++++++++++++++++++++++++++++++ docs/user-guide/geofilter.md | 28 +++++- public/customize-v2.js | 88 ++++++++++++++++++ 5 files changed, 406 insertions(+), 2 deletions(-) diff --git a/cmd/server/db.go b/cmd/server/db.go index aeb09769..f065a72e 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2344,3 +2344,59 @@ func (db *DB) GetSignatureDropCount() int64 { } return count } + +// NodeForGeoPrune holds the minimal fields needed for geo-filter pruning. +type NodeForGeoPrune struct { + PubKey string + Name string + Lat *float64 + Lon *float64 +} + +// GetNodesForGeoPrune returns all nodes with their coordinates for geo-filter evaluation. +func (db *DB) GetNodesForGeoPrune() ([]NodeForGeoPrune, error) { + rows, err := db.conn.Query("SELECT public_key, name, lat, lon FROM nodes ORDER BY name") + if err != nil { + return nil, err + } + defer rows.Close() + + var nodes []NodeForGeoPrune + for rows.Next() { + var pk string + var name sql.NullString + var lat, lon sql.NullFloat64 + if err := rows.Scan(&pk, &name, &lat, &lon); err != nil { + continue + } + n := NodeForGeoPrune{PubKey: pk, Name: name.String} + if lat.Valid { + v := lat.Float64 + n.Lat = &v + } + if lon.Valid { + v := lon.Float64 + n.Lon = &v + } + nodes = append(nodes, n) + } + return nodes, rows.Err() +} + +// DeleteNodesByPubkeys deletes nodes by their public keys and returns the count deleted. +func (db *DB) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { + if len(pubkeys) == 0 { + return 0, nil + } + placeholders := strings.Repeat("?,", len(pubkeys)) + placeholders = placeholders[:len(placeholders)-1] + args := make([]interface{}, len(pubkeys)) + for i, pk := range pubkeys { + args[i] = pk + } + result, err := db.conn.Exec("DELETE FROM nodes WHERE public_key IN ("+placeholders+")", args...) + if err != nil { + return 0, err + } + return result.RowsAffected() +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 6b7c5113..c504b322 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -142,6 +142,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/perf", s.handlePerf).Methods("GET") r.Handle("/api/perf/reset", s.requireAPIKey(http.HandlerFunc(s.handlePerfReset))).Methods("POST") r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST") + r.Handle("/api/admin/prune-geo-filter", s.requireAPIKey(http.HandlerFunc(s.handlePruneGeoFilter))).Methods("POST") r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET") r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET") @@ -2650,6 +2651,69 @@ func (s *Server) handleAdminPrune(w http.ResponseWriter, r *http.Request) { writeJSON(w, results) } +// handlePruneGeoFilter identifies (dry_run=true, default) or deletes (confirm=true) +// nodes whose GPS coordinates fall outside the currently configured geo_filter. +// Nodes with no GPS fix are always kept. Requires geo_filter to be configured. +func (s *Server) handlePruneGeoFilter(w http.ResponseWriter, r *http.Request) { + if s.cfg.GeoFilter == nil || len(s.cfg.GeoFilter.Polygon) < 3 { + writeError(w, http.StatusBadRequest, "no geo_filter configured") + return + } + + nodes, err := s.db.GetNodesForGeoPrune() + if err != nil { + writeError(w, http.StatusInternalServerError, "db error") + return + } + + type nodeResult struct { + PubKey string `json:"pubkey"` + Name string `json:"name"` + Lat *float64 `json:"lat"` + Lon *float64 `json:"lon"` + } + + var outside []nodeResult + for _, n := range nodes { + var lat, lon float64 + if n.Lat != nil { + lat = *n.Lat + } + if n.Lon != nil { + lon = *n.Lon + } + if !NodePassesGeoFilter(lat, lon, s.cfg.GeoFilter) { + outside = append(outside, nodeResult{PubKey: n.PubKey, Name: n.Name, Lat: n.Lat, Lon: n.Lon}) + } + } + + if r.URL.Query().Get("confirm") != "true" { + // Dry run — return preview without deleting + writeJSON(w, map[string]interface{}{ + "dryRun": true, + "count": len(outside), + "nodes": outside, + }) + return + } + + // Confirmed — delete the nodes + pubkeys := make([]string, len(outside)) + for i, n := range outside { + pubkeys[i] = n.PubKey + } + deleted, err := s.db.DeleteNodesByPubkeys(pubkeys) + if err != nil { + writeError(w, http.StatusInternalServerError, "delete failed") + return + } + log.Printf("[geo-prune] deleted %d nodes outside geo filter", deleted) + writeJSON(w, map[string]interface{}{ + "dryRun": false, + "deleted": deleted, + }) +} + // constantTimeEqual compares two strings in constant time to prevent timing attacks. func constantTimeEqual(a, b string) bool { return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index c88e2a84..3e01761e 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -4195,3 +4195,175 @@ func TestSaveGeoFilter(t *testing.T) { } }) } + +// --- prune-geo-filter endpoint tests --- + +func setupPruneGeoFilterServer(t *testing.T, apiKey string, gf *GeoFilterConfig) (*Server, *mux.Router) { + t.Helper() + db := setupTestDB(t) + seedTestData(t, db) + // Add a node clearly outside the geo filter (high lat/lon in Europe) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ('aaaa111122223333', 'OutsideNode', 'repeater', 51.5, 4.5, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`) + // Add a node with no GPS (should always be kept) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('bbbb111122223333', 'NoGPSNode', 'companion', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`) + + cfg := &Config{Port: 3000, APIKey: apiKey, GeoFilter: gf} + hub := NewHub() + srv := NewServer(db, cfg, hub) + store := NewPacketStore(db, nil) + store.Load() + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + return srv, router +} + +func TestPruneGeoFilterEndpoint(t *testing.T) { + const apiKey = "a-strong-api-key-for-testing" + + // Polygon around San Jose — seed nodes are at 37.4–37.6, -122.1 to -121.9 (inside) + // OutsideNode is at 51.5, 4.5 (Europe — outside) + gf := &GeoFilterConfig{ + Polygon: [][2]float64{{37.0, -123.0}, {38.0, -123.0}, {38.0, -121.0}, {37.0, -121.0}}, + BufferKm: 0, + } + + t.Run("dry run returns outside nodes without deleting", func(t *testing.T) { + _, router := setupPruneGeoFilterServer(t, apiKey, gf) + + req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil) + req.Header.Set("X-API-Key", apiKey) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var body map[string]interface{} + json.Unmarshal(w.Body.Bytes(), &body) + if body["dryRun"] != true { + t.Error("expected dryRun=true") + } + count, _ := body["count"].(float64) + if count != 1 { + t.Errorf("expected 1 outside node (OutsideNode), got %v", count) + } + nodes, _ := body["nodes"].([]interface{}) + if len(nodes) != 1 { + t.Fatalf("expected 1 node in preview, got %d", len(nodes)) + } + n, _ := nodes[0].(map[string]interface{}) + if n["name"] != "OutsideNode" { + t.Errorf("expected OutsideNode, got %v", n["name"]) + } + }) + + t.Run("confirm=true deletes outside nodes", func(t *testing.T) { + srv, router := setupPruneGeoFilterServer(t, apiKey, gf) + + req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter?confirm=true", nil) + req.Header.Set("X-API-Key", apiKey) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var body map[string]interface{} + json.Unmarshal(w.Body.Bytes(), &body) + if body["dryRun"] != false { + t.Error("expected dryRun=false") + } + deleted, _ := body["deleted"].(float64) + if deleted != 1 { + t.Errorf("expected 1 deleted, got %v", deleted) + } + + // Verify node is actually gone from DB + var count int + srv.db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key = 'aaaa111122223333'").Scan(&count) + if count != 0 { + t.Error("expected OutsideNode to be deleted from DB") + } + // No-GPS node must still exist + srv.db.conn.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key = 'bbbb111122223333'").Scan(&count) + if count != 1 { + t.Error("expected NoGPSNode to be kept") + } + }) + + t.Run("returns 400 when no geo filter configured", func(t *testing.T) { + _, router := setupPruneGeoFilterServer(t, apiKey, nil) + + req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil) + req.Header.Set("X-API-Key", apiKey) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", w.Code) + } + }) + + t.Run("returns 401 without API key", func(t *testing.T) { + _, router := setupPruneGeoFilterServer(t, apiKey, gf) + + req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", w.Code) + } + }) +} + +func TestGetNodesForGeoPrune(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + + nodes, err := db.GetNodesForGeoPrune() + if err != nil { + t.Fatalf("GetNodesForGeoPrune: %v", err) + } + if len(nodes) == 0 { + t.Error("expected nodes to be returned") + } + // Check that nodes with lat/lon have non-nil fields + for _, n := range nodes { + if n.PubKey == "" { + t.Error("expected non-empty pubkey") + } + } +} + +func TestDeleteNodesByPubkeys(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + + // Count before + var before int + db.conn.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&before) + if before == 0 { + t.Skip("no nodes to delete") + } + + // Delete one node + var pk string + db.conn.QueryRow("SELECT public_key FROM nodes LIMIT 1").Scan(&pk) + n, err := db.DeleteNodesByPubkeys([]string{pk}) + if err != nil { + t.Fatalf("DeleteNodesByPubkeys: %v", err) + } + if n != 1 { + t.Errorf("expected 1 deleted, got %d", n) + } + + var after int + db.conn.QueryRow("SELECT COUNT(*) FROM nodes").Scan(&after) + if after != before-1 { + t.Errorf("expected %d nodes after delete, got %d", before-1, after) + } +} diff --git a/docs/user-guide/geofilter.md b/docs/user-guide/geofilter.md index c61852d6..453d1915 100644 --- a/docs/user-guide/geofilter.md +++ b/docs/user-guide/geofilter.md @@ -101,9 +101,33 @@ Request body: To clear the filter, send `{"polygon": null}`. +``` +POST /api/admin/prune-geo-filter +POST /api/admin/prune-geo-filter?confirm=true +``` + +Requires `X-API-Key` header. Without `?confirm=true`, performs a dry run and returns the list of nodes that would be deleted. With `?confirm=true`, permanently deletes them from the database. + +Response (dry run or confirmed): +```json +{"deleted": 5, "nodes": [{"pubKey": "...", "name": "NodeName", "lat": 51.12, "lon": 4.50}]} +``` + ## Cleaning up historical nodes -The ingestor prevents new out-of-bounds nodes from being ingested, but it does not retroactively remove nodes stored before the filter was configured. For that, use the prune script: +The ingestor prevents new out-of-bounds nodes from being ingested, but it does not retroactively remove nodes stored before the filter was configured. + +### One-click prune from the Customizer (recommended) + +If `writeEnabled` is true (server has a write-capable `apiKey`), the GeoFilter tab shows a **Prune nodes** section at the bottom: + +1. Click **Preview** — the server dry-runs the deletion and lists every node that falls outside the current polygon + buffer. No data is deleted yet. +2. Review the list. It shows the node name (or public key) and coordinates. +3. Click **Confirm delete** to permanently remove those nodes from the database. + +Nodes without GPS coordinates are always kept. + +### CLI alternative (Python script) **File:** `scripts/prune-nodes-outside-geo-filter.py` @@ -125,4 +149,4 @@ docker exec -it meshcore-analyzer \ The script reads `geo_filter.polygon` and `geo_filter.bufferKm` from config, lists nodes that fall outside, then asks for `yes` confirmation before deleting. Nodes without coordinates are always kept. -This is a **one-time migration tool** — run it once after first configuring `geo_filter` to clean up pre-filter data. The ingestor handles all subsequent filtering automatically. +Both the UI button and the script are **one-time migration tools** — run once after first configuring `geo_filter` to clean up pre-filter data. The ingestor handles all subsequent filtering automatically. diff --git a/public/customize-v2.js b/public/customize-v2.js index 003f5514..af5c9ceb 100644 --- a/public/customize-v2.js +++ b/public/customize-v2.js @@ -1193,6 +1193,17 @@ '' + '' + '' + + // Prune section — only shown when a polygon is active (toggled in _initGeoFilterTab) + '' + '' + ''; } @@ -1400,6 +1411,73 @@ }).catch(function (e) { _gfMsg(container, 'Error: ' + e.message, false); }); } + var _gfPruneNodes = []; // nodes returned by last dry-run preview + + function _gfPruneMsg(container, msg, ok) { + var el = container.querySelector('#cv2-gf-prune-msg'); + if (!el) return; + el.textContent = msg; + el.style.display = msg ? '' : 'none'; + el.style.color = ok ? 'var(--status-green)' : 'var(--status-red)'; + } + + function _gfPrunePreview(container) { + var apiKey = (container.querySelector('#cv2-gf-apikey') || {}).value || ''; + if (!apiKey) { _gfPruneMsg(container, 'API key required.', false); return; } + var btn = container.querySelector('#cv2-gf-prune-preview'); + if (btn) btn.textContent = 'Loading…'; + fetch('/api/admin/prune-geo-filter', { + method: 'POST', + headers: { 'X-API-Key': apiKey } + }).then(function (r) { + if (!r.ok) return r.json().then(function (e) { throw new Error(e.error || ('HTTP ' + r.status)); }); + return r.json(); + }).then(function (data) { + if (btn) btn.textContent = 'Preview prune'; + _gfPruneNodes = data.nodes || []; + var count = data.count || 0; + var resultEl = container.querySelector('#cv2-gf-prune-result'); + var listEl = container.querySelector('#cv2-gf-prune-list'); + var confirmBtn = container.querySelector('#cv2-gf-prune-confirm'); + if (!resultEl || !listEl || !confirmBtn) return; + if (count === 0) { + _gfPruneMsg(container, 'No nodes outside the filter. Nothing to prune.', true); + resultEl.style.display = 'none'; + return; + } + listEl.innerHTML = _gfPruneNodes.map(function (n) { + var coords = n.lat != null ? (' · ' + n.lat.toFixed(4) + ', ' + n.lon.toFixed(4)) : ''; + return '
' + (n.name || n.pubkey.slice(0, 12)) + coords + '
'; + }).join(''); + confirmBtn.textContent = 'Delete ' + count + ' node' + (count !== 1 ? 's' : ''); + resultEl.style.display = ''; + _gfPruneMsg(container, '', true); + }).catch(function (e) { + if (btn) btn.textContent = 'Preview prune'; + _gfPruneMsg(container, 'Error: ' + e.message, false); + }); + } + + function _gfPruneConfirm(container) { + if (!_gfPruneNodes.length) { _gfPruneMsg(container, 'Run preview first.', false); return; } + var apiKey = (container.querySelector('#cv2-gf-apikey') || {}).value || ''; + if (!apiKey) { _gfPruneMsg(container, 'API key required.', false); return; } + var count = _gfPruneNodes.length; + if (!confirm('Delete ' + count + ' node' + (count !== 1 ? 's' : '') + ' from the database? This cannot be undone.')) return; + fetch('/api/admin/prune-geo-filter?confirm=true', { + method: 'POST', + headers: { 'X-API-Key': apiKey } + }).then(function (r) { + if (!r.ok) return r.json().then(function (e) { throw new Error(e.error || ('HTTP ' + r.status)); }); + return r.json(); + }).then(function (data) { + _gfPruneNodes = []; + var resultEl = container.querySelector('#cv2-gf-prune-result'); + if (resultEl) resultEl.style.display = 'none'; + _gfPruneMsg(container, 'Deleted ' + data.deleted + ' node' + (data.deleted !== 1 ? 's' : '') + '.', true); + }).catch(function (e) { _gfPruneMsg(container, 'Error: ' + e.message, false); }); + } + function _initGeoFilterTab(container) { var mapEl = container.querySelector('#cv2-gf-map'); if (!mapEl || typeof L === 'undefined') return; @@ -1424,6 +1502,11 @@ _gfRender(); if (_gfPolygon) _gfMap.fitBounds(_gfPolygon.getBounds(), { padding: [20, 20] }); _gfStatus(container, gf.polygon.length + ' points · bufferKm=' + (gf.bufferKm || 0)); + // Show prune section when a polygon is active and write access is available + if (gf.writeEnabled) { + var pruneEl = container.querySelector('#cv2-gf-prune-section'); + if (pruneEl) pruneEl.style.display = ''; + } } else { _gfPoints = []; _gfStatus(container, gf && gf.writeEnabled ? 'No geo filter. Click the map to open the editor.' : 'No geo filter configured.'); @@ -1454,6 +1537,11 @@ container.querySelector('#cv2-gf-save').addEventListener('click', function () { _gfSave(container); }); container.querySelector('#cv2-gf-remove').addEventListener('click', function () { _gfRemove(container); }); + + var prunePreviewBtn = container.querySelector('#cv2-gf-prune-preview'); + var pruneConfirmBtn = container.querySelector('#cv2-gf-prune-confirm'); + if (prunePreviewBtn) prunePreviewBtn.addEventListener('click', function () { _gfPrunePreview(container); }); + if (pruneConfirmBtn) pruneConfirmBtn.addEventListener('click', function () { _gfPruneConfirm(container); }); } function _renderExport() {