mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-09-10 15:26:00 +00:00
Part 1 of #1856. Part 2 (the hash migration reporting false success) is #1958. ## It has never worked `handlePostPacket` writes to the server's DB handle, and that handle is read-only. `cmd/server/db.go:106`: ```go dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL&_busy_timeout=5000", path) ``` Every call answered `500 attempt to write a readonly database`. **This is the second report.** #1196 raised it on 2026-06-13, a fix was merged that corrected v2 column names to v3, and the issue was closed. That fix could not have worked, because the column names were never why the write failed. Its comment is still sitting at `routes.go:1288`, next to code that has never executed successfully in production. ## Why remove rather than build a handoff **It cannot break a caller.** An endpoint that has only ever returned 500 has no working consumer. This is not a breaking API change, it is documentation catching up with reality. Nothing in `public/` calls it. **It was actively misleading.** `openapi.go` advertised it as "Ingest a packet" and it sits behind `requireAPIKey`, which reads as a live, protected write endpoint. **Its test hid the breakage.** `TestPostPacketPersistsV3Schema` asserted the observation row is written and passed for four months, because the test DB is opened read-write while production is not. That is how #1196 came to be closed as fixed. **Ingest is MQTT-only by design since #1283.** Re-adding an HTTP write path re-opens the invariant that change established. If manual injection is wanted later for testing or replay, it belongs on the ingestor side and deserves its own issue. The repository already has the handoff shape for that: the server writes `request-<id>.json` and the ingestor consumes it (`cmd/ingestor/prune_geofilter.go`). ## What went The route, `handlePostPacket` (103 lines), the now-unused `PacketIngestResponse` type, the `openapi.go` entry, the round-trip test, and the section plus table-of-contents line in `docs/api-spec.md`. The `packetpath` import in `routes.go` became unused and went with it. `+4/-225` across 6 files. ## The auth tests The four `requireAPIKey` tests used `"/api/packets"` only as a request path while building their own handler with `s.requireAPIKey(...)`, so they never touched the route. I checked that by **running them**, not by reading the code: ``` --- PASS: TestRequireAPIKey_RejectsWeakKey --- PASS: TestRequireAPIKey_AcceptsStrongKey --- PASS: TestRequireAPIKey_EmptyKeyDisablesEndpoints --- PASS: TestRequireAPIKey_WrongKeyUnauthorized ``` Their paths now point at `/api/admin/prune-geo-filter`, which still exists, so they no longer name a removed endpoint. Re-ran after that change: still 4 of 4. `/api/packets/observations` is a different endpoint and is untouched. Verified: `gofmt` clean, `go vet` clean, `cmd/server` suite ok in 62.9s. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -52,7 +52,7 @@ func TestRequireAPIKey_RejectsWeakKey(t *testing.T) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil)
|
||||
req.Header.Set("X-API-Key", "test")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
@@ -69,7 +69,7 @@ func TestRequireAPIKey_AcceptsStrongKey(t *testing.T) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil)
|
||||
req.Header.Set("X-API-Key", strongKey)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
@@ -85,7 +85,7 @@ func TestRequireAPIKey_EmptyKeyDisablesEndpoints(t *testing.T) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
@@ -100,7 +100,7 @@ func TestRequireAPIKey_WrongKeyUnauthorized(t *testing.T) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets", nil)
|
||||
req := httptest.NewRequest("POST", "/api/admin/prune-geo-filter", nil)
|
||||
req.Header.Set("X-API-Key", "wrong-key-entirely-here")
|
||||
rr := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rr, req)
|
||||
|
||||
@@ -66,7 +66,6 @@ func routeDescriptions() map[string]routeMeta {
|
||||
{Name: "search", Description: "Full-text search", Type: "string"},
|
||||
{Name: "groupByHash", Description: "Group duplicate packets by hash", Type: "boolean"},
|
||||
}},
|
||||
"POST /api/packets": {Summary: "Ingest a packet", Description: "Submit a raw packet for decoding and storage.", Tag: "packets", Auth: true},
|
||||
"GET /api/packets/{id}": {Summary: "Get packet detail", Tag: "packets"},
|
||||
"GET /api/packets/timestamps": {Summary: "Get packet timestamp ranges", Tag: "packets"},
|
||||
"POST /api/packets/observations": {Summary: "Batch submit observations", Description: "Submit multiple observer sightings for existing packets.", Tag: "packets"},
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/meshcore-analyzer/packetpath"
|
||||
"github.com/meshcore-analyzer/prunequeue"
|
||||
)
|
||||
|
||||
@@ -254,7 +253,6 @@ func (s *Server) RegisterRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/api/packets/timestamps", s.handlePacketTimestamps).Methods("GET")
|
||||
r.HandleFunc("/api/packets/{id}", s.handlePacketDetail).Methods("GET")
|
||||
r.HandleFunc("/api/packets", s.handlePackets).Methods("GET")
|
||||
r.Handle("/api/packets", s.requireAPIKey(http.HandlerFunc(s.handlePostPacket))).Methods("POST")
|
||||
|
||||
// Decode endpoint
|
||||
r.HandleFunc("/api/decode", s.handleDecode).Methods("POST")
|
||||
@@ -1232,110 +1230,6 @@ func (s *Server) handleDecode(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handlePostPacket(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Hex string `json:"hex"`
|
||||
Observer *string `json:"observer"`
|
||||
Snr *float64 `json:"snr"`
|
||||
Rssi *float64 `json:"rssi"`
|
||||
Region *string `json:"region"`
|
||||
Hash *string `json:"hash"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, 400, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
hexStr := strings.TrimSpace(body.Hex)
|
||||
if hexStr == "" {
|
||||
writeError(w, 400, "hex is required")
|
||||
return
|
||||
}
|
||||
decoded, err := DecodePacket(hexStr, false)
|
||||
if err != nil {
|
||||
writeError(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
contentHash := ComputeContentHash(hexStr)
|
||||
pathJSON := "[]"
|
||||
// For TRACE packets, path_json must be the payload-decoded route hops
|
||||
// (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values.
|
||||
// For all other packet types, derive path from raw_hex (#886).
|
||||
if !packetpath.PathBytesAreHops(byte(decoded.Header.PayloadType)) {
|
||||
if len(decoded.Path.Hops) > 0 {
|
||||
if pj, e := json.Marshal(decoded.Path.Hops); e == nil {
|
||||
pathJSON = string(pj)
|
||||
}
|
||||
}
|
||||
} else if hops, err := packetpath.DecodePathFromRawHex(hexStr); err == nil && len(hops) > 0 {
|
||||
if pj, e := json.Marshal(hops); e == nil {
|
||||
pathJSON = string(pj)
|
||||
}
|
||||
}
|
||||
decodedJSON := PayloadJSON(&decoded.Payload)
|
||||
now := time.Now().UTC().Format("2006-01-02T15:04:05.000Z")
|
||||
nowEpoch := time.Now().Unix()
|
||||
|
||||
var snr, rssi interface{}
|
||||
if body.Snr != nil {
|
||||
snr = *body.Snr
|
||||
}
|
||||
if body.Rssi != nil {
|
||||
rssi = *body.Rssi
|
||||
}
|
||||
|
||||
// v3 schema (cmd/ingestor/db.go:251-303): transmissions no longer carries
|
||||
// path_json (it lives on observations now), observations uses observer_idx
|
||||
// INTEGER (FK observers.rowid) and timestamp INTEGER (unix epoch).
|
||||
// Fix for #1196 — pre-fix code wrote v2 column names and silently
|
||||
// swallowed the observations insert error.
|
||||
res, dbErr := s.db.conn.Exec(`INSERT INTO transmissions (hash, raw_hex, route_type, payload_type, payload_version, decoded_json, first_seen)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
contentHash, strings.ToUpper(hexStr), decoded.Header.RouteType, decoded.Header.PayloadType,
|
||||
decoded.Header.PayloadVersion, decodedJSON, now)
|
||||
if dbErr != nil {
|
||||
writeError(w, 500, "transmission insert: "+dbErr.Error())
|
||||
return
|
||||
}
|
||||
insertedID, _ := res.LastInsertId()
|
||||
|
||||
// Resolve observer string → observers.rowid. INSERT OR IGNORE then SELECT
|
||||
// mirrors the ingestor's resolver (cmd/ingestor/db.go:778,799,906).
|
||||
var observerIdx interface{}
|
||||
if body.Observer != nil && *body.Observer != "" {
|
||||
obsID := *body.Observer
|
||||
if _, err := s.db.conn.Exec(
|
||||
`INSERT OR IGNORE INTO observers (id, name, last_seen, first_seen) VALUES (?, ?, ?, ?)`,
|
||||
obsID, obsID, now, now); err != nil {
|
||||
writeError(w, 500, "observer upsert: "+err.Error())
|
||||
return
|
||||
}
|
||||
var rowid int64
|
||||
if err := s.db.conn.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, obsID).Scan(&rowid); err != nil {
|
||||
writeError(w, 500, "observer lookup: "+err.Error())
|
||||
return
|
||||
}
|
||||
observerIdx = rowid
|
||||
}
|
||||
|
||||
if _, obsErr := s.db.conn.Exec(
|
||||
`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
insertedID, observerIdx, snr, rssi, pathJSON, nowEpoch); obsErr != nil {
|
||||
writeError(w, 500, "observation insert: "+obsErr.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, PacketIngestResponse{
|
||||
ID: insertedID,
|
||||
Decoded: map[string]interface{}{
|
||||
"header": decoded.Header,
|
||||
"path": decoded.Path,
|
||||
"payload": decoded.Payload,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// --- Node Handlers ---
|
||||
|
||||
func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -4846,72 +4846,6 @@ func TestListLimitsConfigurable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostPacketPersistsV3Schema is the round-trip regression for #1196.
|
||||
// POST /api/packets must write the observation row using the v3 schema
|
||||
// (observer_idx INTEGER, timestamp INTEGER) and surface insert errors.
|
||||
// The pre-fix handler writes v2 columns (observer_id, observer_name,
|
||||
// RFC3339 timestamp) and silently swallows the obs insert error.
|
||||
func TestPostPacketPersistsV3Schema(t *testing.T) {
|
||||
const apiKey = "test-secret-key-strong-enough"
|
||||
srv, router := setupTestServerWithAPIKey(t, apiKey)
|
||||
|
||||
// FLOOD/ADVERT hex (header 0x11, path byte 0x00, payload bytes).
|
||||
// Mirrors TestDecodePacket_FloodHasNoCodes.
|
||||
const rawHex = "110011223344556677889900AABBCCDD"
|
||||
bodyJSON := `{"hex":"` + rawHex + `","observer":"obs1","snr":5.5,"rssi":-72}`
|
||||
|
||||
req := httptest.NewRequest("POST", "/api/packets",
|
||||
bytes.NewReader([]byte(bodyJSON)))
|
||||
req.Header.Set("X-API-Key", apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /api/packets: expected 200, got %d (body: %s)",
|
||||
w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
idF, _ := resp["id"].(float64)
|
||||
txID := int64(idF)
|
||||
if txID <= 0 {
|
||||
t.Fatalf("expected transmission id > 0, got %v (body: %s)",
|
||||
resp["id"], w.Body.String())
|
||||
}
|
||||
|
||||
// Resolve expected observer_idx from the seeded observers table.
|
||||
var wantIdx int64
|
||||
if err := srv.db.conn.QueryRow(
|
||||
"SELECT rowid FROM observers WHERE id = ?", "obs1",
|
||||
).Scan(&wantIdx); err != nil {
|
||||
t.Fatalf("lookup observer rowid: %v", err)
|
||||
}
|
||||
|
||||
// Assert the observation row was written with v3 columns.
|
||||
var (
|
||||
gotIdx int64
|
||||
gotTS int64
|
||||
)
|
||||
err := srv.db.conn.QueryRow(
|
||||
"SELECT observer_idx, timestamp FROM observations WHERE transmission_id = ?",
|
||||
txID,
|
||||
).Scan(&gotIdx, &gotTS)
|
||||
if err != nil {
|
||||
t.Fatalf("observation row missing for tx %d: %v (handler swallowed insert error?)", txID, err)
|
||||
}
|
||||
if gotIdx != wantIdx {
|
||||
t.Errorf("observer_idx: want %d, got %d", wantIdx, gotIdx)
|
||||
}
|
||||
nowSec := time.Now().Unix()
|
||||
if gotTS < nowSec-60 || gotTS > nowSec+60 {
|
||||
t.Errorf("timestamp: want unix int near %d, got %d", nowSec, gotTS)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigThemeTypeColorsLegacyRequestKey verifies the REQUEST→REQ rename
|
||||
// (#1799 PR #1804 r1 item 6) doesn't break operators whose config.json still
|
||||
// carries the legacy `typeColors.REQUEST` key. The GET response must:
|
||||
|
||||
@@ -388,11 +388,6 @@ type PacketDetailResponse struct {
|
||||
Observations []ObservationResp `json:"observations,omitempty"`
|
||||
}
|
||||
|
||||
type PacketIngestResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
Decoded interface{} `json:"decoded"`
|
||||
}
|
||||
|
||||
type DecodeResponse struct {
|
||||
Decoded interface{} `json:"decoded"`
|
||||
}
|
||||
|
||||
@@ -27,7 +27,6 @@
|
||||
- [GET /api/packets](#get-apipackets)
|
||||
- [GET /api/packets/timestamps](#get-apipacketstimestamps)
|
||||
- [GET /api/packets/:id](#get-apipacketsid)
|
||||
- [POST /api/packets](#post-apipackets)
|
||||
- [POST /api/decode](#post-apidecode)
|
||||
- [GET /api/observers](#get-apiobservers)
|
||||
- [GET /api/observers/:id](#get-apiobserversid)
|
||||
@@ -907,48 +906,6 @@ Single packet detail with byte breakdown and observations.
|
||||
|
||||
---
|
||||
|
||||
## POST /api/packets
|
||||
|
||||
Ingest a raw packet. Requires API key.
|
||||
|
||||
### Headers
|
||||
|
||||
- `X-API-Key: <key>` (required if `config.apiKey` is set)
|
||||
|
||||
### Request Body
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"hex": string, // required — raw hex-encoded packet
|
||||
"observer": string | null, // observer ID
|
||||
"snr": number | null,
|
||||
"rssi": number | null,
|
||||
"region": string | null, // IATA code
|
||||
"hash": string | null // pre-computed content hash
|
||||
}
|
||||
```
|
||||
|
||||
### Response `200`
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": number, // packet/observation ID
|
||||
"decoded": { // full decode result
|
||||
"header": DecodedHeader,
|
||||
"path": DecodedPath,
|
||||
"payload": object
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Response `400`
|
||||
|
||||
```json
|
||||
{ "error": "hex is required" }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## POST /api/decode
|
||||
|
||||
Decode a raw packet without storing it.
|
||||
|
||||
Reference in New Issue
Block a user