diff --git a/.badges/e2e-tests.json b/.badges/e2e-tests.json index b21bba6d..5cc52853 100644 --- a/.badges/e2e-tests.json +++ b/.badges/e2e-tests.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"e2e tests","message":"1178 passed","color":"brightgreen"} +{"schemaVersion":1,"label":"e2e tests","message":"2 passed","color":"brightgreen"} diff --git a/.badges/frontend-coverage.json b/.badges/frontend-coverage.json index 0d9df12f..cb4e706b 100644 --- a/.badges/frontend-coverage.json +++ b/.badges/frontend-coverage.json @@ -1 +1 @@ -{"schemaVersion":1,"label":"frontend coverage","message":"39.03%","color":"red"} +{"schemaVersion":1,"label":"frontend coverage","message":"40.02%","color":"red"} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b827d226..2c96355a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -103,6 +103,7 @@ jobs: node test-observer-iata-1188.js node test-pull-to-reconnect-1091.js node test-channel-fluid-layout.js + node test-issue-1279-p2-code-filter.js - name: Verify proto syntax run: | @@ -264,6 +265,9 @@ jobs: CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/AGENTS.md b/AGENTS.md index c50c7a69..0d36255f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,17 @@ scripts/ — Tooling (coverage collector, fixture capture, frontend in 2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket 3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side +### Read/Write Separation Invariant (#1283) +- **All DB writes live in `cmd/ingestor/`.** INSERT / UPDATE / DELETE / VACUUM / + schema migrations / retention all run in the ingestor process. +- **`cmd/server/` is read-only.** It opens SQLite with `mode=ro` and must not + acquire a write lock. Adding a write-side helper (e.g. a `cachedRW`-style + RW connection) regresses this invariant and races the ingestor → SQLITE_BUSY. +- Enforcement: `cmd/server/readonly_invariant_test.go` reflect-asserts that + `PruneOldPackets`, `PruneOldMetrics`, and `RemoveStaleObservers` are NOT + methods on the server's `*DB`. If you need a new write, add it to + `cmd/ingestor/`. + ### What's Deprecated (DO NOT TOUCH) The following were part of the old Node.js backend and have been removed: - `server.js`, `db.js`, `decoder.js`, `server-helpers.js`, `packet-store.js`, `iata-coords.js` diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 0429b23a..8f6b8431 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -99,9 +99,21 @@ func (f *ForeignAdvertConfig) IsDropMode() bool { // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes. type RetentionConfig struct { - NodeDays int `json:"nodeDays"` - ObserverDays int `json:"observerDays"` - MetricsDays int `json:"metricsDays"` + NodeDays int `json:"nodeDays"` + ObserverDays int `json:"observerDays"` + MetricsDays int `json:"metricsDays"` + // PacketDays is the retention window for transmissions (#1283). + // Ownership moved from cmd/server to cmd/ingestor; 0 disables. + PacketDays int `json:"packetDays"` +} + +// PacketDaysOrZero returns the configured retention.packetDays or 0 +// (disabled) if not set. +func (c *Config) PacketDaysOrZero() int { + if c.Retention != nil && c.Retention.PacketDays > 0 { + return c.Retention.PacketDays + } + return 0 } // MetricsConfig controls observer metrics collection. diff --git a/cmd/ingestor/decoder.go b/cmd/ingestor/decoder.go index 7b8f0941..23982fca 100644 --- a/cmd/ingestor/decoder.go +++ b/cmd/ingestor/decoder.go @@ -126,6 +126,11 @@ type Payload struct { ChannelHashHex string `json:"channelHashHex,omitempty"` DecryptionStatus string `json:"decryptionStatus,omitempty"` Channel string `json:"channel,omitempty"` + // GRP_DATA (PAYLOAD_TYPE_GRP_DATA=0x06) inner fields, decoded after + // channel decrypt per firmware/src/helpers/BaseChatMesh.cpp:382-385. + DataType *int `json:"dataType,omitempty"` + DataLen *int `json:"dataLen,omitempty"` + DecryptedBlob string `json:"decryptedBlob,omitempty"` Text string `json:"text,omitempty"` Sender string `json:"sender,omitempty"` SenderTimestamp uint32 `json:"sender_timestamp,omitempty"` @@ -137,6 +142,23 @@ type Payload struct { TraceFlags *int `json:"traceFlags,omitempty"` RawHex string `json:"raw,omitempty"` Error string `json:"error,omitempty"` + // MULTIPART (PAYLOAD_TYPE_MULTIPART=0x0A) inner fields, decoded per + // firmware/src/Mesh.cpp:289 — byte0 = (remaining<<4) | inner_type. + Remaining *int `json:"remaining,omitempty"` + InnerType *int `json:"innerType,omitempty"` + InnerTypeName string `json:"innerTypeName,omitempty"` + InnerAckCrc string `json:"innerAckCrc,omitempty"` + InnerPayload string `json:"innerPayload,omitempty"` + // CONTROL (PAYLOAD_TYPE_CONTROL=0x0B) byte0 flags, per + // firmware/src/Mesh.cpp:69 — byte0 high-bit marks zero-hop direct subset. + CtrlFlags string `json:"ctrlFlags,omitempty"` + CtrlZeroHop *bool `json:"ctrlZeroHop,omitempty"` + CtrlLength *int `json:"ctrlLength,omitempty"` + // RAW_CUSTOM (PAYLOAD_TYPE_RAW_CUSTOM=0x0F) — application-defined per + // firmware/src/Mesh.cpp:577 (createRawData). Exposes the bare envelope + // shape (length + leading tag) so consumers can triage by app id. + RawLength *int `json:"rawLength,omitempty"` + FirstByteTag string `json:"firstByteTag,omitempty"` } // DecodedPacket is the full decoded result. @@ -343,6 +365,17 @@ func decodeAdvert(buf []byte, validateSignatures bool) Payload { // Telemetry bytes after name: battery_mv(2 LE) + temperature_c(2 LE, signed, /100) // Only sensor nodes (advType=4) carry telemetry bytes. + // + // Firmware derivation (see firmware/src/helpers/SensorMesh.h and the + // SensorHost::handleAdvert path in firmware/src/helpers/SensorMesh.cpp: + // the sensor builds appdata as + // followed by two little-endian uint16 fields appended verbatim: + // appdata[name_end+0..1] = battery voltage in millivolts (uint16 LE, + // valid 0 < mv ≤ 10000) + // appdata[name_end+2..3] = temperature × 100 (int16 LE, divide by 100 + // for °C; valid raw -5000..10000 → -50..100 °C) + // We accept only adverts whose flags.Sensor bit is set (firmware + // AdvertDataHelpers.h:7-12, ADV_TYPE_SENSOR=4) before parsing telemetry. if p.Flags.Sensor && off+4 <= len(appdata) { batteryMv := int(binary.LittleEndian.Uint16(appdata[off : off+2])) tempRaw := int16(binary.LittleEndian.Uint16(appdata[off+2 : off+4])) @@ -512,6 +545,185 @@ func decodeGrpTxt(buf []byte, channelKeys map[string]string) Payload { } } +// decodeGrpData decodes PAYLOAD_TYPE_GRP_DATA (0x06). Outer envelope is the +// same shape as GRP_TXT (channel_hash(1)+MAC(2)+ciphertext) — see +// firmware/src/helpers/BaseChatMesh.cpp:476,500. When the channel key matches, +// the decrypted inner is parsed per firmware/src/helpers/BaseChatMesh.cpp:382-385 +// as data_type(uint16 LE) + data_len(1) + blob(data_len). +func decodeGrpData(buf []byte, channelKeys map[string]string) Payload { + if len(buf) < 3 { + return Payload{Type: "GRP_DATA", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + channelHash := int(buf[0]) + channelHashHex := fmt.Sprintf("%02X", buf[0]) + mac := hex.EncodeToString(buf[1:3]) + encryptedData := hex.EncodeToString(buf[3:]) + + hasKeys := len(channelKeys) > 0 + if hasKeys && len(encryptedData) >= 10 { + for name, key := range channelKeys { + plain, err := decryptChannelBlock(encryptedData, mac, key) + if err != nil { + continue + } + // Inner: data_type(uint16 LE) + data_len(1) + blob (firmware:382-385). + if len(plain) < 3 { + return Payload{ + Type: "GRP_DATA", + Channel: name, + ChannelHash: channelHash, + ChannelHashHex: channelHashHex, + DecryptionStatus: "decrypted", + Error: "inner too short", + } + } + dataType := int(binary.LittleEndian.Uint16(plain[0:2])) + dataLen := int(plain[2]) + if 3+dataLen > len(plain) { + return Payload{ + Type: "GRP_DATA", + Channel: name, + ChannelHash: channelHash, + ChannelHashHex: channelHashHex, + DecryptionStatus: "decrypted", + DataType: &dataType, + DataLen: &dataLen, + Error: "inner data_len exceeds buffer", + } + } + blob := hex.EncodeToString(plain[3 : 3+dataLen]) + return Payload{ + Type: "GRP_DATA", + Channel: name, + ChannelHash: channelHash, + ChannelHashHex: channelHashHex, + DecryptionStatus: "decrypted", + DataType: &dataType, + DataLen: &dataLen, + DecryptedBlob: blob, + } + } + return Payload{ + Type: "GRP_DATA", + ChannelHash: channelHash, + ChannelHashHex: channelHashHex, + DecryptionStatus: "decryption_failed", + MAC: mac, + EncryptedData: encryptedData, + } + } + + return Payload{ + Type: "GRP_DATA", + ChannelHash: channelHash, + ChannelHashHex: channelHashHex, + DecryptionStatus: "no_key", + MAC: mac, + EncryptedData: encryptedData, + } +} + +// decodeMultipart decodes PAYLOAD_TYPE_MULTIPART (0x0A) per +// firmware/src/Mesh.cpp:287-310. byte0 = (remaining<<4) | inner_type; +// when inner_type == PAYLOAD_TYPE_ACK the next 4 bytes are an ack_crc. +func decodeMultipart(buf []byte) Payload { + if len(buf) < 1 { + return Payload{Type: "MULTIPART", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + remaining := int(buf[0] >> 4) + innerType := int(buf[0] & 0x0F) + innerName := payloadTypeNames[innerType] + if innerName == "" { + innerName = "UNKNOWN" + } + p := Payload{ + Type: "MULTIPART", + Remaining: &remaining, + InnerType: &innerType, + InnerTypeName: innerName, + } + if innerType == PayloadACK && len(buf) >= 5 { + // ack_crc is little-endian; surface as canonical big-endian hex + // to match decodeAck's extraHash convention. + crc := binary.LittleEndian.Uint32(buf[1:5]) + p.InnerAckCrc = fmt.Sprintf("%08x", crc) + } else if len(buf) > 1 { + p.InnerPayload = hex.EncodeToString(buf[1:]) + } + return p +} + +// decodeControl decodes PAYLOAD_TYPE_CONTROL (0x0B) byte0 flags per +// firmware/src/Mesh.cpp:69 (high-bit set ⇒ zero-hop direct subset). +func decodeControl(buf []byte) Payload { + if len(buf) < 1 { + return Payload{Type: "CONTROL", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + zeroHop := buf[0]&0x80 != 0 + length := len(buf) + return Payload{ + Type: "CONTROL", + CtrlFlags: fmt.Sprintf("%02x", buf[0]), + CtrlZeroHop: &zeroHop, + CtrlLength: &length, + RawHex: hex.EncodeToString(buf), + } +} + +// decodeRawCustom decodes PAYLOAD_TYPE_RAW_CUSTOM (0x0F). Application-defined +// payload per firmware/src/Mesh.cpp:577 (createRawData); we only surface the +// envelope shape (total length + leading tag byte). +func decodeRawCustom(buf []byte) Payload { + length := len(buf) + p := Payload{ + Type: "RAW_CUSTOM", + RawLength: &length, + RawHex: hex.EncodeToString(buf), + } + if length > 0 { + p.FirstByteTag = fmt.Sprintf("%02X", buf[0]) + } + return p +} + +// decryptChannelBlock performs the MAC verify + AES-128-ECB decrypt step shared +// by GRP_TXT and GRP_DATA, returning the raw plaintext block (no further +// parsing). See firmware/src/helpers/BaseChatMesh.cpp:376-391. +func decryptChannelBlock(ciphertextHex, macHex, channelKeyHex string) ([]byte, error) { + channelKey, err := hex.DecodeString(channelKeyHex) + if err != nil || len(channelKey) != 16 { + return nil, fmt.Errorf("invalid channel key") + } + macBytes, err := hex.DecodeString(macHex) + if err != nil || len(macBytes) != 2 { + return nil, fmt.Errorf("invalid MAC") + } + ciphertext, err := hex.DecodeString(ciphertextHex) + if err != nil || len(ciphertext) == 0 { + return nil, fmt.Errorf("invalid ciphertext") + } + channelSecret := make([]byte, 32) + copy(channelSecret, channelKey) + h := hmac.New(sha256.New, channelSecret) + h.Write(ciphertext) + calc := h.Sum(nil) + if calc[0] != macBytes[0] || calc[1] != macBytes[1] { + return nil, fmt.Errorf("MAC verification failed") + } + if len(ciphertext)%aes.BlockSize != 0 { + return nil, fmt.Errorf("ciphertext not aligned to AES block size") + } + block, err := aes.NewCipher(channelKey) + if err != nil { + return nil, err + } + plain := make([]byte, len(ciphertext)) + for i := 0; i < len(ciphertext); i += aes.BlockSize { + block.Decrypt(plain[i:i+aes.BlockSize], ciphertext[i:i+aes.BlockSize]) + } + return plain, nil +} + func decodeAnonReq(buf []byte) Payload { if len(buf) < 35 { return Payload{Type: "ANON_REQ", Error: "too short", RawHex: hex.EncodeToString(buf)} @@ -571,12 +783,20 @@ func decodePayload(payloadType int, buf []byte, channelKeys map[string]string, v return decodeAdvert(buf, validateSignatures) case PayloadGRP_TXT: return decodeGrpTxt(buf, channelKeys) + case PayloadGRP_DATA: + return decodeGrpData(buf, channelKeys) case PayloadANON_REQ: return decodeAnonReq(buf) case PayloadPATH: return decodePathPayload(buf) case PayloadTRACE: return decodeTrace(buf) + case PayloadMULTIPART: + return decodeMultipart(buf) + case PayloadCONTROL: + return decodeControl(buf) + case PayloadRAW_CUSTOM: + return decodeRawCustom(buf) default: return Payload{Type: "UNKNOWN", RawHex: hex.EncodeToString(buf)} } @@ -824,8 +1044,13 @@ func ValidateAdvert(p *Payload) (bool, string) { if p.Flags != nil { role := advertRole(p.Flags) - validRoles := map[string]bool{"repeater": true, "companion": true, "room": true, "sensor": true} - if !validRoles[role] { + // Accept canonical labels plus "none" (ADV_TYPE_NONE=0) and the + // "type-N" placeholders we now return for ADV_TYPE 5-15 (FUTURE) + // — see firmware/src/helpers/AdvertDataHelpers.h:7-12. + validRoles := map[string]bool{ + "repeater": true, "companion": true, "room": true, "sensor": true, "none": true, + } + if !validRoles[role] && !strings.HasPrefix(role, "type-") { return false, fmt.Sprintf("unknown role: %s", role) } } @@ -845,17 +1070,29 @@ func sanitizeName(s string) string { return b.String() } +// advertRole returns a stable role label for an advert. Follows firmware +// ADV_TYPE_* constants in firmware/src/helpers/AdvertDataHelpers.h:7-12: +// 0 NONE, 1 CHAT, 2 REPEATER, 3 ROOM, 4 SENSOR, 5-15 FUTURE. +// Previously this coerced both 0 (NONE) and 5-15 (FUTURE) to "companion", +// silently relabelling unknown/reserved types — see issue #1279 P1 #3. func advertRole(f *AdvertFlags) string { - if f.Repeater { + if f == nil { + return "companion" + } + switch f.Type { + case 0: + return "none" + case 1: + return "companion" + case 2: return "repeater" - } - if f.Room { + case 3: return "room" - } - if f.Sensor { + case 4: return "sensor" + default: + return fmt.Sprintf("type-%d", f.Type) } - return "companion" } func epochToISO(epoch uint32) string { diff --git a/cmd/ingestor/decoder_test.go b/cmd/ingestor/decoder_test.go index 779a9062..fa1cc26b 100644 --- a/cmd/ingestor/decoder_test.go +++ b/cmd/ingestor/decoder_test.go @@ -631,21 +631,28 @@ func TestDecodeEncryptedPayloadValid(t *testing.T) { } func TestDecodePayloadGRPData(t *testing.T) { + // GRP_DATA (0x06) decoder added for #1279 P0 #1 — envelope only when no + // channel key matches (firmware/src/helpers/BaseChatMesh.cpp:500). buf := []byte{0x01, 0x02, 0x03} p := decodePayload(PayloadGRP_DATA, buf, nil, false) - if p.Type != "UNKNOWN" { - t.Errorf("type=%s, want UNKNOWN", p.Type) - } - if p.RawHex != "010203" { - t.Errorf("rawHex=%s, want 010203", p.RawHex) + if p.Type != "GRP_DATA" { + t.Errorf("type=%s, want GRP_DATA", p.Type) } } func TestDecodePayloadRAWCustom(t *testing.T) { + // #1279 P2 #5: RAW_CUSTOM (0x0F) now exposes envelope shape (length + + // first-byte tag) per firmware/src/Mesh.cpp:577 (createRawData). buf := []byte{0xFF, 0xFE} p := decodePayload(PayloadRAW_CUSTOM, buf, nil, false) - if p.Type != "UNKNOWN" { - t.Errorf("type=%s, want UNKNOWN", p.Type) + if p.Type != "RAW_CUSTOM" { + t.Errorf("type=%s, want RAW_CUSTOM", p.Type) + } + if p.RawLength == nil || *p.RawLength != 2 { + t.Errorf("rawLength missing or wrong, want 2") + } + if p.FirstByteTag != "FF" { + t.Errorf("firstByteTag=%q, want FF", p.FirstByteTag) } } @@ -1097,18 +1104,18 @@ func TestDecodeHeaderUnknownTypes(t *testing.T) { } func TestDecodePayloadMultipart(t *testing.T) { - // MULTIPART (0x0A) falls through to default → UNKNOWN + // MULTIPART (0x0A) now decoded — #1279 P0 #2 (firmware/src/Mesh.cpp:289). p := decodePayload(PayloadMULTIPART, []byte{0x01, 0x02}, nil, false) - if p.Type != "UNKNOWN" { - t.Errorf("MULTIPART type=%s, want UNKNOWN", p.Type) + if p.Type != "MULTIPART" { + t.Errorf("MULTIPART type=%s, want MULTIPART", p.Type) } } func TestDecodePayloadControl(t *testing.T) { - // CONTROL (0x0B) falls through to default → UNKNOWN + // CONTROL (0x0B) now decoded — #1279 P1 #4 (firmware/src/Mesh.cpp:69). p := decodePayload(PayloadCONTROL, []byte{0x01, 0x02}, nil, false) - if p.Type != "UNKNOWN" { - t.Errorf("CONTROL type=%s, want UNKNOWN", p.Type) + if p.Type != "CONTROL" { + t.Errorf("CONTROL type=%s, want CONTROL", p.Type) } } diff --git a/cmd/ingestor/issue1279_p2_test.go b/cmd/ingestor/issue1279_p2_test.go new file mode 100644 index 00000000..3a12803b --- /dev/null +++ b/cmd/ingestor/issue1279_p2_test.go @@ -0,0 +1,30 @@ +package main + +// Tests for issue #1279 P2 item 5: ingestor RAW_CUSTOM exposure. + +import ( + "strings" + "testing" +) + +func TestDecodeRawCustomExposesLengthAndTag(t *testing.T) { + // header = (1<<6)|(0x0F<<2)|1 = 0x7D ; path byte = 0x00 ; payload = A5 DE AD BE EF + hexStr := "7D00A5DEADBEEF" + pkt, err := DecodePacket(hexStr, nil, false) + if err != nil { + t.Fatalf("decode: %v", err) + } + if pkt.Payload.Type != "RAW_CUSTOM" { + t.Fatalf("payload type = %q, want RAW_CUSTOM", pkt.Payload.Type) + } + if pkt.Payload.RawLength == nil || *pkt.Payload.RawLength != 5 { + got := -1 + if pkt.Payload.RawLength != nil { + got = *pkt.Payload.RawLength + } + t.Errorf("RawLength=%d, want 5", got) + } + if !strings.EqualFold(pkt.Payload.FirstByteTag, "A5") { + t.Errorf("FirstByteTag=%q, want A5", pkt.Payload.FirstByteTag) + } +} diff --git a/cmd/ingestor/issue1279_test.go b/cmd/ingestor/issue1279_test.go new file mode 100644 index 00000000..da82fce9 --- /dev/null +++ b/cmd/ingestor/issue1279_test.go @@ -0,0 +1,211 @@ +package main + +// Tests for issue #1279 P0+P1 decoder additions. +// +// Each test uses firmware-derived wire vectors: +// - GRP_DATA outer: firmware/src/helpers/BaseChatMesh.cpp:500 (createGroupDatagram) +// - GRP_DATA inner: firmware/src/helpers/BaseChatMesh.cpp:382-385 +// - MULTIPART byte0: firmware/src/Mesh.cpp:289 +// - MULTIPART ACK inner: firmware/src/Mesh.cpp:292-307 +// - CONTROL byte0 flags: firmware/src/Mesh.cpp:69 + createControlData at Mesh.cpp:609 +// - advertRole label rules: firmware/src/helpers/AdvertDataHelpers.h:7-12 + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "testing" +) + +// --- P0 #1: GRP_DATA decoder --- + +// buildChannelEncrypted encrypts arbitrary inner bytes with the channel +// key/MAC scheme firmware uses for both GRP_TXT and GRP_DATA (see +// BaseChatMesh.cpp:376-391: AES-128-ECB, HMAC-SHA256-trunc-2 MAC). +func buildChannelEncrypted(channelKeyHex string, inner []byte) (ctHex, macHex string) { + key, _ := hex.DecodeString(channelKeyHex) + plain := append([]byte{}, inner...) + pad := aes.BlockSize - (len(plain) % aes.BlockSize) + if pad != aes.BlockSize { + plain = append(plain, make([]byte, pad)...) + } + block, _ := aes.NewCipher(key) + ct := make([]byte, len(plain)) + for i := 0; i < len(plain); i += aes.BlockSize { + block.Encrypt(ct[i:i+aes.BlockSize], plain[i:i+aes.BlockSize]) + } + secret := make([]byte, 32) + copy(secret, key) + h := hmac.New(sha256.New, secret) + h.Write(ct) + mac := h.Sum(nil) + return hex.EncodeToString(ct), hex.EncodeToString(mac[:2]) +} + +func TestDecodeGrpDataNoKey(t *testing.T) { + // Envelope alone (no key in store). + buf := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11} + p := decodeGrpData(buf, nil) + if p.Type != "GRP_DATA" { + t.Fatalf("type=%q want GRP_DATA", p.Type) + } + if p.ChannelHash != 0xAA { + t.Errorf("channelHash=%d want 170", p.ChannelHash) + } + if p.ChannelHashHex != "AA" { + t.Errorf("channelHashHex=%q want AA", p.ChannelHashHex) + } + if p.MAC != "bbcc" { + t.Errorf("mac=%q want bbcc", p.MAC) + } + if p.EncryptedData != "ddeeff11" { + t.Errorf("encryptedData=%q want ddeeff11", p.EncryptedData) + } + if p.DecryptionStatus != "no_key" { + t.Errorf("decryptionStatus=%q want no_key", p.DecryptionStatus) + } +} + +func TestDecodeGrpDataDecryptedInner(t *testing.T) { + // Inner per BaseChatMesh.cpp:382-385: data_type(uint16 LE) + data_len(1) + blob. + key := "2cc3d22840e086105ad73443da2cacb8" + blob := []byte{0x10, 0x20, 0x30, 0x40, 0x50} + inner := []byte{0x34, 0x12, byte(len(blob))} // data_type = 0x1234 + inner = append(inner, blob...) + ctHex, macHex := buildChannelEncrypted(key, inner) + + buf := []byte{0xAB} + mb, _ := hex.DecodeString(macHex) + buf = append(buf, mb...) + cb, _ := hex.DecodeString(ctHex) + buf = append(buf, cb...) + + p := decodeGrpData(buf, map[string]string{"test": key}) + if p.Type != "GRP_DATA" { + t.Fatalf("type=%q want GRP_DATA", p.Type) + } + if p.DecryptionStatus != "decrypted" { + t.Fatalf("decryptionStatus=%q want decrypted", p.DecryptionStatus) + } + if p.DataType == nil || *p.DataType != 0x1234 { + t.Errorf("dataType=%v want 0x1234", p.DataType) + } + if p.DataLen == nil || *p.DataLen != 5 { + t.Errorf("dataLen=%v want 5", p.DataLen) + } + if p.DecryptedBlob != hex.EncodeToString(blob) { + t.Errorf("decryptedBlob=%q want %q", p.DecryptedBlob, hex.EncodeToString(blob)) + } + if p.Channel != "test" { + t.Errorf("channel=%q want test", p.Channel) + } +} + +// --- P0 #2: MULTIPART decoder --- + +func TestDecodeMultipartAck(t *testing.T) { + // remaining=3, inner_type=PAYLOAD_TYPE_ACK(0x03), ack_crc=0xDEADBEEF. + // byte0 = (3<<4) | 3 = 0x33; next 4 bytes are LE crc. + buf := []byte{0x33, 0xEF, 0xBE, 0xAD, 0xDE} + p := decodeMultipart(buf) + if p.Type != "MULTIPART" { + t.Fatalf("type=%q want MULTIPART", p.Type) + } + if p.Remaining == nil || *p.Remaining != 3 { + t.Errorf("remaining=%v want 3", p.Remaining) + } + if p.InnerType == nil || *p.InnerType != 0x03 { + t.Errorf("innerType=%v want 3", p.InnerType) + } + if p.InnerTypeName != "ACK" { + t.Errorf("innerTypeName=%q want ACK", p.InnerTypeName) + } + if p.InnerAckCrc != "deadbeef" { + t.Errorf("innerAckCrc=%q want deadbeef", p.InnerAckCrc) + } +} + +func TestDecodeMultipartNonAck(t *testing.T) { + // remaining=2, inner_type=0x02 (TXT_MSG), arbitrary inner payload. + buf := []byte{0x22, 0x01, 0x02, 0x03} + p := decodeMultipart(buf) + if p.Remaining == nil || *p.Remaining != 2 { + t.Errorf("remaining=%v want 2", p.Remaining) + } + if p.InnerType == nil || *p.InnerType != 0x02 { + t.Errorf("innerType=%v want 2", p.InnerType) + } + if p.InnerTypeName != "TXT_MSG" { + t.Errorf("innerTypeName=%q want TXT_MSG", p.InnerTypeName) + } + if p.InnerPayload != "010203" { + t.Errorf("innerPayload=%q want 010203", p.InnerPayload) + } + if p.InnerAckCrc != "" { + t.Errorf("non-ACK should not surface innerAckCrc, got %q", p.InnerAckCrc) + } +} + +// --- P1 #3: advertRole label fix --- + +func TestAdvertRoleLabelsRawType(t *testing.T) { + // Firmware: ADV_TYPE_NONE=0, CHAT=1, REPEATER=2, ROOM=3, SENSOR=4, 5..15 FUTURE. + cases := []struct { + typ int + want string + }{ + {0, "none"}, + {1, "companion"}, + {2, "repeater"}, + {3, "room"}, + {4, "sensor"}, + {5, "type-5"}, + {15, "type-15"}, + } + for _, tc := range cases { + got := advertRole(&AdvertFlags{Type: tc.typ, Repeater: tc.typ == 2, Room: tc.typ == 3, Sensor: tc.typ == 4}) + if got != tc.want { + t.Errorf("advertRole(type=%d) = %q, want %q", tc.typ, got, tc.want) + } + } +} + +// --- P1 #4: CONTROL byte0 flags --- + +func TestDecodeControlZeroHop(t *testing.T) { + // byte0 = 0x81 (high-bit set ⇒ zero-hop), followed by 3 app bytes. + buf := []byte{0x81, 0xAA, 0xBB, 0xCC} + p := decodeControl(buf) + if p.Type != "CONTROL" { + t.Fatalf("type=%q want CONTROL", p.Type) + } + if p.CtrlFlags != "81" { + t.Errorf("ctrlFlags=%q want 81", p.CtrlFlags) + } + if p.CtrlZeroHop == nil || !*p.CtrlZeroHop { + t.Errorf("ctrlZeroHop=%v want true", p.CtrlZeroHop) + } + if p.CtrlLength == nil || *p.CtrlLength != 4 { + t.Errorf("ctrlLength=%v want 4", p.CtrlLength) + } +} + +func TestDecodeControlMultiHop(t *testing.T) { + // byte0 = 0x01 (high-bit clear ⇒ not zero-hop subset). + buf := []byte{0x01, 0x42} + p := decodeControl(buf) + if p.CtrlFlags != "01" { + t.Errorf("ctrlFlags=%q want 01", p.CtrlFlags) + } + if p.CtrlZeroHop == nil || *p.CtrlZeroHop { + t.Errorf("ctrlZeroHop=%v want false", p.CtrlZeroHop) + } + if p.CtrlLength == nil || *p.CtrlLength != 2 { + t.Errorf("ctrlLength=%v want 2", p.CtrlLength) + } +} + +// silence unused-import diagnostics for stub-phase builds +var _ = binary.LittleEndian diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go new file mode 100644 index 00000000..f19936ba --- /dev/null +++ b/cmd/ingestor/issue1283_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "database/sql" + "path/filepath" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// TestIngestorPruneOldPackets enforces #1283: the writer for +// transmissions retention lives on the ingestor's *Store. Before the fix, +// this lived on cmd/server/*DB and raced with ingestor INSERTs. After +// the fix, ingestor owns it and runs it on its own write-locked handle. +func TestIngestorPruneOldPackets(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "prune.db") + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + + old := time.Now().UTC().AddDate(0, 0, -10).Format(time.RFC3339) + new := time.Now().UTC().Format(time.RFC3339) + for i, ts := range []string{old, old, new} { + _, err := store.db.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) + VALUES (?, ?, ?, 0, 1, 1, '{}')`, + "AA", "h"+string(rune('a'+i)), ts, + ) + if err != nil { + t.Fatalf("seed tx: %v", err) + } + } + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != 2 { + t.Fatalf("expected 2 pruned, got %d", n) + } + + var remaining int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&remaining); err != nil { + t.Fatalf("count: %v", err) + } + if remaining != 1 { + t.Fatalf("expected 1 transmission remaining, got %d", remaining) + } +} + +// TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL exercises the +// scenario that originally broke in #1283: a fresh DB with +// auto_vacuum=NONE, vacuumOnStartup=true, no contention from a server +// process. The ingestor must complete the VACUUM and flip auto_vacuum to +// INCREMENTAL. Before the fix, the migration ran inside cmd/server and +// hit SQLITE_BUSY because the ingestor (sharing the container) was +// already writing. +func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "vac.db") + + // Create a NONE-auto_vacuum DB (simulates an older deployment). + seed, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + if err != nil { + t.Fatal(err) + } + seed.SetMaxOpenConns(1) + if _, err := seed.Exec(`CREATE TABLE dummy(id INTEGER PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + var before int + seed.QueryRow("PRAGMA auto_vacuum").Scan(&before) + if before != 0 { + t.Fatalf("precondition: auto_vacuum=%d, want 0", before) + } + seed.Close() + + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + + cfg := &Config{DB: &DBConfig{VacuumOnStartup: true}} + store.CheckAutoVacuum(cfg) + + var after int + if err := store.db.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { + t.Fatal(err) + } + if after != 2 { + t.Fatalf("expected auto_vacuum=2 after ingestor VACUUM, got %d", after) + } +} diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index bfc72e5c..41c000e6 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -77,6 +77,19 @@ func main() { metricsDays := cfg.MetricsRetentionDays() store.PruneOldMetrics(metricsDays) store.PruneDroppedPackets(metricsDays) + + // Packet (transmissions) retention: previously lived in cmd/server, + // moved to ingestor in #1283 to eliminate cross-process write + // contention (SQLITE_BUSY). 0 = disabled. + packetDays := cfg.PacketDaysOrZero() + if packetDays > 0 { + if n, err := store.PruneOldPackets(packetDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + log.Printf("[prune] startup pruned %d transmissions older than %d days", n, packetDays) + } + } + vacuumPages := cfg.IncrementalVacuumPages() store.RunIncrementalVacuum(vacuumPages) @@ -111,6 +124,22 @@ func main() { } }() + // Daily ticker for transmission retention (#1283). + var packetRetentionTicker *time.Ticker + if packetDays > 0 { + packetRetentionTicker = time.NewTicker(24 * time.Hour) + go func() { + for range packetRetentionTicker.C { + if n, err := store.PruneOldPackets(packetDays); err != nil { + log.Printf("[prune] error: %v", err) + } else if n > 0 { + store.RunIncrementalVacuum(vacuumPages) + } + } + }() + log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", packetDays) + } + // Periodic stats logging (every 5 minutes) statsTicker := time.NewTicker(5 * time.Minute) go func() { @@ -253,6 +282,9 @@ func main() { log.Println("Shutting down...") retentionTicker.Stop() metricsRetentionTicker.Stop() + if packetRetentionTicker != nil { + packetRetentionTicker.Stop() + } statsTicker.Stop() stopWatchdog() store.LogStats() // final stats on shutdown diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index ce5b3116..16beb3f3 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -480,18 +480,20 @@ func TestEpochToISO(t *testing.T) { } func TestAdvertRole(t *testing.T) { + // advertRole now keys off AdvertFlags.Type (firmware ADV_TYPE_*) — see + // firmware/src/helpers/AdvertDataHelpers.h:7-12 and issue #1279 P1 #3. tests := []struct { name string flags *AdvertFlags want string }{ - {"repeater", &AdvertFlags{Repeater: true}, "repeater"}, - {"room", &AdvertFlags{Room: true}, "room"}, - {"sensor", &AdvertFlags{Sensor: true}, "sensor"}, - {"companion (default)", &AdvertFlags{Chat: true}, "companion"}, - {"companion (no flags)", &AdvertFlags{}, "companion"}, - {"repeater takes priority", &AdvertFlags{Repeater: true, Room: true}, "repeater"}, - {"room before sensor", &AdvertFlags{Room: true, Sensor: true}, "room"}, + {"none (type 0)", &AdvertFlags{Type: 0}, "none"}, + {"companion (type 1)", &AdvertFlags{Type: 1, Chat: true}, "companion"}, + {"repeater (type 2)", &AdvertFlags{Type: 2, Repeater: true}, "repeater"}, + {"room (type 3)", &AdvertFlags{Type: 3, Room: true}, "room"}, + {"sensor (type 4)", &AdvertFlags{Type: 4, Sensor: true}, "sensor"}, + {"future type-5", &AdvertFlags{Type: 5}, "type-5"}, + {"nil flags falls back to companion", nil, "companion"}, } for _, tt := range tests { diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go new file mode 100644 index 00000000..44c8f7aa --- /dev/null +++ b/cmd/ingestor/maintenance.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "log" + "time" +) + +// PruneOldPackets deletes transmissions (and their child observations) +// older than `days`. Returns count of transmissions deleted. +// +// Owned by the ingestor per #1283: the writer process is the only one +// allowed to hold the DB write lock; previously this lived in +// cmd/server/db.go and raced ingestor INSERTs (SQLITE_BUSY). +func (s *Store) PruneOldPackets(days int) (int64, error) { + if days <= 0 { + return 0, nil + } + cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) + + tx, err := s.db.Begin() + if err != nil { + return 0, fmt.Errorf("prune begin: %w", err) + } + defer tx.Rollback() + + // Delete child observations first (no CASCADE in SQLite). + if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN ( + SELECT id FROM transmissions WHERE first_seen < ? + )`, cutoff); err != nil { + return 0, fmt.Errorf("prune observations: %w", err) + } + + res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff) + if err != nil { + return 0, fmt.Errorf("prune transmissions: %w", err) + } + n, _ := res.RowsAffected() + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("prune commit: %w", err) + } + if n > 0 { + log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + } + return n, nil +} diff --git a/cmd/server/analytics_recomputer.go b/cmd/server/analytics_recomputer.go index 82a51128..fbc13518 100644 --- a/cmd/server/analytics_recomputer.go +++ b/cmd/server/analytics_recomputer.go @@ -146,13 +146,15 @@ func (r *analyticsRecomputer) ComputeRuns() int64 { // per-endpoint recompute interval from config.json. Zero values fall // back to the defaultInterval passed to StartAnalyticsRecomputers. type AnalyticsRecomputeIntervals struct { - Topology time.Duration - RF time.Duration - Distance time.Duration - Channels time.Duration - HashCollisions time.Duration - HashSizes time.Duration - Roles time.Duration + Topology time.Duration + RF time.Duration + Distance time.Duration + Channels time.Duration + HashCollisions time.Duration + HashSizes time.Duration + Roles time.Duration + ObserversClockSkew time.Duration + NodesClockSkew time.Duration } func pickInterval(override, def time.Duration) time.Duration { @@ -224,10 +226,19 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o "roles", pickInterval(ov.Roles, defaultInterval), func() interface{} { return s.computeAnalyticsRoles() }, ) + s.recompObserversClockSkew = newAnalyticsRecomputer( + "observers-clock-skew", pickInterval(ov.ObserversClockSkew, defaultInterval), + func() interface{} { return s.computeObserverCalibrations() }, + ) + s.recompNodesClockSkew = newAnalyticsRecomputer( + "nodes-clock-skew", pickInterval(ov.NodesClockSkew, defaultInterval), + func() interface{} { return s.computeFleetClockSkew() }, + ) all := []*analyticsRecomputer{ s.recompTopology, s.recompRF, s.recompDistance, s.recompChannels, s.recompHashCollisions, s.recompHashSizes, s.recompRoles, + s.recompObserversClockSkew, s.recompNodesClockSkew, } s.analyticsRecomputerMu.Unlock() diff --git a/cmd/server/bridge_handle_nodes_test.go b/cmd/server/bridge_handle_nodes_test.go new file mode 100644 index 00000000..b88b7b1b --- /dev/null +++ b/cmd/server/bridge_handle_nodes_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// TestBridgeScore_HandleNodesSurface verifies that /api/nodes +// includes a `bridge_score` field on repeater rows after the bridge +// recomputer has run. Drives the line-graph A-B-C-D through the full +// pipeline: insert nodes, populate the neighbor graph, force a +// recompute, hit the handler, parse the response. Issue #672 axis 2. +func TestBridgeScore_HandleNodesSurface(t *testing.T) { + db := setupCapabilityTestDB(t) + defer db.conn.Close() + // handleNodes/db.GetNodes selects a foreign_advert column not in + // the minimal capability-test schema. + if _, err := db.conn.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { + t.Fatal(err) + } + + // Four repeater nodes in a line. + pks := []string{ + "aaaa000000000000000000000000000000000000000000000000000000000000", + "bbbb000000000000000000000000000000000000000000000000000000000000", + "cccc000000000000000000000000000000000000000000000000000000000000", + "dddd000000000000000000000000000000000000000000000000000000000000", + } + recent := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") + for _, pk := range pks { + if _, err := db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + pk, "node-"+pk[:4], recent, recent); err != nil { + t.Fatal(err) + } + } + + store := NewPacketStore(db, nil) + // Build neighbor graph with the line A-B-C-D. Add each edge + // `count` times so its time-decayed Score saturates. + g := NewNeighborGraph() + now := time.Now() + obs := "obs-test" + snr := 5.0 + for i := 0; i < 10; i++ { + g.upsertEdge(pks[0], pks[1], "aa", obs, &snr, now) + g.upsertEdge(pks[1], pks[2], "bb", obs, &snr, now) + g.upsertEdge(pks[2], pks[3], "cc", obs, &snr, now) + } + store.graph.Store(g) + + // Direct invocation of the recomputer's compute path — bypassing + // StartBridgeScoreRecomputer's package-level once-flag (which is + // problematic across tests). + recomputeBridgeScoresSafe(store) + + snap := store.GetBridgeScoreMap() + if len(snap) == 0 { + t.Fatalf("expected non-empty bridge score snapshot, got empty") + } + // Sanity: middle nodes b/c must be positive, ends must be zero. + if snap[pks[1]] <= 0 || snap[pks[2]] <= 0 { + t.Errorf("middle nodes should have positive bridge: b=%v c=%v", + snap[pks[1]], snap[pks[2]]) + } + if snap[pks[0]] != 0 || snap[pks[3]] != 0 { + t.Errorf("end nodes should have zero bridge: a=%v d=%v", + snap[pks[0]], snap[pks[3]]) + } + + // Wire a Server, call handleNodes, parse the response. + cfg := &Config{Port: 3000} + hub := NewHub() + srv := NewServer(db, cfg, hub) + srv.store = store + + router := mux.NewRouter() + srv.RegisterRoutes(router) + + req := httptest.NewRequest("GET", "/api/nodes?limit=100", nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + + if rr.Code != 200 { + t.Fatalf("handleNodes status: want 200, got %d body=%s", rr.Code, rr.Body.String()) + } + var resp struct { + Nodes []map[string]interface{} `json:"nodes"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v body=%s", err, rr.Body.String()) + } + gotBy := map[string]map[string]interface{}{} + for _, n := range resp.Nodes { + if pk, _ := n["public_key"].(string); pk != "" { + gotBy[pk] = n + } + } + for _, pk := range pks { + n, ok := gotBy[pk] + if !ok { + t.Errorf("node %s missing from response", pk[:4]) + continue + } + if _, has := n["bridge_score"]; !has { + t.Errorf("node %s: bridge_score field absent from response", pk[:4]) + } + } + // Middle node B must report a non-zero bridge_score; end node A + // must report exactly zero. These two assertions together prevent + // a "field present but always 0" regression. + if v, _ := gotBy[pks[1]]["bridge_score"].(float64); v <= 0 { + t.Errorf("middle node B bridge_score in API response should be > 0, got %v", v) + } + if v, _ := gotBy[pks[0]]["bridge_score"].(float64); v != 0 { + t.Errorf("end node A bridge_score in API response should be 0, got %v", v) + } +} diff --git a/cmd/server/bridge_recomputer.go b/cmd/server/bridge_recomputer.go new file mode 100644 index 00000000..66d25a9a --- /dev/null +++ b/cmd/server/bridge_recomputer.go @@ -0,0 +1,198 @@ +// Package main: bridge-axis recomputer (issue #672 axis 2 of 4). +// +// Steady-state background loop that recomputes the per-pubkey bridge +// centrality score over the in-memory NeighborGraph and stores the +// resulting map atomically. handleNodes reads via a single atomic +// load — no lock contention with ingest or with other recomputers +// (same pattern as #1240 / #1248). +// +// Interval default: 5 minutes. The graph itself rebuilds asynchronously +// on its own schedule (path_inspect.go); a 5-minute cadence here is +// well within the freshness budget for a structural metric (centrality +// changes slowly — a new edge or evicted node nudges scores by +// fractions of a percent). +// +// Cost (Brandes + Dijkstra): O(V · (E + V log V)). Staging-scale ~600 +// nodes / ~2 000 edges ≈ ~4.8M ops, well under 100 ms in practice. On +// host-fleet scale (5 000 nodes / 30 000 edges) it is still seconds, +// running in a background goroutine off the request path. +package main + +import ( + "sync" + "time" +) + +// bridgeRecomputerDefaultInterval is how often the bridge score map is +// rebuilt. 5 minutes mirrors analytics_recomputer (#1240) and +// repeater_enrich_recomputer (#1262); centrality is a slow-moving +// structural signal and does not warrant tighter cadence. +const bridgeRecomputerDefaultInterval = 5 * time.Minute + +// bridgeRecompStartedMu serializes start of the bridge recomputer. +// We do not currently expose Stop publicly — the goroutine lives for +// the lifetime of the process — but keeping the started flag local +// (instead of on PacketStore) avoids further field churn in store.go. +var ( + bridgeRecompStartedMu sync.Mutex + bridgeRecompStarted bool +) + +// StartBridgeScoreRecomputer launches the bridge-centrality recomputer +// (issue #672 axis 2). It performs an initial synchronous compute so +// that the very first /api/nodes after server start hits a populated +// snapshot instead of returning bridge_score=0 for every node, then +// reschedules every `interval` (default 5min if <= 0). +// +// Idempotent: subsequent calls are no-ops and return a no-op stop +// closure. +func (s *PacketStore) StartBridgeScoreRecomputer(interval time.Duration) func() { + if interval <= 0 { + interval = bridgeRecomputerDefaultInterval + } + + bridgeRecompStartedMu.Lock() + if bridgeRecompStarted { + bridgeRecompStartedMu.Unlock() + return func() {} + } + bridgeRecompStarted = true + stop := make(chan struct{}) + done := make(chan struct{}) + bridgeRecompStartedMu.Unlock() + + // Initial synchronous prewarm — see comment above. + recomputeBridgeScoresSafe(s) + + var stopOnce sync.Once + go func() { + defer close(done) + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-t.C: + recomputeBridgeScoresSafe(s) + case <-stop: + return + } + } + }() + + return func() { + stopOnce.Do(func() { + close(stop) + }) + select { + case <-done: + case <-time.After(5 * time.Second): + } + } +} + +// recomputeBridgeScoresSafe runs ComputeBridgeScores over the current +// neighbor graph and installs the result. Panics in compute are +// swallowed (defensive) so the goroutine never dies; the previous +// snapshot remains valid. +func recomputeBridgeScoresSafe(s *PacketStore) { + defer func() { _ = recover() }() + graph := s.graph.Load() + if graph == nil { + // No graph yet — install an empty map so readers get a defined + // zero rather than a nil sentinel (handleNodes treats both as + // 0.0, but an explicit empty snapshot avoids "is this ready + // yet?" confusion in operator-facing tooling). + empty := map[string]float64{} + s.bridgeScoreMap.Store(&empty) + return + } + now := time.Now() + edges := bridgeEdgesFromGraph(graph, now) + scores := ComputeBridgeScores(edges) + s.bridgeScoreMap.Store(&scores) +} + +// bridgeEdgesFromGraph snapshots the NeighborGraph into a flat slice +// of BridgeEdge tuples with weight = Score(now) * Confidence(), per +// the convention established by #1235. Edges with unresolved B +// endpoints (no concrete pubkey yet — only a hop prefix) are skipped: +// they contribute no betweenness signal because the second endpoint +// is unknown. +func bridgeEdgesFromGraph(graph *NeighborGraph, now time.Time) []BridgeEdge { + all := graph.AllEdges() + out := make([]BridgeEdge, 0, len(all)) + for _, e := range all { + if e == nil { + continue + } + if e.NodeA == "" || e.NodeB == "" { + // Unresolved (prefix-only) — no defined second endpoint. + continue + } + w := e.Score(now) * e.Confidence() + if w < bridgeMinWeightEpsilon { + continue + } + out = append(out, BridgeEdge{A: e.NodeA, B: e.NodeB, Weight: w}) + } + return out +} + +// GetBridgeScore returns the bridge centrality score for a pubkey in +// [0, 1], or 0 if the recomputer has not run yet or the pubkey is not +// in the graph. Lookup is case-insensitive (the score map keys are +// lowercase, matching byPathHop convention). +func (s *PacketStore) GetBridgeScore(pubkey string) float64 { + if pubkey == "" { + return 0 + } + snap := s.bridgeScoreMap.Load() + if snap == nil { + return 0 + } + m := *snap + if v, ok := m[pubkey]; ok { + return v + } + // Try lowercase form. + lc := pubkey + for i := 0; i < len(lc); i++ { + if lc[i] >= 'A' && lc[i] <= 'Z' { + b := []byte(pubkey) + for j := i; j < len(b); j++ { + if b[j] >= 'A' && b[j] <= 'Z' { + b[j] += 'a' - 'A' + } + } + lc = string(b) + break + } + } + if v, ok := m[lc]; ok { + return v + } + return 0 +} + +// GetBridgeScoreMap returns a defensive copy-by-reference of the +// current bridge score snapshot. Nil-safe: returns an empty map if +// no snapshot has been installed yet. Map is read-only by convention +// — callers MUST NOT mutate it (the snapshot is shared across all +// concurrent readers). +func (s *PacketStore) GetBridgeScoreMap() map[string]float64 { + snap := s.bridgeScoreMap.Load() + if snap == nil { + return map[string]float64{} + } + return *snap +} + +// resetBridgeRecomputerForTest is a test-only helper to allow the +// integration test to re-Start the recomputer in a fresh process +// (which would otherwise be blocked by the package-level +// bridgeRecompStarted flag). Production code must not call this. +func resetBridgeRecomputerForTest() { + bridgeRecompStartedMu.Lock() + bridgeRecompStarted = false + bridgeRecompStartedMu.Unlock() +} diff --git a/cmd/server/bridge_score.go b/cmd/server/bridge_score.go new file mode 100644 index 00000000..970e17ff --- /dev/null +++ b/cmd/server/bridge_score.go @@ -0,0 +1,206 @@ +// Package main: bridge axis of repeater usefulness score (issue #672, +// axis 2 of 4). The "Bridge" signal is the betweenness centrality of a +// node in the (undirected, weighted) neighbor graph: a high value means +// the node lies on many shortest paths between other pairs and is hence +// structurally important — removing it would force traffic around or +// fragment the mesh. +// +// Algorithm: Brandes' algorithm (1) with Dijkstra for weighted +// shortest paths. Complexity O(V · (E + V log V)). For the staging +// graph (~600 nodes, ~2 000 edges) this is ~4.8M ops — trivial, +// completes in milliseconds. We accumulate raw betweenness across all +// sources, halve (an undirected pair is counted from each endpoint +// once), then normalize by the max observed value so the per-node +// score is in [0, 1]. +// +// Edge weight follows the convention established by #1235: the +// affinity score (count + recency decay) is multiplied by the +// observer-diversity confidence — stronger, more corroborated +// neighborships are preferred when there is a choice of paths. +// Geo-rejected edges are already excluded from the input graph at +// build time (#1230) so we don't have to re-filter here. +// +// For Dijkstra we need a DISTANCE (lower = better) not an affinity +// (higher = better), so we convert: cost = 1 / max(epsilon, weight). +// epsilon avoids divide-by-zero on a degenerate zero-weight edge. +// +// (1) Brandes, "A Faster Algorithm for Betweenness Centrality" (2001). +package main + +import ( + "container/heap" + "math" + "strings" +) + +// BridgeEdge is the algorithm-facing edge tuple consumed by +// ComputeBridgeScores. Endpoints A and B are pubkeys (case preserved +// by caller; we lowercase internally for stable keying). Weight is +// the affinity (higher = stronger connection). Edges with zero or +// negative weight are skipped — they would break Dijkstra's +// relaxation invariant. +type BridgeEdge struct { + A, B string + Weight float64 +} + +// bridgeMinWeightEpsilon is the floor applied to weights before we +// invert them into Dijkstra distances. 1e-9 is small enough that any +// real weight (Score in [0,1] times Confidence in [0,1]) dominates, +// but large enough to avoid Inf when weight is exactly zero. +const bridgeMinWeightEpsilon = 1e-9 + +// ComputeBridgeScores returns a map pubkey → bridge score in [0, 1] +// computed via Brandes' weighted betweenness centrality on the +// undirected graph defined by `edges`. Returned map is keyed by the +// lowercase pubkey form (matching the byPathHop / persisted-edge +// convention). Nodes appearing in the graph but with zero betweenness +// are still present in the map with value 0.0. +// +// Self-loops (A == B) and edges with weight < epsilon are silently +// skipped. Duplicate edges between the same pair keep the cheapest +// (= the highest-weight) version — consistent with shortest-path +// semantics. +// +// Pure (no global state, no locks); safe to call concurrently. +// Cost: O(V · (E + V log V)). +func ComputeBridgeScores(edges []BridgeEdge) map[string]float64 { + // 1. Build adjacency list with distance = 1/weight. + adj := make(map[string]map[string]float64) + addOrMerge := func(a, b string, dist float64) { + m, ok := adj[a] + if !ok { + m = make(map[string]float64) + adj[a] = m + } + if existing, has := m[b]; !has || dist < existing { + m[b] = dist + } + } + for _, e := range edges { + a := strings.ToLower(strings.TrimSpace(e.A)) + b := strings.ToLower(strings.TrimSpace(e.B)) + if a == "" || b == "" || a == b { + continue + } + w := e.Weight + if w < bridgeMinWeightEpsilon { + continue + } + dist := 1.0 / w + addOrMerge(a, b, dist) + addOrMerge(b, a, dist) + } + if len(adj) == 0 { + return map[string]float64{} + } + + nodes := make([]string, 0, len(adj)) + for n := range adj { + nodes = append(nodes, n) + } + + bc := make(map[string]float64, len(nodes)) + for _, n := range nodes { + bc[n] = 0 + } + + // 2. Brandes outer loop: one Dijkstra-based single-source shortest + // path computation per source vertex. + for _, s := range nodes { + stack := make([]string, 0, len(nodes)) + pred := make(map[string][]string, len(nodes)) + sigma := make(map[string]float64, len(nodes)) + dist := make(map[string]float64, len(nodes)) + for _, n := range nodes { + sigma[n] = 0 + dist[n] = math.Inf(1) + } + sigma[s] = 1 + dist[s] = 0 + + pq := &bridgePQ{} + heap.Init(pq) + heap.Push(pq, bridgePQItem{node: s, dist: 0}) + + visited := make(map[string]bool, len(nodes)) + for pq.Len() > 0 { + top := heap.Pop(pq).(bridgePQItem) + v := top.node + if visited[v] { + continue + } + visited[v] = true + stack = append(stack, v) + + for w, edgeDist := range adj[v] { + alt := dist[v] + edgeDist + if alt < dist[w]-1e-12 { + dist[w] = alt + sigma[w] = sigma[v] + pred[w] = append(pred[w][:0], v) + heap.Push(pq, bridgePQItem{node: w, dist: alt}) + } else if math.Abs(alt-dist[w]) <= 1e-12 { + sigma[w] += sigma[v] + pred[w] = append(pred[w], v) + } + } + } + + // 3. Back-propagation: walk the stack in reverse order. + delta := make(map[string]float64, len(nodes)) + for i := len(stack) - 1; i >= 0; i-- { + w := stack[i] + for _, v := range pred[w] { + if sigma[w] == 0 { + continue + } + delta[v] += (sigma[v] / sigma[w]) * (1.0 + delta[w]) + } + if w != s { + bc[w] += delta[w] + } + } + } + + // 4. Undirected graphs double-count each (s,t) pair, so halve. + for k := range bc { + bc[k] /= 2.0 + } + + // 5. Normalize by max so scores live in [0, 1]. If max is 0 + // (clique or single edge) we leave everything at zero. + maxBC := 0.0 + for _, v := range bc { + if v > maxBC { + maxBC = v + } + } + if maxBC > 0 { + for k, v := range bc { + bc[k] = v / maxBC + } + } + return bc +} + +// ─── min-heap for Dijkstra ───────────────────────────────────────────────────── + +type bridgePQItem struct { + node string + dist float64 +} + +type bridgePQ []bridgePQItem + +func (h bridgePQ) Len() int { return len(h) } +func (h bridgePQ) Less(i, j int) bool { return h[i].dist < h[j].dist } +func (h bridgePQ) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *bridgePQ) Push(x interface{}) { *h = append(*h, x.(bridgePQItem)) } +func (h *bridgePQ) Pop() interface{} { + old := *h + n := len(old) + it := old[n-1] + *h = old[:n-1] + return it +} diff --git a/cmd/server/bridge_score_test.go b/cmd/server/bridge_score_test.go new file mode 100644 index 00000000..19577132 --- /dev/null +++ b/cmd/server/bridge_score_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "math" + "testing" +) + +// TestComputeBridgeScores_LineGraph asserts the canonical property of +// betweenness centrality on a 4-node line A-B-C-D: the two middle +// nodes B and C have non-zero centrality (every path between an end +// and a far end traverses them) while the two leaves A and D bridge +// no pairs and score zero. This is the RED test for issue #672 bridge +// axis — it fails on master where ComputeBridgeScores is a stub. +func TestComputeBridgeScores_LineGraph(t *testing.T) { + edges := []BridgeEdge{ + {A: "a", B: "b", Weight: 1.0}, + {A: "b", B: "c", Weight: 1.0}, + {A: "c", B: "d", Weight: 1.0}, + } + scores := ComputeBridgeScores(edges) + + for _, leaf := range []string{"a", "d"} { + if v, ok := scores[leaf]; !ok || v != 0 { + t.Errorf("leaf %q: want score 0 (present), got %v ok=%v", leaf, v, ok) + } + } + for _, mid := range []string{"b", "c"} { + v, ok := scores[mid] + if !ok { + t.Errorf("middle %q: missing from result map", mid) + continue + } + if v <= 0 { + t.Errorf("middle %q: want non-zero centrality, got %v", mid, v) + } + } + // Normalization: max must equal 1.0 exactly when any node has + // non-zero centrality. + maxScore := 0.0 + for _, v := range scores { + if v > maxScore { + maxScore = v + } + } + if math.Abs(maxScore-1.0) > 1e-9 { + t.Errorf("max normalized score: want 1.0, got %v", maxScore) + } +} + +// TestComputeBridgeScores_TriangleNoBridge: in a fully connected +// triangle every node has at least one alternate path so betweenness +// is zero everywhere. The map should still contain all three nodes +// (so callers can distinguish "in graph but unimportant" from +// "not in graph") with explicit zero values. +func TestComputeBridgeScores_TriangleNoBridge(t *testing.T) { + edges := []BridgeEdge{ + {A: "x", B: "y", Weight: 1.0}, + {A: "y", B: "z", Weight: 1.0}, + {A: "z", B: "x", Weight: 1.0}, + } + scores := ComputeBridgeScores(edges) + for _, n := range []string{"x", "y", "z"} { + if v, ok := scores[n]; !ok || v != 0 { + t.Errorf("triangle node %q: want 0 present, got %v ok=%v", n, v, ok) + } + } +} + +// TestComputeBridgeScores_Empty: an empty edge list yields an empty +// (non-nil) map. Defensive check so the recomputer can swap in an +// empty result without crashing the lookup path. +func TestComputeBridgeScores_Empty(t *testing.T) { + scores := ComputeBridgeScores(nil) + if scores == nil { + t.Fatal("want non-nil empty map, got nil") + } + if len(scores) != 0 { + t.Errorf("want empty map, got %d entries", len(scores)) + } +} + +// TestComputeBridgeScores_WeightSensitive verifies the algorithm uses +// edge weights as affinity (higher = preferred). In a graph A-B-D and +// A-C-D where the B-route has weight 1.0 and the C-route has weight +// 0.1, shortest path (max-affinity = min 1/w) goes through B, so B +// has positive centrality and C does not. This is the "mutation +// test" — flip the cost formula (e.g., remove the 1/w inversion) and +// this test inverts. +func TestComputeBridgeScores_WeightSensitive(t *testing.T) { + edges := []BridgeEdge{ + {A: "a", B: "b", Weight: 1.0}, + {A: "b", B: "d", Weight: 1.0}, + {A: "a", B: "c", Weight: 0.1}, + {A: "c", B: "d", Weight: 0.1}, + } + scores := ComputeBridgeScores(edges) + if scores["b"] <= scores["c"] { + t.Errorf("stronger-weight intermediary b should outrank c: b=%v c=%v", + scores["b"], scores["c"]) + } +} diff --git a/cmd/server/clock_skew.go b/cmd/server/clock_skew.go index a6a8c649..31289cd7 100644 --- a/cmd/server/clock_skew.go +++ b/cmd/server/clock_skew.go @@ -54,6 +54,20 @@ const ( // drift rarely exceeds 1 hour, while epoch-0 RTCs produce ~1.7B sec. bimodalSkewThresholdSec = 3600.0 + // rtcResetOutlierThresholdSec is the absolute skew above which a + // sample is treated as obvious sensor garbage — an RTC-reset advert + // where the firmware emitted its factory timestamp (typically off by + // months/years). These samples are excluded from the recent-window + // "good/bad" split (bug #1285 — single RTC-reset advert among 30 + // healthy adverts must not flip a node to bimodal_clock) and from the + // per-hash evidence median (a 700-day median is not actionable for + // operators). They remain in the raw sample stream and the RTC-reset + // badge logic which surfaces them separately. 24h is a generous floor: + // real drift is fractions of a sec/advert, real clock-skew tops out + // in the hours range; anything above a day is structurally not a + // drift signal. + rtcResetOutlierThresholdSec = 24 * 3600.0 + // maxPlausibleSkewJumpSec is the largest skew change between // consecutive samples that we treat as physical drift. Anything larger // (e.g. a GPS sync that jumps the clock by minutes/days) is rejected @@ -560,13 +574,25 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew { // no_clock — goodFraction < 0.10 (essentially no real clock) // bimodal_clock — 0.10 <= goodFraction < 0.80 AND badCount > 0 // ok/warn/etc. — goodFraction >= 0.80 (normal, outliers filtered) + // + // RTC-reset outliers (|skew| > 24h — single advert where the firmware + // emitted its factory timestamp) are EXCLUDED from this split (bug + // #1285): they're not "bimodal-bad real-but-large skew" but obvious + // sensor garbage, surfaced separately via the RTC-reset badge. Counting + // them as bimodal-bad produces a false-alarm warning ("3 of last 5 + // adverts had nonsense timestamps") on otherwise-healthy nodes. var goodSamples []float64 + var rtcResetCount int for _, v := range recentVals { - if math.Abs(v) <= bimodalSkewThresholdSec { + absV := math.Abs(v) + switch { + case absV > rtcResetOutlierThresholdSec: + rtcResetCount++ // ignored for good/bad classification + case absV <= bimodalSkewThresholdSec: goodSamples = append(goodSamples, v) } } - recentSampleCount := len(recentVals) + recentSampleCount := len(recentVals) - rtcResetCount recentBadCount := recentSampleCount - len(goodSamples) var goodFraction float64 if recentSampleCount > 0 { @@ -586,8 +612,9 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew { } } else { // Normal path: if there are good samples, use their median - // (filters out rare outliers in ≥80% good case). - if len(goodSamples) > 0 && recentBadCount > 0 { + // (filters out rare outliers in ≥80% good case, and rejects + // RTC-reset outliers regardless of bimodal/bad counts — #1285). + if len(goodSamples) > 0 { recentSkew = median(goodSamples) } severity = classifySkew(math.Abs(recentSkew)) @@ -668,7 +695,7 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew { recentEvidence = append(recentEvidence, HashEvidence{ Hash: eh.hash, Observers: observers, - MedianCorrectedSkewSec: round(median(corrSkews), 1), + MedianCorrectedSkewSec: round(hashEvidenceMedian(corrSkews), 1), Timestamp: eh.ts, }) } @@ -694,9 +721,27 @@ func (s *PacketStore) getNodeClockSkewLocked(pubkey string) *NodeClockSkew { } } -// GetFleetClockSkew returns clock skew data for all nodes that have skew data. -// Must NOT be called with s.mu held. +// GetFleetClockSkew returns clock skew data for all nodes, preferring +// the steady-state recomputer snapshot (issue #1265). Falls back to an +// on-request compute if the recomputer is not yet running. func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew { + s.analyticsRecomputerMu.RLock() + rc := s.recompNodesClockSkew + s.analyticsRecomputerMu.RUnlock() + if rc != nil { + if v := rc.Load(); v != nil { + if r, ok := v.([]*NodeClockSkew); ok { + return r + } + } + } + return s.computeFleetClockSkew() +} + +// computeFleetClockSkew is the underlying compute used by the +// recomputer and the on-request fallback. Must NOT be called with +// s.mu held. +func (s *PacketStore) computeFleetClockSkew() []*NodeClockSkew { s.mu.RLock() defer s.mu.RUnlock() @@ -707,7 +752,7 @@ func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew { nameMap[ni.PublicKey] = ni } - var results []*NodeClockSkew + var results = []*NodeClockSkew{} for pubkey := range s.byNode { cs := s.getNodeClockSkewLocked(pubkey) if cs == nil { @@ -727,8 +772,26 @@ func (s *PacketStore) GetFleetClockSkew() []*NodeClockSkew { return results } -// GetObserverCalibrations returns the current observer clock offsets. +// GetObserverCalibrations returns the current observer clock offsets, +// preferring the steady-state recomputer snapshot (issue #1265). Falls +// back to an on-request compute when the recomputer is not running. func (s *PacketStore) GetObserverCalibrations() []ObserverCalibration { + s.analyticsRecomputerMu.RLock() + rc := s.recompObserversClockSkew + s.analyticsRecomputerMu.RUnlock() + if rc != nil { + if v := rc.Load(); v != nil { + if r, ok := v.([]ObserverCalibration); ok { + return r + } + } + } + return s.computeObserverCalibrations() +} + +// computeObserverCalibrations is the underlying compute used by the +// recomputer and on-request fallback. Must NOT be called with s.mu held. +func (s *PacketStore) computeObserverCalibrations() []ObserverCalibration { s.mu.RLock() defer s.mu.RUnlock() @@ -768,6 +831,23 @@ func median(vals []float64) float64 { return sorted[n/2] } +// hashEvidenceMedian returns the median corrected skew for a single +// transmission hash, filtering out RTC-reset outliers (|skew| > 24h — +// firmware emitting factory timestamp). Issue #1285: a single outlier +// observer was dragging the displayed median to ~-704d on an otherwise +// healthy node. If filtering leaves zero usable samples (every observer +// of this hash saw a reset-shaped advert), return 0 so the UI can render +// "insufficient data" rather than the garbage outlier value. +func hashEvidenceMedian(vals []float64) float64 { + clean := vals[:0:0] + for _, v := range vals { + if math.Abs(v) <= rtcResetOutlierThresholdSec { + clean = append(clean, v) + } + } + return median(clean) +} + func mean(vals []float64) float64 { if len(vals) == 0 { return 0 diff --git a/cmd/server/clock_skew_issue1285_test.go b/cmd/server/clock_skew_issue1285_test.go new file mode 100644 index 00000000..ae6afe2f --- /dev/null +++ b/cmd/server/clock_skew_issue1285_test.go @@ -0,0 +1,129 @@ +package main + +// Regression tests for #1285: +// +// Bug A: per-hash evidence's MedianCorrectedSkewSec includes 700-day RTC-reset +// outliers, dragging the displayed median into "−704d 18h" garbage even +// though every recent sample is small (< 30s). +// +// Bug B: RecentBadSampleCount counts samples outside the *displayed* recent +// window (or counts against raw skew, not corrected) → "3 of last 5 +// adverts had nonsense timestamps" warning fires on healthy nodes. +// +// Both must pass without disturbing the existing bimodal / no-clock logic. + +import ( + "testing" + "time" +) + +// Synthesizes the repro from issue #1285: +// 30 healthy adverts (skew ~−20s) + 1 historical advert with an RTC reset +// (advertTS = 2024-06-13, observed today → −705d skew). +// Returns the populated store. +func seedIssue1285Repro(t *testing.T) *PacketStore { + t.Helper() + ps := NewPacketStore(nil, nil) + pt := 4 // ADVERT + + const pubkey = "RTCRESET" + const skewSec = int64(-20) // node clock is 20s BEHIND wall-clock + + baseObs := int64(1779000000) // ~mid-2026 + rtcResetAdv := int64(1718281640) // 2024-06-13 (from the issue repro) + + var txs []*StoreTx + + // 30 healthy adverts spanning the older end of the recent window. + for i := 0; i < 30; i++ { + obsTS := baseObs + int64(i)*60 + advTS := obsTS + skewSec + tx := &StoreTx{ + Hash: "healthy-" + formatInt64(int64(i)), + PayloadType: &pt, + DecodedJSON: `{"payload":{"timestamp":` + formatInt64(advTS) + `}}`, + Observations: []*StoreObs{ + {ObserverID: "obs1", Timestamp: time.Unix(obsTS, 0).UTC().Format(time.RFC3339)}, + }, + } + txs = append(txs, tx) + } + + // One RTC-reset packet observed MOST RECENTLY (so it sits in the + // per-hash evidence list AND is included in the recent-window count + // on master). Its advertTS is from 2024 → corrected skew ≈ -60M sec. + rtcResetObs := baseObs + int64(30*60) + 60 + rtcTx := &StoreTx{ + Hash: "rtc-reset-0001", + PayloadType: &pt, + DecodedJSON: `{"payload":{"timestamp":` + formatInt64(rtcResetAdv) + `}}`, + Observations: []*StoreObs{ + {ObserverID: "obs1", Timestamp: time.Unix(rtcResetObs, 0).UTC().Format(time.RFC3339)}, + }, + } + txs = append(txs, rtcTx) + + ps.mu.Lock() + ps.byNode[pubkey] = txs + for _, tx := range txs { + ps.byPayloadType[4] = append(ps.byPayloadType[4], tx) + } + ps.clockSkew.computeInterval = 0 + ps.mu.Unlock() + return ps +} + +// Bug A — per-hash evidence median must EXCLUDE the 705-day RTC-reset outlier. +// On master this asserts on the RTC-reset hash's MedianCorrectedSkewSec being +// ≈ −60M sec ("−704d 18h"); after the fix the field is suppressed (0) or +// otherwise marked insufficient, never displayed as the garbage value. +func TestIssue1285_HashEvidence_OutlierExcludedFromMedian(t *testing.T) { + ps := seedIssue1285Repro(t) + r := ps.GetNodeClockSkew("RTCRESET") + if r == nil { + t.Fatal("expected clock skew result") + } + + // The recent-hash evidence list is the source of the "median corrected: + // −704d 18h" string in the UI. After the fix, NO entry in this list + // should report a |median| above the 24h sanity threshold — the fix is + // to drop outlier samples (or flag the hash as insufficient-data) before + // publishing the median. + const maxSaneAbsSec = float64(24 * 3600) + for _, ev := range r.RecentHashEvidence { + if abs(ev.MedianCorrectedSkewSec) > maxSaneAbsSec { + t.Errorf("hash %s exposes outlier-dominated median %.0fs (~%.1fd); "+ + "expected entry to be filtered out or marked insufficient (|median| <= %.0fs)", + ev.Hash, ev.MedianCorrectedSkewSec, + ev.MedianCorrectedSkewSec/86400, maxSaneAbsSec) + } + } +} + +// Bug B — RecentBadSampleCount must be 0 when every sample in the recent +// window is healthy (<30s |corrected skew|). On master this fires because +// "recent" is computed over the wrong set (or against raw skew). +func TestIssue1285_RecentBadCount_NotPollutedByOldOutlier(t *testing.T) { + ps := seedIssue1285Repro(t) + r := ps.GetNodeClockSkew("RTCRESET") + if r == nil { + t.Fatal("expected clock skew result") + } + if r.RecentBadSampleCount != 0 { + t.Errorf("RecentBadSampleCount = %d, want 0 — recent samples are all "+ + "~−20s (healthy); the historical RTC-reset outlier is outside the "+ + "recent window and must not be counted", r.RecentBadSampleCount) + } + if r.Severity == SkewBimodalClock || r.Severity == SkewNoClock { + t.Errorf("severity = %v, want ok/warning — recent samples are all "+ + "healthy (~−20s skew), one historical outlier must not flip the node "+ + "to bimodal/no-clock", r.Severity) + } +} + +func abs(v float64) float64 { + if v < 0 { + return -v + } + return v +} diff --git a/cmd/server/clock_skew_recompute_test.go b/cmd/server/clock_skew_recompute_test.go new file mode 100644 index 00000000..7cabe9fd --- /dev/null +++ b/cmd/server/clock_skew_recompute_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "sort" + "sync" + "testing" + "time" +) + +// Issue #1265: /api/observers/clock-skew (3.3s) and /api/nodes/clock-skew (8.9s) +// must be wired into the steady-state analytics recomputer so reads serve +// from an atomic-pointer snapshot in <100ms p99 under concurrent load. + +func TestClockSkewRecomputersRegistered(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + + stop := store.StartAnalyticsRecomputers(50 * time.Millisecond) + defer stop() + time.Sleep(100 * time.Millisecond) + + store.analyticsRecomputerMu.RLock() + rcObs := store.recompObserversClockSkew + rcNodes := store.recompNodesClockSkew + store.analyticsRecomputerMu.RUnlock() + + if rcObs == nil { + t.Fatalf("recompObserversClockSkew not registered after StartAnalyticsRecomputers (issue #1265 not fixed)") + } + if rcNodes == nil { + t.Fatalf("recompNodesClockSkew not registered after StartAnalyticsRecomputers (issue #1265 not fixed)") + } + if rcObs.Load() == nil { + t.Fatalf("recompObserversClockSkew snapshot is nil after initial compute") + } + if rcNodes.Load() == nil { + t.Fatalf("recompNodesClockSkew snapshot is nil after initial compute") + } +} + +func TestClockSkewHandlersSteadyStateLatency(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + stop := store.StartAnalyticsRecomputers(50 * time.Millisecond) + defer stop() + time.Sleep(100 * time.Millisecond) + + s := &Server{store: store} + + endpoints := []struct { + name string + path string + handler http.HandlerFunc + }{ + {"observers", "/api/observers/clock-skew", s.handleObserverClockSkew}, + {"nodes", "/api/nodes/clock-skew", s.handleFleetClockSkew}, + } + + for _, ep := range endpoints { + ep := ep + t.Run(ep.name, func(t *testing.T) { + const readers = 8 + const perReader = 25 + var ( + mu sync.Mutex + samples []time.Duration + wg sync.WaitGroup + ) + wg.Add(readers) + for r := 0; r < readers; r++ { + go func() { + defer wg.Done() + for i := 0; i < perReader; i++ { + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, ep.path, nil) + t0 := time.Now() + ep.handler(rr, req) + dt := time.Since(t0) + if rr.Code != http.StatusOK { + t.Errorf("%s status = %d, want 200", ep.path, rr.Code) + } + mu.Lock() + samples = append(samples, dt) + mu.Unlock() + } + }() + } + wg.Wait() + + sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] }) + p99 := samples[int(float64(len(samples))*0.99)] + if p99 > 100*time.Millisecond { + t.Fatalf("%s p99 latency = %v over %d reqs, want <100ms (recomputer snapshot)", ep.path, p99, len(samples)) + } + }) + } +} diff --git a/cmd/server/clock_skew_test.go b/cmd/server/clock_skew_test.go index e64de32c..be309e41 100644 --- a/cmd/server/clock_skew_test.go +++ b/cmd/server/clock_skew_test.go @@ -708,9 +708,9 @@ func TestBimodalClock_845(t *testing.T) { baseObs := int64(1700000000) var txs []*StoreTx - // 6 good samples (-5s each), 4 bad samples (-50000000s each) = 60% good + // 6 good samples (-5s each), 4 bad samples (-7200s each) = 60% good // Interleave so the recent window (last 5) captures both good and bad. - skews := []int64{-5, -5, -50000000, -5, -50000000, -5, -50000000, -5, -50000000, -5} + skews := []int64{-5, -5, -7200, -5, -7200, -5, -7200, -5, -7200, -5} for i := 0; i < 10; i++ { obsTS := baseObs + int64(i)*60 advTS := obsTS + skews[i] @@ -794,14 +794,14 @@ func TestMostlyGood_OK_845(t *testing.T) { baseObs := int64(1700000000) var txs []*StoreTx - // 9 good at -5s, 1 bad at -50000000s + // 9 good at -5s, 1 bad at -7200s for i := 0; i < 10; i++ { obsTS := baseObs + int64(i)*60 var skew int64 if i < 9 { skew = -5 } else { - skew = -50000000 + skew = -7200 } advTS := obsTS + skew tx := &StoreTx{ @@ -882,7 +882,7 @@ func TestFiftyFifty_Bimodal_845(t *testing.T) { if i%2 == 0 { skew = -10 } else { - skew = -50000000 + skew = -7200 } tx := &StoreTx{ Hash: fmt.Sprintf("fifty-%04d", i), diff --git a/cmd/server/config.go b/cmd/server/config.go index a43f8203..93ecc7c5 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -546,7 +546,7 @@ func (c *Config) IsObserverBlacklisted(id string) bool { // data slowly." Lower values give fresher data at higher CPU cost. // // RecomputeIntervalSeconds keys (all optional): -// topology, rf, distance, channels, hashCollisions, hashSizes, roles +// topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew type AnalyticsConfig struct { DefaultIntervalSeconds int `json:"defaultIntervalSeconds,omitempty"` RecomputeIntervalSeconds map[string]int `json:"recomputeIntervalSeconds,omitempty"` @@ -583,5 +583,7 @@ func (c *Config) AnalyticsRecomputeIntervals() AnalyticsRecomputeIntervals { out.HashCollisions = get("hashCollisions") out.HashSizes = get("hashSizes") out.Roles = get("roles") + out.ObserversClockSkew = get("observersClockSkew") + out.NodesClockSkew = get("nodesClockSkew") return out } diff --git a/cmd/server/db.go b/cmd/server/db.go index 1be8ad6a..3fd8abaa 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2032,38 +2032,10 @@ func nullInt(ni sql.NullInt64) interface{} { return nil } -// PruneOldPackets deletes transmissions and their observations older than the -// given number of days. Nodes and observers are never touched. -// Returns the number of transmissions deleted. -// Opens a separate read-write connection since the main connection is read-only. -func (db *DB) PruneOldPackets(days int) (int64, error) { - rw, err := cachedRW(db.path) - if err != nil { - return 0, err - } - - cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - tx, err := rw.Begin() - if err != nil { - return 0, err - } - defer tx.Rollback() - - // Delete observations linked to old transmissions first (no CASCADE in SQLite) - _, err = tx.Exec(`DELETE FROM observations WHERE transmission_id IN ( - SELECT id FROM transmissions WHERE first_seen < ? - )`, cutoff) - if err != nil { - return 0, err - } - - res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff) - if err != nil { - return 0, err - } - n, _ := res.RowsAffected() - return n, tx.Commit() -} +// PruneOldPackets, PruneOldMetrics, and RemoveStaleObservers were +// removed in #1283 — they are write operations and now live on the +// ingestor's *Store (cmd/ingestor/maintenance.go and cmd/ingestor/db.go). +// The server is the read path; it must not hold the SQLite write lock. // MetricsSample represents a single row from observer_metrics with computed deltas. type MetricsSample struct { @@ -2381,52 +2353,8 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { return result, nil } -// PruneOldMetrics deletes observer_metrics rows older than retentionDays. -func (db *DB) PruneOldMetrics(retentionDays int) (int64, error) { - rw, err := cachedRW(db.path) - if err != nil { - return 0, err - } - - cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) - res, err := rw.Exec(`DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff) - if err != nil { - return 0, err - } - n, _ := res.RowsAffected() - if n > 0 { - log.Printf("[metrics] Pruned %d observer_metrics rows older than %d days", n, retentionDays) - } - return n, nil -} - -// RemoveStaleObservers marks observers that have not actively sent data in observerDays -// as inactive (soft-delete). This preserves JOIN integrity for observations.observer_idx -// and observer_metrics.observer_id — historical data still references the correct observer. -// An observer must actively send data to stay listed — being seen by another node does not count. -// observerDays <= -1 means never remove (keep forever). -func (db *DB) RemoveStaleObservers(observerDays int) (int64, error) { - if observerDays <= -1 { - return 0, nil // keep forever - } - rw, err := cachedRW(db.path) - if err != nil { - return 0, err - } - - cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339) - res, err := rw.Exec(`UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff) - if err != nil { - return 0, err - } - n, _ := res.RowsAffected() - if n > 0 { - // Clean up orphaned metrics for now-inactive observers - rw.Exec(`DELETE FROM observer_metrics WHERE observer_id IN (SELECT id FROM observers WHERE inactive = 1)`) - log.Printf("[observers] Marked %d observer(s) as inactive (not seen in %d days)", n, observerDays) - } - return n, nil -} +// (PruneOldMetrics / RemoveStaleObservers removed in #1283 — see note +// above the MetricsSample type. Ingestor owns these writes now.) // TouchNodeLastSeen updates last_seen for a node identified by full public key. // Only updates if the new timestamp is newer than the existing value (or NULL). diff --git a/cmd/server/db_vacuum_test.go b/cmd/server/db_vacuum_test.go deleted file mode 100644 index 6dad269f..00000000 --- a/cmd/server/db_vacuum_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package main - -import ( - "database/sql" - "os" - "path/filepath" - "strings" - "testing" - "time" - - _ "modernc.org/sqlite" -) - -// createFreshIngestorDB creates a SQLite DB using the ingestor's applySchema logic -// (simulated here) with auto_vacuum=INCREMENTAL set before tables. -func createFreshDBWithAutoVacuum(t *testing.T, path string) *sql.DB { - t.Helper() - // auto_vacuum must be set via DSN before journal_mode creates the DB file - db, err := sql.Open("sqlite", path+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") - if err != nil { - t.Fatal(err) - } - db.SetMaxOpenConns(1) - - // Create minimal schema - _, err = db.Exec(` - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - `) - if err != nil { - t.Fatal(err) - } - return db -} - -func TestNewDBHasIncrementalAutoVacuum(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.db") - - db := createFreshDBWithAutoVacuum(t, path) - defer db.Close() - - var autoVacuum int - if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - t.Fatal(err) - } - if autoVacuum != 2 { - t.Fatalf("expected auto_vacuum=2 (INCREMENTAL), got %d", autoVacuum) - } -} - -func TestExistingDBHasAutoVacuumNone(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.db") - - // Create DB WITHOUT setting auto_vacuum (simulates old DB) - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") - if err != nil { - t.Fatal(err) - } - db.SetMaxOpenConns(1) - _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") - if err != nil { - t.Fatal(err) - } - - var autoVacuum int - if err := db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - t.Fatal(err) - } - db.Close() - - if autoVacuum != 0 { - t.Fatalf("expected auto_vacuum=0 (NONE) for old DB, got %d", autoVacuum) - } -} - -func TestVacuumOnStartupMigratesDB(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.db") - - // Create DB without auto_vacuum (old DB) - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") - if err != nil { - t.Fatal(err) - } - db.SetMaxOpenConns(1) - _, err = db.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") - if err != nil { - t.Fatal(err) - } - - var before int - db.QueryRow("PRAGMA auto_vacuum").Scan(&before) - if before != 0 { - t.Fatalf("precondition: expected auto_vacuum=0, got %d", before) - } - db.Close() - - // Simulate vacuumOnStartup migration using openRW - rw, err := openRW(path) - if err != nil { - t.Fatal(err) - } - if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { - t.Fatal(err) - } - if _, err := rw.Exec("VACUUM"); err != nil { - t.Fatal(err) - } - rw.Close() - - // Verify migration - db2, err := sql.Open("sqlite", path+"?mode=ro") - if err != nil { - t.Fatal(err) - } - defer db2.Close() - - var after int - if err := db2.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { - t.Fatal(err) - } - if after != 2 { - t.Fatalf("expected auto_vacuum=2 after VACUUM migration, got %d", after) - } -} - -func TestIncrementalVacuumReducesFreelist(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.db") - - db := createFreshDBWithAutoVacuum(t, path) - - // Insert a bunch of data - now := time.Now().UTC().Format(time.RFC3339) - for i := 0; i < 500; i++ { - _, err := db.Exec( - "INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, ?, ?)", - strings.Repeat("AA", 200), // ~400 bytes each - "hash_"+string(rune('A'+i%26))+string(rune('0'+i/26)), - now, - ) - if err != nil { - t.Fatal(err) - } - } - - // Get file size before delete - db.Close() - infoBefore, _ := os.Stat(path) - sizeBefore := infoBefore.Size() - - // Reopen and delete all - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") - if err != nil { - t.Fatal(err) - } - db.SetMaxOpenConns(1) - defer db.Close() - - _, err = db.Exec("DELETE FROM transmissions") - if err != nil { - t.Fatal(err) - } - - // Check freelist before vacuum - var freelistBefore int64 - db.QueryRow("PRAGMA freelist_count").Scan(&freelistBefore) - if freelistBefore == 0 { - t.Fatal("expected non-zero freelist after DELETE") - } - - // Run incremental vacuum - _, err = db.Exec("PRAGMA incremental_vacuum(10000)") - if err != nil { - t.Fatal(err) - } - - // Check freelist after vacuum - var freelistAfter int64 - db.QueryRow("PRAGMA freelist_count").Scan(&freelistAfter) - if freelistAfter >= freelistBefore { - t.Fatalf("expected freelist to shrink: before=%d after=%d", freelistBefore, freelistAfter) - } - - // Checkpoint WAL and check file size shrunk - db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") - db.Close() - infoAfter, _ := os.Stat(path) - sizeAfter := infoAfter.Size() - if sizeAfter >= sizeBefore { - t.Logf("warning: file did not shrink (before=%d after=%d) — may depend on page reuse", sizeBefore, sizeAfter) - } -} - -func TestCheckAutoVacuumLogs(t *testing.T) { - // This test verifies checkAutoVacuum doesn't panic on various configs - dir := t.TempDir() - path := filepath.Join(dir, "test.db") - - // Create a fresh DB with auto_vacuum=INCREMENTAL - dbConn := createFreshDBWithAutoVacuum(t, path) - db := &DB{conn: dbConn, path: path} - cfg := &Config{} - - // Should not panic - checkAutoVacuum(db, cfg, path) - dbConn.Close() - - // Create a DB without auto_vacuum - path2 := filepath.Join(dir, "test2.db") - dbConn2, _ := sql.Open("sqlite", path2+"?_pragma=journal_mode(WAL)") - dbConn2.SetMaxOpenConns(1) - dbConn2.Exec("CREATE TABLE dummy (id INTEGER PRIMARY KEY)") - db2 := &DB{conn: dbConn2, path: path2} - - // Should log warning but not panic - checkAutoVacuum(db2, cfg, path2) - dbConn2.Close() -} - -func TestConfigIncrementalVacuumPages(t *testing.T) { - // Default - cfg := &Config{} - if cfg.IncrementalVacuumPages() != 1024 { - t.Fatalf("expected default 1024, got %d", cfg.IncrementalVacuumPages()) - } - - // Custom - cfg.DB = &DBConfig{IncrementalVacuumPages: 512} - if cfg.IncrementalVacuumPages() != 512 { - t.Fatalf("expected 512, got %d", cfg.IncrementalVacuumPages()) - } - - // Zero should return default - cfg.DB.IncrementalVacuumPages = 0 - if cfg.IncrementalVacuumPages() != 1024 { - t.Fatalf("expected default 1024 for zero, got %d", cfg.IncrementalVacuumPages()) - } -} diff --git a/cmd/server/decoder.go b/cmd/server/decoder.go index 2e004002..f948a9be 100644 --- a/cmd/server/decoder.go +++ b/cmd/server/decoder.go @@ -109,6 +109,32 @@ type Payload struct { SNRValues []float64 `json:"snrValues,omitempty"` RawHex string `json:"raw,omitempty"` Error string `json:"error,omitempty"` + // GRP_TXT/GRP_DATA channel envelope helpers — see + // firmware/src/helpers/BaseChatMesh.cpp:376-391. + ChannelHashHex string `json:"channelHashHex,omitempty"` + DecryptionStatus string `json:"decryptionStatus,omitempty"` + // GRP_DATA (PAYLOAD_TYPE_GRP_DATA=0x06) inner fields, per + // firmware/src/helpers/BaseChatMesh.cpp:382-385. + DataType *int `json:"dataType,omitempty"` + DataLen *int `json:"dataLen,omitempty"` + DecryptedBlob string `json:"decryptedBlob,omitempty"` + // MULTIPART (PAYLOAD_TYPE_MULTIPART=0x0A) inner fields, per + // firmware/src/Mesh.cpp:289 — byte0 = (remaining<<4) | inner_type. + Remaining *int `json:"remaining,omitempty"` + InnerType *int `json:"innerType,omitempty"` + InnerTypeName string `json:"innerTypeName,omitempty"` + InnerAckCrc string `json:"innerAckCrc,omitempty"` + InnerPayload string `json:"innerPayload,omitempty"` + // CONTROL (PAYLOAD_TYPE_CONTROL=0x0B) byte0 flags, per + // firmware/src/Mesh.cpp:69 — high-bit = zero-hop direct subset. + CtrlFlags string `json:"ctrlFlags,omitempty"` + CtrlZeroHop *bool `json:"ctrlZeroHop,omitempty"` + CtrlLength *int `json:"ctrlLength,omitempty"` + // RAW_CUSTOM (PAYLOAD_TYPE_RAW_CUSTOM=0x0F) — application-defined per + // firmware/src/Mesh.cpp:577 (createRawData). We expose the bare envelope + // shape so consumers can triage by length + leading tag byte. + RawLength *int `json:"rawLength,omitempty"` + FirstByteTag string `json:"firstByteTag,omitempty"` } // DecodedPacket is the full decoded result. @@ -313,6 +339,85 @@ func decodeGrpTxt(buf []byte) Payload { } } +// decodeGrpData decodes PAYLOAD_TYPE_GRP_DATA (0x06). Outer envelope is the +// same shape as GRP_TXT (channel_hash(1)+MAC(2)+ciphertext) — see +// firmware/src/helpers/BaseChatMesh.cpp:476,500. This server-side decoder has +// no channel keys, so it surfaces the envelope only. +func decodeGrpData(buf []byte) Payload { + if len(buf) < 3 { + return Payload{Type: "GRP_DATA", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + return Payload{ + Type: "GRP_DATA", + ChannelHash: int(buf[0]), + ChannelHashHex: fmt.Sprintf("%02X", buf[0]), + MAC: hex.EncodeToString(buf[1:3]), + EncryptedData: hex.EncodeToString(buf[3:]), + } +} + +// decodeMultipart decodes PAYLOAD_TYPE_MULTIPART (0x0A) per +// firmware/src/Mesh.cpp:287-310. byte0 = (remaining<<4) | inner_type; +// when inner_type == PAYLOAD_TYPE_ACK the next 4 bytes are an ack_crc. +func decodeMultipart(buf []byte) Payload { + if len(buf) < 1 { + return Payload{Type: "MULTIPART", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + remaining := int(buf[0] >> 4) + innerType := int(buf[0] & 0x0F) + innerName := payloadTypeNames[innerType] + if innerName == "" { + innerName = "UNKNOWN" + } + p := Payload{ + Type: "MULTIPART", + Remaining: &remaining, + InnerType: &innerType, + InnerTypeName: innerName, + } + if innerType == PayloadACK && len(buf) >= 5 { + crc := binary.LittleEndian.Uint32(buf[1:5]) + p.InnerAckCrc = fmt.Sprintf("%08x", crc) + } else if len(buf) > 1 { + p.InnerPayload = hex.EncodeToString(buf[1:]) + } + return p +} + +// decodeControl decodes PAYLOAD_TYPE_CONTROL (0x0B) byte0 flags per +// firmware/src/Mesh.cpp:69 (high-bit set ⇒ zero-hop direct subset). +func decodeControl(buf []byte) Payload { + if len(buf) < 1 { + return Payload{Type: "CONTROL", Error: "too short", RawHex: hex.EncodeToString(buf)} + } + zeroHop := buf[0]&0x80 != 0 + length := len(buf) + return Payload{ + Type: "CONTROL", + CtrlFlags: fmt.Sprintf("%02x", buf[0]), + CtrlZeroHop: &zeroHop, + CtrlLength: &length, + RawHex: hex.EncodeToString(buf), + } +} + +// decodeRawCustom decodes PAYLOAD_TYPE_RAW_CUSTOM (0x0F). The payload bytes +// are application-defined per firmware/src/Mesh.cpp:577 (createRawData), so +// we only surface the bare envelope shape: total length plus the leading +// byte, which apps commonly use as a tag/type discriminator. +func decodeRawCustom(buf []byte) Payload { + length := len(buf) + p := Payload{ + Type: "RAW_CUSTOM", + RawLength: &length, + RawHex: hex.EncodeToString(buf), + } + if length > 0 { + p.FirstByteTag = fmt.Sprintf("%02X", buf[0]) + } + return p +} + func decodeAnonReq(buf []byte) Payload { if len(buf) < 35 { return Payload{Type: "ANON_REQ", Error: "too short", RawHex: hex.EncodeToString(buf)} @@ -372,12 +477,20 @@ func decodePayload(payloadType int, buf []byte, validateSignatures bool) Payload return decodeAdvert(buf, validateSignatures) case PayloadGRP_TXT: return decodeGrpTxt(buf) + case PayloadGRP_DATA: + return decodeGrpData(buf) case PayloadANON_REQ: return decodeAnonReq(buf) case PayloadPATH: return decodePathPayload(buf) case PayloadTRACE: return decodeTrace(buf) + case PayloadMULTIPART: + return decodeMultipart(buf) + case PayloadCONTROL: + return decodeControl(buf) + case PayloadRAW_CUSTOM: + return decodeRawCustom(buf) default: return Payload{Type: "UNKNOWN", RawHex: hex.EncodeToString(buf)} } @@ -617,8 +730,13 @@ func ValidateAdvert(p *Payload) (bool, string) { if p.Flags != nil { role := advertRole(p.Flags) - validRoles := map[string]bool{"repeater": true, "companion": true, "room": true, "sensor": true} - if !validRoles[role] { + // Accept canonical labels plus "none" (ADV_TYPE_NONE=0) and "type-N" + // placeholders for ADV_TYPE 5-15 (FUTURE) — see + // firmware/src/helpers/AdvertDataHelpers.h:7-12. + validRoles := map[string]bool{ + "repeater": true, "companion": true, "room": true, "sensor": true, "none": true, + } + if !validRoles[role] && !strings.HasPrefix(role, "type-") { return false, fmt.Sprintf("unknown role: %s", role) } } @@ -638,17 +756,29 @@ func sanitizeName(s string) string { return b.String() } +// advertRole returns a stable role label for an advert. Follows firmware +// ADV_TYPE_* constants in firmware/src/helpers/AdvertDataHelpers.h:7-12: +// 0 NONE, 1 CHAT, 2 REPEATER, 3 ROOM, 4 SENSOR, 5-15 FUTURE. +// Previously this coerced both 0 (NONE) and 5-15 (FUTURE) to "companion", +// silently relabelling unknown/reserved types — see issue #1279 P1 #3. func advertRole(f *AdvertFlags) string { - if f.Repeater { + if f == nil { + return "companion" + } + switch f.Type { + case 0: + return "none" + case 1: + return "companion" + case 2: return "repeater" - } - if f.Room { + case 3: return "room" - } - if f.Sensor { + case 4: return "sensor" + default: + return fmt.Sprintf("type-%d", f.Type) } - return "companion" } func epochToISO(epoch uint32) string { diff --git a/cmd/server/issue1279_p2_test.go b/cmd/server/issue1279_p2_test.go new file mode 100644 index 00000000..6de136f0 --- /dev/null +++ b/cmd/server/issue1279_p2_test.go @@ -0,0 +1,62 @@ +package main + +// Tests for issue #1279 P2 items: +// - Item 1: payloadTypeNames map must include ALL 13 firmware payload types. +// - Item 5: RAW_CUSTOM (0x0F) decoder must expose rawLength + firstByteTag. +// +// Firmware refs: +// - firmware/src/Packet.h:19-32 (PAYLOAD_TYPE_*) — 0..0xB plus 0xF (RAW_CUSTOM) +// - firmware/src/Mesh.cpp:577 (createRawData) — application-defined payload + +import ( + "strings" + "testing" +) + +func TestPayloadTypeNamesAll13(t *testing.T) { + // Firmware-defined: 0..9, 0x0A, 0x0B, 0x0F — 13 total. + want := map[int]string{ + 0x00: "REQ", 0x01: "RESPONSE", 0x02: "TXT_MSG", 0x03: "ACK", + 0x04: "ADVERT", 0x05: "GRP_TXT", 0x06: "GRP_DATA", 0x07: "ANON_REQ", + 0x08: "PATH", 0x09: "TRACE", 0x0A: "MULTIPART", 0x0B: "CONTROL", + 0x0F: "RAW_CUSTOM", + } + if len(payloadTypeNames) != len(want) { + t.Errorf("payloadTypeNames has %d entries, want %d", len(payloadTypeNames), len(want)) + } + for code, name := range want { + got, ok := payloadTypeNames[code] + if !ok { + t.Errorf("payloadTypeNames missing 0x%02X (%s)", code, name) + continue + } + if got != name { + t.Errorf("payloadTypeNames[0x%02X] = %q, want %q", code, got, name) + } + } +} + +func TestDecodeRawCustomExposesLengthAndTag(t *testing.T) { + // Build a RAW_CUSTOM packet: header byte = (route<<6 | type<<2 | ver), + // type=0x0F. Route FLOOD (1), version 1: (1<<6)|(0x0F<<2)|1 = 0x7D. + // Path byte: 0 hops, hash_size=1 → upper bits 0, lower 0 → 0x00. + // Payload: first byte tag 0xA5, then arbitrary data. + hexStr := "7D00A5DEADBEEF" + pkt, err := DecodePacket(hexStr, false) + if err != nil { + t.Fatalf("decode: %v", err) + } + if pkt.Payload.Type != "RAW_CUSTOM" { + t.Fatalf("payload type = %q, want RAW_CUSTOM", pkt.Payload.Type) + } + if pkt.Payload.RawLength == nil { + t.Fatal("RawLength should be set for RAW_CUSTOM") + } + // payload = 4 bytes (A5 DE AD BE EF) — wait, A5 DE AD BE EF = 5 bytes. + if *pkt.Payload.RawLength != 5 { + t.Errorf("RawLength=%d, want 5", *pkt.Payload.RawLength) + } + if !strings.EqualFold(pkt.Payload.FirstByteTag, "A5") { + t.Errorf("FirstByteTag=%q, want A5", pkt.Payload.FirstByteTag) + } +} diff --git a/cmd/server/issue1279_test.go b/cmd/server/issue1279_test.go new file mode 100644 index 00000000..59412941 --- /dev/null +++ b/cmd/server/issue1279_test.go @@ -0,0 +1,122 @@ +package main + +// Tests for issue #1279 P0+P1 decoder additions (server-side). +// +// Wire-vector citations identical to the ingestor counterpart: +// - GRP_DATA outer: firmware/src/helpers/BaseChatMesh.cpp:500 +// - MULTIPART byte0: firmware/src/Mesh.cpp:289 +// - MULTIPART ACK inner: firmware/src/Mesh.cpp:292-307 +// - CONTROL byte0 flags: firmware/src/Mesh.cpp:69 + Mesh.cpp:609 +// - advertRole label rules: firmware/src/helpers/AdvertDataHelpers.h:7-12 + +import "testing" + +func TestDecodeGrpDataEnvelopeServer(t *testing.T) { + // Server-side decoder has no channel keys: envelope only. + buf := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x11} + p := decodeGrpData(buf) + if p.Type != "GRP_DATA" { + t.Fatalf("type=%q want GRP_DATA", p.Type) + } + if p.ChannelHash != 0xAA { + t.Errorf("channelHash=%d want 170", p.ChannelHash) + } + if p.ChannelHashHex != "AA" { + t.Errorf("channelHashHex=%q want AA", p.ChannelHashHex) + } + if p.MAC != "bbcc" { + t.Errorf("mac=%q want bbcc", p.MAC) + } + if p.EncryptedData != "ddeeff11" { + t.Errorf("encryptedData=%q want ddeeff11", p.EncryptedData) + } +} + +func TestDecodeMultipartAckServer(t *testing.T) { + buf := []byte{0x33, 0xEF, 0xBE, 0xAD, 0xDE} + p := decodeMultipart(buf) + if p.Type != "MULTIPART" { + t.Fatalf("type=%q want MULTIPART", p.Type) + } + if p.Remaining == nil || *p.Remaining != 3 { + t.Errorf("remaining=%v want 3", p.Remaining) + } + if p.InnerType == nil || *p.InnerType != 0x03 { + t.Errorf("innerType=%v want 3", p.InnerType) + } + if p.InnerTypeName != "ACK" { + t.Errorf("innerTypeName=%q want ACK", p.InnerTypeName) + } + if p.InnerAckCrc != "deadbeef" { + t.Errorf("innerAckCrc=%q want deadbeef", p.InnerAckCrc) + } +} + +func TestDecodeMultipartNonAckServer(t *testing.T) { + buf := []byte{0x22, 0x01, 0x02, 0x03} + p := decodeMultipart(buf) + if p.Remaining == nil || *p.Remaining != 2 { + t.Errorf("remaining=%v want 2", p.Remaining) + } + if p.InnerType == nil || *p.InnerType != 0x02 { + t.Errorf("innerType=%v want 2", p.InnerType) + } + if p.InnerTypeName != "TXT_MSG" { + t.Errorf("innerTypeName=%q want TXT_MSG", p.InnerTypeName) + } + if p.InnerPayload != "010203" { + t.Errorf("innerPayload=%q want 010203", p.InnerPayload) + } +} + +func TestAdvertRoleLabelsRawTypeServer(t *testing.T) { + cases := []struct { + typ int + want string + }{ + {0, "none"}, + {1, "companion"}, + {2, "repeater"}, + {3, "room"}, + {4, "sensor"}, + {5, "type-5"}, + {15, "type-15"}, + } + for _, tc := range cases { + got := advertRole(&AdvertFlags{Type: tc.typ, Repeater: tc.typ == 2, Room: tc.typ == 3, Sensor: tc.typ == 4}) + if got != tc.want { + t.Errorf("advertRole(type=%d) = %q, want %q", tc.typ, got, tc.want) + } + } +} + +func TestDecodeControlZeroHopServer(t *testing.T) { + buf := []byte{0x81, 0xAA, 0xBB, 0xCC} + p := decodeControl(buf) + if p.Type != "CONTROL" { + t.Fatalf("type=%q want CONTROL", p.Type) + } + if p.CtrlFlags != "81" { + t.Errorf("ctrlFlags=%q want 81", p.CtrlFlags) + } + if p.CtrlZeroHop == nil || !*p.CtrlZeroHop { + t.Errorf("ctrlZeroHop=%v want true", p.CtrlZeroHop) + } + if p.CtrlLength == nil || *p.CtrlLength != 4 { + t.Errorf("ctrlLength=%v want 4", p.CtrlLength) + } +} + +func TestDecodeControlMultiHopServer(t *testing.T) { + buf := []byte{0x01, 0x42} + p := decodeControl(buf) + if p.CtrlFlags != "01" { + t.Errorf("ctrlFlags=%q want 01", p.CtrlFlags) + } + if p.CtrlZeroHop == nil || *p.CtrlZeroHop { + t.Errorf("ctrlZeroHop=%v want false", p.CtrlZeroHop) + } + if p.CtrlLength == nil || *p.CtrlLength != 2 { + t.Errorf("ctrlLength=%v want 2", p.CtrlLength) + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 7c0a1fb2..cc7a199b 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -167,8 +167,8 @@ func main() { stats.TotalTransmissions, stats.TotalObservations, stats.TotalNodes, stats.TotalObservers) } - // Check auto_vacuum mode and optionally migrate (#919) - checkAutoVacuum(database, cfg, resolvedDB) + // auto_vacuum is checked + migrated by the ingestor (#1283). The + // server is read-only and must not race the writer for the lock. // Ensure indexes the server's SQL fallback path depends on // (mirrors ingestor schema for DBs created by old server-only builds). @@ -377,120 +377,21 @@ func main() { log.Printf("[repeater-enrich-recompute] background recompute enabled (window=%.1fh, interval=%s)", relayWindowHours, cfg.AnalyticsDefaultRecomputeInterval()) - // Auto-prune old packets if retention.packetDays is configured - vacuumPages := cfg.IncrementalVacuumPages() - var stopPrune func() - if cfg.Retention != nil && cfg.Retention.PacketDays > 0 { - days := cfg.Retention.PacketDays - pruneTicker := time.NewTicker(24 * time.Hour) - pruneDone := make(chan struct{}) - stopPrune = func() { - pruneTicker.Stop() - close(pruneDone) - } - go func() { - defer func() { - if r := recover(); r != nil { - log.Printf("[prune] panic recovered: %v", r) - } - }() - time.Sleep(1 * time.Minute) - if n, err := database.PruneOldPackets(days); err != nil { - log.Printf("[prune] error: %v", err) - } else { - log.Printf("[prune] deleted %d transmissions older than %d days", n, days) - if n > 0 { - runIncrementalVacuum(resolvedDB, vacuumPages) - } - } - for { - select { - case <-pruneTicker.C: - if n, err := database.PruneOldPackets(days); err != nil { - log.Printf("[prune] error: %v", err) - } else { - log.Printf("[prune] deleted %d transmissions older than %d days", n, days) - if n > 0 { - runIncrementalVacuum(resolvedDB, vacuumPages) - } - } - case <-pruneDone: - return - } - } - }() - log.Printf("[prune] auto-prune enabled: packets older than %d days will be removed daily", days) - } + // Steady-state bridge-centrality recomputer (issue #672 axis 2). + // Computes betweenness centrality over the in-memory neighbor + // graph and stores the per-pubkey score map atomically. Read by + // handleNodes via a single atomic load. + stopBridgeRecomp := store.StartBridgeScoreRecomputer( + cfg.AnalyticsDefaultRecomputeInterval(), + ) + defer stopBridgeRecomp() + log.Printf("[bridge-recompute] background recompute enabled (interval=%s)", + cfg.AnalyticsDefaultRecomputeInterval()) - // Auto-prune old metrics - var stopMetricsPrune func() - { - metricsDays := cfg.MetricsRetentionDays() - metricsPruneTicker := time.NewTicker(24 * time.Hour) - metricsPruneDone := make(chan struct{}) - stopMetricsPrune = func() { - metricsPruneTicker.Stop() - close(metricsPruneDone) - } - go func() { - defer func() { - if r := recover(); r != nil { - log.Printf("[metrics-prune] panic recovered: %v", r) - } - }() - time.Sleep(2 * time.Minute) // stagger after packet prune - database.PruneOldMetrics(metricsDays) - runIncrementalVacuum(resolvedDB, vacuumPages) - for { - select { - case <-metricsPruneTicker.C: - database.PruneOldMetrics(metricsDays) - runIncrementalVacuum(resolvedDB, vacuumPages) - case <-metricsPruneDone: - return - } - } - }() - log.Printf("[metrics-prune] auto-prune enabled: metrics older than %d days", metricsDays) - } - - // Auto-prune stale observers - var stopObserverPrune func() - { - observerDays := cfg.ObserverDaysOrDefault() - if observerDays <= -1 { - // -1 means keep forever, skip - } else { - observerPruneTicker := time.NewTicker(24 * time.Hour) - observerPruneDone := make(chan struct{}) - stopObserverPrune = func() { - observerPruneTicker.Stop() - close(observerPruneDone) - } - go func() { - defer func() { - if r := recover(); r != nil { - log.Printf("[observer-prune] panic recovered: %v", r) - } - }() - time.Sleep(3 * time.Minute) // stagger after metrics prune - database.RemoveStaleObservers(observerDays) - runIncrementalVacuum(resolvedDB, vacuumPages) - for { - select { - case <-observerPruneTicker.C: - database.RemoveStaleObservers(observerDays) - runIncrementalVacuum(resolvedDB, vacuumPages) - case <-observerPruneDone: - return - } - } - }() - log.Printf("[observer-prune] auto-prune enabled: observers not seen in %d days will be removed", observerDays) - } - } - - // Auto-prune old neighbor edges + // Packet / metrics / observer retention moved to the ingestor in + // #1283 (writes only belong on the writer process). The server no + // longer schedules any of these; the ingestor's tickers handle them. + _ = cfg.IncrementalVacuumPages() // kept reachable for config validation; not used here var stopEdgePrune func() { maxAgeDays := cfg.NeighborMaxAgeDays() @@ -509,13 +410,11 @@ func main() { time.Sleep(4 * time.Minute) // stagger after metrics prune g := store.graph.Load() PruneNeighborEdges(dbPath, g, maxAgeDays) - runIncrementalVacuum(resolvedDB, vacuumPages) for { select { case <-edgePruneTicker.C: g := store.graph.Load() PruneNeighborEdges(dbPath, g, maxAgeDays) - runIncrementalVacuum(resolvedDB, vacuumPages) case <-edgePruneDone: return } @@ -542,16 +441,8 @@ func main() { // 1. Stop accepting new WebSocket/poll data poller.Stop() - // 1b. Stop auto-prune ticker - if stopPrune != nil { - stopPrune() - } - if stopMetricsPrune != nil { - stopMetricsPrune() - } - if stopObserverPrune != nil { - stopObserverPrune() - } + // 1b. Stop auto-prune ticker (server-side packet/metrics/observer + // prunes were removed in #1283; only neighbor-edge prune remains.) if stopEdgePrune != nil { stopEdgePrune() } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 650d50c2..c17099fa 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -43,7 +43,7 @@ func routeDescriptions() map[string]routeMeta { "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true}, - "POST /api/admin/prune": {Summary: "Prune old data", Description: "Deletes packets and nodes older than the configured retention period.", Tag: "admin", Auth: true}, + // "POST /api/admin/prune" removed in #1283 (ingestor owns prune). "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true}, "GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-.db.", Tag: "admin", Auth: true}, diff --git a/cmd/server/paths_anchor_bias_test.go b/cmd/server/paths_anchor_bias_test.go new file mode 100644 index 00000000..e1313f35 --- /dev/null +++ b/cmd/server/paths_anchor_bias_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// TestHandleNodePaths_AnchorBiasInconsistency_Issue1278 reproduces #1278: +// /api/nodes/{pk}/paths returns a tx whose CANONICAL persisted resolved_path +// (the one the packets page reads via fetchResolvedPathForTxBest) does NOT +// contain the queried pubkey. +// +// Two nodes share the 1-byte prefix "c0": +// - nodeNoGPS ("c0dedad…") — no GPS (staging: Kpa Roof Solar) +// - nodeGPS ("c0ffeec…") — has GPS (staging: West SoMa Repeater) +// +// A transmission has TWO observations of the same raw path ["c0"]: +// - obs1 (short path_json): persisted resolved_path = [nodeNoGPSPK] +// (e.g. a region where context picked nodeNoGPS at ingest time) +// - obs2 (longer path_json): persisted resolved_path = [nodeGPSPK] +// (best-obs picks this one — it's the canonical answer the packets page shows) +// +// The membership index has BOTH pubkeys → /api/nodes/{nodeNoGPS}/paths +// passes the candidacy gate (obs1's resolved_path mentions nodeNoGPS), then +// re-resolves with the anchor-biased context and reports the tx — even +// though the CANONICAL ("best") resolved_path picked nodeGPS. +// +// Acceptance: /api/nodes/{nodeNoGPS}/paths MUST exclude this tx because +// the best-obs canonical resolved_path doesn't contain it. Conversely +// /api/nodes/{nodeGPS}/paths MUST include it. +func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { + db := setupTestDB(t) + recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) + recentEpoch := time.Now().Add(-1 * time.Hour).Unix() + + nodeNoGPSPK := "c0dedad4208acb6cbe44b848943fc6d3c5d43cf38a21e48b43826a70862980e4" + nodeGPSPK := "c0ffeec700000000000000000000000000000000000000000000000000000001" + + if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES (?, 'NodeNoGPS', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeNoGPSPK, recent); err != nil { + t.Fatalf("insert nodeNoGPS: %v", err) + } + if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES (?, 'NodeGPS', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeGPSPK, recent); err != nil { + t.Fatalf("insert nodeGPS: %v", err) + } + + if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) + VALUES (100, 'AA', 'hash_collision', ?)`, recent); err != nil { + t.Fatalf("insert tx: %v", err) + } + // obs1: SHORTER path_json (single hop), resolved → nodeNoGPS. + // (Without this row, the membership index wouldn't list nodeNoGPS at all + // and the tx would be cleanly excluded — the bug needs the index hit.) + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) + VALUES (100, NULL, '["c0"]', ?, ?)`, recentEpoch, `["`+nodeNoGPSPK+`"]`); err != nil { + t.Fatalf("insert obs1: %v", err) + } + // obs2: LONGER path_json (two hops, first hop is what packets page shows + // as resolved_path[0]). fetchResolvedPathForTxBest picks this obs as + // canonical because it has the longer path_json. Its resolved_path + // picks nodeGPS for "c0", NOT nodeNoGPS. + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) + VALUES (100, NULL, '["c0","ee"]', ?, ?)`, recentEpoch, `["`+nodeGPSPK+`","ee00000000000000000000000000000000000000000000000000000000000000"]`); err != nil { + t.Fatalf("insert obs2: %v", err) + } + + cfg := &Config{Port: 3000} + hub := NewHub() + srv := NewServer(db, cfg, hub) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + + doGet := func(pk string) NodePathsResponse { + t.Helper() + req := httptest.NewRequest("GET", "/api/nodes/"+pk+"/paths", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("GET /paths for %s: code=%d body=%s", pk, w.Code, w.Body.String()) + } + var resp NodePathsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return resp + } + + // Acceptance #1: nodeGPS (the canonical / best-obs pick) MUST include tx. + respGPS := doGet(nodeGPSPK) + if respGPS.TotalTransmissions != 1 { + t.Errorf("nodeGPS /paths: expected 1 transmission (canonical owner), got %d", respGPS.TotalTransmissions) + } + + // Acceptance #2: nodeNoGPS MUST NOT include the tx — its CANONICAL + // (best-obs) resolved_path picked nodeGPS, so the packets page would + // show nodeGPS. Consistency requires the same here. + respNoGPS := doGet(nodeNoGPSPK) + if respNoGPS.TotalTransmissions != 0 { + var hashes []string + for _, p := range respNoGPS.Paths { + hashes = append(hashes, p.SampleHash) + } + t.Errorf("nodeNoGPS /paths: expected 0 transmissions (canonical/best-obs resolved_path picked NodeGPS, not NodeNoGPS) — anchor-bias inconsistency, got %d; sample hashes: %s", + respNoGPS.TotalTransmissions, strings.Join(hashes, ",")) + } +} diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go new file mode 100644 index 00000000..629009d8 --- /dev/null +++ b/cmd/server/readonly_invariant_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "database/sql" + "fmt" + "reflect" + "testing" + + _ "modernc.org/sqlite" +) + +// TestServerDBHasNoWriteMethods enforces the architectural invariant from +// issue #1283: cmd/server is the read path. All write/maintenance methods +// (PruneOldPackets, PruneOldMetrics, RemoveStaleObservers) MUST live on +// the ingestor's *Store, not on the server's *DB. +// +// Before the fix, these methods existed on cmd/server/*DB and used +// cachedRW(db.path) to acquire a write lock, racing with the ingestor's +// concurrent INSERTs and producing SQLITE_BUSY (the bug in #1283). +// After the fix, this test passes because the methods are gone. +func TestServerDBHasNoWriteMethods(t *testing.T) { + forbidden := []string{ + "PruneOldPackets", + "PruneOldMetrics", + "RemoveStaleObservers", + } + typ := reflect.TypeOf((*DB)(nil)) + for _, name := range forbidden { + if _, ok := typ.MethodByName(name); ok { + t.Errorf("server *DB exposes forbidden write method %q — must be relocated to ingestor (#1283)", name) + } + } +} + +// TestServerDBConnIsReadOnly asserts that the *sql.DB the server opens +// cannot acquire a write lock. The server has always opened mode=ro, but +// before #1283 it routed around that by calling cachedRW(path) to get a +// second RW handle. After the fix, server-side writes are impossible +// because there is no helper to open a writable connection. +func TestServerDBConnIsReadOnly(t *testing.T) { + dir := t.TempDir() + path := dir + "/ro_invariant.db" + + // Bootstrap a minimal DB with the ingestor-style WAL opener so the + // server can attach in read-only mode. + if err := bootstrapMinimalDB(path); err != nil { + t.Fatalf("bootstrap: %v", err) + } + + d, err := OpenDB(path) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + defer d.conn.Close() + + _, err = d.conn.Exec(`INSERT INTO nodes (public_key, name) VALUES ('x','y')`) + if err == nil { + t.Fatalf("expected INSERT via server *DB to fail (read-only invariant)") + } +} + +// bootstrapMinimalDB creates a tiny DB with the columns these tests +// need, opened with WAL so the read-only opener in OpenDB can attach. +// Kept in *_test.go so it does NOT add any write capability to the +// production server binary. +func bootstrapMinimalDB(path string) error { + dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) + rw, err := sql.Open("sqlite", dsn) + if err != nil { + return err + } + defer rw.Close() + if _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT)`); err != nil { + return err + } + return nil +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 3571cc4e..3fb146b5 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -154,7 +154,9 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET") r.HandleFunc("/api/perf/write-sources", s.handlePerfWriteSources).Methods("GET") r.Handle("/api/perf/reset", s.requireAPIKey(http.HandlerFunc(s.handlePerfReset))).Methods("POST") - r.Handle("/api/admin/prune", s.requireAPIKey(http.HandlerFunc(s.handleAdminPrune))).Methods("POST") + // /api/admin/prune removed in #1283 — pruning is owned by the + // ingestor process (scheduled tickers + startup pass). Operators + // who want an ad-hoc prune can restart the ingestor. r.Handle("/api/debug/affinity", s.requireAPIKey(http.HandlerFunc(s.handleDebugAffinity))).Methods("GET") r.Handle("/api/dropped-packets", s.requireAPIKey(http.HandlerFunc(s.handleDroppedPackets))).Methods("GET") r.Handle("/api/backup", s.requireAPIKey(http.HandlerFunc(s.handleBackup))).Methods("GET") @@ -1206,6 +1208,10 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { relayMap = s.store.GetRepeaterRelayInfoMap(relayWindow) usefulMap = s.store.GetRepeaterUsefulnessScoreMap() } + // Bridge axis (#672 axis 2 of 4). Snapshot is an atomic load + // — safe to call regardless of needsRelay, and we want the + // score on repeater rows specifically. + bridgeMap := s.store.GetBridgeScoreMap() for _, node := range nodes { if pk, ok := node["public_key"].(string); ok { EnrichNodeWithHashSize(node, hashInfo[pk]) @@ -1220,6 +1226,7 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { node["relay_count_1h"] = info.RelayCount1h node["relay_count_24h"] = info.RelayCount24h node["usefulness_score"] = lookupUsefulnessScore(usefulMap, pk) + node["bridge_score"] = lookupUsefulnessScore(bridgeMap, pk) } } } @@ -1335,6 +1342,7 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { node["relay_count_1h"] = info.RelayCount1h node["relay_count_24h"] = info.RelayCount24h node["usefulness_score"] = s.store.GetRepeaterUsefulnessScore(pubkey) + node["bridge_score"] = s.store.GetBridgeScore(pubkey) } } @@ -1511,6 +1519,27 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) { } candidates = filtered + // #1278: Read the CANONICAL persisted resolved_path for each surviving + // candidate OUTSIDE s.mu (fetchResolvedPathForTxBest takes lruMu; the + // lock-ordering contract forbids acquiring lruMu under s.mu). + // + // Option A from the issue: the packets page renders each tx via + // fetchResolvedPathForTxBest. For /api/nodes/{pk}/paths to stay + // CONSISTENT with the packets page, BOTH the containsTarget membership + // decision AND the displayed hop names must come from that same + // canonical resolved_path — not a re-resolution biased by passing the + // queried node as hopContext anchor. + // + // Falls back to biased re-resolve only when a tx has no persisted + // resolved_path (older data / async backfill incomplete); in that case + // there's no canonical answer to be consistent with. + canonicalRP := make(map[int][]*string, len(candidates)) + for _, tx := range candidates { + if rp := s.store.fetchResolvedPathForTxBest(tx); rp != nil { + canonicalRP[tx.ID] = rp + } + } + // Re-acquire read lock for the aggregation phase that reads store data. s.store.mu.RLock() @@ -1528,6 +1557,11 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) { // (handleNodePaths aggregates paths terminating at lowerPK). Passing nil // here re-introduced regression #1197 in production. See // resolve_context_callsites_test.go. + // + // NOTE (#1278): this biased resolver is only consulted for the FALLBACK + // path — txs with no persisted resolved_path. Txs with a canonical + // resolved_path use the persisted pubkeys directly (see canonicalRP), + // which keeps results consistent with the packets page. hopContext := []string{lowerPK} resolveHop := func(hop string) *nodeInfo { if cached, ok := hopCache[hop]; ok { @@ -1537,38 +1571,96 @@ func (s *Server) handleNodePaths(w http.ResponseWriter, r *http.Request) { hopCache[hop] = r return r } + // nodeByPK caches pubkey → *nodeInfo lookups when rendering canonical + // resolved_path entries. Cheap O(1) hit against pm.m (the prefix map + // stores the full pubkey as a key for pubkeys >= maxPrefixLen). + nodeByPK := make(map[string]*nodeInfo) + lookupNode := func(pk string) *nodeInfo { + key := strings.ToLower(pk) + if cached, ok := nodeByPK[key]; ok { + return cached + } + // Use plain resolve(); we have the full pubkey, no ambiguity. + n := pm.resolve(key) + if n == nil || !strings.EqualFold(n.PublicKey, key) { + // Full pubkey may not be present in pm (role filter, eviction). + // Fall through with nil; caller renders prefix-only entry. + nodeByPK[key] = nil + return nil + } + nodeByPK[key] = n + return n + } for _, tx := range candidates { hops := txGetParsedPath(tx) resolvedHops := make([]PathHopResp, len(hops)) sigParts := make([]string, len(hops)) - // For candidates not confirmed via full-pubkey index or SQL, verify that at - // least one hop actually resolves to the target. This catches prefix collisions - // (e.g. two nodes sharing a "7a" 1-byte prefix) that slipped through the - // conservative resolved_path fallback. - containsTarget := confirmedByFullKey[tx.ID] || confirmedBySQL[tx.ID] - for i, hop := range hops { - resolved := resolveHop(hop) - entry := PathHopResp{Prefix: hop, Name: hop} - if resolved != nil { - entry.Name = resolved.Name - entry.Pubkey = resolved.PublicKey - if resolved.HasGPS { - entry.Lat = resolved.Lat - entry.Lon = resolved.Lon + containsTarget := false + + if rp, ok := canonicalRP[tx.ID]; ok { + // Option A: render hops + decide membership from the CANONICAL + // persisted resolved_path. resolved_path is parallel to the + // best-obs path_json which may be longer than tx.PathJSON used by + // txGetParsedPath; align by the shorter length. + rpLen := len(rp) + for i, hop := range hops { + entry := PathHopResp{Prefix: hop, Name: hop} + var resolvedPK string + if i < rpLen && rp[i] != nil { + resolvedPK = strings.ToLower(*rp[i]) } - sigParts[i] = resolved.PublicKey - if strings.ToLower(resolved.PublicKey) == lowerPK { - containsTarget = true - } - } else { - sigParts[i] = hop - // Unresolvable hop: keep conservative if prefix could be the target. - if strings.HasPrefix(lowerPK, strings.ToLower(hop)) { - containsTarget = true + if resolvedPK != "" { + if n := lookupNode(resolvedPK); n != nil { + entry.Name = n.Name + entry.Pubkey = n.PublicKey + if n.HasGPS { + entry.Lat = n.Lat + entry.Lon = n.Lon + } + sigParts[i] = n.PublicKey + } else { + entry.Pubkey = resolvedPK + sigParts[i] = resolvedPK + } + if resolvedPK == lowerPK { + containsTarget = true + } + } else { + sigParts[i] = hop } + resolvedHops[i] = entry + } + } else { + // Fallback: no canonical resolved_path persisted (older data / + // async backfill incomplete). Use biased re-resolve and the + // legacy containsTarget heuristics (preserves #1197 behavior + // and the #929 prefix-collision exclusion test). + containsTarget = confirmedByFullKey[tx.ID] || confirmedBySQL[tx.ID] + for i, hop := range hops { + resolved := resolveHop(hop) + entry := PathHopResp{Prefix: hop, Name: hop} + if resolved != nil { + entry.Name = resolved.Name + entry.Pubkey = resolved.PublicKey + if resolved.HasGPS { + entry.Lat = resolved.Lat + entry.Lon = resolved.Lon + } + sigParts[i] = resolved.PublicKey + if strings.ToLower(resolved.PublicKey) == lowerPK { + containsTarget = true + } + } else { + sigParts[i] = hop + // Unresolvable hop: keep conservative if prefix could be the target. + if strings.HasPrefix(lowerPK, strings.ToLower(hop)) { + containsTarget = true + } + } + resolvedHops[i] = entry } - resolvedHops[i] = entry } + if !containsTarget { continue } @@ -2761,45 +2853,8 @@ func parseWindowDuration(window string) (time.Duration, error) { return time.ParseDuration(window) } -func (s *Server) handleAdminPrune(w http.ResponseWriter, r *http.Request) { - days := 0 - if d := r.URL.Query().Get("days"); d != "" { - fmt.Sscanf(d, "%d", &days) - } - if days <= 0 && s.cfg.Retention != nil { - days = s.cfg.Retention.PacketDays - } - if days <= 0 { - writeError(w, 400, "days parameter required (or set retention.packetDays in config)") - return - } - - results := map[string]interface{}{} - - // Prune old packets - n, err := s.db.PruneOldPackets(days) - if err != nil { - writeError(w, 500, err.Error()) - return - } - log.Printf("[prune] deleted %d transmissions older than %d days", n, days) - results["packets_deleted"] = n - results["deleted"] = n // legacy alias - - // Also mark stale observers as inactive if observerDays is configured - observerDays := s.cfg.ObserverDaysOrDefault() - if observerDays > 0 { - obsN, obsErr := s.db.RemoveStaleObservers(observerDays) - if obsErr != nil { - log.Printf("[prune] observer prune error: %v", obsErr) - } else { - results["observers_inactive"] = obsN - } - } - - results["days"] = days - writeJSON(w, results) -} +// handleAdminPrune was removed in #1283. Prune now runs in the ingestor +// process (server is read-only). The function and route are gone. // constantTimeEqual compares two strings in constant time to prevent timing attacks. func constantTimeEqual(a, b string) bool { diff --git a/cmd/server/store.go b/cmd/server/store.go index cd382844..e6292aff 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -18,9 +18,12 @@ import ( ) // payloadTypeNames maps payload_type int → human-readable name (firmware-standard). +// Must stay in sync with the canonical map in cmd/ingestor/decoder.go and +// cmd/server/decoder.go. Source of truth: firmware/src/Packet.h:19-32. var payloadTypeNames = map[int]string{ 0: "REQ", 1: "RESPONSE", 2: "TXT_MSG", 3: "ACK", 4: "ADVERT", - 5: "GRP_TXT", 7: "ANON_REQ", 8: "PATH", 9: "TRACE", 11: "CONTROL", + 5: "GRP_TXT", 6: "GRP_DATA", 7: "ANON_REQ", 8: "PATH", 9: "TRACE", + 10: "MULTIPART", 11: "CONTROL", 15: "RAW_CUSTOM", } // StoreTx is an in-memory transmission with embedded observations. @@ -165,6 +168,8 @@ type PacketStore struct { recompHashCollisions *analyticsRecomputer recompHashSizes *analyticsRecomputer recompRoles *analyticsRecomputer + recompObserversClockSkew *analyticsRecomputer + recompNodesClockSkew *analyticsRecomputer cacheHits int64 cacheMisses int64 // Rate-limited invalidation (fixes #533: caches cleared faster than hit) @@ -234,6 +239,14 @@ type PacketStore struct { repeaterEnrichRecompStop chan struct{} repeaterEnrichRecompDone chan struct{} + // Bridge axis (issue #672 axis 2 of 4): atomic snapshot of pubkey + // → 0..1 betweenness-centrality score over the current neighbor + // graph. Populated by the bridge recomputer (bridge_recomputer.go); + // nil until the first compute lands. Read path is a single atomic + // pointer load — no lock contention with the per-request enrichment + // path in handleNodes (same discipline as #1248). + bridgeScoreMap atomic.Pointer[map[string]float64] + // Precomputed distinct advert pubkey count (refcounted for eviction correctness). // Updated incrementally during Load/Ingest/Evict — avoids JSON parsing in GetPerfStoreStats. advertPubkeys map[string]int // pubkey → number of advert packets referencing it diff --git a/cmd/server/vacuum.go b/cmd/server/vacuum.go deleted file mode 100644 index bea629dc..00000000 --- a/cmd/server/vacuum.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "fmt" - "log" - "time" -) - -// checkAutoVacuum inspects the current auto_vacuum mode and logs a warning -// if it's not INCREMENTAL. Optionally performs a one-time full VACUUM if -// the operator has set db.vacuumOnStartup: true in config (#919). -func checkAutoVacuum(db *DB, cfg *Config, dbPath string) { - var autoVacuum int - if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - log.Printf("[db] warning: could not read auto_vacuum: %v", err) - return - } - - if autoVacuum == 2 { - log.Printf("[db] auto_vacuum=INCREMENTAL") - return - } - - modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} - mode := modes[autoVacuum] - if mode == "" { - mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) - } - - log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ - "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ - "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) - - if cfg.DB != nil && cfg.DB.VacuumOnStartup { - // WARNING: Full VACUUM creates a temporary copy of the entire DB file. - // Requires ~2× the DB file size in free disk space or it will fail. - log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") - start := time.Now() - - rw, err := cachedRW(dbPath) - if err != nil { - log.Printf("[db] VACUUM failed: could not open RW connection: %v", err) - return - } - - if _, err := rw.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil { - log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) - return - } - if _, err := rw.Exec("VACUUM"); err != nil { - log.Printf("[db] VACUUM failed: %v", err) - return - } - - elapsed := time.Since(start) - log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) - - // Re-check - var newMode int - if err := db.conn.QueryRow("PRAGMA auto_vacuum").Scan(&newMode); err == nil { - if newMode == 2 { - log.Printf("[db] auto_vacuum=INCREMENTAL (confirmed after VACUUM)") - } else { - log.Printf("[db] warning: auto_vacuum=%d after VACUUM — expected 2", newMode) - } - } - } -} - -// runIncrementalVacuum runs PRAGMA incremental_vacuum(N) on a read-write -// connection. Safe to call on auto_vacuum=NONE databases (noop). -func runIncrementalVacuum(dbPath string, pages int) { - rw, err := cachedRW(dbPath) - if err != nil { - log.Printf("[vacuum] could not open RW connection: %v", err) - return - } - - if _, err := rw.Exec(fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { - log.Printf("[vacuum] incremental_vacuum error: %v", err) - } -} diff --git a/config.example.json b/config.example.json index 91076d5d..ae15cab0 100644 --- a/config.example.json +++ b/config.example.json @@ -9,12 +9,12 @@ "nodeDays": 7, "observerDays": 14, "packetDays": 30, - "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled)." + "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). packetDays: transmissions older than N days are deleted (0 = disabled). NOTE (#1283): all four retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { "vacuumOnStartup": false, "incrementalVacuumPages": 1024, - "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs (blocks startup for minutes on large DBs; requires 2x DB file size in free disk space). incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919." + "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs. Executed by the INGESTOR at startup, BEFORE the MQTT subscriber starts (#1283), so there is no contention with concurrent writes. Blocks ingestor startup for minutes on large DBs; requires 2x DB file size in free disk space. incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919." }, "_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.", "https": { @@ -259,8 +259,10 @@ "channels": 300, "hashCollisions": 300, "hashSizes": 300, - "roles": 300 + "roles": 300, + "observersClockSkew": 300, + "nodesClockSkew": 300 } }, - "_comment_analytics": "Issue #1240 + #1256. Each analytics endpoint (topology, rf, distance, channels, hashCollisions, hashSizes, roles) is recomputed in the background on the configured interval and served from an atomic-pointer cache. Reads never block on compute. Default 300s (5 min) per endpoint reflects the operator principle: serving slightly stale data quickly beats real-time data slowly. Lower values = fresher data at higher CPU cost. Only the default query (no region/window) is precomputed; region- and window-filtered requests fall back to the legacy on-request compute + 60s TTL cache." + "_comment_analytics": "Issue #1240 + #1256 + #1265. Each analytics endpoint (topology, rf, distance, channels, hashCollisions, hashSizes, roles, observersClockSkew, nodesClockSkew) is recomputed in the background on the configured interval and served from an atomic-pointer cache. Reads never block on compute. Default 300s (5 min) per endpoint reflects the operator principle: serving slightly stale data quickly beats real-time data slowly. Lower values = fresher data at higher CPU cost. Only the default query (no region/window) is precomputed; region- and window-filtered requests fall back to the legacy on-request compute + 60s TTL cache." } diff --git a/public/analytics.js b/public/analytics.js index b6ce1048..c998938a 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2720,9 +2720,15 @@ function destroy() { _stopRolesRefresh(); _analyticsData = {}; _channelData = nu const rq = RegionFilter.regionQueryString(); const regionLabel = rq ? (new URLSearchParams(rq.slice(1)).get('region') || '') : ''; - let nodesResp; + let nodesResp, hashSizesResp; try { - nodesResp = await api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }); + [nodesResp, hashSizesResp] = await Promise.all([ + api('/nodes?limit=10000&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }), + // #1270: fetch CONFIGURED-hash-size counts so the Network Overview + // tells the operational story (matching Hash Stats "By Repeaters"), + // not just a math-only count of unique pubkey slices. + api('/analytics/hash-sizes' + rq, { ttl: CLIENT_TTL.analyticsRF }).catch(() => null), + ]); } catch (e) { el.innerHTML = ``; return; @@ -2766,6 +2772,34 @@ function destroy() { _stopRolesRefresh(); _analyticsData = {}; _channelData = nu }; }); + // #1270: CONFIGURED-hash-size counts (operational truth) from + // /api/analytics/hash-sizes — same source the Hash Stats tab uses. + // distributionByRepeaters keys are stringified ints ("1","2","3"). + // "0" = no adverts observed yet, so the configured size is unknown + // and that node doesn't count for any tier. + const distByRepeaters = (hashSizesResp && hashSizesResp.distributionByRepeaters) || {}; + const configuredCount = { + 1: Number(distByRepeaters['1'] || 0), + 2: Number(distByRepeaters['2'] || 0), + 3: Number(distByRepeaters['3'] || 0), + }; + const totalConfigured = configuredCount[1] + configuredCount[2] + configuredCount[3]; + + // Operational collisions per tier: only consider repeaters CONFIGURED + // for this hash size — same definition the Hash Issues tab uses. + // n.hash_size is enriched on the /nodes payload from GetNodeHashSizeInfo. + const opCollisions = { 1: 0, 2: 0, 3: 0 }; + [1, 2, 3].forEach(b => { + const sub = new Map(); + nodes.forEach(n => { + if (n.hash_size !== b) return; + const p = n.public_key.toUpperCase().slice(0, b * 2); + if (!sub.has(p)) sub.set(p, 0); + sub.set(p, sub.get(p) + 1); + }); + opCollisions[b] = [...sub.values()].filter(c => c > 1).length; + }); + // Recommendation by network size const totalNodes = nodes.length; let rec, recDetail; @@ -2798,29 +2832,45 @@ function destroy() { _stopRolesRefresh(); _analyticsData = {}; _channelData = nu
Total repeaters
${totalNodes.toLocaleString()}
+
+ ${totalConfigured.toLocaleString()} with known hash size +
- ${[1, 2, 3].map(b => ` -
+ ${[1, 2, 3].map(b => { + // #1270: PRIMARY = configured-for-this-size repeater count + // (operational truth, matches Hash Stats "By Repeaters"). + // SECONDARY = math fact (unique pubkey slices). OPERATIONAL + // COLLISIONS = colliding slices among configured-for-this-size repeaters only. + const cfg = configuredCount[b]; + const opC = opCollisions[b]; + const borderVar = opC > 0 ? 'var(--status-red)' : 'var(--border)'; + const opLine = opC === 0 + ? `✅ No operational collisions` + : `⚠️ ${opC} operational collision${opC !== 1 ? 's' : ''}`; + return ` +
${b}-byte prefixes
-
- ${stats[b].usedPrefixes.toLocaleString()} - / ${spaceSizes[b].toLocaleString()} +
+ ${cfg.toLocaleString()} + of ${totalNodes.toLocaleString()} repeaters configured
-
- ${stats[b].collidingPrefixes === 0 - ? '✅ No collisions' - : `⚠️ ${stats[b].collidingPrefixes} prefix${stats[b].collidingPrefixes !== 1 ? 'es' : ''} collide`} +
${opLine}
+
+ Theoretical: ${stats[b].usedPrefixes.toLocaleString()} unique ${b}-byte slice${stats[b].usedPrefixes !== 1 ? 's' : ''} + across all repeater pubkeys (of ${spaceSizes[b].toLocaleString()} possible)
-
`).join('')} +
`; + }).join('')}
Recommendation: ${rec} prefixes — ${recDetail} Hash size is configured per-node in firmware. Changing requires reflashing.
- ℹ️ About these numbers: This tool checks repeater public key prefixes regardless of their configured hash size. Only repeaters are included because they are the nodes that relay packets using hash-based addressing. - The Hash Issues tab shows only operational collisions — nodes that actually use the same hash size and are repeaters. - A collision shown here may not appear in Hash Issues if the nodes use a different hash size. + ℹ️ About these numbers: The primary count is how many repeaters are configured for each hash size (their advertised path hash byte length), matching the + Hash Stats tab. Operational collisions count only repeaters configured for the same hash size whose actual prefix slices collide — same definition the + Hash Issues tab uses. The theoretical line shows the math fact: how many distinct slices appear when every repeater pubkey is truncated to N bytes, regardless of configured hash size.
diff --git a/public/live.css b/public/live.css index ece53334..b4b2477b 100644 --- a/public/live.css +++ b/public/live.css @@ -351,6 +351,28 @@ box-shadow: 0 0 4px currentColor; } +/* #1274: marker-style swatches — mirror the live map circleMarker ring + * convention (bright white ring = repeater, faded ring = other roles). + * Background uses --role-repeater / --text-muted via CSS variables so + * theming remains consistent. */ +.live-ring { + display: inline-block; + width: 10px; + height: 10px; + border-radius: 50%; + margin-right: 6px; + vertical-align: middle; + background: var(--text-muted); +} +.live-ring--repeater { + border: 1.5px solid #fff; + opacity: 0.95; +} +.live-ring--other { + border: 1px solid #fff; + opacity: 0.45; +} + /* ---- Tooltip ---- */ .live-tooltip { background: color-mix(in srgb, var(--surface-1) 95%, transparent) !important; diff --git a/public/live.js b/public/live.js index a29a5271..afe360bc 100644 --- a/public/live.js +++ b/public/live.js @@ -121,7 +121,9 @@ const TYPE_COLORS = window.TYPE_COLORS || { ADVERT: '#22c55e', GRP_TXT: '#3b82f6', TXT_MSG: '#f59e0b', ACK: '#6b7280', - REQUEST: '#a855f7', RESPONSE: '#06b6d4', TRACE: '#ec4899', PATH: '#14b8a6' + REQUEST: '#a855f7', RESPONSE: '#06b6d4', TRACE: '#ec4899', PATH: '#14b8a6', + ANON_REQ: '#f43f5e', GRP_DATA: '#8b5cf6', MULTIPART: '#0d9488', + CONTROL: '#b45309', RAW_CUSTOM: '#c026d3' }; const PAYLOAD_ICONS = { @@ -236,7 +238,32 @@ // Set live-page height from JS — most reliable across all mobile browsers const page = document.querySelector('.live-page'); const appEl = document.getElementById('app'); - const h = window.innerHeight; + // #1267: the CSS rule for .live-page subtracts --bottom-nav-reserve + // (0px desktop, 56px+safe-area at ≤768px) so the fixed .bottom-nav + // (z-index 1200) does not occlude the VCR bar (position:absolute; + // bottom:0; z-index 1000). Mirror that subtraction here — otherwise + // this JS override clobbers the CSS height with raw window.innerHeight + // and the VCR bar slides under the bottom-nav (issue #1267). + // Prefer the bottom-nav's measured rendered height so we also cover + // the 1px top border and any visual chrome the --bottom-nav-reserve + // token doesn't account for; fall back to the token-resolved value. + const reserve = (() => { + const bn = document.querySelector('.bottom-nav'); + if (bn) { + const cs = getComputedStyle(bn); + if (cs.display !== 'none') { + const r = bn.getBoundingClientRect().height; + if (r > 0) return r; + } + } + const probe = document.createElement('div'); + probe.style.cssText = 'position:absolute;visibility:hidden;height:var(--bottom-nav-reserve,0px);pointer-events:none;'; + (document.body || document.documentElement).appendChild(probe); + const px = probe.getBoundingClientRect().height || 0; + probe.remove(); + return px; + })(); + const h = Math.max(0, window.innerHeight - reserve); if (page) page.style.height = h + 'px'; if (appEl) appEl.style.height = h + 'px'; if (map) { @@ -1010,10 +1037,23 @@
  • Message — Group text
  • Direct — Direct message
  • Request — Data request
    • +
  • Response — Data response
  • Trace — Route trace
    • +
  • Path — Path discovery
    • +
  • Anon Req — Anonymous request
    • +
  • Grp Data — Group datagram
    • +
  • Multipart — Multi-fragment payload
    • +
  • Control — Control plane
    • +
  • Raw Custom — Application-defined payload
    • +
  • Ack / Other — Acknowledgment or unknown type

    • NODE ROLES

      +

      MARKER STYLES

      +
        +
      • Bright white ring — repeater
        • +
      • Faded ring — companion / sensor / room
        • +
      diff --git a/public/nodes.js b/public/nodes.js index c33eaf6a..0c8ecf10 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -176,6 +176,13 @@ if (n.hash_size) { html += ` ${n.public_key.slice(0, n.hash_size * 2).toUpperCase()}`; } + // #1279 P2 #4: multibyte capability badge — surfaced from the observable + // multibyte hash_size (firmware Feat1/Feat2 carry the wire capability bits + // per AdvertDataHelpers.h:14-16, but Feat1/Feat2 aren't persisted per-node + // in CoreScope today; hash_size is the observed effective capability). + if (n.hash_size && Number(n.hash_size) >= 2) { + html += ` Multibyte: ${Number(n.hash_size)}-byte`; + } if (n.hash_size_inconsistent) { html += ` ⚠️ variable hash size`; } @@ -547,6 +554,24 @@ const barWidth = Math.max(2, Math.round(s * 100)); return `Usefulness${pct}% ${label}`; })() : ''} + ${(n.role === 'repeater' || n.role === 'room') && n.bridge_score != null ? (() => { + // Bridge axis (issue #672 axis 2 of 4): normalized betweenness + // centrality from the neighbor-edges graph. Distinct from the + // Traffic-based Usefulness score above — bridge measures + // STRUCTURAL importance (how many shortest paths between + // other node pairs go through this one) regardless of + // current traffic. + const b = Number(n.bridge_score) || 0; + const bpct = (b * 100).toFixed(1); + let blabel, bcolor; + if (b >= 0.5) { blabel = 'Critical bridge'; bcolor = 'var(--status-green, #2ecc71)'; } + else if (b >= 0.2) { blabel = 'Important'; bcolor = 'var(--status-green, #2ecc71)'; } + else if (b >= 0.05) { blabel = 'Some role'; bcolor = 'var(--status-yellow, #f1c40f)'; } + else if (b > 0) { blabel = 'Marginal'; bcolor = 'var(--status-orange, #e67e22)'; } + else { blabel = 'No bridge role'; bcolor = 'var(--text-muted)'; } + const bbarWidth = Math.max(2, Math.round(b * 100)); + return `Bridge${bpct}% ${blabel}`; + })() : ''} First Seen${renderNodeTimestampHtml(n.first_seen)} Total Packets${stats.totalTransmissions || stats.totalPackets || n.advert_count || 0}${stats.totalObservations && stats.totalObservations !== (stats.totalTransmissions || stats.totalPackets) ? ' (seen ' + stats.totalObservations + '×)' : ''} Packets Today${stats.packetsToday || 0} @@ -918,7 +943,25 @@ var hashBlocks = evidence.map(function(ev) { var shortHash = (ev.hash || '').substring(0, 8) + '…'; var obsCount = ev.observers ? ev.observers.length : 0; - var header = '
      Hash ' + shortHash + ' · ' + obsCount + ' observer' + (obsCount !== 1 ? 's' : '') + ' · median corrected: ' + formatSkew(ev.medianCorrectedSkewSec) + '
      '; + // #1285: per-hash median is server-side filtered to exclude RTC-reset + // outliers (|corrected skew| > 24h). Compute the same on the client so + // we can label hashes whose observers ALL saw a reset-shaped advert as + // "insufficient data — N outliers excluded" instead of rendering 0 or + // a misleading post-filter value. + var OUTLIER_SEC = 86400; + var outlierObs = 0; + (ev.observers || []).forEach(function(o) { + if (Math.abs(o.correctedSkewSec || 0) > OUTLIER_SEC) outlierObs++; + }); + var medianLabel; + if (outlierObs > 0 && outlierObs === obsCount) { + medianLabel = 'insufficient data (' + outlierObs + ' RTC-reset outlier' + (outlierObs !== 1 ? 's' : '') + ' excluded)'; + } else if (outlierObs > 0) { + medianLabel = formatSkew(ev.medianCorrectedSkewSec) + ' (' + outlierObs + ' RTC-reset outlier' + (outlierObs !== 1 ? 's' : '') + ' excluded)'; + } else { + medianLabel = formatSkew(ev.medianCorrectedSkewSec); + } + var header = '
      Hash ' + shortHash + ' · ' + obsCount + ' observer' + (obsCount !== 1 ? 's' : '') + ' · median corrected: ' + medianLabel + '
      '; var lines = (ev.observers || []).map(function(o) { var name = o.observerName || o.observerID; return '
      ' + diff --git a/public/packet-filter.js b/public/packet-filter.js index 42154b3a..5cfa71d9 100644 --- a/public/packet-filter.js +++ b/public/packet-filter.js @@ -283,6 +283,16 @@ if (field === 'payload_hex') { return packet.raw_hex ? packet.raw_hex.slice(4) : ''; } + // TransportCodes fields surfaced from decoded_json.transportCodes + // (firmware/src/Packet.h:46, parsed at cmd/server/decoder.go:492-498). + if (field === 'code1' || field === 'code2') { + try { + var dc = typeof packet.decoded_json === 'string' ? JSON.parse(packet.decoded_json) : packet.decoded_json; + if (!dc || !dc.transportCodes) return null; + var v = field === 'code1' ? dc.transportCodes.code1 : dc.transportCodes.code2; + return v ? String(v).toUpperCase() : null; + } catch (e) { return null; } + } // Decoded payload fields (dot notation) if (field.startsWith('payload.')) { try { @@ -449,6 +459,8 @@ { name: 'payload.flags.repeater', desc: 'Decoded payload: advert flag (repeater role)' }, { name: 'payload.flags.room', desc: 'Decoded payload: advert flag (room server)' }, { name: 'payload.flags.hasLocation', desc: 'Decoded payload: advert has location' }, + { name: 'code1', desc: 'Transport route Code1 (hex, e.g. AABB) — present on TRANSPORT_FLOOD/DIRECT' }, + { name: 'code2', desc: 'Transport route Code2 (hex, e.g. CCDD) — present on TRANSPORT_FLOOD/DIRECT' }, ]; var OPERATORS = [ diff --git a/public/packets.js b/public/packets.js index b222e290..7ffd1435 100644 --- a/public/packets.js +++ b/public/packets.js @@ -2841,15 +2841,18 @@ } } - // Location: from ADVERT lat/lon, or from known node via pubkey/sender name - let locationHtml = '—'; + // Location: from ADVERT lat/lon, or from known node via pubkey/sender name. + // Issue #1281: only render the row when we actually have transmitter GPS. + // Non-ADVERT packets don't carry GPS in the unencrypted payload, so the row + // would otherwise render as "—" and waste a slot on ~90% of packet types. + let locationHtml = ''; let locationNodeKey = null; if (decoded.lat != null && decoded.lon != null && !(decoded.lat === 0 && decoded.lon === 0)) { locationNodeKey = decoded.pubKey || decoded.srcPubKey || ''; const nodeName = decoded.name || ''; locationHtml = `${decoded.lat.toFixed(5)}, ${decoded.lon.toFixed(5)}`; if (nodeName) locationHtml = `${escapeHtml(nodeName)} — ${locationHtml}`; - if (locationNodeKey) locationHtml += ` 📍map`; + if (locationNodeKey) locationHtml += ` 📍map`; } else { // Try to resolve sender node location from nodes list const senderKey = decoded.pubKey || decoded.srcPubKey; @@ -2861,7 +2864,7 @@ locationNodeKey = nodeData.node.public_key; locationHtml = `${nodeData.node.lat.toFixed(5)}, ${nodeData.node.lon.toFixed(5)}`; if (nodeData.node.name) locationHtml = `${escapeHtml(nodeData.node.name)} — ${locationHtml}`; - locationHtml += ` 📍map`; + locationHtml += ` 📍map`; } else if (senderName && !senderKey) { // Search by name const searchData = await api(`/nodes/search?q=${encodeURIComponent(senderName)}`, { ttl: 30000 }).catch(() => null); @@ -2870,7 +2873,7 @@ locationNodeKey = match.public_key; locationHtml = `${match.lat.toFixed(5)}, ${match.lon.toFixed(5)}`; locationHtml = `${escapeHtml(match.name)} — ${locationHtml}`; - locationHtml += ` 📍map`; + locationHtml += ` 📍map`; } } } catch {} @@ -2889,6 +2892,27 @@ ? `(observation ${observations.indexOf(currentObs) + 1} of ${observations.length})` : ''; + // #1279 P2 #3 — Transport codes detail row (firmware/src/Packet.h:46, + // parsed at cmd/server/decoder.go:492-498). Present on TRANSPORT_FLOOD/ + // TRANSPORT_DIRECT routes only. + var tcCode1 = '—', tcCode2 = '—', tcShow = false; + if (decoded.transportCodes) { + tcShow = true; + if (decoded.transportCodes.code1) tcCode1 = String(decoded.transportCodes.code1).toUpperCase(); + if (decoded.transportCodes.code2) tcCode2 = String(decoded.transportCodes.code2).toUpperCase(); + } + var transportCodesRow = tcShow + ? `
      Transport Codes
      Code1: ${escapeHtml(tcCode1)} · Code2: ${escapeHtml(tcCode2)}
      ` + : ''; + + // #1279 P2 #5 — RAW_CUSTOM detail row (firmware/src/Mesh.cpp:577). + var rawCustomRow = ''; + if (pkt.payload_type === 15 && decoded.type === 'RAW_CUSTOM') { + var rl = decoded.rawLength != null ? decoded.rawLength + ' byte' + (decoded.rawLength === 1 ? '' : 's') : '—'; + var ft = decoded.firstByteTag ? String(decoded.firstByteTag).toUpperCase() : '—'; + rawCustomRow = `
      Raw Custom
      Length: ${escapeHtml(rl)} · First byte tag: ${escapeHtml(ft)}
      `; + } + panel.innerHTML = ` ${anomalyBanner}
      ${hasRawHex ? `Packet Byte Breakdown (${size} bytes)` : typeName + ' Packet'}
      @@ -2896,7 +2920,7 @@ ${messageHtml}
      Observer
      ${obsNameOnly(effectivePkt.observer_id)}${obsIataBadge(effectivePkt)}
      -
      Location
      ${locationHtml}
      + ${locationHtml ? `
      Location
      ${locationHtml}
      ` : ''}
      SNR / RSSI
      ${snr != null ? snr + ' dB' : '—'} / ${rssi != null ? rssi + ' dBm' : '—'}
      Route Type
      ${routeTypeName(pkt.route_type)}
      Payload Type
      ${typeName}
      @@ -2904,6 +2928,8 @@
      Timestamp
      ${renderTimestampCell(effectivePkt.timestamp)}
      Propagation
      ${propagationHtml}
      Path
      ${displayHopCount > 0 ? `${displayHopCount} hop${displayHopCount !== 1 ? 's' : ''} ` + renderPath(pathHops, effectivePkt.observer_id) : '— (direct)'}
      + ${transportCodesRow} + ${rawCustomRow} ${effectivePkt.direction ? `
      Direction
      ${escapeHtml(effectivePkt.direction)}
      ` : ''}
      diff --git a/public/style.css b/public/style.css index fb302327..8bc92257 100644 --- a/public/style.css +++ b/public/style.css @@ -917,6 +917,10 @@ body.scroll-locked { overflow: hidden; } } .detail-meta dt { color: var(--text-muted); font-size: 11px; text-transform: uppercase; letter-spacing: .3px; } .detail-meta dd { font-weight: 500; margin-bottom: 4px; } +/* #1281: 📍map link inside detail-meta — UA-default blue is unreadable on + * dark backgrounds. Force theme-aware color via --accent. */ +.loc-map-link { color: var(--accent); font-size: 0.85em; text-decoration: none; } +.loc-map-link:hover { text-decoration: underline; } .observation-current { background: var(--accent-bg, rgba(0,122,255,0.1)); font-weight: 600; } .detail-obs-row:hover { background: var(--hover-bg, rgba(255,255,255,0.05)); } .detail-obs-table th { font-size: 0.8em; text-transform: uppercase; color: var(--text-muted); } @@ -2170,12 +2174,12 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); } } .observer-selector { display: flex; gap: 4px; margin-bottom: 12px; flex-wrap: wrap; } .node-qr { text-align: center; margin-top: 8px; } -.node-qr svg { max-width: 100px; border-radius: 4px; } +.node-qr svg { max-width: 100px; height: auto; border-radius: 4px; } [data-theme="dark"] .node-qr svg rect[fill="#ffffff"] { fill: var(--card-bg); } [data-theme="dark"] .node-qr svg rect[fill="#000000"] { fill: var(--text); } .node-map-qr-wrap { position: relative; } .node-map-qr-overlay { position: absolute; bottom: 8px; right: 8px; z-index: 400; background: rgba(255,255,255,0.5); border-radius: 4px; padding: 4px; line-height: 0; margin: 0; text-align: center; } -.node-map-qr-overlay svg { max-width: 56px !important; display: block; margin: 0; } +.node-map-qr-overlay svg { max-width: 56px !important; height: auto; display: block; margin: 0; } [data-theme="dark"] .node-map-qr-overlay { background: rgba(255,255,255,0.4); } /* Replay on Live Map button in packet detail */ @@ -2462,7 +2466,8 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); } .node-top-row { display: flex; gap: 16px; margin-bottom: 12px; } .node-top-row .node-map-wrap { flex: 3; min-height: 200px; border-radius: 8px; overflow: hidden; } .node-top-row .node-map-wrap .node-detail-map { height: 100%; } -.node-top-row .node-qr-wrap { flex: 1; min-width: 120px; max-width: 160px; display: flex; flex-direction: column; align-items: center; justify-content: center; background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; padding: 12px; } +.node-top-row .node-qr-wrap { flex: 1; align-self: flex-start; min-width: 120px; max-width: 160px; display: flex; flex-direction: column; align-items: center; justify-content: center; background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; padding: 8px; } +.node-top-row .node-qr-wrap .node-qr { margin-top: 0; } .node-qr-wrap--full { max-width: 240px; margin: 0 auto; } .node-stats-table { width: 100%; border-collapse: collapse; font-size: 13px; background: var(--card-bg); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; margin-bottom: 12px; } .node-stats-table td { padding: 6px 12px; border-bottom: 1px solid var(--border); } @@ -2500,7 +2505,7 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); } line-height: 0; margin: 0; } - .node-top-row .node-qr-wrap .node-qr svg { max-width: 72px; } + .node-top-row .node-qr-wrap .node-qr svg { max-width: 72px; height: auto; } /* Hide the redundant pubkey caption inside the overlay QR — it's already shown in the card above and would push the overlay too large. */ .node-top-row .node-qr-wrap .mono { display: none; } diff --git a/test-e2e-1267-mobile-vcr.js b/test-e2e-1267-mobile-vcr.js new file mode 100644 index 00000000..f4029440 --- /dev/null +++ b/test-e2e-1267-mobile-vcr.js @@ -0,0 +1,112 @@ +/** + * #1267 — VCR bar invisible on mobile /live (iOS Safari ~375x812). + * + * RED first: at a 375x812 mobile viewport, the `.vcr-bar` must be visible + * between the map and bottom-nav. Asserts measured height > 0, display !== + * 'none', visibility !== 'hidden', and that its top edge is within the + * viewport (not pushed below the visible area). + * + * Usage: BASE_URL=http://localhost:13581 node test-e2e-1267-mobile-vcr.js + */ +const { chromium, devices } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || '/usr/bin/chromium', + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] + }); + const context = await browser.newContext({ + viewport: { width: 375, height: 812 }, + deviceScaleFactor: 2, + isMobile: true, + hasTouch: true, + userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1', + }); + const page = await context.newPage(); + page.setDefaultTimeout(15000); + + let failed = false; + function fail(msg) { failed = true; console.log(' \u274c', msg); } + function pass(msg) { console.log(' \u2705', msg); } + + console.log(`\n#1267 mobile VCR-bar visibility against ${BASE} (375x812)\n`); + + await page.goto(`${BASE}/#/live`, { waitUntil: 'domcontentloaded' }); + // Allow live page mount + initial render + VCR bar ResizeObserver publish. + await page.waitForSelector('.live-page', { timeout: 15000 }); + await page.waitForSelector('#vcrBar', { timeout: 15000 }); + // Wait until markers populate — #1267 only manifests after marker render. + // We poll for >=1 leaflet-marker-icon to appear in the DOM, then settle. + await page.waitForFunction( + () => document.querySelectorAll('#liveMap .leaflet-marker-icon, #liveMap .leaflet-marker-pane > *').length > 0, + null, + { timeout: 15000 } + ).catch(() => {}); + await page.waitForTimeout(4000); + + const info = await page.evaluate(() => { + const bar = document.getElementById('vcrBar'); + if (!bar) return { missing: true }; + const r = bar.getBoundingClientRect(); + const cs = getComputedStyle(bar); + const page = document.querySelector('.live-page'); + const pageR = page ? page.getBoundingClientRect() : null; + const bn = document.querySelector('.bottom-nav'); + const bnR = bn ? bn.getBoundingClientRect() : null; + const bnCs = bn ? getComputedStyle(bn) : null; + const rootCs = getComputedStyle(document.documentElement); + return { + rect: { top: r.top, bottom: r.bottom, height: r.height, width: r.width, left: r.left, right: r.right }, + display: cs.display, + visibility: cs.visibility, + opacity: cs.opacity, + position: cs.position, + zIndex: cs.zIndex, + viewportH: window.innerHeight, + viewportW: window.innerWidth, + pageRect: pageR ? { top: pageR.top, bottom: pageR.bottom, height: pageR.height } : null, + bottomNav: bnR ? { top: bnR.top, bottom: bnR.bottom, height: bnR.height, display: bnCs.display } : null, + bottomNavReserve: rootCs.getPropertyValue('--bottom-nav-reserve'), + vcrBarHeightVar: getComputedStyle(page || document.body).getPropertyValue('--vcr-bar-height'), + }; + }); + + console.log('VCR bar measurement:', JSON.stringify(info, null, 2)); + + if (info.missing) { + fail('#vcrBar element not in DOM'); + } else { + if (info.display === 'none') fail(`display:none on .vcr-bar`); + else pass(`display is ${info.display}`); + + if (info.visibility === 'hidden') fail(`visibility:hidden on .vcr-bar`); + else pass(`visibility is ${info.visibility}`); + + if (info.rect.height <= 0) fail(`getBoundingClientRect().height = ${info.rect.height} (expected > 0)`); + else pass(`height ${info.rect.height}px > 0`); + + if (info.rect.top >= info.viewportH) fail(`bar top ${info.rect.top} >= viewport height ${info.viewportH} (pushed off-screen)`); + else pass(`bar top ${info.rect.top} < viewport height ${info.viewportH}`); + + // #1267 root assertion: the VCR bar must not be occluded by the + // fixed bottom-nav (z=1200 > vcr-bar z=1000). The bar's bottom edge + // must sit AT OR ABOVE the bottom-nav's top edge. + if (info.bottomNav && info.bottomNav.display !== 'none') { + if (info.rect.bottom > info.bottomNav.top + 0.5) { + fail(`VCR bar bottom ${info.rect.bottom} overlaps bottom-nav top ${info.bottomNav.top} (hidden behind bottom-nav — #1267)`); + } else { + pass(`VCR bar bottom ${info.rect.bottom} ≤ bottom-nav top ${info.bottomNav.top}`); + } + } + + // Sanity: the bar should occupy width across most of the viewport. + if (info.rect.width < info.viewportW * 0.5) fail(`bar width ${info.rect.width} < 50% of viewport ${info.viewportW}`); + else pass(`bar width ${info.rect.width} spans >50% viewport`); + } + + await browser.close(); + process.exit(failed ? 1 : 0); +})().catch(err => { console.error(err); process.exit(2); }); diff --git a/test-e2e-playwright.js b/test-e2e-playwright.js index 0003659d..31643c86 100644 --- a/test-e2e-playwright.js +++ b/test-e2e-playwright.js @@ -3102,6 +3102,55 @@ async function run() { await page.setViewportSize({ width: 1280, height: 800 }); }); + // Issue #1270: The Prefix Tool's Network Overview must report + // CONFIGURED-hash-size repeater counts (the operational truth) as the + // primary number for each tier, agreeing with the Hash Stats tab's + // "By Repeaters" panel. The math-only "unique slices of every pubkey" + // number is allowed only as a secondary/educational stat. Before the + // fix, Prefix Tool showed "168 / 65536" for 2-byte while Hash Stats + // showed only 20 repeaters actually configured for 2-byte hashing. + await test('#1270 Prefix Tool primary counts match Hash Stats By Repeaters', async () => { + // 1) Read configured-by-hash-size counts straight from the API + // (this is what the Hash Stats tab renders). + const distByRepeaters = await page.evaluate(async () => { + const r = await fetch('/api/analytics/hash-sizes'); + const j = await r.json(); + return j.distributionByRepeaters || {}; + }); + const expected = { + 1: Number(distByRepeaters['1'] || 0), + 2: Number(distByRepeaters['2'] || 0), + 3: Number(distByRepeaters['3'] || 0), + }; + + // 2) Visit the Prefix Tool tab, open Network Overview, scrape + // the primary stat values for each tier. + await page.goto(`${BASE}/#/analytics?tab=prefix-tool`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#ptOverview', { timeout: 15000 }); + // Open the overview accordion if collapsed. + await page.evaluate(() => { + const body = document.getElementById('ptOverviewBody'); + if (body && getComputedStyle(body).display === 'none') { + document.getElementById('ptOverviewToggle').click(); + } + }); + await page.waitForSelector('[data-pt-configured="1"]', { timeout: 5000 }); + const got = await page.evaluate(() => { + const read = (b) => { + const el = document.querySelector(`[data-pt-configured="${b}"]`); + return el ? Number(el.getAttribute('data-value')) : null; + }; + return { 1: read(1), 2: read(2), 3: read(3) }; + }); + + assert(got[1] === expected[1], + `#1270 1-byte: prefix-tool shows ${got[1]}, hash-sizes API shows ${expected[1]}`); + assert(got[2] === expected[2], + `#1270 2-byte: prefix-tool shows ${got[2]}, hash-sizes API shows ${expected[2]}`); + assert(got[3] === expected[3], + `#1270 3-byte: prefix-tool shows ${got[3]}, hash-sizes API shows ${expected[3]}`); + }); + await browser.close(); // Summary diff --git a/test-issue-1273-qr-overlay-height-e2e.js b/test-issue-1273-qr-overlay-height-e2e.js new file mode 100644 index 00000000..d8e237f4 --- /dev/null +++ b/test-issue-1273-qr-overlay-height-e2e.js @@ -0,0 +1,125 @@ +/** + * #1273 — QR overlay container is 2-3× taller than the QR canvas on the + * full node detail page (`#/nodes/`). + * + * On mobile (<=640px) and desktop, `.node-top-row .node-qr-wrap` (the QR + * overlay box) must NOT have meaningful empty translucent space below the + * QR canvas. The wrap's bounding-rect height must be ≤ the inner QR + * canvas/svg height + 32px (covers padding + caption + minor rounding). + * + * Asserted on: + * - 375x800 (mobile — overlay style applies) + * - 1280x800 (desktop guard — existing flex layout must not regress) + * + * RED on master (mobile): the absolute-positioned overlay inherits the + * column flex layout with `justify-content: center` and the caption hidden + * but still allocates space because the wrap doesn't have a content-fit + * height. + * + * Usage: BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' \u2713 ' + name); } + catch (e) { failed++; console.error(' \u2717 ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +async function pickPubkey(page) { + await page.goto(BASE + '/', { waitUntil: 'domcontentloaded' }); + return await page.evaluate(async () => { + const r = await fetch('/api/nodes?limit=20'); + const d = await r.json(); + return (d.nodes || [])[0] && (d.nodes || [])[0].public_key; + }); +} + +async function measureOverlay(page, pubkey) { + await page.goto(BASE + '/#/nodes/' + encodeURIComponent(pubkey), + { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('.node-top-row .node-qr-wrap', { timeout: 10000 }); + // Wait until the QR svg is actually painted inside the wrap. + await page.waitForFunction(() => { + const wrap = document.querySelector('.node-top-row .node-qr-wrap'); + return wrap && wrap.querySelector('.node-qr svg'); + }, { timeout: 10000 }); + await page.waitForTimeout(150); // allow layout to settle + return await page.evaluate(() => { + const wrap = document.querySelector('.node-top-row .node-qr-wrap'); + const svg = wrap.querySelector('.node-qr svg'); + const wr = wrap.getBoundingClientRect(); + const sr = svg.getBoundingClientRect(); + const cap = wrap.querySelector('.mono'); + const capH = cap && getComputedStyle(cap).display !== 'none' + ? Math.round(cap.getBoundingClientRect().height) : 0; + // QR is always square — the visible/intended QR height is the SMALLER + // of svg width vs svg height. Any "extra" svg height beyond that is + // wasted intrinsic-sizing space that bloats the wrap. + const qrVisibleH = Math.min(Math.round(sr.width), Math.round(sr.height)); + return { + wrapH: Math.round(wr.height), + wrapW: Math.round(wr.width), + svgH: Math.round(sr.height), + svgW: Math.round(sr.width), + qrVisibleH, + capH, + position: getComputedStyle(wrap).position, + top: Math.round(wr.top), + right: Math.round(window.innerWidth - wr.right), + }; + }); +} + +(async () => { + const launchOpts = { headless: true, args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] }; + if (process.env.CHROMIUM_PATH) launchOpts.executablePath = process.env.CHROMIUM_PATH; + const browser = await chromium.launch(launchOpts); + + console.log(`\n=== #1273 QR overlay height E2E against ${BASE} ===`); + + // ── pick a real pubkey from the API ── + const ctxBoot = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const probe = await ctxBoot.newPage(); + const pubkey = await pickPubkey(probe); + await ctxBoot.close(); + assert(pubkey, 'No pubkey returned from /api/nodes'); + console.log(' → probe pubkey: ' + pubkey.slice(0, 12) + '…'); + + // ── Mobile 375x800 ── + const m = await browser.newContext({ viewport: { width: 375, height: 800 } }); + const mp = await m.newPage(); + await step('mobile 375x800: .node-qr-wrap height \u2264 visible QR + 32px', async () => { + const d = await measureOverlay(mp, pubkey); + console.log(' mobile measurements: ' + JSON.stringify(d)); + assert(d.qrVisibleH > 0, 'QR svg has zero visible square (not rendered)'); + assert(d.position === 'absolute', + 'mobile overlay must remain position:absolute, got ' + d.position); + assert(d.wrapH <= d.qrVisibleH + d.capH + 32, + `wrap height ${d.wrapH}px must be \u2264 visible QR ${d.qrVisibleH}px + caption ${d.capH}px + 32px (delta ${d.wrapH - d.qrVisibleH - d.capH}px)`); + }); + await m.close(); + + // ── Desktop 1280x800 (regression guard) ── + const dctx = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const dp = await dctx.newPage(); + await step('desktop 1280x800: .node-qr-wrap height \u2264 visible QR + caption + 32px', async () => { + const d = await measureOverlay(dp, pubkey); + console.log(' desktop measurements: ' + JSON.stringify(d)); + assert(d.qrVisibleH > 0, 'QR svg has zero visible square (not rendered)'); + assert(d.wrapH <= d.qrVisibleH + d.capH + 32, + `wrap height ${d.wrapH}px must be \u2264 visible QR ${d.qrVisibleH}px + caption ${d.capH}px + 32px (delta ${d.wrapH - d.qrVisibleH - d.capH}px)`); + }); + await dctx.close(); + + await browser.close(); + + console.log('\n' + passed + '/' + (passed + failed) + ' tests passed' + + (failed ? ', ' + failed + ' failed' : '')); + process.exit(failed > 0 ? 1 : 0); +} +)().catch(err => { console.error('Fatal:', err); process.exit(1); }); diff --git a/test-issue-1274-legend-coverage-e2e.js b/test-issue-1274-legend-coverage-e2e.js new file mode 100644 index 00000000..415c67f5 --- /dev/null +++ b/test-issue-1274-legend-coverage-e2e.js @@ -0,0 +1,88 @@ +/** + * E2E for #1274 — Live legend must document gray packets (ACK + unknown + * payload types), the RESPONSE and PATH colors, AND the white-ring + * repeater convention. See issue #1274 acceptance criteria. + * + * Run: BASE_URL=http://localhost:13581 node test-issue-1274-legend-coverage-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +async function gotoLive(page) { + await page.goto(BASE + '/#/live', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#liveLegend', { timeout: 8000, state: 'attached' }); + await page.waitForTimeout(400); + // Ensure the legend is expanded (it persists collapsed state via localStorage). + const hidden = await page.evaluate(() => { + const el = document.getElementById('liveLegend'); + return !!el && el.classList.contains('hidden'); + }); + if (hidden) { + await page.evaluate(() => { + try { localStorage.removeItem('live-legend-hidden'); } catch (_) {} + const el = document.getElementById('liveLegend'); + if (el) el.classList.remove('hidden'); + }); + } +} + +async function legendText(page) { + return page.evaluate(() => { + const el = document.getElementById('liveLegend'); + return el ? (el.textContent || '').toLowerCase() : ''; + }); +} + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + + console.log(`\n=== #1274 legend documents ACK/RESPONSE/PATH + white-ring — E2E against ${BASE} ===`); + + for (const vp of [ + { w: 1440, h: 900, tag: '[1440x900 desktop]' }, + { w: 375, h: 800, tag: '[375x800 mobile]' }, + ]) { + const ctx = await browser.newContext({ viewport: { width: vp.w, height: vp.h } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(8000); + page.on('pageerror', (e) => console.error('[pageerror]', e.message)); + await step(vp.tag + ' navigate to /live', async () => { await gotoLive(page); }); + + await step(vp.tag + ' legend lists ACK row', async () => { + const t = await legendText(page); + assert(/\back\b/.test(t), 'legend missing ACK row; text=' + t.slice(0, 400)); + }); + await step(vp.tag + ' legend lists RESPONSE row', async () => { + const t = await legendText(page); + assert(/response/.test(t), 'legend missing RESPONSE row'); + }); + await step(vp.tag + ' legend lists PATH row', async () => { + const t = await legendText(page); + assert(/path/.test(t), 'legend missing PATH row'); + }); + await step(vp.tag + ' legend documents white-ring / repeater convention', async () => { + const t = await legendText(page); + assert(/repeater/.test(t) && /ring/.test(t), + 'legend missing repeater white-ring documentation; text=' + t.slice(0, 600)); + }); + + await ctx.close(); + } + + await browser.close(); + console.log(`\n=== ${passed} passed, ${failed} failed ===`); + process.exit(failed === 0 ? 0 : 1); +})().catch((e) => { console.error(e); process.exit(1); }); diff --git a/test-issue-1279-legend-p2-e2e.js b/test-issue-1279-legend-p2-e2e.js new file mode 100644 index 00000000..a37fb86b --- /dev/null +++ b/test-issue-1279-legend-p2-e2e.js @@ -0,0 +1,75 @@ +/** + * E2E for #1279 P2 #2 — Live legend covers all remaining named payload types. + * After PR #1276 the legend already lists Advert/Message/Direct/Request/ + * Response/Trace/Path/Ack; this PR adds Anon Req, Grp Data, Multipart, + * Control and Raw Custom. + * + * Run: BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +async function gotoLive(page) { + await page.goto(BASE + '/#/live', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#liveLegend', { timeout: 8000, state: 'attached' }); + await page.waitForTimeout(400); + const hidden = await page.evaluate(() => { + const el = document.getElementById('liveLegend'); + return !!el && el.classList.contains('hidden'); + }); + if (hidden) { + await page.evaluate(() => { + try { localStorage.removeItem('live-legend-hidden'); } catch (_) {} + const el = document.getElementById('liveLegend'); + if (el) el.classList.remove('hidden'); + }); + } +} + +async function legendText(page) { + return page.evaluate(() => { + const el = document.getElementById('liveLegend'); + return el ? (el.textContent || '').toLowerCase() : ''; + }); +} + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + + console.log(`\n=== #1279 P2 legend covers all 13 payload types — E2E against ${BASE} ===`); + + const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(8000); + page.on('pageerror', (e) => console.error('[pageerror]', e.message)); + + await step('navigate to /live', async () => { await gotoLive(page); }); + + // Types already covered by #1274/#1276: Advert/Message/Direct/Request/ + // Response/Trace/Path/Ack. New ones added by #1279 P2: + const newRows = ['anon req', 'grp data', 'multipart', 'control', 'raw custom']; + for (const label of newRows) { + await step(`legend lists "${label}"`, async () => { + const t = await legendText(page); + assert(t.indexOf(label) !== -1, 'legend missing row: ' + label); + }); + } + + await ctx.close(); + await browser.close(); + console.log(`\n=== ${passed} passed, ${failed} failed ===`); + process.exit(failed === 0 ? 0 : 1); +})().catch((e) => { console.error(e); process.exit(1); }); diff --git a/test-issue-1279-p2-code-filter.js b/test-issue-1279-p2-code-filter.js new file mode 100644 index 00000000..cb17c32d --- /dev/null +++ b/test-issue-1279-p2-code-filter.js @@ -0,0 +1,50 @@ +/* Unit tests for issue #1279 P2 #3 — code1/code2 filter grammar */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); + +const code = fs.readFileSync('public/packet-filter.js', 'utf8'); +const ctx = { window: {}, console }; +vm.createContext(ctx); +vm.runInContext(code, ctx); +const PF = ctx.window.PacketFilter; + +let pass = 0, fail = 0; +function test(name, fn) { + try { fn(); pass++; console.log(' ✓ ' + name); } + catch (e) { console.log(' ✗ ' + name + ' — ' + e.message); fail++; } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +const withCodes = { + route_type: 0, payload_type: 2, raw_hex: '00aabbccdd', + decoded_json: JSON.stringify({ transportCodes: { code1: 'AABB', code2: 'CCDD' } }) +}; +const noCodes = { + route_type: 1, payload_type: 2, raw_hex: '04', + decoded_json: JSON.stringify({}) +}; + +test('code1 == AABB matches transport packet', () => { + assert(PF.compile('code1 == AABB').filter(withCodes)); +}); +test('code1 == aabb (case-insensitive)', () => { + assert(PF.compile('code1 == aabb').filter(withCodes)); +}); +test('code1 == AABB does NOT match packet without transportCodes', () => { + assert(!PF.compile('code1 == AABB').filter(noCodes)); +}); +test('code2 == CCDD matches', () => { + assert(PF.compile('code2 == CCDD').filter(withCodes)); +}); +test('code1 != AABB false when code1 is AABB', () => { + assert(!PF.compile('code1 != AABB').filter(withCodes)); +}); +test('FIELDS metadata includes code1 and code2', () => { + var names = PF.FIELDS.map(function(f){return f.name;}); + assert(names.indexOf('code1') !== -1, 'code1 missing from FIELDS'); + assert(names.indexOf('code2') !== -1, 'code2 missing from FIELDS'); +}); + +console.log('\n' + pass + ' passed, ' + fail + ' failed'); +process.exit(fail === 0 ? 0 : 1); diff --git a/test-issue-1281-location-row-e2e.js b/test-issue-1281-location-row-e2e.js new file mode 100644 index 00000000..006d13aa --- /dev/null +++ b/test-issue-1281-location-row-e2e.js @@ -0,0 +1,155 @@ +/** + * #1281 — Packet detail Location row + 📍map link contrast. + * + * Bug: + * A)
      Location
      —
      renders unconditionally on every packet, + * wasting a row on ~90% of packet types (only ADVERT carries unencrypted + * transmitter GPS). + * B) The trailing `📍map` link has no class/color → inherits UA-default + * blue → unreadable in dark mode. + * + * Asserts: + * 1. Some non-ADVERT packet detail does NOT contain
      Location
      . + * 2. Some ADVERT packet detail DOES contain
      Location
      with coords. + * 3. The 📍map link uses class="loc-map-link" with color = --accent + * (NOT the default UA blue rgb(0,0,238)). + * + * Usage: BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +function normRgb(s) { + const m = s && s.match(/rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)/); + if (!m) return null; + return `rgb(${m[1]}, ${m[2]}, ${m[3]})`; +} + +async function gotoPackets(page) { + await page.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => { + localStorage.removeItem('meshcore-groupbyhash'); + localStorage.setItem('meshcore-time-window', '525600'); + }); + await page.reload({ waitUntil: 'load' }); + await page.waitForSelector('table tbody tr[data-hash]', { timeout: 15000 }); +} + +// Click rows until detail pane's Payload Type matches `wantType` (e.g. "Advert" +// or any non-"Advert"). Returns true on hit, false if exhausted. +async function findPacketDetailByType(page, predicate, maxRows = 40) { + await page.waitForTimeout(400); + const rows = await page.$$('table tbody tr[data-hash][data-action]'); + for (let i = 0; i < Math.min(rows.length, maxRows); i++) { + await rows[i].click({ timeout: 3000 }).catch(() => null); + await page.waitForTimeout(350); + const meta = await page.evaluate(() => { + const dts = document.querySelectorAll('dl.detail-meta dt'); + let typeName = null; + let hasLocation = false; + let locationText = ''; + for (const dt of dts) { + const label = dt.textContent.trim(); + const dd = dt.nextElementSibling; + if (label === 'Payload Type') typeName = dd ? dd.textContent.trim() : null; + if (label === 'Location') { hasLocation = true; locationText = dd ? dd.textContent.trim() : ''; } + } + return { typeName, hasLocation, locationText }; + }); + if (predicate(meta)) return meta; + } + return null; +} + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || '/usr/bin/chromium', + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(15000); + page.on('pageerror', (e) => console.error('[pageerror]', e.message)); + + console.log(`\n=== #1281 Location row + map link contrast E2E against ${BASE} ===`); + + await step('Non-ADVERT packet detail does NOT render
      Location
      ', async () => { + await gotoPackets(page); + // Filter to a non-ADVERT type to make the search efficient. + const meta = await findPacketDetailByType( + page, + (m) => m.typeName && m.typeName !== 'Advert', + 40 + ); + assert(meta, 'No non-ADVERT packet found in first 40 rows'); + assert(!meta.hasLocation, + `Expected NO
      Location
      for type "${meta.typeName}", but found one with text "${meta.locationText}"`); + }); + + await step('ADVERT packet detail STILL renders
      Location
      with GPS coords', async () => { + await gotoPackets(page); + // Filter UI to ADVERTs to guarantee we find one. + const fInput = await page.$('#packetFilterInput'); + if (fInput) { + await fInput.fill('type == ADVERT'); + await page.keyboard.press('Enter'); + await page.waitForTimeout(600); + } + const meta = await findPacketDetailByType( + page, + (m) => m.typeName === 'Advert' && m.hasLocation, + 40 + ); + assert(meta, 'No ADVERT packet with Location row found in first 40 ADVERT rows'); + assert(/-?\d+\.\d+\s*,\s*-?\d+\.\d+/.test(meta.locationText), + `ADVERT Location should contain GPS coords, got: "${meta.locationText}"`); + }); + + await step('📍map link uses class="loc-map-link" with color = var(--accent)', async () => { + // Reuse the ADVERT detail pane left open from the previous step. + const result = await page.evaluate(() => { + const link = document.querySelector('dl.detail-meta a.loc-map-link'); + if (!link) return { missing: true }; + const cs = getComputedStyle(link); + const accentRaw = getComputedStyle(document.documentElement).getPropertyValue('--accent').trim(); + // Resolve --accent value to its computed rgb() via a probe element. + const probe = document.createElement('span'); + probe.style.color = `var(--accent)`; + document.body.appendChild(probe); + const accentRgb = getComputedStyle(probe).color; + probe.remove(); + return { + linkColor: cs.color, + accentRgb, + accentRaw, + href: link.getAttribute('href'), + text: link.textContent.trim(), + }; + }); + assert(!result.missing, + '       not found in detail pane — implementation must apply the class'); + const link = normRgb(result.linkColor); + const accent = normRgb(result.accentRgb); + console.log(` link.color=${result.linkColor} --accent→${result.accentRgb} (raw "${result.accentRaw}")`); + assert(link === accent, + `📍map link color ${result.linkColor} must equal --accent (${result.accentRgb}); ` + + `default UA blue (rgb(0, 0, 238)) is not acceptable`); + assert(link !== 'rgb(0, 0, 238)', + 'Link color is UA-default blue — class is missing or CSS rule does not match'); + }); + + await browser.close(); + + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed === 0 ? 0 : 1); +})().catch((e) => { console.error(e); process.exit(1); });