mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-10-12 11:17:19 +00:00
Closes #2177. ## What Node notifications can mail alerts computed outside CoreScope. With `userManagement.notifications.externalAlerts` configured, users can choose a new event, `node.external`, which is off by default and only offered when configured: ```json "notifications": { "enabled": true, "externalAlerts": { "url": "http://coverage-job/alerts.json", "maxAgeHours": 48, "label": "Coverage" } } ``` The feed: ```json { "generatedAt": "2026-10-11T01:35:12Z", "alerts": [ { "pubkey": "<64 hex>", "key": "SE", "text": "never heard to the south-east", "url": "https://example.org/coverage" } ] } ``` Mail lines: `<name>: Coverage: never heard to the south-east, <time>` when an alert appears, `<name>: Coverage resolved (SE), <time>` when it is gone from a fresh feed. ## How - `cmd/server/notify_external.go`: feed types, `parseExternalFeed` (RFC 3339 `generatedAt`, age check, per-alert validation), `fetchExternalFeed` (200 only, 1 MB cap, 10 s timeout). - `notify_eval.go`: `evalExternal`. Subjects are `<pubkey>/<key>`; a `<pubkey>/*` row is the per-node baseline, so the first fresh feed after watching stores the current alerts without a mail (like `foreign.new`), and later alerts mail at once. `External == nil` (not configured, nobody chose it, or stale) leaves every `node.external` state as it is. - `notify_service.go`: the feed is fetched only when configured and at least one user chose the event. Each new reason the feed is unusable is logged once, and so is the recovery. Mail text and link: the alert's `url`, otherwise the node page. - `notify_handlers.go`: `availableEvents` and the new `externalLabel` include the event only when configured; PUT answers 400 for `node.external` otherwise. - `internal/users/notify.go`: event constant, `OptionalNodeNotifyEvents` (canonical order: node, optional, admin), `RemoveWatch` also deletes the node's `<pubkey>/...` rows. - Account page label, OpenAPI descriptions, `config.example.json` comment, `docs/user-guide/accounts.md` section. Decisions on the three open questions in #2177: the generic name `node.external`; the resolved mail names the key instead of repeating the text, so there is **no `users.db` schema change** (a bump would also make older binaries refuse the file); one feed. ## Invariants - Read/write separation (#1283): the server only reads the feed over HTTP. State rows go to `users.db` through `internal/users`, like the other events. - A dead, garbled or old feed never mails "resolved": `TestEvaluateExternalStaleFeedChangesNothing` and `TestNotifierMailsExternalAlertsAndSkipsAStaleFeed` (HTTP 502 and a 72 h old feed in a row, then a fresh empty feed resolves). ## Performance The evaluator adds one pass over the stored states (already read every tick) plus, per user with the event on and per watched node, the node's current alerts. Bounded by `maxWatchesPerUser` (50) times the alerts per node. The feed is fetched once per tick (default 5 minutes) only when someone chose the event, capped at 1 MB. No packet-store lock is taken for it. Nothing changes for instances without `externalAlerts`. ## Tests - `cmd/server`: `go test` green locally (Windows; `TestSaveGeoFilterPreservesFileMode` skipped as it needs Unix mode bits). New: `notify_external_test.go` (feed parsing and limits, HTTP fetch 200/404/oversized, config resolution, evaluator baseline/new/resolved/returning/stale/not chosen/notifications off, notifier end to end with a fake feed, account routes). - `internal/users`: green, 2 new tests (optional event and canonical order, `RemoveWatch` deletes external subjects only for that node). - Frontend: `test-all.sh` green, 1 new test in `test-notifications-ui.js` (label from `externalLabel`, escaped). - `scripts/check-xss-sinks.sh --file public/notifications.js`: clean. ## Not done - No E2E test: the account page change is one label, covered by the unit test. - Not run against a live instance yet. The first feed in use is the coverage job of on8ar.eu (`alerts.json`, 29 alerts at the moment). Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
356 lines
13 KiB
Go
356 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/meshcore-analyzer/users"
|
|
)
|
|
|
|
// Node notification evaluator (docs/specs/2026-10-07-node-notifications-design.md).
|
|
// evaluateNotifications is a pure function: everything it reads is in
|
|
// notifyInput, the time included, so it is tested without a clock or a
|
|
// database. The notifier (notify_service.go) gathers the input and acts on
|
|
// the result.
|
|
|
|
// notifyNode is one analyzer node row the evaluator needs.
|
|
type notifyNode struct {
|
|
Pubkey string // lowercase hex
|
|
Name string
|
|
Role string // as stored; compared lowercase
|
|
LastSeen string // nodes.last_seen, "" when NULL
|
|
BatteryMv *int // latest advert telemetry; nil when the node reports none
|
|
Foreign bool // nodes.foreign_advert = 1
|
|
}
|
|
|
|
// notifyObserver is one active (not soft-deleted) observer.
|
|
type notifyObserver struct {
|
|
ID string
|
|
Name string
|
|
LastSeen string // observers.last_seen, "" when NULL
|
|
}
|
|
|
|
// notifyInput is everything one evaluation reads.
|
|
type notifyInput struct {
|
|
Now time.Time
|
|
Health HealthThresholds // resolved, as Config.GetHealthThresholds returns it
|
|
LowMv int // Config.LowBatteryMv
|
|
Accounts map[int64]users.User
|
|
Prefs []users.NotifyPrefs
|
|
Watches []users.NotifyWatch
|
|
States []users.NotifyState
|
|
Nodes map[string]notifyNode // watched nodes, plus every foreign node when an admin chose foreign.new
|
|
Heard map[string]time.Time // newest packet involving the node in the packet store (node page "Last Heard")
|
|
Relayed map[string]time.Time // last relay hop, used for repeaters and rooms only (#1598)
|
|
Observers []notifyObserver // nil unless an admin chose observer.offline
|
|
// IngestStale: the newest packet in the store is old, so a silent node
|
|
// and a silent feed look the same. node.offline and observer.offline
|
|
// are then not compared (states unchanged, no changes).
|
|
IngestStale bool
|
|
// IngestResumedAt: when the last stale period ended (zero: none since
|
|
// startup). For a node or observer whose stored state is good, evidence
|
|
// older than this counts as heard at this time, so nothing goes offline
|
|
// until a full silent window has passed after the feed came back.
|
|
IngestResumedAt time.Time
|
|
// External is the fresh externalAlerts feed; nil when it is not
|
|
// configured, nobody chose node.external, or the feed is stale. nil
|
|
// leaves every node.external state unchanged.
|
|
External *externalFeed
|
|
}
|
|
|
|
// notifyChange is one transition for one user.
|
|
type notifyChange struct {
|
|
Event string
|
|
Subject string // node pubkey or observer id
|
|
Name string // display name at evaluation time
|
|
To string // users.NotifyGood, users.NotifyBad or users.NotifyTold
|
|
BatteryMv *int // node.battery only
|
|
Text string // node.external only: the alert text ("" when resolved)
|
|
URL string // node.external only: the alert's link, "" for the node page
|
|
At time.Time
|
|
}
|
|
|
|
// notifyResult: Changes per user, ordered by event then name; States are
|
|
// the rows to write (new or changed), disabled users included; Drop are the
|
|
// rows to delete (admin-event rows of users who are no longer admin).
|
|
type notifyResult struct {
|
|
Changes map[int64][]notifyChange
|
|
States []users.NotifyState
|
|
Drop []users.NotifyKey
|
|
}
|
|
|
|
const (
|
|
// batteryHysteresisMv: a low battery is good again only at lowMv + 100.
|
|
batteryHysteresisMv = 100
|
|
// foreignBaselineSubject marks that an admin's foreign.new baseline
|
|
// exists: the first evaluation after opting in stores every current
|
|
// foreign node as told without mailing; afterwards a foreign node
|
|
// without a row is new.
|
|
foreignBaselineSubject = "*"
|
|
)
|
|
|
|
var notifyEventRank = map[string]int{
|
|
users.NotifyNodeOffline: 0, users.NotifyNodeBattery: 1, users.NotifyNodeExternal: 2,
|
|
users.NotifyForeignNew: 3, users.NotifyObserverOffline: 4,
|
|
}
|
|
|
|
// notifyInfra reports the roles whose relay activity counts as heard
|
|
// (GetHealthMs and roles.js use the same two).
|
|
func notifyInfra(role string) bool {
|
|
r := strings.ToLower(role)
|
|
return r == "repeater" || r == "room"
|
|
}
|
|
|
|
// nodeLabel is the node's name made safe for a mail line (mailSafeText),
|
|
// or the first 8 hex characters of its key.
|
|
func nodeLabel(n notifyNode, known bool, pk string) string {
|
|
if name := mailSafeText(n.Name); known && name != "" {
|
|
return name
|
|
}
|
|
if len(pk) > 8 {
|
|
return pk[:8]
|
|
}
|
|
return pk
|
|
}
|
|
|
|
// nodeOnlineState is good when the node was heard within its role's silent
|
|
// window, like roles.js getNodeStatus: the latest of last_seen, the packet
|
|
// store's last heard and, for repeaters and rooms, the last relay hop. A
|
|
// node missing from the analyzer database is bad. While prev is good, the
|
|
// latest evidence is never older than in.IngestResumedAt (the outage grace).
|
|
func nodeOnlineState(pk string, n notifyNode, known bool, prev string, in *notifyInput) string {
|
|
if !known {
|
|
return users.NotifyBad
|
|
}
|
|
last, _ := parseRelayTS(n.LastSeen)
|
|
if t, ok := in.Heard[pk]; ok && t.After(last) {
|
|
last = t
|
|
}
|
|
if notifyInfra(n.Role) {
|
|
if t, ok := in.Relayed[pk]; ok && t.After(last) {
|
|
last = t
|
|
}
|
|
}
|
|
if prev == users.NotifyGood && last.Before(in.IngestResumedAt) {
|
|
last = in.IngestResumedAt
|
|
}
|
|
_, silentMs := in.Health.GetHealthMs(strings.ToLower(n.Role))
|
|
if last.IsZero() || in.Now.Sub(last) >= time.Duration(silentMs)*time.Millisecond {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// batteryState applies the low-battery threshold with hysteresis.
|
|
func batteryState(mv, lowMv int, prev string) string {
|
|
if prev == users.NotifyBad {
|
|
if mv >= lowMv+batteryHysteresisMv {
|
|
return users.NotifyGood
|
|
}
|
|
return users.NotifyBad
|
|
}
|
|
if mv < lowMv {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// observerState: bad when last_seen is older than observerStaleMinutes,
|
|
// good again within observerOnlineMinutes. While prev is good, last_seen
|
|
// is never older than resumed (the outage grace, zero for none).
|
|
func observerState(o notifyObserver, h HealthThresholds, now, resumed time.Time, prev string) string {
|
|
t, ok := parseRelayTS(o.LastSeen)
|
|
if prev == users.NotifyGood && !resumed.IsZero() && (!ok || t.Before(resumed)) {
|
|
t, ok = resumed, true
|
|
}
|
|
if !ok {
|
|
return users.NotifyBad
|
|
}
|
|
age := now.Sub(t)
|
|
if prev == users.NotifyBad {
|
|
if age <= time.Duration(h.ObserverOnlineMinutes)*time.Minute {
|
|
return users.NotifyGood
|
|
}
|
|
return users.NotifyBad
|
|
}
|
|
if age > time.Duration(h.ObserverStaleMinutes)*time.Minute {
|
|
return users.NotifyBad
|
|
}
|
|
return users.NotifyGood
|
|
}
|
|
|
|
// evalExternal compares one watched node's current external alerts with the
|
|
// stored node.external states of one user. The first fresh feed after the
|
|
// node is watched (no baseline row) stores the current alerts silently;
|
|
// afterwards a new or returning alert is a bad change and a stored bad alert
|
|
// missing from the feed is a good (resolved) change.
|
|
func evalExternal(res *notifyResult, prev map[users.NotifyKey]string, bad []string, current map[string]externalAlert,
|
|
uid int64, pk, name string, mailing bool, now time.Time) {
|
|
base := users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + externalBaselineKey}
|
|
_, baselined := prev[base]
|
|
if !baselined {
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: base, State: users.NotifyGood, ChangedAt: now})
|
|
}
|
|
emit := func(key, state string, a externalAlert) {
|
|
k := users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + key}
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: k, State: state, ChangedAt: now})
|
|
if baselined && mailing {
|
|
res.Changes[uid] = append(res.Changes[uid], notifyChange{Event: users.NotifyNodeExternal, Subject: k.Subject,
|
|
Name: name, To: state, Text: a.Text, URL: a.URL, At: now})
|
|
}
|
|
}
|
|
for _, key := range sortedExternalKeys(current) {
|
|
if prev[users.NotifyKey{UserID: uid, Event: users.NotifyNodeExternal, Subject: pk + "/" + key}] != users.NotifyBad {
|
|
emit(key, users.NotifyBad, current[key])
|
|
}
|
|
}
|
|
sort.Strings(bad)
|
|
for _, key := range bad {
|
|
if _, still := current[key]; !still {
|
|
emit(key, users.NotifyGood, externalAlert{})
|
|
}
|
|
}
|
|
}
|
|
|
|
// evaluateNotifications computes every chosen (user, event, subject) state
|
|
// and compares it with the stored one. A subject without a stored state is
|
|
// stored without a change (a restart or a new watch never mails); a stored
|
|
// state that differs is a change. Users missing from Accounts are skipped.
|
|
// A user with watches but no preferences row gets the defaults (enabled,
|
|
// node events). A disabled user's states are kept current without changes,
|
|
// so turning notifications on again never mails what happened while off.
|
|
// Admin events are evaluated only for admins; a non-admin's admin-event
|
|
// rows are dropped, like opting out, so a re-promotion starts silently.
|
|
func evaluateNotifications(in notifyInput) notifyResult {
|
|
prev := make(map[users.NotifyKey]string, len(in.States))
|
|
// extBad: per user and node, the node.external keys stored as bad.
|
|
extBad := map[int64]map[string][]string{}
|
|
for _, s := range in.States {
|
|
prev[s.NotifyKey] = s.State
|
|
if s.Event == users.NotifyNodeExternal && s.State == users.NotifyBad {
|
|
if pk, key, ok := strings.Cut(s.Subject, "/"); ok && key != externalBaselineKey {
|
|
if extBad[s.UserID] == nil {
|
|
extBad[s.UserID] = map[string][]string{}
|
|
}
|
|
extBad[s.UserID][pk] = append(extBad[s.UserID][pk], key)
|
|
}
|
|
}
|
|
}
|
|
watched := map[int64][]string{}
|
|
for _, w := range in.Watches {
|
|
watched[w.UserID] = append(watched[w.UserID], w.Pubkey)
|
|
}
|
|
var foreign []string
|
|
for pk, n := range in.Nodes {
|
|
if n.Foreign {
|
|
foreign = append(foreign, pk)
|
|
}
|
|
}
|
|
sort.Strings(foreign)
|
|
|
|
res := notifyResult{Changes: map[int64][]notifyChange{}}
|
|
for _, s := range in.States {
|
|
if u, ok := in.Accounts[s.UserID]; ok && u.Role != users.RoleAdmin && users.IsAdminNotifyEvent(s.Event) {
|
|
res.Drop = append(res.Drop, s.NotifyKey)
|
|
}
|
|
}
|
|
prefs := make([]users.NotifyPrefs, 0, len(in.Prefs)+len(watched))
|
|
hasPrefs := make(map[int64]bool, len(in.Prefs))
|
|
for _, p := range in.Prefs {
|
|
prefs = append(prefs, p)
|
|
hasPrefs[p.UserID] = true
|
|
}
|
|
var defaulted []int64
|
|
for uid := range watched {
|
|
if !hasPrefs[uid] {
|
|
defaulted = append(defaulted, uid)
|
|
}
|
|
}
|
|
sort.Slice(defaulted, func(i, j int) bool { return defaulted[i] < defaulted[j] })
|
|
for _, uid := range defaulted {
|
|
prefs = append(prefs, users.DefaultNotifyPrefs(uid))
|
|
}
|
|
mailing := false // Enabled of the user being evaluated
|
|
store := func(k users.NotifyKey, state string) {
|
|
res.States = append(res.States, users.NotifyState{NotifyKey: k, State: state, ChangedAt: in.Now})
|
|
}
|
|
compare := func(k users.NotifyKey, name, state string, mv *int) {
|
|
old, had := prev[k]
|
|
if had && old == state {
|
|
return
|
|
}
|
|
store(k, state)
|
|
if had && mailing {
|
|
res.Changes[k.UserID] = append(res.Changes[k.UserID],
|
|
notifyChange{Event: k.Event, Subject: k.Subject, Name: name, To: state, BatteryMv: mv, At: in.Now})
|
|
}
|
|
}
|
|
|
|
for _, p := range prefs {
|
|
u, ok := in.Accounts[p.UserID]
|
|
if !ok {
|
|
continue
|
|
}
|
|
mailing = p.Enabled
|
|
for _, pk := range watched[p.UserID] {
|
|
n, known := in.Nodes[pk]
|
|
name := nodeLabel(n, known, pk)
|
|
if p.Has(users.NotifyNodeOffline) && !in.IngestStale {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeOffline, Subject: pk}
|
|
compare(k, name, nodeOnlineState(pk, n, known, prev[k], &in), nil)
|
|
}
|
|
if p.Has(users.NotifyNodeBattery) && known && n.BatteryMv != nil {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeBattery, Subject: pk}
|
|
compare(k, name, batteryState(*n.BatteryMv, in.LowMv, prev[k]), n.BatteryMv)
|
|
}
|
|
if p.Has(users.NotifyNodeExternal) && in.External != nil {
|
|
evalExternal(&res, prev, extBad[p.UserID][pk], in.External.Alerts[pk], p.UserID, pk, name, mailing, in.Now)
|
|
}
|
|
}
|
|
if u.Role != users.RoleAdmin {
|
|
continue
|
|
}
|
|
if p.Has(users.NotifyForeignNew) {
|
|
base := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: foreignBaselineSubject}
|
|
_, baselined := prev[base]
|
|
if !baselined {
|
|
store(base, users.NotifyGood)
|
|
}
|
|
for _, pk := range foreign {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: pk}
|
|
if _, told := prev[k]; told {
|
|
continue
|
|
}
|
|
store(k, users.NotifyTold)
|
|
if baselined && mailing {
|
|
res.Changes[p.UserID] = append(res.Changes[p.UserID], notifyChange{Event: users.NotifyForeignNew, Subject: pk,
|
|
Name: nodeLabel(in.Nodes[pk], true, pk), To: users.NotifyTold, At: in.Now})
|
|
}
|
|
}
|
|
}
|
|
if p.Has(users.NotifyObserverOffline) && !in.IngestStale {
|
|
for _, o := range in.Observers {
|
|
k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyObserverOffline, Subject: o.ID}
|
|
name := mailSafeText(o.Name)
|
|
if name == "" {
|
|
name = o.ID
|
|
}
|
|
compare(k, name, observerState(o, in.Health, in.Now, in.IngestResumedAt, prev[k]), nil)
|
|
}
|
|
}
|
|
}
|
|
for _, list := range res.Changes {
|
|
sort.SliceStable(list, func(i, j int) bool {
|
|
if a, b := notifyEventRank[list[i].Event], notifyEventRank[list[j].Event]; a != b {
|
|
return a < b
|
|
}
|
|
if list[i].Name != list[j].Name {
|
|
return list[i].Name < list[j].Name
|
|
}
|
|
return list[i].Subject < list[j].Subject
|
|
})
|
|
}
|
|
return res
|
|
}
|