mirror of
https://github.com/Kpa-clawbot/meshcore-analyzer.git
synced 2026-09-26 02:43:44 +00:00
chore: exclude meshguide sync script + ping-scores backfill from master
Two dborup-specific, already-used pieces held back from the generic master branch: - ops/meshguide-sync/sync_areas.py: fetches Danish region boundaries from meshguide.dk, useless for any other CoreScope deployment. This fast-forward merge brought it in again (areas-meshguide-sync had it restored after an earlier accidental deletion); excluding it here, same as every previous merge round. - cmd/ingestor/db.go's ping_triggers_backfill_v1 async migration: a one-time historical scan that already ran and completed on stg (298 pings recorded from before the feature existed). Per dborup: other deployments merging from master didn't ask for a full-table CHAN-message scan on their own database. The ping-scores feature itself (detection, scoring, leaderboards, frontend) stays -- only the backfill-specific function + its tests are removed. Both pieces remain intact on areas-meshguide-sync, where they've already served their purpose. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
dborup
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
bcb190bffa
commit
e6cc7a5d99
@@ -141,10 +141,3 @@ imprecision than a polygon since there's usually nothing on the other side
|
||||
of an unclaimed edge to misclassify — but two adjacent countries sharing a
|
||||
long border will bleed into each other badly with simple boxes, which is
|
||||
why Denmark/Sweden/Norway/Germany all ended up as polygons.
|
||||
|
||||
## `ops/meshguide-sync/`
|
||||
|
||||
That directory holds a script specific to this deployment (fetches Danish
|
||||
region boundaries from a community-run site, meshguide.dk) — not part of
|
||||
the generic areas feature, and not expected to be useful for any other
|
||||
CoreScope deployment. See its own docstring for details.
|
||||
|
||||
@@ -185,72 +185,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error)
|
||||
log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err)
|
||||
}
|
||||
|
||||
// Ping-score highscore backfill: ping_triggers only started getting
|
||||
// written at InsertTransmission's isNew branch going forward (#1865-
|
||||
// style pattern, see ping_triggers.go) -- CHAN messages that predate
|
||||
// this feature never triggered that write. Scan history once so the
|
||||
// highscore board isn't empty for pings sent before today.
|
||||
// PREFLIGHT: async=true reason="full scan of payload_type=5 transmissions with a LIKE prefilter -- bounded by channel-message volume, not total transmissions, but still a full-table scan so kept off the boot path"
|
||||
if err := s.RunAsyncMigration(context.Background(), "ping_triggers_backfill_v1", backfillPingTriggers); err != nil {
|
||||
log.Printf("[migration/async] scheduling ping_triggers_backfill_v1 failed: %v", err)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// backfillPingTriggers is the ping_triggers_backfill_v1 async migration
|
||||
// body -- pulled into its own named function (rather than inline like its
|
||||
// siblings above) so tests can call it directly without needing to fake
|
||||
// the whole marker-row/goroutine dance RunAsyncMigration wraps it in.
|
||||
func backfillPingTriggers(ctx context.Context, d *sql.DB) error {
|
||||
log.Println("[migration/async] Backfilling ping_triggers from historical CHAN messages...")
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
SELECT id, hash, channel_hash, decoded_json, first_seen FROM transmissions
|
||||
WHERE payload_type = 5 AND decoded_json LIKE '%ping%'
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type candidate struct {
|
||||
txID int64
|
||||
hash, channelHash, decodedJSON, firstSeen string
|
||||
}
|
||||
var candidates []candidate
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
var channelHash sql.NullString
|
||||
if err := rows.Scan(&c.txID, &c.hash, &channelHash, &c.decodedJSON, &c.firstSeen); err != nil {
|
||||
continue
|
||||
}
|
||||
c.channelHash = channelHash.String
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
var inserted int
|
||||
for _, c := range candidates {
|
||||
sender, displayText, ok := pingTriggerSenderAndText(c.decodedJSON)
|
||||
if !ok || !isPingTrigger(displayText) {
|
||||
continue
|
||||
}
|
||||
res, err := d.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO ping_triggers (tx_id, hash, channel_hash, sender, first_seen) VALUES (?, ?, ?, ?, ?)`,
|
||||
c.txID, c.hash, nilIfEmpty(c.channelHash), nilIfEmpty(sender), c.firstSeen)
|
||||
if err != nil {
|
||||
log.Printf("[migration/async] ping_triggers backfill insert (non-fatal): %v", err)
|
||||
continue
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n > 0 {
|
||||
inserted++
|
||||
}
|
||||
}
|
||||
if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('ping_triggers_backfill_v1')`); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[migration/async] ping_triggers backfill complete: %d historical ping(s) recorded (scanned %d candidates)", inserted, len(candidates))
|
||||
return nil
|
||||
}
|
||||
|
||||
func applySchema(db *sql.DB) error {
|
||||
// auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode).
|
||||
// Logging of current mode is handled by CheckAutoVacuum — no duplicate log here.
|
||||
|
||||
@@ -3,15 +3,9 @@ package main
|
||||
// Tests for the ping-score highscore/leaderboard feature's detection side:
|
||||
// isPingTrigger/pingTriggerSenderAndText mirror cmd/server/db.go's copies
|
||||
// exactly, and InsertTransmission writes exactly one ping_triggers row per
|
||||
// new ping-triggering CHAN transmission. Also covers backfillPingTriggers,
|
||||
// the one-time async migration that catches CHAN messages sent before this
|
||||
// feature existed (which never went through InsertTransmission's isNew
|
||||
// detection hook).
|
||||
// new ping-triggering CHAN transmission.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
func TestIsPingTrigger(t *testing.T) {
|
||||
cases := []struct {
|
||||
@@ -127,88 +121,6 @@ func TestInsertTransmission_RepeatObservationDoesNotDuplicate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// insertHistoricalChanTxDirect inserts a transmission row directly via SQL,
|
||||
// bypassing InsertTransmission entirely -- simulating a CHAN message that
|
||||
// was ingested before the ping-score feature existed, so it never went
|
||||
// through the isNew detection hook and has no ping_triggers row.
|
||||
func insertHistoricalChanTxDirect(t *testing.T, s *Store, hash, text, channelHash string) int64 {
|
||||
t.Helper()
|
||||
res, err := s.db.Exec(
|
||||
`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, last_seen)
|
||||
VALUES ('AABB', ?, '2026-01-01T00:00:00Z', 1, 5, 1, ?, ?, 0)`,
|
||||
hash, `{"type":"CHAN","channel":"`+channelHash+`","text":"`+text+`"}`, channelHash,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("insert historical tx: %v", err)
|
||||
}
|
||||
txID, _ := res.LastInsertId()
|
||||
return txID
|
||||
}
|
||||
|
||||
func TestBackfillPingTriggers_FindsHistoricalPings(t *testing.T) {
|
||||
s := openNeighborsStore(t)
|
||||
// OpenStore already schedules this same migration in the background;
|
||||
// let its (empty-DB, no-op) first pass finish before inserting test
|
||||
// data, or the automatic run could race the inserts below and find
|
||||
// them itself before this test's own explicit call does.
|
||||
s.WaitForAsyncMigrations()
|
||||
|
||||
insertHistoricalChanTxDirect(t, s, "histping0000001", "Alice: ping", "#test")
|
||||
insertHistoricalChanTxDirect(t, s, "histchat0000001", "Alice: just chatting", "#test")
|
||||
|
||||
if got := countPingTriggers(t, s); got != 0 {
|
||||
t.Fatalf("ping_triggers count before backfill = %d, want 0 (historical rows bypass InsertTransmission)", got)
|
||||
}
|
||||
|
||||
if err := backfillPingTriggers(context.Background(), s.db); err != nil {
|
||||
t.Fatalf("backfillPingTriggers: %v", err)
|
||||
}
|
||||
|
||||
if got := countPingTriggers(t, s); got != 1 {
|
||||
t.Fatalf("ping_triggers count after backfill = %d, want 1 (only the historical ping, not the chat message)", got)
|
||||
}
|
||||
var hash string
|
||||
if err := s.db.QueryRow(`SELECT hash FROM ping_triggers`).Scan(&hash); err != nil {
|
||||
t.Fatalf("read backfilled row: %v", err)
|
||||
}
|
||||
if hash != "histping0000001" {
|
||||
t.Errorf("backfilled hash = %q, want histping0000001", hash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillPingTriggers_IdempotentOnRerun(t *testing.T) {
|
||||
s := openNeighborsStore(t)
|
||||
s.WaitForAsyncMigrations() // let OpenStore's own (empty-DB) pass finish first
|
||||
insertHistoricalChanTxDirect(t, s, "histping0000002", "Bob: /ping", "#test")
|
||||
|
||||
if err := backfillPingTriggers(context.Background(), s.db); err != nil {
|
||||
t.Fatalf("first backfill: %v", err)
|
||||
}
|
||||
if err := backfillPingTriggers(context.Background(), s.db); err != nil {
|
||||
t.Fatalf("second backfill: %v", err)
|
||||
}
|
||||
|
||||
if got := countPingTriggers(t, s); got != 1 {
|
||||
t.Errorf("ping_triggers count after two backfill runs = %d, want 1 (tx_id PRIMARY KEY + INSERT OR IGNORE must dedupe)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenStore_SchedulesPingTriggersBackfill confirms the migration is
|
||||
// actually wired into OpenStore's boot path (registered + completed),
|
||||
// not just directly callable in isolation like the tests above.
|
||||
func TestOpenStore_SchedulesPingTriggersBackfill(t *testing.T) {
|
||||
s := openNeighborsStore(t)
|
||||
s.WaitForAsyncMigrations()
|
||||
|
||||
status, err := s.AsyncMigrationStatus("ping_triggers_backfill_v1")
|
||||
if err != nil {
|
||||
t.Fatalf("AsyncMigrationStatus: %v", err)
|
||||
}
|
||||
if status != "done" {
|
||||
t.Errorf("ping_triggers_backfill_v1 status = %q, want %q -- OpenStore must schedule and complete this migration on every boot", status, "done")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertTransmission_NonChanPayloadNotChecked(t *testing.T) {
|
||||
s := openNeighborsStore(t)
|
||||
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync CoreScope's config.json "areas" from meshguide.dk's community-run
|
||||
region/city dataset (polygons + hashRegions channel-scope links).
|
||||
|
||||
This is dborup/meshview.dk-specific tooling -- meshguide.dk doesn't exist for
|
||||
other CoreScope deployments, so this lives outside the Go binary/repo core and
|
||||
is meant to be run manually or via cron/systemd timer, never as part of the
|
||||
application itself.
|
||||
|
||||
Usage:
|
||||
sync_areas.py --config /opt/corescope/data/config.json [--dry-run]
|
||||
|
||||
What it does:
|
||||
1. Fetches https://meshguide.dk/regions.json (polygon per scope) and
|
||||
cities.json (scope confirmation + human names).
|
||||
2. For areas already in config.json's "areas" that we're confident match a
|
||||
meshguide region (see CROSSWALK below -- hand-verified, never guessed),
|
||||
adds the scope to regionScopes (an area's scope list -- CoreScope reads
|
||||
this field, not the singular "regionScope") and replaces the polygon
|
||||
with meshguide's more precise one.
|
||||
3. Adds any meshguide region we don't already have as a new area entry, as
|
||||
long as it has a real (non-empty) scope assigned.
|
||||
4. Anything not in CROSSWALK and not clearly a new region is left alone and
|
||||
reported as a warning, never silently linked -- e.g. "dk-sdk" (Syddanmark)
|
||||
is NOT the same place as our existing DK_SJ (Sønderjylland) area, so it's
|
||||
added as its own new area instead of being merged into DK_SJ.
|
||||
|
||||
A timestamped backup of config.json is written before any change.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
DEFAULT_BASE = "https://meshguide.dk"
|
||||
|
||||
# Hand-verified area-key -> meshguide scope mappings. Only pairs we've
|
||||
# actually confirmed refer to the same place go here.
|
||||
CROSSWALK = {
|
||||
"DK": "dk",
|
||||
"JYL": "dk-jylland",
|
||||
"DK_NJ": "dk-nj",
|
||||
"DK_MJ": "dk-mj",
|
||||
"DK_OJ": "dk-oj",
|
||||
"DK_3K": "dk-3kant",
|
||||
"AAR": "dk-aarhus",
|
||||
"AAL": "dk-aalborg",
|
||||
"FYN": "dk-fyn",
|
||||
"ODE": "dk-fyn-odense",
|
||||
"SJL": "dk-sjl",
|
||||
"DK_NSJ": "dk-nordsjaelland",
|
||||
"DK_LF": "dk-lo-fa",
|
||||
"RNN": "dk-bhm",
|
||||
}
|
||||
|
||||
# Areas we deliberately did NOT auto-link, and why -- printed as a reminder
|
||||
# each run so the mismatch doesn't get silently forgotten.
|
||||
KNOWN_GAPS = {
|
||||
"DK_VJ": "no matching meshguide region found (Vestjylland)",
|
||||
"DK_SJ": 'meshguide\'s dk-sdk is "Syddanmark" (a different, broader region than Sønderjylland) -- not linked',
|
||||
"CPH": "no matching meshguide region found (Storkøbenhavn)",
|
||||
"SE_SKA": 'meshguide\'s se12 is "SydSverige" -- close but not confirmed identical to Skåne -- not linked',
|
||||
}
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
with urllib.request.urlopen(url, timeout=20) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def normalize_key(scope):
|
||||
"""dk-fyn-odense -> DK_FYN_ODENSE, se12 -> SE12"""
|
||||
return re.sub(r"[^A-Za-z0-9]+", "_", scope).strip("_").upper()
|
||||
|
||||
|
||||
def geojson_ring_to_polygon(geometry):
|
||||
"""First ring of a GeoJSON Polygon: [lon,lat] -> [lat,lon], closing point dropped."""
|
||||
if not geometry or geometry.get("type") != "Polygon":
|
||||
return None
|
||||
coords = geometry.get("coordinates") or []
|
||||
if not coords:
|
||||
return None
|
||||
ring = coords[0]
|
||||
if len(ring) > 1 and ring[0] == ring[-1]:
|
||||
ring = ring[:-1]
|
||||
return [[round(lat, 6), round(lon, 6)] for lon, lat in ring]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument("--config", required=True, help="Path to CoreScope config.json")
|
||||
ap.add_argument("--base-url", default=DEFAULT_BASE)
|
||||
ap.add_argument(
|
||||
"--dry-run", action="store_true", help="Print what would change, write nothing"
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
regions = fetch_json(args.base_url.rstrip("/") + "/regions.json")
|
||||
cities = fetch_json(args.base_url.rstrip("/") + "/cities.json")
|
||||
|
||||
# meshguide region keys confirmed to have NO real scope assigned yet
|
||||
# (cities.json lists them with scope: "") -- never auto-link or add these.
|
||||
no_scope_keys = {k for k, v in cities.items() if not v.get("scope")}
|
||||
|
||||
with open(args.config, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
areas = cfg.setdefault("areas", {})
|
||||
|
||||
changed = []
|
||||
warnings = []
|
||||
|
||||
# scopes confirmed real even without a regions.json polygon (e.g. "dk"
|
||||
# itself only has a cities.json point, no drawn boundary)
|
||||
confirmed_scopes = {v.get("scope") for v in cities.values() if v.get("scope")}
|
||||
confirmed_scopes |= set(regions.keys())
|
||||
|
||||
# 0) one-time migration: an earlier version of this script wrote the
|
||||
# singular "regionScope" key, which CoreScope's config schema never
|
||||
# reads (it reads regionScopes, a list) -- silently invisible to the
|
||||
# app. Areas created by that buggy run (not in CROSSWALK, so step 1
|
||||
# below never touches them, and already present, so step 2's "add new
|
||||
# area" skips them too) would stay broken forever without this pass.
|
||||
for area_key, entry in areas.items():
|
||||
legacy_scope = entry.pop("regionScope", None)
|
||||
if legacy_scope:
|
||||
scopes_list = entry.setdefault("regionScopes", [])
|
||||
if legacy_scope not in scopes_list:
|
||||
scopes_list.append(legacy_scope)
|
||||
changed.append(f"migrated {area_key} ({entry.get('label')}) regionScope -> regionScopes")
|
||||
|
||||
# 1) enrich existing crosswalked areas
|
||||
for area_key, scope in CROSSWALK.items():
|
||||
entry = areas.get(area_key)
|
||||
if entry is None:
|
||||
warnings.append(
|
||||
f'CROSSWALK references area "{area_key}" which no longer exists in config.json -- skipped'
|
||||
)
|
||||
continue
|
||||
if scope not in confirmed_scopes:
|
||||
warnings.append(
|
||||
f'CROSSWALK maps {area_key} -> "{scope}" but meshguide no longer has that scope -- skipped'
|
||||
)
|
||||
continue
|
||||
before = json.dumps(entry, sort_keys=True)
|
||||
# regionScopes is the field CoreScope's config schema actually reads
|
||||
# (cmd/server/routes.go's handleConfigAreas -> Config.Areas[k].RegionScopes)
|
||||
# -- a stray singular "regionScope" here would be silently invisible
|
||||
# to the app, so append to the list instead (preserving any other
|
||||
# manually-configured scopes) and drop the wrong key if a prior buggy
|
||||
# run of this script left one behind.
|
||||
entry.pop("regionScope", None)
|
||||
scopes_list = entry.setdefault("regionScopes", [])
|
||||
if scope not in scopes_list:
|
||||
scopes_list.append(scope)
|
||||
polygon = geojson_ring_to_polygon((regions.get(scope) or {}).get("geometry"))
|
||||
if polygon:
|
||||
entry["polygon"] = polygon
|
||||
for k in ("latMin", "latMax", "lonMin", "lonMax"):
|
||||
entry.pop(k, None)
|
||||
if json.dumps(entry, sort_keys=True) != before:
|
||||
changed.append(
|
||||
f"enriched {area_key} ({entry.get('label')}) with regionScopes+={scope}"
|
||||
+ (" + polygon" if polygon else "")
|
||||
)
|
||||
|
||||
# 2) add new areas for meshguide regions we don't have yet
|
||||
linked_scopes = set(CROSSWALK.values())
|
||||
existing_scopes = {s for v in areas.values() for s in v.get("regionScopes", [])}
|
||||
for scope, region in regions.items():
|
||||
if scope in no_scope_keys:
|
||||
continue
|
||||
if scope in linked_scopes or scope in existing_scopes:
|
||||
continue
|
||||
new_key = normalize_key(scope)
|
||||
if new_key in areas:
|
||||
continue
|
||||
polygon = geojson_ring_to_polygon(region.get("geometry"))
|
||||
if not polygon:
|
||||
warnings.append(f'meshguide region "{scope}" has no polygon geometry -- skipped')
|
||||
continue
|
||||
areas[new_key] = {
|
||||
"label": region.get("name", scope),
|
||||
"polygon": polygon,
|
||||
"regionScopes": [scope],
|
||||
}
|
||||
changed.append(f"added new area {new_key} ({region.get('name')}) regionScopes=[{scope}]")
|
||||
|
||||
for area_key, reason in KNOWN_GAPS.items():
|
||||
if area_key in areas and not areas[area_key].get("regionScopes"):
|
||||
warnings.append(f"{area_key}: {reason}")
|
||||
|
||||
print(f"{len(changed)} change(s):")
|
||||
for c in changed:
|
||||
print(" -", c)
|
||||
if warnings:
|
||||
print(f"\n{len(warnings)} warning(s):")
|
||||
for w in warnings:
|
||||
print(" !", w)
|
||||
|
||||
if not changed:
|
||||
print("\nNo changes -- config.json left untouched.")
|
||||
return
|
||||
|
||||
if args.dry_run:
|
||||
print("\n--dry-run: not writing changes.")
|
||||
return
|
||||
|
||||
backup_path = f"{args.config}.bak-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}"
|
||||
with open(args.config, "r", encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
with open(backup_path, "w", encoding="utf-8") as f:
|
||||
f.write(raw)
|
||||
print(f"\nBackup written to {backup_path}")
|
||||
|
||||
with open(args.config, "w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
print(f"Wrote changes to {args.config}")
|
||||
print("\nRestart corescope for the change to take effect (config is only read at startup).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user