diff --git a/AREAS.md b/AREAS.md index 58ec539d..2b24eb42 100644 --- a/AREAS.md +++ b/AREAS.md @@ -141,10 +141,3 @@ imprecision than a polygon since there's usually nothing on the other side of an unclaimed edge to misclassify — but two adjacent countries sharing a long border will bleed into each other badly with simple boxes, which is why Denmark/Sweden/Norway/Germany all ended up as polygons. - -## `ops/meshguide-sync/` - -That directory holds a script specific to this deployment (fetches Danish -region boundaries from a community-run site, meshguide.dk) — not part of -the generic areas feature, and not expected to be useful for any other -CoreScope deployment. See its own docstring for details. diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 1fe7d7d8..a70eb804 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -185,72 +185,9 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err) } - // Ping-score highscore backfill: ping_triggers only started getting - // written at InsertTransmission's isNew branch going forward (#1865- - // style pattern, see ping_triggers.go) -- CHAN messages that predate - // this feature never triggered that write. Scan history once so the - // highscore board isn't empty for pings sent before today. - // PREFLIGHT: async=true reason="full scan of payload_type=5 transmissions with a LIKE prefilter -- bounded by channel-message volume, not total transmissions, but still a full-table scan so kept off the boot path" - if err := s.RunAsyncMigration(context.Background(), "ping_triggers_backfill_v1", backfillPingTriggers); err != nil { - log.Printf("[migration/async] scheduling ping_triggers_backfill_v1 failed: %v", err) - } - return s, nil } -// backfillPingTriggers is the ping_triggers_backfill_v1 async migration -// body -- pulled into its own named function (rather than inline like its -// siblings above) so tests can call it directly without needing to fake -// the whole marker-row/goroutine dance RunAsyncMigration wraps it in. -func backfillPingTriggers(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Backfilling ping_triggers from historical CHAN messages...") - rows, err := d.QueryContext(ctx, ` - SELECT id, hash, channel_hash, decoded_json, first_seen FROM transmissions - WHERE payload_type = 5 AND decoded_json LIKE '%ping%' - `) - if err != nil { - return err - } - type candidate struct { - txID int64 - hash, channelHash, decodedJSON, firstSeen string - } - var candidates []candidate - for rows.Next() { - var c candidate - var channelHash sql.NullString - if err := rows.Scan(&c.txID, &c.hash, &channelHash, &c.decodedJSON, &c.firstSeen); err != nil { - continue - } - c.channelHash = channelHash.String - candidates = append(candidates, c) - } - rows.Close() - - var inserted int - for _, c := range candidates { - sender, displayText, ok := pingTriggerSenderAndText(c.decodedJSON) - if !ok || !isPingTrigger(displayText) { - continue - } - res, err := d.ExecContext(ctx, - `INSERT OR IGNORE INTO ping_triggers (tx_id, hash, channel_hash, sender, first_seen) VALUES (?, ?, ?, ?, ?)`, - c.txID, c.hash, nilIfEmpty(c.channelHash), nilIfEmpty(sender), c.firstSeen) - if err != nil { - log.Printf("[migration/async] ping_triggers backfill insert (non-fatal): %v", err) - continue - } - if n, _ := res.RowsAffected(); n > 0 { - inserted++ - } - } - if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('ping_triggers_backfill_v1')`); err != nil { - return err - } - log.Printf("[migration/async] ping_triggers backfill complete: %d historical ping(s) recorded (scanned %d candidates)", inserted, len(candidates)) - return nil -} - func applySchema(db *sql.DB) error { // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. diff --git a/cmd/ingestor/ping_triggers_test.go b/cmd/ingestor/ping_triggers_test.go index c1669b8e..27629d1f 100644 --- a/cmd/ingestor/ping_triggers_test.go +++ b/cmd/ingestor/ping_triggers_test.go @@ -3,15 +3,9 @@ package main // Tests for the ping-score highscore/leaderboard feature's detection side: // isPingTrigger/pingTriggerSenderAndText mirror cmd/server/db.go's copies // exactly, and InsertTransmission writes exactly one ping_triggers row per -// new ping-triggering CHAN transmission. Also covers backfillPingTriggers, -// the one-time async migration that catches CHAN messages sent before this -// feature existed (which never went through InsertTransmission's isNew -// detection hook). +// new ping-triggering CHAN transmission. -import ( - "context" - "testing" -) +import "testing" func TestIsPingTrigger(t *testing.T) { cases := []struct { @@ -127,88 +121,6 @@ func TestInsertTransmission_RepeatObservationDoesNotDuplicate(t *testing.T) { } } -// insertHistoricalChanTxDirect inserts a transmission row directly via SQL, -// bypassing InsertTransmission entirely -- simulating a CHAN message that -// was ingested before the ping-score feature existed, so it never went -// through the isNew detection hook and has no ping_triggers row. -func insertHistoricalChanTxDirect(t *testing.T, s *Store, hash, text, channelHash string) int64 { - t.Helper() - res, err := s.db.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, last_seen) - VALUES ('AABB', ?, '2026-01-01T00:00:00Z', 1, 5, 1, ?, ?, 0)`, - hash, `{"type":"CHAN","channel":"`+channelHash+`","text":"`+text+`"}`, channelHash, - ) - if err != nil { - t.Fatalf("insert historical tx: %v", err) - } - txID, _ := res.LastInsertId() - return txID -} - -func TestBackfillPingTriggers_FindsHistoricalPings(t *testing.T) { - s := openNeighborsStore(t) - // OpenStore already schedules this same migration in the background; - // let its (empty-DB, no-op) first pass finish before inserting test - // data, or the automatic run could race the inserts below and find - // them itself before this test's own explicit call does. - s.WaitForAsyncMigrations() - - insertHistoricalChanTxDirect(t, s, "histping0000001", "Alice: ping", "#test") - insertHistoricalChanTxDirect(t, s, "histchat0000001", "Alice: just chatting", "#test") - - if got := countPingTriggers(t, s); got != 0 { - t.Fatalf("ping_triggers count before backfill = %d, want 0 (historical rows bypass InsertTransmission)", got) - } - - if err := backfillPingTriggers(context.Background(), s.db); err != nil { - t.Fatalf("backfillPingTriggers: %v", err) - } - - if got := countPingTriggers(t, s); got != 1 { - t.Fatalf("ping_triggers count after backfill = %d, want 1 (only the historical ping, not the chat message)", got) - } - var hash string - if err := s.db.QueryRow(`SELECT hash FROM ping_triggers`).Scan(&hash); err != nil { - t.Fatalf("read backfilled row: %v", err) - } - if hash != "histping0000001" { - t.Errorf("backfilled hash = %q, want histping0000001", hash) - } -} - -func TestBackfillPingTriggers_IdempotentOnRerun(t *testing.T) { - s := openNeighborsStore(t) - s.WaitForAsyncMigrations() // let OpenStore's own (empty-DB) pass finish first - insertHistoricalChanTxDirect(t, s, "histping0000002", "Bob: /ping", "#test") - - if err := backfillPingTriggers(context.Background(), s.db); err != nil { - t.Fatalf("first backfill: %v", err) - } - if err := backfillPingTriggers(context.Background(), s.db); err != nil { - t.Fatalf("second backfill: %v", err) - } - - if got := countPingTriggers(t, s); got != 1 { - t.Errorf("ping_triggers count after two backfill runs = %d, want 1 (tx_id PRIMARY KEY + INSERT OR IGNORE must dedupe)", got) - } -} - -// TestOpenStore_SchedulesPingTriggersBackfill confirms the migration is -// actually wired into OpenStore's boot path (registered + completed), -// not just directly callable in isolation like the tests above. -func TestOpenStore_SchedulesPingTriggersBackfill(t *testing.T) { - s := openNeighborsStore(t) - s.WaitForAsyncMigrations() - - status, err := s.AsyncMigrationStatus("ping_triggers_backfill_v1") - if err != nil { - t.Fatalf("AsyncMigrationStatus: %v", err) - } - if status != "done" { - t.Errorf("ping_triggers_backfill_v1 status = %q, want %q -- OpenStore must schedule and complete this migration on every boot", status, "done") - } -} - func TestInsertTransmission_NonChanPayloadNotChecked(t *testing.T) { s := openNeighborsStore(t) diff --git a/ops/meshguide-sync/sync_areas.py b/ops/meshguide-sync/sync_areas.py deleted file mode 100755 index f307b049..00000000 --- a/ops/meshguide-sync/sync_areas.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 -"""Sync CoreScope's config.json "areas" from meshguide.dk's community-run -region/city dataset (polygons + hashRegions channel-scope links). - -This is dborup/meshview.dk-specific tooling -- meshguide.dk doesn't exist for -other CoreScope deployments, so this lives outside the Go binary/repo core and -is meant to be run manually or via cron/systemd timer, never as part of the -application itself. - -Usage: - sync_areas.py --config /opt/corescope/data/config.json [--dry-run] - -What it does: - 1. Fetches https://meshguide.dk/regions.json (polygon per scope) and - cities.json (scope confirmation + human names). - 2. For areas already in config.json's "areas" that we're confident match a - meshguide region (see CROSSWALK below -- hand-verified, never guessed), - adds the scope to regionScopes (an area's scope list -- CoreScope reads - this field, not the singular "regionScope") and replaces the polygon - with meshguide's more precise one. - 3. Adds any meshguide region we don't already have as a new area entry, as - long as it has a real (non-empty) scope assigned. - 4. Anything not in CROSSWALK and not clearly a new region is left alone and - reported as a warning, never silently linked -- e.g. "dk-sdk" (Syddanmark) - is NOT the same place as our existing DK_SJ (Sønderjylland) area, so it's - added as its own new area instead of being merged into DK_SJ. - -A timestamped backup of config.json is written before any change. -""" -import argparse -import json -import re -import sys -import urllib.request -from datetime import datetime, timezone - -DEFAULT_BASE = "https://meshguide.dk" - -# Hand-verified area-key -> meshguide scope mappings. Only pairs we've -# actually confirmed refer to the same place go here. -CROSSWALK = { - "DK": "dk", - "JYL": "dk-jylland", - "DK_NJ": "dk-nj", - "DK_MJ": "dk-mj", - "DK_OJ": "dk-oj", - "DK_3K": "dk-3kant", - "AAR": "dk-aarhus", - "AAL": "dk-aalborg", - "FYN": "dk-fyn", - "ODE": "dk-fyn-odense", - "SJL": "dk-sjl", - "DK_NSJ": "dk-nordsjaelland", - "DK_LF": "dk-lo-fa", - "RNN": "dk-bhm", -} - -# Areas we deliberately did NOT auto-link, and why -- printed as a reminder -# each run so the mismatch doesn't get silently forgotten. -KNOWN_GAPS = { - "DK_VJ": "no matching meshguide region found (Vestjylland)", - "DK_SJ": 'meshguide\'s dk-sdk is "Syddanmark" (a different, broader region than Sønderjylland) -- not linked', - "CPH": "no matching meshguide region found (Storkøbenhavn)", - "SE_SKA": 'meshguide\'s se12 is "SydSverige" -- close but not confirmed identical to Skåne -- not linked', -} - - -def fetch_json(url): - with urllib.request.urlopen(url, timeout=20) as r: - return json.load(r) - - -def normalize_key(scope): - """dk-fyn-odense -> DK_FYN_ODENSE, se12 -> SE12""" - return re.sub(r"[^A-Za-z0-9]+", "_", scope).strip("_").upper() - - -def geojson_ring_to_polygon(geometry): - """First ring of a GeoJSON Polygon: [lon,lat] -> [lat,lon], closing point dropped.""" - if not geometry or geometry.get("type") != "Polygon": - return None - coords = geometry.get("coordinates") or [] - if not coords: - return None - ring = coords[0] - if len(ring) > 1 and ring[0] == ring[-1]: - ring = ring[:-1] - return [[round(lat, 6), round(lon, 6)] for lon, lat in ring] - - -def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--config", required=True, help="Path to CoreScope config.json") - ap.add_argument("--base-url", default=DEFAULT_BASE) - ap.add_argument( - "--dry-run", action="store_true", help="Print what would change, write nothing" - ) - args = ap.parse_args() - - regions = fetch_json(args.base_url.rstrip("/") + "/regions.json") - cities = fetch_json(args.base_url.rstrip("/") + "/cities.json") - - # meshguide region keys confirmed to have NO real scope assigned yet - # (cities.json lists them with scope: "") -- never auto-link or add these. - no_scope_keys = {k for k, v in cities.items() if not v.get("scope")} - - with open(args.config, "r", encoding="utf-8") as f: - cfg = json.load(f) - areas = cfg.setdefault("areas", {}) - - changed = [] - warnings = [] - - # scopes confirmed real even without a regions.json polygon (e.g. "dk" - # itself only has a cities.json point, no drawn boundary) - confirmed_scopes = {v.get("scope") for v in cities.values() if v.get("scope")} - confirmed_scopes |= set(regions.keys()) - - # 0) one-time migration: an earlier version of this script wrote the - # singular "regionScope" key, which CoreScope's config schema never - # reads (it reads regionScopes, a list) -- silently invisible to the - # app. Areas created by that buggy run (not in CROSSWALK, so step 1 - # below never touches them, and already present, so step 2's "add new - # area" skips them too) would stay broken forever without this pass. - for area_key, entry in areas.items(): - legacy_scope = entry.pop("regionScope", None) - if legacy_scope: - scopes_list = entry.setdefault("regionScopes", []) - if legacy_scope not in scopes_list: - scopes_list.append(legacy_scope) - changed.append(f"migrated {area_key} ({entry.get('label')}) regionScope -> regionScopes") - - # 1) enrich existing crosswalked areas - for area_key, scope in CROSSWALK.items(): - entry = areas.get(area_key) - if entry is None: - warnings.append( - f'CROSSWALK references area "{area_key}" which no longer exists in config.json -- skipped' - ) - continue - if scope not in confirmed_scopes: - warnings.append( - f'CROSSWALK maps {area_key} -> "{scope}" but meshguide no longer has that scope -- skipped' - ) - continue - before = json.dumps(entry, sort_keys=True) - # regionScopes is the field CoreScope's config schema actually reads - # (cmd/server/routes.go's handleConfigAreas -> Config.Areas[k].RegionScopes) - # -- a stray singular "regionScope" here would be silently invisible - # to the app, so append to the list instead (preserving any other - # manually-configured scopes) and drop the wrong key if a prior buggy - # run of this script left one behind. - entry.pop("regionScope", None) - scopes_list = entry.setdefault("regionScopes", []) - if scope not in scopes_list: - scopes_list.append(scope) - polygon = geojson_ring_to_polygon((regions.get(scope) or {}).get("geometry")) - if polygon: - entry["polygon"] = polygon - for k in ("latMin", "latMax", "lonMin", "lonMax"): - entry.pop(k, None) - if json.dumps(entry, sort_keys=True) != before: - changed.append( - f"enriched {area_key} ({entry.get('label')}) with regionScopes+={scope}" - + (" + polygon" if polygon else "") - ) - - # 2) add new areas for meshguide regions we don't have yet - linked_scopes = set(CROSSWALK.values()) - existing_scopes = {s for v in areas.values() for s in v.get("regionScopes", [])} - for scope, region in regions.items(): - if scope in no_scope_keys: - continue - if scope in linked_scopes or scope in existing_scopes: - continue - new_key = normalize_key(scope) - if new_key in areas: - continue - polygon = geojson_ring_to_polygon(region.get("geometry")) - if not polygon: - warnings.append(f'meshguide region "{scope}" has no polygon geometry -- skipped') - continue - areas[new_key] = { - "label": region.get("name", scope), - "polygon": polygon, - "regionScopes": [scope], - } - changed.append(f"added new area {new_key} ({region.get('name')}) regionScopes=[{scope}]") - - for area_key, reason in KNOWN_GAPS.items(): - if area_key in areas and not areas[area_key].get("regionScopes"): - warnings.append(f"{area_key}: {reason}") - - print(f"{len(changed)} change(s):") - for c in changed: - print(" -", c) - if warnings: - print(f"\n{len(warnings)} warning(s):") - for w in warnings: - print(" !", w) - - if not changed: - print("\nNo changes -- config.json left untouched.") - return - - if args.dry_run: - print("\n--dry-run: not writing changes.") - return - - backup_path = f"{args.config}.bak-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}" - with open(args.config, "r", encoding="utf-8") as f: - raw = f.read() - with open(backup_path, "w", encoding="utf-8") as f: - f.write(raw) - print(f"\nBackup written to {backup_path}") - - with open(args.config, "w", encoding="utf-8") as f: - json.dump(cfg, f, indent=2, ensure_ascii=False) - f.write("\n") - print(f"Wrote changes to {args.config}") - print("\nRestart corescope for the change to take effect (config is only read at startup).") - - -if __name__ == "__main__": - sys.exit(main())