From 79da3f60e331c8b7298b96ddcf69bb4759068fb9 Mon Sep 17 00:00:00 2001 From: sh-agents <350651734+sh-agents@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:27:03 +0000 Subject: [PATCH] store: run the names migration after badges --- apps/simplex-store-service/README.md | 5 + .../StoreService/Store/Postgres/Migrations.hs | 182 ++++++++++++++++- .../StoreService/Store/SQLite/Migrations.hs | 183 +++++++++++++++++- tests/Bots/StoreService/WebTests.hs | 17 +- 4 files changed, 381 insertions(+), 6 deletions(-) diff --git a/apps/simplex-store-service/README.md b/apps/simplex-store-service/README.md index cfe7366b15..9ab62a0128 100644 --- a/apps/simplex-store-service/README.md +++ b/apps/simplex-store-service/README.md @@ -449,6 +449,11 @@ running beside the new one, on the same port and database. The database `badge_s `BADGE_DB_CONN`, the `sx_badge_service_` tables and the chat database prefix `simplex_badge_service` are unchanged. +A badge-service database upgrades in place: the store's own migrations run after every badge +migration, `20261012_badge_subscriptions` included. A store database created before the names +migration became `20261013_names` (it was `20261010_names`) does not start, as the service refuses +migrations applied out of the order it lists. Recreate such a database. + ## Issuing codes Operators issue codes two ways: from the service's own command line in `--run-cli` mode, and from the diff --git a/apps/simplex-store-service/src/StoreService/Store/Postgres/Migrations.hs b/apps/simplex-store-service/src/StoreService/Store/Postgres/Migrations.hs index 7a18a807f0..1f4166e33f 100644 --- a/apps/simplex-store-service/src/StoreService/Store/Postgres/Migrations.hs +++ b/apps/simplex-store-service/src/StoreService/Store/Postgres/Migrations.hs @@ -2,7 +2,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} -module StoreService.Store.Postgres.Migrations (storeServiceSchemaMigrations, servicePrefix, withPrefix) where +module StoreService.Store.Postgres.Migrations (storeServiceSchemaMigrations, schemaMigrations, servicePrefix, withPrefix) where import Data.List (sortOn) import Data.Text (Text) @@ -20,7 +20,8 @@ schemaMigrations = [ ("20260915_badge_service_schema", m20260915_badge_service_schema, Just down_m20260915_badge_service_schema), ("20260918_badge_group_ops", m20260918_badge_group_ops, Just down_m20260918_badge_group_ops), ("20260925_badge_store_receipts", m20260925_badge_store_receipts, Just down_m20260925_badge_store_receipts), - ("20261012_badge_subscriptions", m20261012_badge_subscriptions, Just down_m20261012_badge_subscriptions) + ("20261012_badge_subscriptions", m20261012_badge_subscriptions, Just down_m20261012_badge_subscriptions), + ("20261013_names", m20261013_names, Just down_m20261013_names) ] -- | The client tables share this database, so the service tables are the same names behind a prefix. @@ -272,6 +273,183 @@ ALTER TABLE @badge_ledger DROP COLUMN charge_id; DROP TABLE @subscription_charges; |] +-- A registration is live as Names.Store's orderLive decides. +-- reserved_by is added after the registrations table, which references the codes table. +m20261013_names :: Text +m20261013_names = + withPrefix + servicePrefix + [r| +CREATE TABLE @name_prepaid_codes( + name_prepaid_code_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + code_hash BYTEA NOT NULL, + min_length INTEGER NOT NULL, + years INTEGER NOT NULL, + code_payment_status TEXT NOT NULL, + reserved_at TIMESTAMPTZ, + spent_at TIMESTAMPTZ, + revoked_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + UNIQUE(code_hash) +); + +CREATE TABLE @name_registrations( + registration_id TEXT NOT NULL PRIMARY KEY, + version BIGINT NOT NULL, + label_hash BYTEA NOT NULL, + length INTEGER NOT NULL, + duration BIGINT NOT NULL, + owner BYTEA NOT NULL, + commitment BYTEA NOT NULL, + sealed TEXT NOT NULL, + phase TEXT NOT NULL, + years INTEGER NOT NULL, + price INTEGER NOT NULL, + method TEXT, + name_prepaid_code_id BIGINT REFERENCES @name_prepaid_codes, + label TEXT, + secret BYTEA, + contact_link TEXT, + channel_link TEXT, + intent_nonce BIGINT, + intent_deadline BIGINT, + intent_signature BYTEA, + commit_tx BYTEA, + commit_block BIGINT, + register_tx BYTEA, + register_block BIGINT, + reveal_after BIGINT, + reveal_before BIGINT, + expires_at BIGINT, + issued_code_id BIGINT REFERENCES @name_prepaid_codes, + registered_at TIMESTAMPTZ, + failed_at TIMESTAMPTZ, + error TEXT, + live SMALLINT NOT NULL, + order_hash BYTEA NOT NULL, + moves INTEGER NOT NULL DEFAULT 0, + links_refused_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +ALTER TABLE @name_prepaid_codes ADD COLUMN reserved_by TEXT REFERENCES @name_registrations; + +CREATE UNIQUE INDEX @idx_name_prepaid_codes_reserved_by ON @name_prepaid_codes(reserved_by); + +CREATE UNIQUE INDEX @idx_name_registrations_live_commitment ON @name_registrations(commitment) WHERE live = 1; + +CREATE UNIQUE INDEX @idx_name_registrations_live_owner_label ON @name_registrations(owner, label_hash) WHERE live = 1; + +CREATE INDEX @idx_name_registrations_label_phase ON @name_registrations(label_hash, phase); + +CREATE INDEX @idx_name_registrations_owner ON @name_registrations(owner, created_at); + +CREATE INDEX @idx_name_registrations_phase ON @name_registrations(phase, created_at); + +CREATE INDEX @idx_name_registrations_order_hash ON @name_registrations(order_hash); + +CREATE INDEX @idx_name_registrations_prepaid_code ON @name_registrations(name_prepaid_code_id); + +CREATE INDEX @idx_name_registrations_issued_code ON @name_registrations(issued_code_id); + +CREATE TABLE @name_invoices( + invoice_id TEXT NOT NULL PRIMARY KEY REFERENCES @invoices ON DELETE CASCADE, + registration_id TEXT NOT NULL REFERENCES @name_registrations ON DELETE CASCADE, + provider_ref TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE UNIQUE INDEX @idx_name_invoices_registration ON @name_invoices(registration_id); + +CREATE UNIQUE INDEX @idx_name_invoices_provider_ref ON @name_invoices(provider_ref); + +CREATE TABLE @name_record_edits( + name_record_edit_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + label_hash BYTEA NOT NULL, + owner BYTEA NOT NULL, + nonce BIGINT NOT NULL, + record_key TEXT, + record_value TEXT, + deadline BIGINT NOT NULL, + signature BYTEA NOT NULL, + body_hash BYTEA NOT NULL, + state TEXT NOT NULL, + revert TEXT, + mined_tx BYTEA, + created_at TIMESTAMPTZ NOT NULL +); + +-- A reverted edit keeps its row for its repeats while a new body reuses the nonce. +CREATE UNIQUE INDEX @idx_name_record_edits_owner_nonce ON @name_record_edits(owner, nonce) WHERE state IN ('pending', 'mined'); + +CREATE INDEX @idx_name_record_edits_body_hash ON @name_record_edits(body_hash); + +CREATE INDEX @idx_name_record_edits_label_created ON @name_record_edits(label_hash, created_at); + +CREATE TABLE @name_transactions( + name_transaction_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + registration_id TEXT REFERENCES @name_registrations, + name_record_edit_id BIGINT REFERENCES @name_record_edits, + kind TEXT NOT NULL, + raw BYTEA NOT NULL, + tx_hash BYTEA NOT NULL, + registrar_nonce BIGINT NOT NULL, + sent_at TIMESTAMPTZ NOT NULL, + mined_block BIGINT, + status TEXT NOT NULL, + CHECK ((registration_id IS NULL) <> (name_record_edit_id IS NULL)), + UNIQUE(tx_hash) +); + +CREATE INDEX @idx_name_transactions_registration ON @name_transactions(registration_id); + +CREATE INDEX @idx_name_transactions_edit ON @name_transactions(name_record_edit_id); + +CREATE INDEX @idx_name_transactions_status ON @name_transactions(status, registrar_nonce); +|] + +-- A table another table's foreign key references cannot be dropped, so reserved_by goes first. +-- The name orders' invoices go with them, except those with a payment, which stay for the accounts. +down_m20261013_names :: Text +down_m20261013_names = + withPrefix + servicePrefix + [r| +DROP INDEX @idx_name_transactions_status; +DROP INDEX @idx_name_transactions_edit; +DROP INDEX @idx_name_transactions_registration; +DROP TABLE @name_transactions; + +DROP INDEX @idx_name_record_edits_label_created; +DROP INDEX @idx_name_record_edits_body_hash; +DROP INDEX @idx_name_record_edits_owner_nonce; +DROP TABLE @name_record_edits; + +DELETE FROM @invoices WHERE invoice_id IN ( + SELECT ni.invoice_id FROM @name_invoices ni + WHERE NOT EXISTS (SELECT 1 FROM @payments p WHERE p.invoice_id = ni.invoice_id) +); +DROP INDEX @idx_name_invoices_provider_ref; +DROP INDEX @idx_name_invoices_registration; +DROP TABLE @name_invoices; + +DROP INDEX @idx_name_registrations_issued_code; +DROP INDEX @idx_name_registrations_prepaid_code; +DROP INDEX @idx_name_registrations_order_hash; +DROP INDEX @idx_name_registrations_phase; +DROP INDEX @idx_name_registrations_owner; +DROP INDEX @idx_name_registrations_label_phase; +DROP INDEX @idx_name_registrations_live_owner_label; +DROP INDEX @idx_name_registrations_live_commitment; +DROP INDEX @idx_name_prepaid_codes_reserved_by; +ALTER TABLE @name_prepaid_codes DROP COLUMN reserved_by; +DROP TABLE @name_registrations; + +DROP TABLE @name_prepaid_codes; +|] + {- TODO [badges] deferred with the draft in M20260915_user_badges, service only. ALTER TABLE @payments ADD COLUMN receipt_hash BYTEA; diff --git a/apps/simplex-store-service/src/StoreService/Store/SQLite/Migrations.hs b/apps/simplex-store-service/src/StoreService/Store/SQLite/Migrations.hs index 396429f39e..600d625a61 100644 --- a/apps/simplex-store-service/src/StoreService/Store/SQLite/Migrations.hs +++ b/apps/simplex-store-service/src/StoreService/Store/SQLite/Migrations.hs @@ -2,7 +2,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} -module StoreService.Store.SQLite.Migrations (storeServiceSchemaMigrations, servicePrefix, withPrefix) where +module StoreService.Store.SQLite.Migrations (storeServiceSchemaMigrations, schemaMigrations, servicePrefix, withPrefix) where import Data.List (sortOn) import Data.Text (Text) @@ -21,7 +21,8 @@ schemaMigrations = [ ("20260915_badge_service_schema", m20260915_badge_service_schema, Just down_m20260915_badge_service_schema), ("20260918_badge_group_ops", m20260918_badge_group_ops, Just down_m20260918_badge_group_ops), ("20260925_badge_store_receipts", m20260925_badge_store_receipts, Just down_m20260925_badge_store_receipts), - ("20261012_badge_subscriptions", m20261012_badge_subscriptions, Just down_m20261012_badge_subscriptions) + ("20261012_badge_subscriptions", m20261012_badge_subscriptions, Just down_m20261012_badge_subscriptions), + ("20261013_names", m20261013_names, Just down_m20261013_names) ] -- | The client tables share this database, so the service tables are the same names behind a prefix. @@ -273,6 +274,184 @@ ALTER TABLE @badge_ledger DROP COLUMN charge_id; DROP TABLE @subscription_charges; |] +-- A registration is live as Names.Store's orderLive decides. +-- reserved_by is added after the registrations table, which references the codes table. +m20261013_names :: Query +m20261013_names = + withPrefix + servicePrefix + [sql| +CREATE TABLE @name_prepaid_codes( + name_prepaid_code_id INTEGER PRIMARY KEY AUTOINCREMENT, + code_hash BLOB NOT NULL, + min_length INTEGER NOT NULL, + years INTEGER NOT NULL, + code_payment_status TEXT NOT NULL, + reserved_at TEXT, + spent_at TEXT, + revoked_at TEXT, + expires_at TEXT, + created_at TEXT NOT NULL, + UNIQUE(code_hash) +) STRICT; + +CREATE TABLE @name_registrations( + registration_id TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + label_hash BLOB NOT NULL, + length INTEGER NOT NULL, + duration INTEGER NOT NULL, + owner BLOB NOT NULL, + commitment BLOB NOT NULL, + sealed TEXT NOT NULL, + phase TEXT NOT NULL, + years INTEGER NOT NULL, + price INTEGER NOT NULL, + method TEXT, + name_prepaid_code_id INTEGER REFERENCES @name_prepaid_codes, + label TEXT, + secret BLOB, + contact_link TEXT, + channel_link TEXT, + intent_nonce INTEGER, + intent_deadline INTEGER, + intent_signature BLOB, + commit_tx BLOB, + commit_block INTEGER, + register_tx BLOB, + register_block INTEGER, + reveal_after INTEGER, + reveal_before INTEGER, + expires_at INTEGER, + issued_code_id INTEGER REFERENCES @name_prepaid_codes, + registered_at TEXT, + failed_at TEXT, + error TEXT, + live INTEGER NOT NULL, + order_hash BLOB NOT NULL, + moves INTEGER NOT NULL DEFAULT 0, + links_refused_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +) STRICT; + +ALTER TABLE @name_prepaid_codes ADD COLUMN reserved_by TEXT REFERENCES @name_registrations; + +CREATE UNIQUE INDEX @idx_name_prepaid_codes_reserved_by ON @name_prepaid_codes(reserved_by); + +CREATE UNIQUE INDEX @idx_name_registrations_live_commitment ON @name_registrations(commitment) WHERE live = 1; + +CREATE UNIQUE INDEX @idx_name_registrations_live_owner_label ON @name_registrations(owner, label_hash) WHERE live = 1; + +CREATE INDEX @idx_name_registrations_label_phase ON @name_registrations(label_hash, phase); + +CREATE INDEX @idx_name_registrations_owner ON @name_registrations(owner, created_at); + +CREATE INDEX @idx_name_registrations_phase ON @name_registrations(phase, created_at); + +CREATE INDEX @idx_name_registrations_order_hash ON @name_registrations(order_hash); + +CREATE INDEX @idx_name_registrations_prepaid_code ON @name_registrations(name_prepaid_code_id); + +CREATE INDEX @idx_name_registrations_issued_code ON @name_registrations(issued_code_id); + +CREATE TABLE @name_invoices( + invoice_id TEXT NOT NULL PRIMARY KEY REFERENCES @invoices ON DELETE CASCADE, + registration_id TEXT NOT NULL REFERENCES @name_registrations ON DELETE CASCADE, + provider_ref TEXT NOT NULL, + created_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX @idx_name_invoices_registration ON @name_invoices(registration_id); + +CREATE UNIQUE INDEX @idx_name_invoices_provider_ref ON @name_invoices(provider_ref); + +CREATE TABLE @name_record_edits( + name_record_edit_id INTEGER PRIMARY KEY AUTOINCREMENT, + label_hash BLOB NOT NULL, + owner BLOB NOT NULL, + nonce INTEGER NOT NULL, + record_key TEXT, + record_value TEXT, + deadline INTEGER NOT NULL, + signature BLOB NOT NULL, + body_hash BLOB NOT NULL, + state TEXT NOT NULL, + revert TEXT, + mined_tx BLOB, + created_at TEXT NOT NULL +) STRICT; + +-- A reverted edit keeps its row for its repeats while a new body reuses the nonce. +CREATE UNIQUE INDEX @idx_name_record_edits_owner_nonce ON @name_record_edits(owner, nonce) WHERE state IN ('pending', 'mined'); + +CREATE INDEX @idx_name_record_edits_body_hash ON @name_record_edits(body_hash); + +CREATE INDEX @idx_name_record_edits_label_created ON @name_record_edits(label_hash, created_at); + +CREATE TABLE @name_transactions( + name_transaction_id INTEGER PRIMARY KEY AUTOINCREMENT, + registration_id TEXT REFERENCES @name_registrations, + name_record_edit_id INTEGER REFERENCES @name_record_edits, + kind TEXT NOT NULL, + raw BLOB NOT NULL, + tx_hash BLOB NOT NULL, + registrar_nonce INTEGER NOT NULL, + sent_at TEXT NOT NULL, + mined_block INTEGER, + status TEXT NOT NULL, + CHECK ((registration_id IS NULL) <> (name_record_edit_id IS NULL)), + UNIQUE(tx_hash) +) STRICT; + +CREATE INDEX @idx_name_transactions_registration ON @name_transactions(registration_id); + +CREATE INDEX @idx_name_transactions_edit ON @name_transactions(name_record_edit_id); + +CREATE INDEX @idx_name_transactions_status ON @name_transactions(status, registrar_nonce); +|] + +-- SQLite refuses to drop a column in a foreign key, and dropping a table deletes its rows first, +-- so the codes' references to registrations are cleared before the registrations go. +-- The name orders' invoices go with them, except those with a payment, which stay for the accounts. +down_m20261013_names :: Query +down_m20261013_names = + withPrefix + servicePrefix + [sql| +DROP INDEX @idx_name_transactions_status; +DROP INDEX @idx_name_transactions_edit; +DROP INDEX @idx_name_transactions_registration; +DROP TABLE @name_transactions; + +DROP INDEX @idx_name_record_edits_label_created; +DROP INDEX @idx_name_record_edits_body_hash; +DROP INDEX @idx_name_record_edits_owner_nonce; +DROP TABLE @name_record_edits; + +DELETE FROM @invoices WHERE invoice_id IN ( + SELECT ni.invoice_id FROM @name_invoices ni + WHERE NOT EXISTS (SELECT 1 FROM @payments p WHERE p.invoice_id = ni.invoice_id) +); +DROP INDEX @idx_name_invoices_provider_ref; +DROP INDEX @idx_name_invoices_registration; +DROP TABLE @name_invoices; + +DROP INDEX @idx_name_registrations_issued_code; +DROP INDEX @idx_name_registrations_prepaid_code; +DROP INDEX @idx_name_registrations_order_hash; +DROP INDEX @idx_name_registrations_phase; +DROP INDEX @idx_name_registrations_owner; +DROP INDEX @idx_name_registrations_label_phase; +DROP INDEX @idx_name_registrations_live_owner_label; +DROP INDEX @idx_name_registrations_live_commitment; +DROP INDEX @idx_name_prepaid_codes_reserved_by; +UPDATE @name_prepaid_codes SET reserved_by = NULL; +DROP TABLE @name_registrations; + +DROP TABLE @name_prepaid_codes; +|] + {- TODO [badges] deferred with the draft in M20260915_user_badges, service only. ALTER TABLE @payments ADD COLUMN receipt_hash BLOB; diff --git a/tests/Bots/StoreService/WebTests.hs b/tests/Bots/StoreService/WebTests.hs index c900f7f8f6..2bd6e0fe6a 100644 --- a/tests/Bots/StoreService/WebTests.hs +++ b/tests/Bots/StoreService/WebTests.hs @@ -88,12 +88,12 @@ import Text.Read (readMaybe) import UnliftIO.Temporary (withTempDirectory) #if defined(dbPostgres) -import StoreService.Store.Postgres.Migrations (storeServiceSchemaMigrations) +import StoreService.Store.Postgres.Migrations (schemaMigrations, storeServiceSchemaMigrations) import ChatClient (testDBConnstr) import Database.PostgreSQL.Simple (Only (..)) import qualified Simplex.Messaging.Agent.Store.Postgres.Migrations as Migrations #else -import StoreService.Store.SQLite.Migrations (storeServiceSchemaMigrations) +import StoreService.Store.SQLite.Migrations (schemaMigrations, storeServiceSchemaMigrations) import Data.String (fromString) import Database.SQLite.Simple (Only (..)) import qualified Database.SQLite.Simple as SQL @@ -149,6 +149,7 @@ badgeWebTests = do it "refuses a duplicate provider_ref" testProviderRefUnique it "M20260918 adds the multi-use columns and the group table" testGroupOpsColumns it "M20260918 defaults a fresh code to one use with none spent" testGroupOpsRedeemCounts + it "declares its migrations in name order, master's badge migrations before the names ones" testMigrationOrder it "migrates all the way down and up again" testSchemaDownUpCycle it "rolls back only the group migration and re-applies it, keeping a code redeemed twice spent" testGroupOpsDownUp it "rolls back only the subscription migration, restoring the columns it dropped, and re-applies it" testSubscriptionsDownUp @@ -365,6 +366,18 @@ runMigrations st = Migrations.run st Nothing runMigrations st = Migrations.run st Nothing True #endif +-- | Applied migrations must be a prefix of this list by name, so a migration of ours dated +-- before one of master's breaks upgrading a badge-service database. +testMigrationOrder :: IO () +testMigrationOrder = do + let declared = map (\(n, _, _) -> n) schemaMigrations + [(a, b) | (a, b) <- zip declared (drop 1 declared), a >= b] `shouldBe` [] + let (badges, ours) = span (`elem` badgeMigrations) (map name storeServiceSchemaMigrations) + badges `shouldBe` badgeMigrations + ours `shouldBe` ["20261013_names"] + where + badgeMigrations = ["20260915_badge_service_schema", "20260918_badge_group_ops", "20260925_badge_store_receipts", "20261012_badge_subscriptions"] + testSchemaDownUpCycle :: IO () testSchemaDownUpCycle = withServiceStore $ \st -> do let downMigrations = mapMaybe toDownMigration storeServiceSchemaMigrations