diff --git a/bots/src/API/Docs/Commands.hs b/bots/src/API/Docs/Commands.hs index 3435bdb6e5..3e1dd00908 100644 --- a/bots/src/API/Docs/Commands.hs +++ b/bots/src/API/Docs/Commands.hs @@ -370,6 +370,7 @@ undocumentedCommands = "APICreateWallet", "APIDeleteChatTag", "APIDeleteWallet", + "APIScanWallet", "APIDeleteMemberSupportChat", "APIDeleteReceivedReports", "APIDeleteStorage", diff --git a/cabal.project b/cabal.project index 49f4cb95d0..3d7bef9d2a 100644 --- a/cabal.project +++ b/cabal.project @@ -21,7 +21,7 @@ constraints: zip +disable-bzip2 +disable-zstd source-repository-package type: git location: https://github.com/simplex-chat/simplexmq.git - tag: 7dcd89d30bdf06913b03f3884bad987b75a6744a + tag: 3ae71f06b44e70d25b7eb7c604c252bd9b7afdf5 source-repository-package type: git diff --git a/scripts/nix/sha256map.nix b/scripts/nix/sha256map.nix index 69aca6ab3e..53eadcbbbe 100644 --- a/scripts/nix/sha256map.nix +++ b/scripts/nix/sha256map.nix @@ -1,5 +1,5 @@ { - "https://github.com/simplex-chat/simplexmq.git"."7dcd89d30bdf06913b03f3884bad987b75a6744a" = "0357ak8sxdm6vh103gjdvd073w6lsa2yc8mv1iis937w270siwqn"; + "https://github.com/simplex-chat/simplexmq.git"."3ae71f06b44e70d25b7eb7c604c252bd9b7afdf5" = "05fxyx5xxyngg68kd6zpmp5vvk15gk6bwab04sbljkgkvyfvnn5w"; "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38"; "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d"; "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl"; diff --git a/src/Simplex/Chat/Controller.hs b/src/Simplex/Chat/Controller.hs index ae873662e0..b252cfd0aa 100644 --- a/src/Simplex/Chat/Controller.hs +++ b/src/Simplex/Chat/Controller.hs @@ -443,6 +443,7 @@ data ChatCommand | APIGetWalletAddress {accountIndex_ :: Maybe AccountIndex} | APIExportWalletMnemonic | APIExportWalletAccount {accountIndex :: AccountIndex} + | APIScanWallet | APIDeleteWallet | APISendCallInvitation ContactId CallType | SendCallInvitation ContactName CallType diff --git a/src/Simplex/Chat/Library/Commands.hs b/src/Simplex/Chat/Library/Commands.hs index dceee7b794..0681cd27b9 100644 --- a/src/Simplex/Chat/Library/Commands.hs +++ b/src/Simplex/Chat/Library/Commands.hs @@ -35,7 +35,7 @@ import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy.Char8 as LB import Data.Char import Data.Constraint (Dict (..)) -import Data.Either (fromRight, partitionEithers, rights) +import Data.Either (fromRight, isLeft, partitionEithers, rights) import Data.Foldable (foldr') import Data.Functor (($>)) import Data.Functor.Identity (Identity (..), runIdentity) @@ -65,9 +65,10 @@ import Simplex.Chat.Badges.Types (BadgeAlert (..), BadgeAlertKind (..), BadgeSta import Simplex.Chat.Badges.Code (badgeCodeText, parseBadgeCode) import Simplex.Chat.Badges.Service (BadgeBalance (..), BadgeServiceCommand (..), BadgeServiceErrorCode (..), BadgeServiceRequest (..), BadgeServiceResponse (..), BadgeStatement (..), StatementDebitType (..), StatementEntry (..), StatementEntryType (..), currentBadgeServiceVersion) import Simplex.Chat.Names (SimplexDomainProof (..), SimplexDomainClaim (..), claimDomain, mkDomainClaim) -import Simplex.Chat.Store.Wallets (WalletSeed (..), accountHeldByOther, bindAccount, createWalletSeed, deleteWalletSeed, getUserAccounts, getWalletSeed, resolveAccount) -import Simplex.Chat.Wallet (AccountIndex, AccountKey, WalletAddress (..), WalletError (..), accountSecret, deriveAccountKey, entropyFromMnemonic, newSeedEntropy, renderAccountPath, seedMaster, seedMnemonic) +import Simplex.Chat.Store.Wallets (WalletSeed (..), accountHeldByOther, bindAccount, createWalletSeed, deleteWalletSeed, getUserAccounts, getWalletSeed, recordScan, resolveAccount) +import Simplex.Chat.Wallet (AccountIndex, AccountKey, WalletAddress (..), WalletError (..), accountSecret, checkAccountIndex, deriveAccountKey, entropyFromMnemonic, newSeedEntropy, renderAccountPath, scanGapLimit, seedMaster, seedMnemonic) import Simplex.Messaging.Eth.Address (addressFromPrivateKey) +import Simplex.Messaging.Names.Record (OwnedNames (..)) import Simplex.Chat.Call import Simplex.Chat.Controller import Simplex.Chat.Delivery (DeliveryJobScope (..), DeliveryJobSpec (..), DeliveryWorkerScope (..)) @@ -1526,6 +1527,11 @@ processChatCommand cxt nm = \case when heldByOther $ throwWalletError WEAccountBound k <- accountKey seed n pure $ CRWalletAccountSecret user (accountAddress n k) (accountSecret k) + APIScanWallet -> withUser $ \user -> do + seed <- walletSeed + (inUse, next) <- scanAccounts nm user seed + withFastStore' $ \db -> recordScan db (wsId seed) inUse next + processChatCommand cxt nm APIGetWallet APIDeleteWallet -> withUser $ \_ -> do deleted <- withFastStore' deleteWalletSeed unless deleted $ throwWalletError WENoMaster @@ -5963,6 +5969,26 @@ liftWallet = either throwWalletError pure accountKey :: WalletSeed -> AccountIndex -> CM AccountKey accountKey seed n = liftWallet $ seedMaster (wsEntropy seed) >>= (`deriveAccountKey` n) +-- | Walk the seed's accounts until 'scanGapLimit' in a row are untouched, +-- asking each account on a relay the scan has not used yet: one relay seeing +-- every address would learn they are one wallet. The counter lands past the +-- last account in use, so the next name takes a free one. +scanAccounts :: NetworkRequestMode -> User -> WalletSeed -> CM ([AccountIndex], AccountIndex) +scanAccounts nm user seed = go 0 [] [] 0 + where + go n used found gap + | gap >= scanGapLimit || isLeft (checkAccountIndex n) = pure (reverse found, next) + | otherwise = do + addr <- addressFromPrivateKey <$> accountKey seed n + (srv, owned) <- withAgent $ \a -> ownedSimplexNames a nm (aUserId user) used addr 0 + if ownInUse owned + then go (n + 1) (srv : used) (n : found) 0 + else go (n + 1) (srv : used) found (gap + 1) + where + next = case found of + latest : _ -> latest + 1 + [] -> 0 + accountAddress :: AccountIndex -> AccountKey -> WalletAddress accountAddress n k = WalletAddress {accountIndex = n, keyPath = renderAccountPath n, address = decodeLatin1 . strEncode $ addressFromPrivateKey k} @@ -6097,6 +6123,7 @@ chatCommandP = "/_wallet address" $> APIGetWalletAddress Nothing, "/_wallet export master" $> APIExportWalletMnemonic, "/_wallet export account " *> (APIExportWalletAccount <$> accountIndexP), + "/_wallet scan" $> APIScanWallet, "/_wallet delete" $> APIDeleteWallet, "/_wallet" $> APIGetWallet, "/_call invite @" *> (APISendCallInvitation <$> A.decimal <* A.space <*> jsonP), diff --git a/src/Simplex/Chat/Store/Wallets.hs b/src/Simplex/Chat/Store/Wallets.hs index d6589b464c..726224cf40 100644 --- a/src/Simplex/Chat/Store/Wallets.hs +++ b/src/Simplex/Chat/Store/Wallets.hs @@ -16,6 +16,7 @@ module Simplex.Chat.Store.Wallets getUserAccounts, accountHeldByOther, bindAccount, + recordScan, ) where @@ -144,6 +145,28 @@ setAccountUser db sId userId n = accountHeldBy :: DB.Connection -> SeedId -> UserId -> AccountIndex -> IO Bool accountHeldBy db sId userId n = (== Just (Just userId)) <$> accountUser db sId n +-- | What a scan found: the accounts in use, bound to the first profile because the chain does not say whose they are, and the counter past them. +recordScan :: DB.Connection -> SeedId -> [AccountIndex] -> AccountIndex -> IO () +recordScan db sId inUse next = do + mapM_ insertScanned inUse + DB.execute + db + [sql| + UPDATE wallet_seeds SET next_account_index = ? + WHERE wallet_seed_id = ? AND (next_account_index IS NULL OR next_account_index < ?) + |] + (accountIndexCol next, sId, accountIndexCol next) + where + insertScanned n = + DB.execute + db + [sql| + INSERT INTO wallet_accounts (wallet_seed_id, account_index, user_id) + VALUES (?, ?, (SELECT min(user_id) FROM users)) + ON CONFLICT (wallet_seed_id, account_index) DO NOTHING + |] + (sId, accountIndexCol n) + -- | Keep the counter a high-water mark. Never lowers it, never gives one to an imported phrase that has none. raiseNextAccount :: DB.Connection -> SeedId -> AccountIndex -> IO () raiseNextAccount db sId n = diff --git a/src/Simplex/Chat/Wallet.hs b/src/Simplex/Chat/Wallet.hs index 2e7fef8a33..2324b2070d 100644 --- a/src/Simplex/Chat/Wallet.hs +++ b/src/Simplex/Chat/Wallet.hs @@ -15,6 +15,7 @@ module Simplex.Chat.Wallet deriveAccountKey, accountSecret, checkAccountIndex, + scanGapLimit, ) where @@ -57,6 +58,11 @@ data WalletError | WEDerivation {derivationError :: String} -- BIP-32 or BIP-39 said no deriving (Eq, Show) +-- | BIP-44's gap limit: a scan of a recovered seed stops after this many +-- accounts in a row that nothing on chain has used. +scanGapLimit :: Int +scanGapLimit = 20 + -- | Refuse an index at or above 2^31: BIP-32 would harden it onto another index's key. checkAccountIndex :: AccountIndex -> Either WalletError () checkAccountIndex n = if n >= B32.hardenedOffset then Left WEIndexTooLarge else Right () diff --git a/tests/NameResolver.hs b/tests/NameResolver.hs index 3020e95833..5f61524684 100644 --- a/tests/NameResolver.hs +++ b/tests/NameResolver.hs @@ -9,6 +9,7 @@ module NameResolver ( NameRegistry, withNameResolver, registerName, + ownedName, contactNameRecord, channelNameRecord, contactAndChannelNameRecord, @@ -26,9 +27,10 @@ import Network.HTTP.Types (hContentType, notFound404, ok200) import Network.Wai (Application, pathInfo, responseLBS) import qualified Network.Wai.Handler.Warp as Warp import Simplex.Messaging.Encoding.String (strEncode) -import Simplex.Messaging.Names.Record (NamePricing (..), NameRecord (..), NameRegistration (..), NameResponse (..), USDCents (..)) +import Simplex.Messaging.Names.Record (NamePricing (..), NameRecord (..), NameRegistration (..), NameResponse (..), OwnedName (..), OwnedNames (..), USDCents (..)) import Simplex.Messaging.Server.Names (NamesConfig (..)) import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexNameInfo (..), labelHash) +import Simplex.Messaging.SystemTime (RoundedSystemTime (..)) type NameRegistry = TVar (Map Text NameRecord) @@ -44,16 +46,25 @@ withNameResolver action = do (st, body) <- case pathInfo req of ["health"] -> pure (ok200, "{}") ["v2", "resolve", q] -> (\r -> (ok200, J.encode $ nameResponse r)) . M.lookup q <$> readTVarIO reg + ["v2", "owned-by", addr] -> (\rs -> (ok200, J.encode $ ownedNames addr rs)) . M.elems <$> readTVarIO reg _ -> pure (notFound404, "{}") send $ responseLBS st [(hContentType, "application/json")] body nameResponse (Just nameRecord) = NameResponse {lastBlockTs = Nothing, registration = NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Nothing, nameRecord}} nameResponse Nothing = NameResponse {lastBlockTs = Nothing, registration = NRAvailable {pricing = NamePricing {registrationPrices = M.empty, basePrice = USDCents 1000, minLabelLength = 1}}} + -- an account is in use when it owns a name, the only thing this resolver knows about + ownedNames addr rs = + let ns = [OwnedName {onName = Just nrName, onLabelhash = "", onExpires = RoundedSystemTime 0, onStatus = "registered"} | NameRecord {nrName, nrOwner} <- rs, nrOwner == addr] + in OwnedNames {ownNames = ns, ownInUse = not (null ns), ownNextOffset = Nothing} -- | Register a name's domain to resolve to the given record. registerName :: TVar (Map Text NameRecord) -> SimplexNameInfo -> NameRecord -> IO () registerName reg SimplexNameInfo {nameDomain = SimplexDomain {nameTLD, domain}} r = atomically $ modifyTVar' reg $ M.insert (decodeLatin1 $ strEncode (labelHash domain) <> strEncode nameTLD) r +-- | Register a name an address owns, for a wallet scan to find. +ownedName :: NameRegistry -> Text -> Text -> IO () +ownedName reg name owner = atomically $ modifyTVar' reg $ M.insert name (emptyRecord name) {nrOwner = owner} + contactNameRecord :: Text -> Text -> NameRecord contactNameRecord name link = (emptyRecord name) {nrSimplexContact = [link]} diff --git a/tests/Test.hs b/tests/Test.hs index 580eea5944..4acbc500df 100644 --- a/tests/Test.hs +++ b/tests/Test.hs @@ -77,6 +77,7 @@ main = do describe "Random servers" randomServersTests #if !defined(dbPostgres) around (tmpTestBracket chatQueryStats agentQueryStats) $ describe "names tests" chatNamesTests + around (tmpTestBracket chatQueryStats agentQueryStats) $ describe "Wallet scan" walletScanTests around (tmpTestBracket chatQueryStats agentQueryStats) $ xdescribe'' "SimpleX Directory names" directoryNameTests #endif #if defined(dbPostgres) diff --git a/tests/WalletTests.hs b/tests/WalletTests.hs index c134f6bb80..d94d8ba6e1 100644 --- a/tests/WalletTests.hs +++ b/tests/WalletTests.hs @@ -13,7 +13,9 @@ import qualified Data.ByteString.Char8 as B import Data.Char (toUpper) import Data.Either (isRight) import Data.List (nub) +import Data.Text (Text) import qualified Data.Text as T +import NameResolver (ownedName) import Simplex.Chat.Wallet (AccountIndex, WalletError (..), accountSecret, deriveAccountKey, entropyFromMnemonic, renderAccountPath, seedMaster, seedMnemonic) import qualified Simplex.Messaging.Crypto.BIP39 as B39 import qualified Simplex.Messaging.Crypto.Secp256k1 as S @@ -42,6 +44,10 @@ addressFromSecret secret = show . addressFromPrivateKey . either error id . S.mkPrivateKey . either error id $ BAE.convertFromBase BAE.Base16 (B.drop 2 $ B.pack secret) +-- | The address of an account of the imported phrase, as the chain would hold it. +accountAddress :: AccountIndex -> Text +accountAddress = T.pack . show . addressFromPrivateKey . accountKey (seedEntropy testPhrase24) + -- | An @export account@ row: the index, the path, the address, the secret. exportRow :: HasCallStack => String -> (String, String, String, String) exportRow row = case words row of @@ -94,6 +100,23 @@ walletTests = do it "will not export an account another profile holds" testWalletExportNotMine it "refuses an index BIP-32 cannot harden, on every command" testWalletIndexTooLarge +-- | Its own group: the scan needs a names resolver, so it runs without the SMP server the other wallet tests share. +walletScanTests :: SpecWith TestParams +walletScanTests = + it "a scan of a recovered phrase finds the accounts in use" testWalletScan + +testWalletScan :: HasCallStack => TestParams -> IO () +testWalletScan ps = withSmpServerAndNames $ \reg -> withNewTestChat ps "alice" aliceProfile $ \alice -> do + enableNamesRole alice + alice ##> ("/_wallet create mnemonic=" <> B.unpack testPhrase24) + alice <## "wallet, no accounts for this profile" + ownedName reg "alice.simplex" (accountAddress 1) + alice ##> "/_wallet scan" + alice <## "accounts: 1" + -- the scan gives the imported phrase the counter it had none of + alice ##> "/_wallet bind" + alice <## "accounts: 1, 2" + testWalletCreate :: HasCallStack => TestParams -> IO () testWalletCreate ps = withNewTestChat ps "alice" aliceProfile $ \alice -> do alice ##> "/_wallet"