scan for names owned by wallet

bots/src/API/Docs/Commands.hs

@@ -370,6 +370,7 @@ undocumentedCommands =
     "APICreateWallet",
     "APIDeleteChatTag",
     "APIDeleteWallet",
+    "APIScanWallet",
     "APIDeleteMemberSupportChat",
     "APIDeleteReceivedReports",
     "APIDeleteStorage",

cabal.project

@@ -21,7 +21,7 @@ constraints: zip +disable-bzip2 +disable-zstd
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: 7dcd89d30bdf06913b03f3884bad987b75a6744a
+    tag: 3ae71f06b44e70d25b7eb7c604c252bd9b7afdf5
 
 source-repository-package
     type: git

scripts/nix/sha256map.nix

@@ -1,5 +1,5 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."7dcd89d30bdf06913b03f3884bad987b75a6744a" = "0357ak8sxdm6vh103gjdvd073w6lsa2yc8mv1iis937w270siwqn";
+  "https://github.com/simplex-chat/simplexmq.git"."3ae71f06b44e70d25b7eb7c604c252bd9b7afdf5" = "05fxyx5xxyngg68kd6zpmp5vvk15gk6bwab04sbljkgkvyfvnn5w";
   "https://github.com/simplex-chat/hs-socks.git"."a30cc7a79a08d8108316094f8f2f82a0c5e1ac51" = "0yasvnr7g91k76mjkamvzab2kvlb1g5pspjyjn2fr6v83swjhj38";
   "https://github.com/simplex-chat/direct-sqlcipher.git"."f814ee68b16a9447fbb467ccc8f29bdd3546bfd9" = "1ql13f4kfwkbaq7nygkxgw84213i0zm7c1a8hwvramayxl38dq5d";
   "https://github.com/simplex-chat/sqlcipher-simple.git"."a46bd361a19376c5211f1058908fc0ae6bf42446" = "1z0r78d8f0812kxbgsm735qf6xx8lvaz27k1a0b4a2m0sshpd5gl";

src/Simplex/Chat/Controller.hs

@@ -443,6 +443,7 @@ data ChatCommand
   | APIGetWalletAddress {accountIndex_ :: Maybe AccountIndex}
   | APIExportWalletMnemonic
   | APIExportWalletAccount {accountIndex :: AccountIndex}
+  | APIScanWallet
   | APIDeleteWallet
   | APISendCallInvitation ContactId CallType
   | SendCallInvitation ContactName CallType

src/Simplex/Chat/Library/Commands.hs

@@ -35,7 +35,7 @@ import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Char
 import Data.Constraint (Dict (..))
-import Data.Either (fromRight, partitionEithers, rights)
+import Data.Either (fromRight, isLeft, partitionEithers, rights)
 import Data.Foldable (foldr')
 import Data.Functor (($>))
 import Data.Functor.Identity (Identity (..), runIdentity)
@@ -65,9 +65,10 @@ import Simplex.Chat.Badges.Types (BadgeAlert (..), BadgeAlertKind (..), BadgeSta
 import Simplex.Chat.Badges.Code (badgeCodeText, parseBadgeCode)
 import Simplex.Chat.Badges.Service (BadgeBalance (..), BadgeServiceCommand (..), BadgeServiceErrorCode (..), BadgeServiceRequest (..), BadgeServiceResponse (..), BadgeStatement (..), StatementDebitType (..), StatementEntry (..), StatementEntryType (..), currentBadgeServiceVersion)
 import Simplex.Chat.Names (SimplexDomainProof (..), SimplexDomainClaim (..), claimDomain, mkDomainClaim)
-import Simplex.Chat.Store.Wallets (WalletSeed (..), accountHeldByOther, bindAccount, createWalletSeed, deleteWalletSeed, getUserAccounts, getWalletSeed, resolveAccount)
-import Simplex.Chat.Wallet (AccountIndex, AccountKey, WalletAddress (..), WalletError (..), accountSecret, deriveAccountKey, entropyFromMnemonic, newSeedEntropy, renderAccountPath, seedMaster, seedMnemonic)
+import Simplex.Chat.Store.Wallets (WalletSeed (..), accountHeldByOther, bindAccount, createWalletSeed, deleteWalletSeed, getUserAccounts, getWalletSeed, recordScan, resolveAccount)
+import Simplex.Chat.Wallet (AccountIndex, AccountKey, WalletAddress (..), WalletError (..), accountSecret, checkAccountIndex, deriveAccountKey, entropyFromMnemonic, newSeedEntropy, renderAccountPath, scanGapLimit, seedMaster, seedMnemonic)
 import Simplex.Messaging.Eth.Address (addressFromPrivateKey)
+import Simplex.Messaging.Names.Record (OwnedNames (..))
 import Simplex.Chat.Call
 import Simplex.Chat.Controller
 import Simplex.Chat.Delivery (DeliveryJobScope (..), DeliveryJobSpec (..), DeliveryWorkerScope (..))
@@ -1526,6 +1527,11 @@ processChatCommand cxt nm = \case
     when heldByOther $ throwWalletError WEAccountBound
     k <- accountKey seed n
     pure $ CRWalletAccountSecret user (accountAddress n k) (accountSecret k)
+  APIScanWallet -> withUser $ \user -> do
+    seed <- walletSeed
+    (inUse, next) <- scanAccounts nm user seed
+    withFastStore' $ \db -> recordScan db (wsId seed) inUse next
+    processChatCommand cxt nm APIGetWallet
   APIDeleteWallet -> withUser $ \_ -> do
     deleted <- withFastStore' deleteWalletSeed
     unless deleted $ throwWalletError WENoMaster
@@ -5963,6 +5969,26 @@ liftWallet = either throwWalletError pure
 accountKey :: WalletSeed -> AccountIndex -> CM AccountKey
 accountKey seed n = liftWallet $ seedMaster (wsEntropy seed) >>= (`deriveAccountKey` n)
 
+-- | Walk the seed's accounts until 'scanGapLimit' in a row are untouched,
+-- asking each account on a relay the scan has not used yet: one relay seeing
+-- every address would learn they are one wallet. The counter lands past the
+-- last account in use, so the next name takes a free one.
+scanAccounts :: NetworkRequestMode -> User -> WalletSeed -> CM ([AccountIndex], AccountIndex)
+scanAccounts nm user seed = go 0 [] [] 0
+  where
+    go n used found gap
+      | gap >= scanGapLimit || isLeft (checkAccountIndex n) = pure (reverse found, next)
+      | otherwise = do
+          addr <- addressFromPrivateKey <$> accountKey seed n
+          (srv, owned) <- withAgent $ \a -> ownedSimplexNames a nm (aUserId user) used addr 0
+          if ownInUse owned
+            then go (n + 1) (srv : used) (n : found) 0
+            else go (n + 1) (srv : used) found (gap + 1)
+      where
+        next = case found of
+          latest : _ -> latest + 1
+          [] -> 0
+
 accountAddress :: AccountIndex -> AccountKey -> WalletAddress
 accountAddress n k =
   WalletAddress {accountIndex = n, keyPath = renderAccountPath n, address = decodeLatin1 . strEncode $ addressFromPrivateKey k}
@@ -6097,6 +6123,7 @@ chatCommandP =
       "/_wallet address" $> APIGetWalletAddress Nothing,
       "/_wallet export master" $> APIExportWalletMnemonic,
       "/_wallet export account " *> (APIExportWalletAccount <$> accountIndexP),
+      "/_wallet scan" $> APIScanWallet,
       "/_wallet delete" $> APIDeleteWallet,
       "/_wallet" $> APIGetWallet,
       "/_call invite @" *> (APISendCallInvitation <$> A.decimal <* A.space <*> jsonP),

src/Simplex/Chat/Store/Wallets.hs

@@ -16,6 +16,7 @@ module Simplex.Chat.Store.Wallets
     getUserAccounts,
     accountHeldByOther,
     bindAccount,
+    recordScan,
   )
 where
 
@@ -144,6 +145,28 @@ setAccountUser db sId userId n =
 accountHeldBy :: DB.Connection -> SeedId -> UserId -> AccountIndex -> IO Bool
 accountHeldBy db sId userId n = (== Just (Just userId)) <$> accountUser db sId n
 
+-- | What a scan found: the accounts in use, bound to the first profile because the chain does not say whose they are, and the counter past them.
+recordScan :: DB.Connection -> SeedId -> [AccountIndex] -> AccountIndex -> IO ()
+recordScan db sId inUse next = do
+  mapM_ insertScanned inUse
+  DB.execute
+    db
+    [sql|
+      UPDATE wallet_seeds SET next_account_index = ?
+      WHERE wallet_seed_id = ? AND (next_account_index IS NULL OR next_account_index < ?)
+    |]
+    (accountIndexCol next, sId, accountIndexCol next)
+  where
+    insertScanned n =
+      DB.execute
+        db
+        [sql|
+          INSERT INTO wallet_accounts (wallet_seed_id, account_index, user_id)
+          VALUES (?, ?, (SELECT min(user_id) FROM users))
+          ON CONFLICT (wallet_seed_id, account_index) DO NOTHING
+        |]
+        (sId, accountIndexCol n)
+
 -- | Keep the counter a high-water mark. Never lowers it, never gives one to an imported phrase that has none.
 raiseNextAccount :: DB.Connection -> SeedId -> AccountIndex -> IO ()
 raiseNextAccount db sId n =

src/Simplex/Chat/Wallet.hs

@@ -15,6 +15,7 @@ module Simplex.Chat.Wallet
     deriveAccountKey,
     accountSecret,
     checkAccountIndex,
+    scanGapLimit,
   )
 where
 
@@ -57,6 +58,11 @@ data WalletError
   | WEDerivation {derivationError :: String} -- BIP-32 or BIP-39 said no
   deriving (Eq, Show)
 
+-- | BIP-44's gap limit: a scan of a recovered seed stops after this many
+-- accounts in a row that nothing on chain has used.
+scanGapLimit :: Int
+scanGapLimit = 20
+
 -- | Refuse an index at or above 2^31: BIP-32 would harden it onto another index's key.
 checkAccountIndex :: AccountIndex -> Either WalletError ()
 checkAccountIndex n = if n >= B32.hardenedOffset then Left WEIndexTooLarge else Right ()

tests/NameResolver.hs

@@ -9,6 +9,7 @@ module NameResolver
   ( NameRegistry,
     withNameResolver,
     registerName,
+    ownedName,
     contactNameRecord,
     channelNameRecord,
     contactAndChannelNameRecord,
@@ -26,9 +27,10 @@ import Network.HTTP.Types (hContentType, notFound404, ok200)
 import Network.Wai (Application, pathInfo, responseLBS)
 import qualified Network.Wai.Handler.Warp as Warp
 import Simplex.Messaging.Encoding.String (strEncode)
-import Simplex.Messaging.Names.Record (NamePricing (..), NameRecord (..), NameRegistration (..), NameResponse (..), USDCents (..))
+import Simplex.Messaging.Names.Record (NamePricing (..), NameRecord (..), NameRegistration (..), NameResponse (..), OwnedName (..), OwnedNames (..), USDCents (..))
 import Simplex.Messaging.Server.Names (NamesConfig (..))
 import Simplex.Messaging.SimplexName (SimplexDomain (..), SimplexNameInfo (..), labelHash)
+import Simplex.Messaging.SystemTime (RoundedSystemTime (..))
 
 type NameRegistry = TVar (Map Text NameRecord)
 
@@ -44,16 +46,25 @@ withNameResolver action = do
       (st, body) <- case pathInfo req of
         ["health"] -> pure (ok200, "{}")
         ["v2", "resolve", q] -> (\r -> (ok200, J.encode $ nameResponse r)) . M.lookup q <$> readTVarIO reg
+        ["v2", "owned-by", addr] -> (\rs -> (ok200, J.encode $ ownedNames addr rs)) . M.elems <$> readTVarIO reg
         _ -> pure (notFound404, "{}")
       send $ responseLBS st [(hContentType, "application/json")] body
     nameResponse (Just nameRecord) = NameResponse {lastBlockTs = Nothing, registration = NRRegistered {expires = Nothing, graceUntil = Nothing, reservedReason_ = Nothing, nameRecord}}
     nameResponse Nothing = NameResponse {lastBlockTs = Nothing, registration = NRAvailable {pricing = NamePricing {registrationPrices = M.empty, basePrice = USDCents 1000, minLabelLength = 1}}}
+    -- an account is in use when it owns a name, the only thing this resolver knows about
+    ownedNames addr rs =
+      let ns = [OwnedName {onName = Just nrName, onLabelhash = "", onExpires = RoundedSystemTime 0, onStatus = "registered"} | NameRecord {nrName, nrOwner} <- rs, nrOwner == addr]
+       in OwnedNames {ownNames = ns, ownInUse = not (null ns), ownNextOffset = Nothing}
 
 -- | Register a name's domain to resolve to the given record.
 registerName :: TVar (Map Text NameRecord) -> SimplexNameInfo -> NameRecord -> IO ()
 registerName reg SimplexNameInfo {nameDomain = SimplexDomain {nameTLD, domain}} r =
   atomically $ modifyTVar' reg $ M.insert (decodeLatin1 $ strEncode (labelHash domain) <> strEncode nameTLD) r
 
+-- | Register a name an address owns, for a wallet scan to find.
+ownedName :: NameRegistry -> Text -> Text -> IO ()
+ownedName reg name owner = atomically $ modifyTVar' reg $ M.insert name (emptyRecord name) {nrOwner = owner}
+
 contactNameRecord :: Text -> Text -> NameRecord
 contactNameRecord name link = (emptyRecord name) {nrSimplexContact = [link]}
 

tests/Test.hs

@@ -77,6 +77,7 @@ main = do
       describe "Random servers" randomServersTests
 #if !defined(dbPostgres)
       around (tmpTestBracket chatQueryStats agentQueryStats) $ describe "names tests" chatNamesTests
+      around (tmpTestBracket chatQueryStats agentQueryStats) $ describe "Wallet scan" walletScanTests
       around (tmpTestBracket chatQueryStats agentQueryStats) $ xdescribe'' "SimpleX Directory names" directoryNameTests
 #endif
 #if defined(dbPostgres)

tests/WalletTests.hs

@@ -13,7 +13,9 @@ import qualified Data.ByteString.Char8 as B
 import Data.Char (toUpper)
 import Data.Either (isRight)
 import Data.List (nub)
+import Data.Text (Text)
 import qualified Data.Text as T
+import NameResolver (ownedName)
 import Simplex.Chat.Wallet (AccountIndex, WalletError (..), accountSecret, deriveAccountKey, entropyFromMnemonic, renderAccountPath, seedMaster, seedMnemonic)
 import qualified Simplex.Messaging.Crypto.BIP39 as B39
 import qualified Simplex.Messaging.Crypto.Secp256k1 as S
@@ -42,6 +44,10 @@ addressFromSecret secret =
   show . addressFromPrivateKey . either error id . S.mkPrivateKey . either error id $
     BAE.convertFromBase BAE.Base16 (B.drop 2 $ B.pack secret)
 
+-- | The address of an account of the imported phrase, as the chain would hold it.
+accountAddress :: AccountIndex -> Text
+accountAddress = T.pack . show . addressFromPrivateKey . accountKey (seedEntropy testPhrase24)
+
 -- | An @export account@ row: the index, the path, the address, the secret.
 exportRow :: HasCallStack => String -> (String, String, String, String)
 exportRow row = case words row of
@@ -94,6 +100,23 @@ walletTests = do
   it "will not export an account another profile holds" testWalletExportNotMine
   it "refuses an index BIP-32 cannot harden, on every command" testWalletIndexTooLarge
 
+-- | Its own group: the scan needs a names resolver, so it runs without the SMP server the other wallet tests share.
+walletScanTests :: SpecWith TestParams
+walletScanTests =
+  it "a scan of a recovered phrase finds the accounts in use" testWalletScan
+
+testWalletScan :: HasCallStack => TestParams -> IO ()
+testWalletScan ps = withSmpServerAndNames $ \reg -> withNewTestChat ps "alice" aliceProfile $ \alice -> do
+  enableNamesRole alice
+  alice ##> ("/_wallet create mnemonic=" <> B.unpack testPhrase24)
+  alice <## "wallet, no accounts for this profile"
+  ownedName reg "alice.simplex" (accountAddress 1)
+  alice ##> "/_wallet scan"
+  alice <## "accounts: 1"
+  -- the scan gives the imported phrase the counter it had none of
+  alice ##> "/_wallet bind"
+  alice <## "accounts: 1, 2"
+
 testWalletCreate :: HasCallStack => TestParams -> IO ()
 testWalletCreate ps = withNewTestChat ps "alice" aliceProfile $ \alice -> do
   alice ##> "/_wallet"