Files
simplex-chat/apps/simplex-store-service/src/StoreService/Web/Http.hs
T
ad75fe706f store: sell and register SimpleX names (#7979)
* core: add prepaid name codes and length bounds

* store: run the names migration after badges

* store: issue and revoke prepaid name codes

* store: document issuing prepaid name codes

* store: move the ABI helpers to Names.Abi

* store: read the [names] and [chain] settings

* store: add the names chain client and reads

* store: refuse group name codes while names are off

* store: search names and preview links

* store: serve each site on its own port

* scripts: deploy the badges and names sites

* store: document search and the two sites

* store: add the drop_taken_codes migration

* store: read the names codes_only setting

* store: add the names store and order signatures

* store: take name orders by card, crypto or code

* store: document name orders and codes-only

* store: add the names transaction sender

* store: commit and register queued orders

* store: reveal and retry an owner's order

* store: document the gas key

* store: add the name balance migrations

* store: read crypto units, rates and confirmations

* store: keep a taken order's money as a balance

* store: spend a balance on another name

* store: record refunds with //refund

* store: document balances and refunds

* store: add the agreed terms migration

* store: read the terms and community names

* store: answer community and terms in search

* store: sign and store the agreed terms

* store: document the terms and community names

* store: add the invoiced units migration

* store: read the crypto an invoice asks

* store: credit late and overpaid crypto by share

* store: document crediting late crypto by share

* store: add the record changes migration

* store: find an owner's orders and move one

* store: relay owners' link edits on chain

* store: document link edits

* store: move isLoopbackHost to Config

* store: trust dev issuer keys on the local chain

* scripts: build the names site from a submodule

* scripts: add the dev setup and contract run

* store: document the dev setup and names image

* store: send a Stripe idempotency key for names

* tests: wait until a relay lets go of its edit

* store: refuse a link edit deadline under 10 min

* store: count reverted link edits in the year limit

* store: require https for a remote chain node

* store: refuse zero chain confirmations

* store: set up the dev stack from the names branch

* store: rerun a failed unpaid cleanup next pass

* store: never replace a succeeded Stripe intent

* store: refuse an ini setting written as key: value

* store: note the webapp submodule's pinned branch

* store: document confirmLanded judging by ownerOf

* store: update the names webapp submodule

* store-service: register names under the name policy, every name accepting its terms on chain (#7889)

* store: accept each name's terms on chain under the name policy

* ci: run the reduced build on sh/names-deploy

* store: address review: policy re-read, terms reads and startup log

* store: never credit an unconfirmed extra

* store: alert money sent to a paid name invoice

* store: refuse to start on another chain's node

* store: keep orders while the allowance is used up

* store: replace a stalled transaction after 3 blocks

* store: judge a name taken at confirmation depth

* store: bind the template's listener to loopback

* store: scan only the ids of active name orders

* store: refuse unknown [names] and [chain] settings

* store: fetch the webapp submodule in upgrade steps

* store: delete definitions only tests use

* store: lock only name invoices at settlement

* store: retry an unexpired commitment as usual

* store: share the helpers defined twice

* store: test route limits on the running service

* store: fix tests that could not fail

* ci: test the store service on Postgres

* store: count IPv6 clients by their /64

* store: sweep invoices read alone past a list cap

* store: cap checkouts of all clients together

* store: cap the long polls held at once

* store: stop the old badge project at upgrade

* store: keep search lookups free of previews

* store: read the ini as UTF-8 in any locale

* ci: run Postgres store tests in a UTF-8 locale

* store: cap the transactions one order sends

* store: cancel a card intent after three declines

* store: refuse a name price under a cent

* store: send each transaction with gas headroom

* store: cap searches and read terms once per block

* store: never log the group join link

* store: mask the database password at startup

* store: run one instance per database

* store: limit name codes to admins, 100 a day

* store: refuse HighSpeed settlement with names

* store: no unverified receipts with a trusted key

* store: cap the log lines of refused webhooks

* store: check Apple certificates at the signed date

* store: refuse a malformed Stripe publishable key

* store: check the owner before a refund

* store: fold the names migrations into one

* store: read onion name links through the proxy

* store: refund money after paid once confirmed

* store: limit the allowance wait to a dry run

* store: bind loopback when upgrading

* ci: cache and widen the store Postgres job

* store: fix stale and duplicate names tests

* store: fix the submodule path in the upgrade

* store: refuse unpaid name codes on the CLI

* store: apply the review's names test list

* store: finish the review's names test list

* store: restore the base registration tests

* store: pass link hosts that do not resolve here

* store: refuse a move once its sends are used up

* store: take the shared checkout at the provider

* store: hold a search slot for chain reads only

* store: state which refunds are safe on a reference

* store: check a failed order's name before refusing its move (#8060)

* store: test and document the shared checkout cap for name orders

* store: check a failed order's name on chain before refusing its move

* store: test name links whose host lookup fails

* store: test that a search holds its slot through the chain reads only

* store: restore the signed transaction check in the chain tests

* store: remove the chain tests' copy of the signed transaction check

* store: correct the search slot texts and the README's balance endpoints

* docs: system and sequence diagrams of the names service (#8068)

---------

Co-authored-by: sh-agents <350651734+sh-agents@users.noreply.github.com>
Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com>
Co-authored-by: brenzi-agent <brenzi-agent@brenzi.ch>
Co-authored-by: Evgeny @ SimpleX Chat <259188159+evgeny-simplex@users.noreply.github.com>
2026-11-07 10:45:19 +00:00

466 lines
17 KiB
Haskell

{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
module StoreService.Web.Http
( HttpEnv (..),
newHttpEnv,
Respond,
Limit (..),
Bucket (..),
takeFrom,
takeToken,
limited,
only,
clientKey,
maxBuckets,
readLimit,
createLimit,
searchLimit,
previewLimit,
codeLimit,
findLimit,
recordsLimit,
invoicesLimit,
limitedForAll,
holdMicros,
maxHolds,
withHoldSlot,
maxSearches,
searchSlot,
holdsFull,
jsonResponse,
jsonError,
notFound,
badRequest,
invalidName,
invalidLink,
codeConflict,
providerUnavailable,
termsUnavailable,
rateLimited,
methodNotAllowed,
internalError,
readBoundedBody,
maxBodyBytes,
nameBodyBytes,
parseCodeHash,
invoicePairs,
invoiceAt,
expiryMoment,
destinationPairs,
amountJSON,
confirmationsFor,
invoiceWindow,
providerFor,
providerNamed,
)
where
import Control.Concurrent.STM
import Control.Exception (bracket)
import Control.Monad (when)
import Data.Aeson ((.=))
import qualified Data.Aeson as J
import Data.Aeson.Types (Pair)
import Data.ByteString (ByteString)
import qualified Data.ByteString as BS
import qualified Data.ByteString.Base64.URL as B64U
import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy as LB
import Data.List (find, sortOn)
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as Map
import Data.Maybe (fromMaybe, listToMaybe)
import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding (decodeUtf8', encodeUtf8)
import Data.Time.Clock (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime)
import Data.Word (Word16)
import Network.HTTP.Types (Header, Method, Status, hCacheControl, hContentType, status400, status404, status405, status409, status429, status500, status503)
import Network.Socket (SockAddr (..), hostAddress6ToTuple, hostAddressToTuple)
import Network.Wai
import Numeric (readHex, showHex)
import Simplex.Chat.PaymentService.Types (CardProvider (..), CurrencyAmount (..), InvoiceStatus (..), PaymentProvider (..), ServicePaymentDestination (..), ServicePaymentMethod (..))
import Simplex.Messaging.Util (safeDecodeUtf8)
import StoreService.Config (BTCPayConfig (..), ListenerConfig (..), ServiceConfig (..), SpeedPolicy (..), StripeConfig (..), defaultExpiryMinutes, defaultSessionMinutes)
import StoreService.Providers (Provider (..))
import StoreService.Store.Invoices (InvoicePayment (..), InvoiceRow (..), cryptoCurrencyText, invoiceStatusText, paymentHolds)
import Text.Read (readMaybe)
data HttpEnv = HttpEnv
{ heBuckets :: TVar (Map Text Bucket),
heTrustForwardedFor :: Bool,
-- | the long polls held now, of every client
heHolds :: TVar Int,
-- | the name searches reading the chain now, of every client
heSearches :: TVar Int
}
newHttpEnv :: ListenerConfig -> IO HttpEnv
newHttpEnv ListenerConfig {lTrustForwardedFor} = do
heBuckets <- newTVarIO Map.empty
heHolds <- newTVarIO 0
heSearches <- newTVarIO 0
pure HttpEnv {heBuckets, heTrustForwardedFor = lTrustForwardedFor, heHolds, heSearches}
type Respond = Response -> IO ResponseReceived
-- | Short enough to stay under the idle timeout of any proxy in front of us.
holdMicros :: Int
holdMicros = 30 * 1000000
-- | Held long polls of every client together, each a connection and a waiter.
maxHolds :: Int
maxHolds = 500
-- | When every hold is taken, how soon a poll is asked to come back.
holdsFullSeconds :: Int
holdsFullSeconds = 5
-- | The hold in one of the shared slots, Nothing when every slot is taken.
withHoldSlot :: HttpEnv -> IO a -> IO (Maybe a)
withHoldSlot HttpEnv {heHolds} = inSlot heHolds maxHolds
holdsFull :: Response
holdsFull = rateLimited holdsFullSeconds
-- | Name searches of every client reading the chain at once, so a flood cannot multiply into the chain node's load.
maxSearches :: Int
maxSearches = 16
-- | A search's chain reads, run while a slot is free; Nothing when none is.
searchSlot :: HttpEnv -> IO a -> IO (Maybe a)
searchSlot HttpEnv {heSearches} = inSlot heSearches maxSearches
inSlot :: TVar Int -> Int -> IO a -> IO (Maybe a)
inSlot slots most action = bracket takeSlot releaseSlot $ \taken -> if taken then Just <$> action else pure Nothing
where
takeSlot = atomically $ stateTVar slots $ \n -> if n < most then (True, n + 1) else (False, n)
releaseSlot taken = when taken $ atomically $ modifyTVar' slots (subtract 1)
readLimit, createLimit, searchLimit, previewLimit, codeLimit, findLimit, recordsLimit :: Limit
readLimit = Limit {lmName = "read", lmPerMinute = 60}
createLimit = Limit {lmName = "create", lmPerMinute = 5}
searchLimit = Limit {lmName = "search", lmPerMinute = 30}
previewLimit = Limit {lmName = "preview", lmPerMinute = 20}
codeLimit = Limit {lmName = "code", lmPerMinute = 10}
findLimit = Limit {lmName = "find", lmPerMinute = 5}
recordsLimit = Limit {lmName = "records", lmPerMinute = 10}
-- | Checkouts of every client together, each a provider invoice or card PaymentIntent, so a flood from many addresses
-- still meets a ceiling.
invoicesLimit :: Limit
invoicesLimit = Limit {lmName = "invoices", lmPerMinute = 30}
maxBodyBytes :: Int
maxBodyBytes = 8192
-- | The cap of the names endpoints whose bodies carry a sealed name code.
nameBodyBytes :: Int
nameBodyBytes = 16384
sha256Bytes :: Int
sha256Bytes = 32
codeHashChars :: Int
codeHashChars = 43
data Limit = Limit {lmName :: Text, lmPerMinute :: Int}
deriving (Eq, Show)
data Bucket = Bucket {bkCount :: Int, bkStarted :: UTCTime}
limitWindow :: NominalDiffTime
limitWindow = 60
maxBuckets :: Int
maxBuckets = 32768
reclaim :: UTCTime -> Map Text Bucket -> Map Text Bucket
reclaim now buckets
| Map.size buckets < maxBuckets = buckets
| Map.size live * 2 <= maxBuckets = live
| otherwise = Map.fromList (drop (Map.size live `div` 2) (sortOn (bkCount . snd) (Map.toList live)))
where
live = Map.filter (fresh now) buckets
fresh :: UTCTime -> Bucket -> Bool
fresh now Bucket {bkStarted} = diffUTCTime now bkStarted < limitWindow
-- | A request against a minute's limit: the bucket after it, or how long until the minute ends.
takeFrom :: Int -> UTCTime -> Maybe Bucket -> Either NominalDiffTime Bucket
takeFrom limit now = \case
Just b@Bucket {bkCount, bkStarted}
| fresh now b -> if bkCount < limit then Right b {bkCount = bkCount + 1} else Left (limitWindow - diffUTCTime now bkStarted)
_ -> Right Bucket {bkCount = 1, bkStarted = now}
takeToken :: HttpEnv -> Limit -> Text -> IO (Maybe Int)
takeToken HttpEnv {heBuckets} Limit {lmName, lmPerMinute} client = do
now <- getCurrentTime
atomically $ do
buckets <- reclaim now <$> readTVar heBuckets
let key = lmName <> "\t" <> client
count bucket = writeTVar heBuckets (Map.insert key bucket buckets) >> pure Nothing
-- write the reclaimed map back here too, or a refused request redoes the filtering every time and the map stays at the limit
refuse seconds = writeTVar heBuckets buckets >> pure (Just seconds)
either (refuse . max 1 . ceiling) count (takeFrom lmPerMinute now (Map.lookup key buckets))
limited :: HttpEnv -> Limit -> Request -> Respond -> IO ResponseReceived -> IO ResponseReceived
limited env limit req respond action =
takeToken env limit (clientKey env req) >>= \case
Nothing -> action
Just seconds -> respond (rateLimited seconds)
-- | A limit every client shares, taken just before the provider is asked, so refused requests do not use it.
limitedForAll :: HttpEnv -> Limit -> (Response -> IO a) -> IO a -> IO a
limitedForAll env limit refuse action = takeToken env limit "" >>= maybe action (refuse . rateLimited)
only :: Method -> Request -> Respond -> IO ResponseReceived -> IO ResponseReceived
only method req respond action
| requestMethod req == method = action
| otherwise = respond (methodNotAllowed method)
-- | Every line is joined first, so a second header line a caller adds cannot leave its own line the one we read.
-- An IPv6 client is keyed by its /64, as one host holds a whole /64.
clientKey :: HttpEnv -> Request -> Text
clientKey HttpEnv {heTrustForwardedFor} req = fromMaybe (peerText (remoteHost req)) forwarded
where
forwarded
| not heTrustForwardedFor = Nothing
| otherwise = case reverse (concatMap entries (requestHeaders req)) of
(ip : _)
| isIPv4 ip -> Just ip
| isIPv6 ip -> v6Key <$> v6Groups ip
_ -> Nothing
entries (name, raw)
| name /= "x-forwarded-for" = []
| otherwise = either (const []) (filter (not . T.null) . map T.strip . T.splitOn ",") (decodeUtf8' raw)
asciiDigit :: Char -> Bool
asciiDigit c = c >= '0' && c <= '9'
-- Data.Char.isDigit and isHexDigit accept Unicode digits, which are not addresses.
asciiHexDigit :: Char -> Bool
asciiHexDigit c = asciiDigit c || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
-- | A leading zero is refused, so "010.0.0.1" cannot be a second key for "10.0.0.1".
isIPv4 :: Text -> Bool
isIPv4 t = case T.splitOn "." t of
[a, b, c, d] -> all octet [a, b, c, d]
_ -> False
where
octet s =
not (T.null s)
&& T.length s <= 3
&& T.all asciiDigit s
&& (T.length s == 1 || T.head s /= '0')
&& maybe False (<= 255) (readMaybe (T.unpack s) :: Maybe Int)
isIPv6 :: Text -> Bool
isIPv6 t
| T.any (== '%') t = False
| otherwise = case T.splitOn "::" t of
[full] -> groupCount True (segments full) == Just 8
[before, after] -> case (groupCount False (segments before), groupCount True (segments after)) of
(Just b, Just a) -> b + a <= 7 -- "::" stands for at least one omitted group
_ -> False
_ -> False
where
segments s = if T.null s then [] else T.splitOn ":" s
groupCount :: Bool -> [Text] -> Maybe Int
groupCount endsAddress gs
| null gs = Just 0
| endsAddress && isIPv4 (last gs) = (+ 2) <$> plainGroups (init gs)
| otherwise = plainGroups gs
where
plainGroups ps = if all hexGroup ps then Just (length ps) else Nothing
hexGroup g = not (T.null g) && T.length g <= 4 && T.all asciiHexDigit g
peerText :: SockAddr -> Text
peerText = \case
SockAddrInet _ addr ->
let (a, b, c, d) = hostAddressToTuple addr
in v4Text [a, b, c, d]
SockAddrInet6 _ _ addr _ ->
let (a, b, c, d, e, f, g, h) = hostAddress6ToTuple addr
in v6Key [a, b, c, d, e, f, g, h]
SockAddrUnix p -> "unix:" <> T.pack p
-- | The eight groups of an address isIPv6 accepts.
v6Groups :: Text -> Maybe [Word16]
v6Groups t = case T.splitOn "::" t of
[full] -> exactly 8 (parts full)
[before, after] -> do
b <- parts before
a <- parts after
pure $ b <> replicate (8 - length b - length a) 0 <> a
_ -> Nothing
where
parts s = concat <$> mapM part (if T.null s then [] else T.splitOn ":" s)
part g
| isIPv4 g = case mapM (readMaybe . T.unpack) (T.splitOn "." g) :: Maybe [Word16] of
Just [a, b, c, d] -> Just [a * 256 + b, c * 256 + d]
_ -> Nothing
| otherwise = (: []) . fst <$> listToMaybe (readHex $ T.unpack g)
exactly n gs = gs >>= \xs -> if length xs == n then Just xs else Nothing
-- | The /64 an address is in, or the IPv4 address an IPv4-mapped one carries.
v6Key :: [Word16] -> Text
v6Key gs = case splitAt 6 gs of
([0, 0, 0, 0, 0, 0xffff], [ab, cd]) -> v4Text [ab `div` 256, ab `mod` 256, cd `div` 256, cd `mod` 256]
_ -> T.intercalate ":" (map (T.pack . (`showHex` "")) (take 4 gs)) <> "::/64"
v4Text :: Show a => [a] -> Text
v4Text = T.intercalate "." . map (T.pack . show)
apiHeaders :: [Header]
apiHeaders = [(hContentType, "application/json"), (hCacheControl, "no-store")]
jsonResponse :: Status -> J.Value -> Response
jsonResponse st = responseLBS st apiHeaders . J.encode
jsonError :: Status -> Text -> Response
jsonError st code = jsonResponse st (J.object ["error" .= code])
notFound :: Response
notFound = jsonError status404 "not_found"
badRequest :: Response
badRequest = jsonError status400 "bad_request"
invalidName :: Response
invalidName = jsonError status400 "invalid_name"
invalidLink :: Response
invalidLink = jsonError status400 "invalid_link"
codeConflict :: Response
codeConflict = jsonError status409 "code_conflict"
providerUnavailable :: Response
providerUnavailable = jsonError status503 "provider_unavailable"
termsUnavailable :: Int -> Response
termsUnavailable = retryAfterError status503 "terms_unavailable"
rateLimited :: Int -> Response
rateLimited = retryAfterError status429 "rate_limited"
retryAfterError :: Status -> Text -> Int -> Response
retryAfterError st code seconds =
responseLBS
st
(("Retry-After", B.pack (show seconds)) : apiHeaders)
(J.encode (J.object ["error" .= code]))
methodNotAllowed :: Method -> Response
methodNotAllowed allow = responseLBS status405 (("Allow", allow) : apiHeaders) (J.encode (J.object ["error" .= ("method_not_allowed" :: Text)]))
internalError :: Response
internalError = jsonError status500 "internal"
-- | Returns the bytes exactly as they arrived, since the webhook signature is over those bytes.
readBoundedBody :: Int -> Request -> IO (Maybe LB.ByteString)
readBoundedBody cap req = go 0 []
where
go :: Int -> [ByteString] -> IO (Maybe LB.ByteString)
go read' acc = do
chunk <- getRequestBodyChunk req
if BS.null chunk
then pure (Just (LB.fromChunks (reverse acc)))
else
let read'' = read' + BS.length chunk
in if read'' > cap then pure Nothing else go read'' (chunk : acc)
-- | The last character of a 43-character base64 string has two bits no digest byte uses, so a lax
-- decoder would accept four spellings of the same digest, which the re-encode check rejects.
parseCodeHash :: Text -> Maybe ByteString
parseCodeHash t
| T.length t /= codeHashChars = Nothing
| otherwise = case B64U.decode (encodeUtf8 (t <> "=")) of
Right bytes | BS.length bytes == sha256Bytes, canonical bytes == t -> Just bytes
_ -> Nothing
where
canonical = T.filter (/= '=') . safeDecodeUtf8 . B64U.encode
-- | An open name invoice past its expiry with no money received reads expired before the sweep marks it.
invoiceAt :: UTCTime -> InvoiceRow -> InvoiceRow
invoiceAt now row = case expiryMoment row of
Just expiry | now > expiry -> row {irStatus = ISExpired}
_ -> row
-- | A funded open invoice stays open until it settles.
expiryMoment :: InvoiceRow -> Maybe UTCTime
expiryMoment InvoiceRow {irStatus, irExpiresAt, irPayment}
| irStatus == ISOpen && not (maybe False paymentHolds irPayment) = Just irExpiresAt
| otherwise = Nothing
-- | The fields every invoice view has, a badge's and a name's.
invoicePairs :: Maybe Int -> InvoiceRow -> [Pair]
invoicePairs confirmations InvoiceRow {irAmount, irCurrency, irDestination, irExpiresAt, irStatus, irPayment} =
[ "status" .= invoiceStatusText irStatus,
"amount" .= amountJSON irAmount,
"currency" .= irCurrency,
"expiresAt" .= irExpiresAt
]
<> destinationPairs confirmations irDestination
<> maybe [] paymentPairs irPayment
where
paymentPairs p =
concat
[ maybe [] (\a -> ["amountPaid" .= amountJSON a]) (ipAmount p),
maybe [] (\a -> ["cryptoAmountPaid" .= a]) (ipCryptoPaid p),
maybe [] (\a -> ["cryptoAmountDue" .= a]) (ipCryptoDue p),
["paidInFull" .= ipPaidInFull p],
["settledAt" .= ipUpdatedAt p | irStatus == ISPaid]
]
destinationPairs :: Maybe Int -> ServicePaymentDestination -> [Pair]
destinationPairs confirmations = \case
SPDCard _ url -> ["clientSecret" .= url]
SPDCrypto currency address cryptoAmount ->
[ "address" .= address,
"cryptoAmount" .= cryptoAmount,
"cryptoCurrency" .= cryptoCurrencyText currency
]
<> maybe [] (\n -> ["requiredConfirmations" .= n]) confirmations
amountJSON :: CurrencyAmount -> J.Value
amountJSON (CurrencyAmount a) = J.toJSON a
-- | Greenfield reports no confirmation count, so this comes from the store's speed policy.
confirmationsFor :: ServiceConfig -> Maybe Int
confirmationsFor cfg = speedPolicyConfirmations . bSpeedPolicy <$> btcpay cfg
-- | BTCPay's own store setting, whose numbering is not in speed order.
speedPolicyConfirmations :: SpeedPolicy -> Int
speedPolicyConfirmations = \case
HighSpeed -> 0
MediumSpeed -> 1
LowMediumSpeed -> 2
LowSpeed -> 6
secondsPerMinute :: Int
secondsPerMinute = 60
-- | The expiry shown and stored must come from the same key the provider sets the invoice's real expiry from.
invoiceWindow :: ServiceConfig -> ServicePaymentMethod -> NominalDiffTime
invoiceWindow cfg = \case
SPMCard CPStripe -> minutes (maybe defaultSessionMinutes sSessionMinutes (stripe cfg))
SPMCrypto _ -> minutes (maybe defaultExpiryMinutes bExpiryMinutes (btcpay cfg))
where
minutes n = fromIntegral (secondsPerMinute * n)
providerFor :: [Provider] -> ServicePaymentMethod -> Maybe Provider
providerFor providers method = providerNamed providers (providerOf method)
providerNamed :: [Provider] -> PaymentProvider -> Maybe Provider
providerNamed providers provider = find ((== provider) . pProvider) providers
providerOf :: ServicePaymentMethod -> PaymentProvider
providerOf = \case
SPMCard CPStripe -> PPStripe
SPMCrypto _ -> PPCrypto