diff --git a/apps/simplex-store-service/src/StoreService/Poller.hs b/apps/simplex-store-service/src/StoreService/Poller.hs index 10542fb4e4..626965bfa8 100644 --- a/apps/simplex-store-service/src/StoreService/Poller.hs +++ b/apps/simplex-store-service/src/StoreService/Poller.hs @@ -35,11 +35,11 @@ import StoreService.Waiters (Waiters, publish, waitingCount, waitingCountSTM) import Control.Concurrent.STM import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try) import StoreService.Log (logError, logInfo, logWarn) -import Control.Monad (filterM, forever, unless, void, when) +import Control.Monad (filterM, forM_, forever, unless, void, when) import Data.List (find, partition, sortOn) import Data.Map.Strict (Map) import qualified Data.Map.Strict as M -import Data.Maybe (fromMaybe, isJust) +import Data.Maybe (catMaybes, fromMaybe, isJust) import Data.Set (Set) import qualified Data.Set as S import Data.Text (Text) @@ -109,25 +109,25 @@ newPollerEnv peStore peWaiters peHints peProviders pePoll = do peCancelFailed <- newTVarIO M.empty pure PollerEnv {peStore, peWaiters, peHints, peProviders, pePoll, peSkipped, peListedAt, peStrayEvery = strayListInterval, peCancelFailed} --- | Read providers before the sweep, and skip the sweep entirely when a read failed, so an --- invoice with money in it is never expired. +-- | Read providers before the sweep, and leave out of the sweep every invoice whose read or settlement failed, +-- so an invoice with money in it is never expired. A list that cannot account for every invoice skips the sweep. runOnePass :: PollerEnv -> IO () runOnePass env@PollerEnv {peStore, peProviders} = do now <- getCurrentTime rows <- unpaidRefs peStore (addUTCTime (negate settleWindow) now) let bulk = length rows > readsPerPass listNow <- listDue env now bulk - accounted <- + missed <- if bulk then do - covered <- and <$> mapM (rowIsCovered env now) rows - (covered &&) . and <$> mapM (listPass env now) peProviders + uncovered <- S.fromList . map snd <$> filterM (fmap not . rowIsCovered env now) rows + fmap (S.unions . (uncovered :)) . sequence <$> mapM (listPass env now) peProviders else do - read' <- readRows env now rows + missed <- readRows env now rows when listNow $ mapM_ (listPass env now) peProviders - pure read' + pure (Just missed) pruneSkipLog env now - when (accounted && not (null peProviders)) $ sweepExpired env now + unless (null peProviders) $ forM_ missed $ sweepExpired env now -- | The cases the stray list catches are rare and not urgent, so minutes, not the pass cadence. strayListInterval :: NominalDiffTime @@ -156,9 +156,10 @@ coveringProvider env@PollerEnv {peProviders} now (provider, ref) = rowIsCovered :: PollerEnv -> UTCTime -> (Text, Text) -> IO Bool rowIsCovered env now row = isJust <$> coveringProvider env now row -readRows :: PollerEnv -> UTCTime -> [(Text, Text)] -> IO Bool +-- | The refs whose read or settlement failed. +readRows :: PollerEnv -> UTCTime -> [(Text, Text)] -> IO (Set Text) readRows env now rows = - and <$> mapM (\r -> safelyWith (readWhat r) False (readRow r)) rows + S.fromList . map snd <$> filterM (\r -> not <$> safelyWith (readWhat r) False (readRow r)) rows where readWhat (provider, ref) = "reading " <> provider <> " invoice " <> ref readRow row@(_, ref) = coveringProvider env now row >>= maybe (pure False) (`readOne` ref) @@ -171,20 +172,24 @@ readRows env now rows = Right Nothing -> pure True Right (Just signal) -> True <$ settleMoved env (pProvider p) now (ref, signal) --- | False when the pass cannot account for every invoice sold, so the sweep does not expire one --- over money it missed. -listPass :: PollerEnv -> UTCTime -> Provider -> IO Bool +-- | The refs of ours the list did not account for, so the sweep does not expire one over money it missed; +-- Nothing when it cannot name them all. +listPass :: PollerEnv -> UTCTime -> Provider -> IO (Maybe (Set Text)) listPass env now p = pListOpen p >>= \case Left (ProviderError e) -> do due <- dueToWarn env now ("list failed: " <> tshow (pProvider p)) when due $ logWarn ("badge poller: " <> tshow (pProvider p) <> " list failed; every invoice waits for the next pass: " <> e) - pure False + pure Nothing Right ListPass {lpMoved, lpSkipped} -> do owners <- mapM (\s -> safelyWith (skipWhat s) SkipUnaccounted (reportSkip env (pProvider p) now s)) lpSkipped reportStrangers env (pProvider p) now [reason | (SkipStranger, (_, reason)) <- zip owners lpSkipped] - settled <- mapM (\m -> safely (settleWhat m) (settleMoved env (pProvider p) now m)) lpMoved - pure (all (== SkipStranger) owners && and settled) + unsettled <- filterM (\m -> not <$> safely (settleWhat m) (settleMoved env (pProvider p) now m)) lpMoved + let skippedOurs = [ref | (SkipOurs, (ref, _)) <- zip owners lpSkipped] + pure $ + if SkipUnaccounted `elem` owners + then Nothing + else Just $ S.fromList $ map fst unsettled <> catMaybes skippedOurs where settleWhat (ref, _) = "settling " <> tshow (pProvider p) <> " invoice " <> ref skipWhat (ref, _) = "reading the skipped " <> tshow (pProvider p) <> " invoice " <> fromMaybe "the provider did not name" ref @@ -227,10 +232,11 @@ readHint env@PollerEnv {peStore, peProviders} now ref = Right Nothing -> pure () Right (Just signal) -> settleMoved env irProvider now (ref, signal) -sweepExpired :: PollerEnv -> UTCTime -> IO () -sweepExpired env@PollerEnv {peStore, peWaiters, peCancelFailed} now = do +-- | Leaves out the invoices whose read or settlement failed this pass. +sweepExpired :: PollerEnv -> UTCTime -> Set Text -> IO () +sweepExpired env@PollerEnv {peStore, peWaiters, peCancelFailed} now missed = do let cutoff = addUTCTime (negate expiryGrace) now - (selfExpiring, payable) <- partition (expiresItself . oiProvider) <$> overdueInvoices peStore cutoff + (selfExpiring, payable) <- partition (expiresItself . oiProvider) . filter ((`S.notMember` missed) . oiProviderRef) <$> overdueInvoices peStore cutoff atomically $ modifyTVar' peCancelFailed (`M.restrictKeys` S.fromList (map oiProviderRef payable)) -- Cancel only a few orders per pass, so a long list does not block other work. cancelled <- filterM (closedAtProvider env now) (take readsPerPass payable) diff --git a/tests/Bots/StoreService/WebTests.hs b/tests/Bots/StoreService/WebTests.hs index a0b6906838..7367168d75 100644 --- a/tests/Bots/StoreService/WebTests.hs +++ b/tests/Bots/StoreService/WebTests.hs @@ -269,7 +269,7 @@ badgeWebTests = do it "warns once for all the invoices it did not sell, not once per invoice" testStrangerSkipsShareOneWarning it "raises a skip naming an invoice this service sold" testSkipNamingOurInvoiceIsRaised it "holds the sweep back until a pass has accounted for every invoice" testSweepWaitsForAPassThatSawEverything - it "settles the rest of the pass around an invoice that throws" testOneBadInvoiceDoesNotStopThePass + it "settles and sweeps the rest of the pass around an invoice that throws" testOneBadInvoiceDoesNotStopThePass it "pages past a hundred open invoices, a request per hundred" testListPaginates it "stops paging at its ceiling rather than looping on a server that ignores take" testPagingStopsAtTheCeiling describe "store service webhook" $ do @@ -3074,12 +3074,8 @@ testOneBadInvoiceDoesNotStopThePass = bounded "one bad invoice" $ withStubPoller setSignals ref [("p-bad", settledSignal), ("p-good", settledSignal)] runOnePass poller invoiceStatus (weStore env) good `shouldReturn` ISPaid - invoiceStatus (weStore env) overdue `shouldReturn` ISOpen - - setSignals ref [] - setSkipped ref [(Just "p-bad", "some reason the provider gave")] - runOnePass poller - invoiceStatus (weStore env) overdue `shouldReturn` ISOpen + -- only the invoice that failed is kept out of the sweep + invoiceStatus (weStore env) overdue `shouldReturn` ISExpired testSweepWaitsForAPassThatSawEverything :: IO () testSweepWaitsForAPassThatSawEverything = bounded "sweep guards" $ withStubPoller raceHold $ \ref poller env _ -> do