store: sweep around invoices a pass failed to read

This commit is contained in:
sh-agents committed 2026-11-01 15:27:03 +00:00
1 parent 18c3103153
commit 2c8050740e
2 files changed
+30 -28

No files matched your search

@@ -35,11 +35,11 @@ import StoreService.Waiters (Waiters, publish, waitingCount, waitingCountSTM)
import Control.Concurrent.STM
import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try)
import StoreService.Log (logError, logInfo, logWarn)
import Control.Monad (filterM, forever, unless, void, when)
import Control.Monad (filterM, forM_, forever, unless, void, when)
import Data.List (find, partition, sortOn)
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as M
import Data.Maybe (fromMaybe, isJust)
import Data.Maybe (catMaybes, fromMaybe, isJust)
import Data.Set (Set)
import qualified Data.Set as S
import Data.Text (Text)
@@ -109,25 +109,25 @@ newPollerEnv peStore peWaiters peHints peProviders pePoll = do
peCancelFailed <- newTVarIO M.empty
pure PollerEnv {peStore, peWaiters, peHints, peProviders, pePoll, peSkipped, peListedAt, peStrayEvery = strayListInterval, peCancelFailed}
-- | Read providers before the sweep, and skip the sweep entirely when a read failed, so an
-- invoice with money in it is never expired.
-- | Read providers before the sweep, and leave out of the sweep every invoice whose read or settlement failed,
-- so an invoice with money in it is never expired. A list that cannot account for every invoice skips the sweep.
runOnePass :: PollerEnv -> IO ()
runOnePass env@PollerEnv {peStore, peProviders} = do
now <- getCurrentTime
rows <- unpaidRefs peStore (addUTCTime (negate settleWindow) now)
let bulk = length rows > readsPerPass
listNow <- listDue env now bulk
accounted <-
missed <-
if bulk
then do
covered <- and <$> mapM (rowIsCovered env now) rows
(covered &&) . and <$> mapM (listPass env now) peProviders
uncovered <- S.fromList . map snd <$> filterM (fmap not . rowIsCovered env now) rows
fmap (S.unions . (uncovered :)) . sequence <$> mapM (listPass env now) peProviders
else do
read' <- readRows env now rows
missed <- readRows env now rows
when listNow $ mapM_ (listPass env now) peProviders
pure read'
pure (Just missed)
pruneSkipLog env now
when (accounted && not (null peProviders)) $ sweepExpired env now
unless (null peProviders) $ forM_ missed $ sweepExpired env now
-- | The cases the stray list catches are rare and not urgent, so minutes, not the pass cadence.
strayListInterval :: NominalDiffTime
@@ -156,9 +156,10 @@ coveringProvider env@PollerEnv {peProviders} now (provider, ref) =
rowIsCovered :: PollerEnv -> UTCTime -> (Text, Text) -> IO Bool
rowIsCovered env now row = isJust <$> coveringProvider env now row
readRows :: PollerEnv -> UTCTime -> [(Text, Text)] -> IO Bool
-- | The refs whose read or settlement failed.
readRows :: PollerEnv -> UTCTime -> [(Text, Text)] -> IO (Set Text)
readRows env now rows =
and <$> mapM (\r -> safelyWith (readWhat r) False (readRow r)) rows
S.fromList . map snd <$> filterM (\r -> not <$> safelyWith (readWhat r) False (readRow r)) rows
where
readWhat (provider, ref) = "reading " <> provider <> " invoice " <> ref
readRow row@(_, ref) = coveringProvider env now row >>= maybe (pure False) (`readOne` ref)
@@ -171,20 +172,24 @@ readRows env now rows =
Right Nothing -> pure True
Right (Just signal) -> True <$ settleMoved env (pProvider p) now (ref, signal)
-- | False when the pass cannot account for every invoice sold, so the sweep does not expire one
-- over money it missed.
listPass :: PollerEnv -> UTCTime -> Provider -> IO Bool
-- | The refs of ours the list did not account for, so the sweep does not expire one over money it missed;
-- Nothing when it cannot name them all.
listPass :: PollerEnv -> UTCTime -> Provider -> IO (Maybe (Set Text))
listPass env now p =
pListOpen p >>= \case
Left (ProviderError e) -> do
due <- dueToWarn env now ("list failed: " <> tshow (pProvider p))
when due $ logWarn ("badge poller: " <> tshow (pProvider p) <> " list failed; every invoice waits for the next pass: " <> e)
pure False
pure Nothing
Right ListPass {lpMoved, lpSkipped} -> do
owners <- mapM (\s -> safelyWith (skipWhat s) SkipUnaccounted (reportSkip env (pProvider p) now s)) lpSkipped
reportStrangers env (pProvider p) now [reason | (SkipStranger, (_, reason)) <- zip owners lpSkipped]
settled <- mapM (\m -> safely (settleWhat m) (settleMoved env (pProvider p) now m)) lpMoved
pure (all (== SkipStranger) owners && and settled)
unsettled <- filterM (\m -> not <$> safely (settleWhat m) (settleMoved env (pProvider p) now m)) lpMoved
let skippedOurs = [ref | (SkipOurs, (ref, _)) <- zip owners lpSkipped]
pure $
if SkipUnaccounted `elem` owners
then Nothing
else Just $ S.fromList $ map fst unsettled <> catMaybes skippedOurs
where
settleWhat (ref, _) = "settling " <> tshow (pProvider p) <> " invoice " <> ref
skipWhat (ref, _) = "reading the skipped " <> tshow (pProvider p) <> " invoice " <> fromMaybe "the provider did not name" ref
@@ -227,10 +232,11 @@ readHint env@PollerEnv {peStore, peProviders} now ref =
Right Nothing -> pure ()
Right (Just signal) -> settleMoved env irProvider now (ref, signal)
sweepExpired :: PollerEnv -> UTCTime -> IO ()
sweepExpired env@PollerEnv {peStore, peWaiters, peCancelFailed} now = do
-- | Leaves out the invoices whose read or settlement failed this pass.
sweepExpired :: PollerEnv -> UTCTime -> Set Text -> IO ()
sweepExpired env@PollerEnv {peStore, peWaiters, peCancelFailed} now missed = do
let cutoff = addUTCTime (negate expiryGrace) now
(selfExpiring, payable) <- partition (expiresItself . oiProvider) <$> overdueInvoices peStore cutoff
(selfExpiring, payable) <- partition (expiresItself . oiProvider) . filter ((`S.notMember` missed) . oiProviderRef) <$> overdueInvoices peStore cutoff
atomically $ modifyTVar' peCancelFailed (`M.restrictKeys` S.fromList (map oiProviderRef payable))
-- Cancel only a few orders per pass, so a long list does not block other work.
cancelled <- filterM (closedAtProvider env now) (take readsPerPass payable)
+3 -7
View File
@@ -269,7 +269,7 @@ badgeWebTests = do
it "warns once for all the invoices it did not sell, not once per invoice" testStrangerSkipsShareOneWarning
it "raises a skip naming an invoice this service sold" testSkipNamingOurInvoiceIsRaised
it "holds the sweep back until a pass has accounted for every invoice" testSweepWaitsForAPassThatSawEverything
it "settles the rest of the pass around an invoice that throws" testOneBadInvoiceDoesNotStopThePass
it "settles and sweeps the rest of the pass around an invoice that throws" testOneBadInvoiceDoesNotStopThePass
it "pages past a hundred open invoices, a request per hundred" testListPaginates
it "stops paging at its ceiling rather than looping on a server that ignores take" testPagingStopsAtTheCeiling
describe "store service webhook" $ do
@@ -3074,12 +3074,8 @@ testOneBadInvoiceDoesNotStopThePass = bounded "one bad invoice" $ withStubPoller
setSignals ref [("p-bad", settledSignal), ("p-good", settledSignal)]
runOnePass poller
invoiceStatus (weStore env) good `shouldReturn` ISPaid
invoiceStatus (weStore env) overdue `shouldReturn` ISOpen
setSignals ref []
setSkipped ref [(Just "p-bad", "some reason the provider gave")]
runOnePass poller
invoiceStatus (weStore env) overdue `shouldReturn` ISOpen
-- only the invoice that failed is kept out of the sweep
invoiceStatus (weStore env) overdue `shouldReturn` ISExpired
testSweepWaitsForAPassThatSawEverything :: IO ()
testSweepWaitsForAPassThatSawEverything = bounded "sweep guards" $ withStubPoller raceHold $ \ref poller env _ -> do