mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-11-22 19:58:39 +00:00
servers: read one message per queue in SUB, omit DB connection string (#2007)
* smp server: plan to deliver messages sent while a subscription is registered * tests: reproduce subscription races with SEND and DEL * smp server: move subscriber registration functions to top level * smp server: deliver messages sent while a subscription is registered * smp server: deliver later messages after SUB while a delivery waits * smp server: correct the subscription race plan and a comment * smp server: remove delivery thread state change and its test * smp server: remove PostgreSQL performance changes of SUB batches * smp server: record the split of the subscription race fix in the plan * smp server: link the split pull requests in the subscription race plan * smp server: shorten row locks of PostgreSQL SUB batches * smp server: read first messages of SUB batches without queue lookups * servers: omit database connection string from output, save stats first * smp server: remove the plan of the closed subscription race fix
This commit is contained in:
1 parent
052b123247
commit
71179c4d4e
7 files changed
+62
-21
No files matched your search
@@ -161,7 +161,7 @@ ntfServer cfg@NtfServerConfig {transports, transportConfig = tCfg, startOptions}
|
||||
>>= mapM (deRefWeak . workerThreadId >=> mapM_ killThread)
|
||||
|
||||
saveServer :: M ()
|
||||
saveServer = asks store >>= liftIO . closeNtfDbStore >> saveServerStats
|
||||
saveServer = saveServerStats >> asks store >>= liftIO . closeNtfDbStore
|
||||
|
||||
serverStatsThread_ :: NtfServerConfig -> [M ()]
|
||||
serverStatsThread_ NtfServerConfig {logStatsInterval = Just interval, logStatsStartTime, serverStatsLogFile} =
|
||||
|
||||
@@ -227,8 +227,8 @@ ntfServerCLI cfgPath logPath =
|
||||
iniDeletedTTL ini = readIniDefault (86400 * defaultDeletedTTL) "STORE_LOG" "db_deleted_ttl" ini
|
||||
|
||||
printNtfServerConfig :: [(ServiceName, ASrvTransport, AddHTTP)] -> PostgresStoreCfg -> IO ()
|
||||
printNtfServerConfig transports PostgresStoreCfg {dbOpts = DBOpts {connstr, schema}, dbStoreLogPath} = do
|
||||
B.putStrLn $ "PostgreSQL database: " <> connstr <> ", schema: " <> schema
|
||||
printNtfServerConfig transports PostgresStoreCfg {dbOpts = DBOpts {schema}, dbStoreLogPath} = do
|
||||
B.putStrLn $ "PostgreSQL database schema: " <> schema
|
||||
printServerConfig "NTF" transports dbStoreLogPath
|
||||
|
||||
data CliCommand
|
||||
|
||||
@@ -458,8 +458,8 @@ printSMPServerConfig transports st = case dbStoreCfg st of
|
||||
Just cfg -> printDBConfig cfg
|
||||
Nothing -> printServerConfig "SMP" transports $ storeLogFile' st
|
||||
where
|
||||
printDBConfig PostgresStoreCfg {dbOpts = DBOpts {connstr, schema}} = do
|
||||
B.putStrLn $ "PostgreSQL database: " <> connstr <> ", schema: " <> schema
|
||||
printDBConfig PostgresStoreCfg {dbOpts = DBOpts {schema}} = do
|
||||
B.putStrLn $ "PostgreSQL database schema: " <> schema
|
||||
printServerTransports "SMP" transports
|
||||
|
||||
deleteDirIfExists :: FilePath -> IO ()
|
||||
|
||||
@@ -158,7 +158,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
unless msgsFileExists $ putStrLn $ storeMsgsFilePath <> " file does not exist."
|
||||
exitFailure
|
||||
(SCImport, DTQueues)
|
||||
| schemaExists && storeLogExists -> exitConfigureQueueStore connstr schema
|
||||
| schemaExists && storeLogExists -> exitConfigureQueueStore schema
|
||||
| schemaExists -> do
|
||||
putStrLn $ "Schema " <> B.unpack schema <> " already exists in PostrgreSQL database: " <> B.unpack connstr
|
||||
exitFailure
|
||||
@@ -216,7 +216,7 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
when msgsFileExists $ putStrLn $ storeMsgsFilePath <> " file already exists."
|
||||
exitFailure
|
||||
(SCExport, DTQueues)
|
||||
| schemaExists && storeLogExists -> exitConfigureQueueStore connstr schema
|
||||
| schemaExists && storeLogExists -> exitConfigureQueueStore schema
|
||||
| not schemaExists -> do
|
||||
putStrLn $ "Schema " <> B.unpack schema <> " does not exist in PostrgreSQL database: " <> B.unpack connstr
|
||||
exitFailure
|
||||
@@ -590,26 +590,26 @@ smpServerCLI_ generateSite serveStaticFiles attachStaticFiles cfgPath logPath =
|
||||
schemaExists <- checkSchemaExists connstr schema
|
||||
case enableDbStoreLog' ini of
|
||||
Just ()
|
||||
| not schemaExists -> noDatabaseSchema connstr schema
|
||||
| not schemaExists -> noDatabaseSchema schema
|
||||
| not storeLogExists -> do
|
||||
putStrLn $ "Error: db_store_log is `on`, " <> storeLogFilePath <> " does not exist"
|
||||
exitFailure
|
||||
| otherwise -> pure ()
|
||||
Nothing
|
||||
| storeLogExists && schemaExists -> exitConfigureQueueStore connstr schema
|
||||
| storeLogExists && schemaExists -> exitConfigureQueueStore schema
|
||||
| storeLogExists -> do
|
||||
putStrLn $ "Error: store_queues is `database` with " <> storeLogFilePath <> " file present."
|
||||
putStrLn "Set store_queues to `memory` or use `smp-server database import` to migrate."
|
||||
exitFailure
|
||||
| not schemaExists -> noDatabaseSchema connstr schema
|
||||
| not schemaExists -> noDatabaseSchema schema
|
||||
| otherwise -> pure ()
|
||||
where
|
||||
noDatabaseSchema connstr schema = do
|
||||
putStrLn $ "Error: store_queues is `database`, create schema " <> B.unpack schema <> " in PostgreSQL database " <> B.unpack connstr
|
||||
noDatabaseSchema schema = do
|
||||
putStrLn $ "Error: store_queues is `database`, create schema " <> B.unpack schema <> " in PostgreSQL database"
|
||||
exitFailure
|
||||
|
||||
exitConfigureQueueStore connstr schema = do
|
||||
putStrLn $ "Error: both " <> storeLogFilePath <> " file and " <> B.unpack schema <> " schema are present (database: " <> B.unpack connstr <> ")."
|
||||
exitConfigureQueueStore schema = do
|
||||
putStrLn $ "Error: both " <> storeLogFilePath <> " file and " <> B.unpack schema <> " schema are present."
|
||||
putStrLn "Configure queue storage."
|
||||
exitFailure
|
||||
|
||||
|
||||
@@ -39,11 +39,12 @@ import Data.Int (Int64)
|
||||
import Data.List (intersperse)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Time.Clock.System (SystemTime (..))
|
||||
import Database.PostgreSQL.Simple (Binary (..), In (..), Only (..), (:.) (..))
|
||||
import Database.PostgreSQL.Simple (Binary (..), Only (..), (:.) (..))
|
||||
import qualified Database.PostgreSQL.Simple as DB
|
||||
import qualified Database.PostgreSQL.Simple.Copy as DB
|
||||
import Database.PostgreSQL.Simple.SqlQQ (sql)
|
||||
import Database.PostgreSQL.Simple.ToField (ToField (..))
|
||||
import Database.PostgreSQL.Simple.Types (PGArray (..))
|
||||
import Simplex.Messaging.Agent.Store.Postgres.Common
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Protocol
|
||||
@@ -203,13 +204,17 @@ instance MsgStoreClass PostgresMsgStore where
|
||||
DB.query
|
||||
db
|
||||
[sql|
|
||||
SELECT DISTINCT ON (recipient_id)
|
||||
recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
FROM messages
|
||||
WHERE recipient_id IN ?
|
||||
ORDER BY recipient_id, message_id ASC
|
||||
SELECT r.recipient_id, m.msg_id, m.msg_ts, m.msg_quota, m.msg_ntf_flag, m.msg_body
|
||||
FROM unnest(?::bytea[]) AS r(recipient_id)
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
FROM messages
|
||||
WHERE recipient_id = r.recipient_id
|
||||
ORDER BY message_id ASC
|
||||
LIMIT 1
|
||||
) m
|
||||
|]
|
||||
(Only (In (map recipientId' qs)))
|
||||
(Only (PGArray (map recipientId' qs)))
|
||||
where
|
||||
toRcvMsg (Only rId :. msg) = (rId, toMessage msg)
|
||||
|
||||
|
||||
@@ -254,6 +254,7 @@ ntfServerTest storeLog = do
|
||||
r `shouldContain` ["SMP notifications server v" <> simplexmqVersionCommit]
|
||||
r `shouldContain` (if storeLog then ["Store log: " <> ntfLogPath <> "/ntf-server-store.log"] else ["Store log disabled."])
|
||||
r `shouldContain` ["Serving NTF protocol on port 443 (TLS)..."]
|
||||
r `shouldNotSatisfy` any (B.unpack ntfTestServerDBConnstr `isInfixOf`)
|
||||
capture_ (withStdin "Y" . withArgs ["delete"] $ ntfServerCLI ntfCfgPath ntfLogPath)
|
||||
>>= (`shouldSatisfy` ("WARNING: deleting the server will make all queues inaccessible" `isPrefixOf`))
|
||||
doesFileExist (cfgPath <> "/ca.key") `shouldReturn` False
|
||||
|
||||
@@ -75,6 +75,7 @@ msgStoreTests = do
|
||||
someMsgStoreTests cfg = do
|
||||
it "should get queue and store/read messages" $ withMsgStore cfg testGetQueue
|
||||
it "should write/ack messages" $ withMsgStore cfg testWriteAckMessages
|
||||
it "should peek the first message of each queue in a batch" $ withMsgStore cfg testPeekMsgs
|
||||
it "should not add link data to secured messaging queue" $ withMsgStore cfg testLinkDataSecuredQueue
|
||||
|
||||
-- TODO constrain to STM stores?
|
||||
@@ -214,6 +215,40 @@ testWriteAckMessages ms = do
|
||||
void $ ExceptT $ deleteQueue ms q1
|
||||
void $ ExceptT $ deleteQueue ms q2
|
||||
|
||||
testPeekMsgs :: MsgStoreClass s => s -> IO ()
|
||||
testPeekMsgs ms = do
|
||||
g <- C.newRandom
|
||||
(rId1, qr1) <- testNewQueueRec g QMMessaging
|
||||
(rId2, qr2) <- testNewQueueRec g QMMessaging
|
||||
(rId3, qr3) <- testNewQueueRec g QMMessaging
|
||||
runRight_ $ do
|
||||
q1 <- ExceptT $ addQueue ms rId1 qr1
|
||||
q2 <- ExceptT $ addQueue ms rId2 qr2
|
||||
q3 <- ExceptT $ addQueue ms rId3 qr3
|
||||
let write q s = writeMsg ms q True =<< mkMessage s
|
||||
peek qs = M.map body <$> tryPeekMsgs ms qs
|
||||
Just (Message {msgId = mId1}, True) <- write q1 "message 1"
|
||||
Just _ <- write q1 "message 2"
|
||||
Just (Message {msgId = mId3}, True) <- write q2 "message 3"
|
||||
Just (Message {msgId = mId4}, False) <- write q2 "message 4"
|
||||
Just (Message {msgId = mId5}, False) <- write q2 "message 5"
|
||||
Nothing <- write q2 "message 6"
|
||||
msgs <- peek [q1, q2, q3, q1]
|
||||
liftIO $ msgs `shouldBe` M.fromList [(rId1, Just "message 1"), (rId2, Just "message 3")]
|
||||
Msg "message 1" <- tryDelMsg ms q1 mId1
|
||||
Msg "message 3" <- tryDelMsg ms q2 mId3
|
||||
Msg "message 4" <- tryDelMsg ms q2 mId4
|
||||
Msg "message 5" <- tryDelMsg ms q2 mId5
|
||||
msgs' <- peek [q2, q1, q3]
|
||||
liftIO $ msgs' `shouldBe` M.fromList [(rId1, Just "message 2"), (rId2, Nothing)]
|
||||
void $ ExceptT $ deleteQueue ms q1
|
||||
void $ ExceptT $ deleteQueue ms q2
|
||||
void $ ExceptT $ deleteQueue ms q3
|
||||
where
|
||||
body :: Message -> Maybe ByteString
|
||||
body Message {msgBody = MaxLenBS s} = Just s
|
||||
body MessageQuota {} = Nothing
|
||||
|
||||
testLinkDataSecuredQueue :: MsgStoreClass s => s -> IO ()
|
||||
testLinkDataSecuredQueue ms = do
|
||||
g <- C.newRandom
|
||||
|
||||
Reference in new issue
Block a user