core: update simplemq (with PostgreSQL support) (#5444)

src/Simplex/Chat.hs

@@ -36,13 +36,14 @@ import Simplex.Chat.Protocol
 import Simplex.Chat.Store
 import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Types
-import Simplex.Chat.Util (encryptFile, liftIOEither, shuffle)
+import Simplex.Chat.Util (shuffle)
 import Simplex.FileTransfer.Client.Presets (defaultXFTPServers)
 import Simplex.Messaging.Agent as Agent
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), InitialAgentServers (..), ServerCfg (..), ServerRoles (..), allRoles, createAgentStore, defaultAgentConfig, presetServerCfg)
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew))
+import Simplex.Messaging.Agent.Store.Common (DBStore (dbNew))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..), MigrationError)
 import Simplex.Messaging.Client (defaultNetworkConfig)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Protocol (ProtoServerWithAuth (..), ProtocolType (..), SProtocolType (..), SubscriptionMode (..), UserProtocol)

src/Simplex/Chat/Archive.hs

@@ -28,7 +28,8 @@ import qualified Database.SQLite3 as SQL
 import Simplex.Chat.Controller
 import Simplex.Chat.Util ()
 import Simplex.Messaging.Agent.Client (agentClientStore)
-import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), closeSQLiteStore, keyString, sqlString, storeKey)
+import Simplex.Messaging.Agent.Store.SQLite (closeDBStore, keyString, sqlString, storeKey)
+import Simplex.Messaging.Agent.Store.SQLite.Common (DBStore (..))
 import Simplex.Messaging.Util
 import System.FilePath
 import UnliftIO.Directory
@@ -74,7 +75,7 @@ importArchive cfg@ArchiveConfig {archivePath} =
   withTempDir cfg "simplex-chat." $ \dir -> do
     Z.withArchive archivePath $ Z.unpackInto dir
     fs@StorageFiles {chatStore, agentStore, filesPath, assetsPath} <- storageFiles
-    liftIO $ closeSQLiteStore `withStores` fs
+    liftIO $ closeDBStore `withStores` fs
     backup `withDBs` fs
     copyFile (dir </> archiveChatDbFile) $ dbFilePath chatStore
     copyFile (dir </> archiveAgentDbFile) $ dbFilePath agentStore
@@ -122,7 +123,7 @@ copyValidDirectoryFiles isFileError fromDir toDir = do
 deleteStorage :: CM ()
 deleteStorage = do
   fs <- lift storageFiles
-  liftIO $ closeSQLiteStore `withStores` fs
+  liftIO $ closeDBStore `withStores` fs
   remove `withDBs` fs
   mapM_ removeDir $ filesPath fs
   mapM_ removeDir $ assetsPath fs
@@ -132,8 +133,8 @@ deleteStorage = do
     removeDir d = whenM (doesDirectoryExist d) $ removePathForcibly d
 
 data StorageFiles = StorageFiles
-  { chatStore :: SQLiteStore,
-    agentStore :: SQLiteStore,
+  { chatStore :: DBStore,
+    agentStore :: DBStore,
     filesPath :: Maybe FilePath,
     assetsPath :: Maybe FilePath
   }
@@ -156,20 +157,20 @@ sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = D
     removeExported `withDBs` fs
     export `withDBs` fs
     -- closing after encryption prevents closing in case wrong encryption key was passed
-    liftIO $ closeSQLiteStore `withStores` fs
+    liftIO $ closeDBStore `withStores` fs
     (moveExported `withStores` fs)
       `catchChatError` \e -> (restore `withDBs` fs) >> throwError e
   where
     backup f = copyFile f (f <> ".bak")
     restore f = copyFile (f <> ".bak") f
     checkFile f = unlessM (doesFileExist f) $ throwDBError $ DBErrorNoFile f
-    checkEncryption SQLiteStore {dbKey} = do
+    checkEncryption DBStore {dbKey} = do
       enc <- maybe True (not . BA.null) <$> readTVarIO dbKey
       when (enc && BA.null key) $ throwDBError DBErrorEncrypted
       when (not enc && not (BA.null key)) $ throwDBError DBErrorPlaintext
     exported = (<> ".exported")
     removeExported f = whenM (doesFileExist $ exported f) $ removeFile (exported f)
-    moveExported SQLiteStore {dbFilePath = f, dbKey} = do
+    moveExported DBStore {dbFilePath = f, dbKey} = do
       renameFile (exported f) f
       atomically $ writeTVar dbKey $ storeKey key' (fromMaybe False keepKey)
     export f = do
@@ -219,5 +220,5 @@ sqlCipherTestKey (DBEncryptionKey key) = do
 withDBs :: Monad m => (FilePath -> m b) -> StorageFiles -> m b
 action `withDBs` StorageFiles {chatStore, agentStore} = action (dbFilePath chatStore) >> action (dbFilePath agentStore)
 
-withStores :: Monad m => (SQLiteStore -> m b) -> StorageFiles -> m b
+withStores :: Monad m => (DBStore -> m b) -> StorageFiles -> m b
 action `withStores` StorageFiles {chatStore, agentStore} = action chatStore >> action agentStore

src/Simplex/Chat/Controller.hs

@@ -73,9 +73,10 @@ import Simplex.Messaging.Agent.Client (AgentLocks, AgentQueuesInfo (..), AgentWo
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig, NetworkConfig, ServerCfg)
 import Simplex.Messaging.Agent.Lock
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation, SQLiteStore, UpMigration, withTransaction, withTransactionPriority)
+import Simplex.Messaging.Agent.Store.SQLite.Common (DBStore, withTransaction, withTransactionPriority)
 import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation, UpMigration)
 import Simplex.Messaging.Client (HostMode (..), SMPProxyFallback (..), SMPProxyMode (..), SocksMode (..))
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..))
@@ -203,7 +204,7 @@ defaultInlineFilesConfig =
       receiveInstant = True -- allow receiving instant files, within receiveChunks limit
     }
 
-data ChatDatabase = ChatDatabase {chatStore :: SQLiteStore, agentStore :: SQLiteStore}
+data ChatDatabase = ChatDatabase {chatStore :: DBStore, agentStore :: DBStore}
 
 data ChatController = ChatController
   { currentUser :: TVar (Maybe User),
@@ -213,7 +214,7 @@ data ChatController = ChatController
     firstTime :: Bool,
     smpAgent :: AgentClient,
     agentAsync :: TVar (Maybe (Async (), Maybe (Async ()))),
-    chatStore :: SQLiteStore,
+    chatStore :: DBStore,
     chatStoreChanged :: TVar Bool, -- if True, chat should be fully restarted
     random :: TVar ChaChaDRG,
     eventSeq :: TVar Int,

src/Simplex/Chat/Core.hs

@@ -26,7 +26,8 @@ import Simplex.Chat.Options (ChatOpts (..), CoreChatOpts (..))
 import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Types
 import Simplex.Chat.View (serializeChatResponse)
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), SQLiteStore, withTransaction)
+import Simplex.Messaging.Agent.Store.SQLite.Common (DBStore (..), withTransaction)
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..))
 import System.Exit (exitFailure)
 import System.IO (hFlush, stdout)
 import Text.Read (readMaybe)
@@ -66,7 +67,7 @@ sendChatCmdStr cc s = runReaderT (execChatCommand Nothing . encodeUtf8 $ T.pack
 sendChatCmd :: ChatController -> ChatCommand -> IO ChatResponse
 sendChatCmd cc cmd = runReaderT (execChatCommand' cmd) cc
 
-getSelectActiveUser :: SQLiteStore -> IO (Maybe User)
+getSelectActiveUser :: DBStore -> IO (Maybe User)
 getSelectActiveUser st = do
   users <- withTransaction st getUsers
   case find activeUser users of

src/Simplex/Chat/Library/Commands.hs

@@ -90,10 +90,12 @@ import Simplex.Messaging.Agent as Agent
 import Simplex.Messaging.Agent.Client (SubInfo (..), agentClientStore, getAgentQueuesInfo, getAgentWorkersDetails, getAgentWorkersSummary)
 import Simplex.Messaging.Agent.Env.SQLite (ServerCfg (..), ServerRoles (..), allRoles)
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (execSQL, upMigration, withConnection)
+import Simplex.Messaging.Agent.Store.SQLite (execSQL)
+import Simplex.Messaging.Agent.Store.SQLite.Common (withConnection)
 import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Agent.Store.SQLite.Migrations as Migrations
+import Simplex.Messaging.Agent.Store.Shared (upMigration)
 import Simplex.Messaging.Client (NetworkConfig (..), SocksMode (SMAlways), textToHostMode)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
@@ -2311,7 +2313,7 @@ processChatCommand' vr = \case
   ShowVersion -> do
     -- simplexmqCommitQ makes iOS builds crash m(
     let versionInfo = coreVersionInfo ""
-    chatMigrations <- map upMigration <$> withFastStore' (Migrations.getCurrent . DB.conn)
+    chatMigrations <- map upMigration <$> withFastStore' Migrations.getCurrent
     agentMigrations <- withAgent getAgentMigrations
     pure $ CRVersionInfo {versionInfo, chatMigrations, agentMigrations}
   DebugLocks -> lift $ do

src/Simplex/Chat/Mobile.hs

@@ -48,7 +48,8 @@ import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Types
 import Simplex.Messaging.Agent.Client (agentClientStore)
 import Simplex.Messaging.Agent.Env.SQLite (createAgentStore)
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, closeSQLiteStore, reopenSQLiteStore)
+import Simplex.Messaging.Agent.Store.SQLite (closeDBStore, reopenSQLiteStore)
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation (..), MigrationError)
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String
 import Simplex.Messaging.Parsers (defaultJSON, dropPrefix, sumTypeJSON)
@@ -226,7 +227,7 @@ defaultMobileConfig =
       deviceNameForRemote = "Mobile"
     }
 
-getActiveUser_ :: SQLiteStore -> IO (Maybe User)
+getActiveUser_ :: DBStore -> IO (Maybe User)
 getActiveUser_ st = find activeUser <$> withTransaction st getUsers
 
 chatMigrateInit :: String -> ScrubbedBytes -> String -> IO (Either DBMigrationResult ChatController)
@@ -255,8 +256,8 @@ chatMigrateInitKey dbFilePrefix dbKey keepKey confirm backgroundMode = runExcept
 
 chatCloseStore :: ChatController -> IO String
 chatCloseStore ChatController {chatStore, smpAgent} = handleErr $ do
-  closeSQLiteStore chatStore
-  closeSQLiteStore $ agentClientStore smpAgent
+  closeDBStore chatStore
+  closeDBStore $ agentClientStore smpAgent
 
 chatReopenStore :: ChatController -> IO String
 chatReopenStore ChatController {chatStore, smpAgent} = handleErr $ do

src/Simplex/Chat/Store.hs

@@ -1,5 +1,5 @@
 module Simplex.Chat.Store
-  ( SQLiteStore,
+  ( DBStore,
     StoreError (..),
     ChatLockEntity (..),
     UserMsgReceiptSettings (..),
@@ -17,10 +17,12 @@ import Data.ByteArray (ScrubbedBytes)
 import Simplex.Chat.Store.Migrations
 import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Store.Shared
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation, MigrationError, SQLiteStore (..), createSQLiteStore, withTransaction)
+import Simplex.Messaging.Agent.Store.SQLite (createDBStore)
+import Simplex.Messaging.Agent.Store.SQLite.Common (DBStore (..), withTransaction)
+import Simplex.Messaging.Agent.Store.Shared (MigrationConfirmation, MigrationError)
 
-createChatStore :: FilePath -> ScrubbedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError SQLiteStore)
-createChatStore dbPath key keepKey = createSQLiteStore dbPath key keepKey migrations
+createChatStore :: FilePath -> ScrubbedBytes -> Bool -> MigrationConfirmation -> IO (Either MigrationError DBStore)
+createChatStore dbPath key keepKey = createDBStore dbPath key keepKey migrations
 
 chatStoreFile :: FilePath -> FilePath
 chatStoreFile = (<> "_chat.db")

src/Simplex/Chat/Store/AppSettings.hs

@@ -8,7 +8,7 @@ import qualified Data.Aeson as J
 import Data.Maybe (fromMaybe)
 import Database.SQLite.Simple (Only (..))
 import Simplex.Chat.AppSettings (AppSettings (..), combineAppSettings, defaultAppSettings, defaultParseAppSettings)
-import Simplex.Messaging.Agent.Store.SQLite (maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 
 saveAppSettings :: DB.Connection -> AppSettings -> IO ()

src/Simplex/Chat/Store/Connections.hs

@@ -35,7 +35,7 @@ import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Store.Shared
 import Simplex.Chat.Types
 import Simplex.Messaging.Agent.Protocol (ConnId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, firstRow', maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, firstRow', maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import Simplex.Messaging.Util (eitherToMaybe)
 

src/Simplex/Chat/Store/Direct.hs

@@ -101,7 +101,7 @@ import Simplex.Chat.Types
 import Simplex.Chat.Types.Preferences
 import Simplex.Chat.Types.UITheme
 import Simplex.Messaging.Agent.Protocol (ConnId, InvitationId, UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import Simplex.Messaging.Crypto.Ratchet (PQSupport)
 import Simplex.Messaging.Protocol (SubscriptionMode (..))

src/Simplex/Chat/Store/Files.hs

@@ -109,7 +109,7 @@ import Simplex.Chat.Store.Shared
 import Simplex.Chat.Types
 import Simplex.Chat.Util (week)
 import Simplex.Messaging.Agent.Protocol (AgentMsgId, ConnId, UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, firstRow', maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, firstRow', maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))

src/Simplex/Chat/Store/Groups.hs

@@ -151,7 +151,7 @@ import Simplex.Chat.Types.Preferences
 import Simplex.Chat.Types.Shared
 import Simplex.Chat.Types.UITheme
 import Simplex.Messaging.Agent.Protocol (ConnId, UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.Ratchet (pattern PQEncOff, pattern PQSupportOff)

src/Simplex/Chat/Store/Messages.hs

@@ -152,7 +152,7 @@ import Simplex.Chat.Store.NoteFolders
 import Simplex.Chat.Store.Shared
 import Simplex.Chat.Types
 import Simplex.Messaging.Agent.Protocol (AgentMsgId, ConnId, MsgMeta (..), UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, firstRow', maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, firstRow', maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))

src/Simplex/Chat/Store/Migrations.hs

@@ -121,7 +121,7 @@ import Simplex.Chat.Migrations.M20241128_business_chats
 import Simplex.Chat.Migrations.M20241205_business_chat_members
 import Simplex.Chat.Migrations.M20241222_operator_conditions
 import Simplex.Chat.Migrations.M20241223_chat_tags
-import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
+import Simplex.Messaging.Agent.Store.Shared (Migration (..))
 
 schemaMigrations :: [(String, Query, Maybe Query)]
 schemaMigrations =

src/Simplex/Chat/Store/NoteFolders.hs

@@ -15,7 +15,7 @@ import Database.SQLite.Simple.QQ (sql)
 import Simplex.Chat.Store.Shared (StoreError (..))
 import Simplex.Chat.Types (NoteFolder (..), NoteFolderId, User (..))
 import Simplex.Messaging.Agent.Protocol (UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 
 createNoteFolder :: DB.Connection -> User -> ExceptT StoreError IO ()

src/Simplex/Chat/Store/Profiles.hs

@@ -100,7 +100,7 @@ import Simplex.Chat.Types.Shared
 import Simplex.Chat.Types.UITheme
 import Simplex.Messaging.Agent.Env.SQLite (ServerRoles (..))
 import Simplex.Messaging.Agent.Protocol (ACorrId, ConnId, UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import qualified Simplex.Messaging.Crypto.Ratchet as CR

src/Simplex/Chat/Store/Remote.hs

@@ -15,7 +15,7 @@ import qualified Database.SQLite.Simple as SQL
 import Database.SQLite.Simple.QQ (sql)
 import Simplex.Chat.Remote.Types
 import Simplex.Chat.Store.Shared
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Encoding.String (StrEncoding (..))

src/Simplex/Chat/Store/Shared.hs

@@ -38,7 +38,7 @@ import Simplex.Chat.Types.Preferences
 import Simplex.Chat.Types.Shared
 import Simplex.Chat.Types.UITheme
 import Simplex.Messaging.Agent.Protocol (ConnId, UserId)
-import Simplex.Messaging.Agent.Store.SQLite (firstRow, maybeFirstRow)
+import Simplex.Messaging.Agent.Store.AgentStore (firstRow, maybeFirstRow)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Crypto as C
 import Simplex.Messaging.Crypto.Ratchet (PQEncryption (..), PQSupport (..))

src/Simplex/Chat/Terminal/Input.hs

@@ -36,7 +36,7 @@ import Simplex.Chat.Messages.CIContent
 import Simplex.Chat.Styled
 import Simplex.Chat.Terminal.Output
 import Simplex.Chat.Types (User (..))
-import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore, withTransaction)
+import Simplex.Messaging.Agent.Store.SQLite.Common (DBStore, withTransaction)
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import Simplex.Messaging.Util (catchAll_, safeDecodeUtf8, whenM)
 import System.Exit (exitSuccess)
@@ -223,7 +223,7 @@ data AutoComplete
   | ACCommand Text
   | ACNone
 
-updateTermState :: Maybe User -> SQLiteStore -> String -> Bool -> Int -> (Key, Modifiers) -> TerminalState -> IO TerminalState
+updateTermState :: Maybe User -> DBStore -> String -> Bool -> Int -> (Key, Modifiers) -> TerminalState -> IO TerminalState
 updateTermState user_ st chatPrefix live tw (key, ms) ts@TerminalState {inputString = s, inputPosition = p, autoComplete = acp} = case key of
   CharKey c
     | ms == mempty || ms == shiftKey -> pure $ insertChars $ charsWithContact [c]