Merge branch 'master' into master-ghc8107

2026-08-28 18:48:16 +00:00 · 2023-09-29 13:14:57 +01:00
parent e273bd1239 70a65e8969
commit 915b53054c
20 changed files with 264 additions and 194 deletions
@@ -30,7 +30,6 @@ import qualified Data.ByteString.Base64 as B64
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
 import Data.Char (isSpace, toLower)
-import Data.Composition ((.:))
 import Data.Constraint (Dict (..))
 import Data.Either (fromRight, rights)
 import Data.Fixed (div')
@@ -80,7 +79,7 @@ import Simplex.Messaging.Agent.Client (AgentStatsKey (..), SubInfo (..), agentCl
 import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (..), InitialAgentServers (..), createAgentStore, defaultAgentConfig)
 import Simplex.Messaging.Agent.Lock
 import Simplex.Messaging.Agent.Protocol
-import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection, closeSQLiteStore, openSQLiteStore)
+import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError, SQLiteStore (dbNew), execSQL, upMigration, withConnection)
 import Simplex.Messaging.Agent.Store.SQLite.DB (SlowQueryStats (..))
 import qualified Simplex.Messaging.Agent.Store.SQLite.DB as DB
 import qualified Simplex.Messaging.Agent.Store.SQLite.Migrations as Migrations
@@ -248,7 +247,7 @@ cfgServers = \case

 startChatController :: forall m. ChatMonad' m => Bool -> Bool -> Bool -> m (Async ())
 startChatController subConns enableExpireCIs startXFTPWorkers = do
-  resumeAgentClient =<< asks smpAgent
+  asks smpAgent >>= resumeAgentClient
  unless subConns $
    chatWriteVar subscriptionMode SMOnlyCreate
  users <- fromRight [] <$> runExceptT (withStoreCtx' (Just "startChatController, getUsers") getUsers)
@@ -320,8 +319,8 @@ restoreCalls = do
  calls <- asks currentCalls
  atomically $ writeTVar calls callsMap

-stopChatController :: forall m. MonadUnliftIO m => ChatController -> Bool -> m ()
-stopChatController ChatController {chatStore, smpAgent, agentAsync = s, sndFiles, rcvFiles, expireCIFlags} closeStore = do
+stopChatController :: forall m. MonadUnliftIO m => ChatController -> m ()
+stopChatController ChatController {smpAgent, agentAsync = s, sndFiles, rcvFiles, expireCIFlags} = do
  disconnectAgentClient smpAgent
  readTVarIO s >>= mapM_ (\(a1, a2) -> uninterruptibleCancel a1 >> mapM_ uninterruptibleCancel a2)
  closeFiles sndFiles
@@ -330,9 +329,6 @@ stopChatController ChatController {chatStore, smpAgent, agentAsync = s, sndFiles
    keys <- M.keys <$> readTVar expireCIFlags
    forM_ keys $ \k -> TM.insert k False expireCIFlags
    writeTVar s Nothing
-  when closeStore $ liftIO $ do
-    closeSQLiteStore chatStore
-    closeSQLiteStore $ agentClientStore smpAgent
  where
    closeFiles :: TVar (Map Int64 Handle) -> m ()
    closeFiles files = do
@@ -462,19 +458,12 @@ processChatCommand = \case
    checkDeleteChatUser user'
    withChatLock "deleteUser" . procCmd $ deleteChatUser user' delSMPQueues
  DeleteUser uName delSMPQueues viewPwd_ -> withUserName uName $ \userId -> APIDeleteUser userId delSMPQueues viewPwd_
-  APIStartChat ChatCtrlCfg {subConns, enableExpireCIs, startXFTPWorkers, openDBWithKey} -> withUser' $ \_ ->
+  StartChat subConns enableExpireCIs startXFTPWorkers -> withUser' $ \_ ->
    asks agentAsync >>= readTVarIO >>= \case
      Just _ -> pure CRChatRunning
-      _ -> checkStoreNotChanged $ do
-        forM_ openDBWithKey $ \(DBEncryptionKey dbKey) -> do
-          ChatController {chatStore, smpAgent} <- ask
-          open chatStore dbKey
-          open (agentClientStore smpAgent) dbKey
-        startChatController subConns enableExpireCIs startXFTPWorkers $> CRChatStarted
-        where
-          open = handleDBError DBErrorOpen .: openSQLiteStore
-  APIStopChat closeStore -> do
-    ask >>= (`stopChatController` closeStore)
+      _ -> checkStoreNotChanged $ startChatController subConns enableExpireCIs startXFTPWorkers $> CRChatStarted
+  APIStopChat -> do
+    ask >>= stopChatController
    pure CRChatStopped
  APIActivateChat -> withUser $ \_ -> do
    restoreCalls
@@ -2562,7 +2551,7 @@ subscribeUserConnections onlyNeeded agentBatchSubscribe user@User {userId} = do
    getContactConns :: m ([ConnId], Map ConnId Contact)
    getContactConns = do
      cts <- withStore_ ("subscribeUserConnections " <> show userId <> ", getUserContacts") getUserContacts
-      let connIds = map contactConnId cts
+      let connIds = map contactConnId (filter contactActive cts)
      pure (connIds, M.fromList $ zip connIds cts)
    getUserContactLinkConns :: m ([ConnId], Map ConnId UserContact)
    getUserContactLinkConns = do
@@ -2572,7 +2561,7 @@ subscribeUserConnections onlyNeeded agentBatchSubscribe user@User {userId} = do
    getGroupMemberConns :: m ([Group], [ConnId], Map ConnId GroupMember)
    getGroupMemberConns = do
      gs <- withStore_ ("subscribeUserConnections " <> show userId <> ", getUserGroups") getUserGroups
-      let mPairs = concatMap (\(Group _ ms) -> mapMaybe (\m -> (,m) <$> memberConnId m) ms) gs
+      let mPairs = concatMap (\(Group _ ms) -> mapMaybe (\m -> (,m) <$> memberConnId m) (filter (not . memberRemoved) ms)) gs
      pure (gs, map fst mPairs, M.fromList mPairs)
    getSndFileTransferConns :: m ([ConnId], Map ConnId SndFileTransfer)
    getSndFileTransferConns = do
@@ -4243,16 +4232,22 @@ processAgentMessageConn user@User {userId} corrId agentConnId agentMessage = do
    xInfo c p' = void $ processContactProfileUpdate c p' True

    xDirectDel :: Contact -> RcvMessage -> MsgMeta -> m ()
-    xDirectDel c msg msgMeta = do
-      checkIntegrityCreateItem (CDDirectRcv c) msgMeta
-      ct' <- withStore' $ \db -> updateContactStatus db user c CSDeleted
-      contactConns <- withStore $ \db -> getContactConnections db userId ct'
-      deleteAgentConnectionsAsync user $ map aConnId contactConns
-      forM_ contactConns $ \conn -> withStore' $ \db -> updateConnectionStatus db conn ConnDeleted
-      let ct'' = ct' {activeConn = (contactConn ct') {connStatus = ConnDeleted}} :: Contact
-      ci <- saveRcvChatItem user (CDDirectRcv ct'') msg msgMeta (CIRcvDirectEvent RDEContactDeleted)
-      toView $ CRNewChatItem user (AChatItem SCTDirect SMDRcv (DirectChat ct'') ci)
-      toView $ CRContactDeletedByContact user ct''
+    xDirectDel c msg msgMeta =
+      if directOrUsed c
+        then do
+          checkIntegrityCreateItem (CDDirectRcv c) msgMeta
+          ct' <- withStore' $ \db -> updateContactStatus db user c CSDeleted
+          contactConns <- withStore $ \db -> getContactConnections db userId ct'
+          deleteAgentConnectionsAsync user $ map aConnId contactConns
+          forM_ contactConns $ \conn -> withStore' $ \db -> updateConnectionStatus db conn ConnDeleted
+          let ct'' = ct' {activeConn = (contactConn ct') {connStatus = ConnDeleted}} :: Contact
+          ci <- saveRcvChatItem user (CDDirectRcv ct'') msg msgMeta (CIRcvDirectEvent RDEContactDeleted)
+          toView $ CRNewChatItem user (AChatItem SCTDirect SMDRcv (DirectChat ct'') ci)
+          toView $ CRContactDeletedByContact user ct''
+        else do
+          contactConns <- withStore $ \db -> getContactConnections db userId c
+          deleteAgentConnectionsAsync user $ map aConnId contactConns
+          withStore' $ \db -> deleteContact db user c

    processContactProfileUpdate :: Contact -> Profile -> Bool -> m Contact
    processContactProfileUpdate c@Contact {profile = p} p' createItems
@@ -5396,9 +5391,9 @@ chatCommandP =
      "/_delete user " *> (APIDeleteUser <$> A.decimal <* " del_smp=" <*> onOffP <*> optional (A.space *> jsonP)),
      "/delete user " *> (DeleteUser <$> displayName <*> pure True <*> optional (A.space *> pwdP)),
      ("/user" <|> "/u") $> ShowActiveUser,
-      "/_start" *> (APIStartChat <$> ((A.space *> jsonP) <|> chatCtrlCfgP)),
-      "/_stop close" $> APIStopChat {closeStore = True},
-      "/_stop" $> APIStopChat False,
+      "/_start subscribe=" *> (StartChat <$> onOffP <* " expire=" <*> onOffP <* " xftp=" <*> onOffP),
+      "/_start" $> StartChat True True True,
+      "/_stop" $> APIStopChat,
      "/_app activate" $> APIActivateChat,
      "/_app suspend " *> (APISuspendChat <$> A.decimal),
      "/_resubscribe all" $> ResubscribeAllConnections,
@@ -5626,12 +5621,6 @@ chatCommandP =
    ]
  where
    choice = A.choice . map (\p -> p <* A.takeWhile (== ' ') <* A.endOfInput)
-    chatCtrlCfgP = do
-      subConns <- (" subscribe=" *> onOffP) <|> pure True
-      enableExpireCIs <- (" expire=" *> onOffP) <|> pure True
-      startXFTPWorkers <- (" xftp=" *> onOffP) <|> pure True
-      openDBWithKey <- optional $ " key=" *> dbKeyP
-      pure ChatCtrlCfg {subConns, enableExpireCIs, startXFTPWorkers, openDBWithKey}
    incognitoP = (A.space *> ("incognito" <|> "i")) $> True <|> pure False
    incognitoOnOffP = (A.space *> "incognito=" *> onOffP) <|> pure False
    imagePrefix = (<>) <$> "data:" <*> ("image/png;base64," <|> "image/jpg;base64,")
@@ -9,7 +9,6 @@ module Simplex.Chat.Archive
    importArchive,
    deleteStorage,
    sqlCipherExport,
-    handleDBError,
  )
 where

@@ -21,7 +20,7 @@ import qualified Data.Text as T
 import qualified Database.SQLite3 as SQL
 import Simplex.Chat.Controller
 import Simplex.Messaging.Agent.Client (agentClientStore)
-import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), sqlString)
+import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), sqlString, closeSQLiteStore)
 import Simplex.Messaging.Util
 import System.FilePath
 import UnliftIO.Directory
@@ -42,9 +41,9 @@ archiveFilesFolder = "simplex_v1_files"
 exportArchive :: ChatMonad m => ArchiveConfig -> m ()
 exportArchive cfg@ArchiveConfig {archivePath, disableCompression} =
  withTempDir cfg "simplex-chat." $ \dir -> do
-    StorageFiles {chatDb, agentDb, filesPath} <- storageFiles
-    copyFile chatDb $ dir </> archiveChatDbFile
-    copyFile agentDb $ dir </> archiveAgentDbFile
+    StorageFiles {chatStore, agentStore, filesPath} <- storageFiles
+    copyFile (dbFilePath chatStore) $ dir </> archiveChatDbFile
+    copyFile (dbFilePath agentStore) $ dir </> archiveAgentDbFile
    forM_ filesPath $ \fp ->
      copyDirectoryFiles fp $ dir </> archiveFilesFolder
    let method = if disableCompression == Just True then Z.Store else Z.Deflate
@@ -54,11 +53,11 @@ importArchive :: ChatMonad m => ArchiveConfig -> m [ArchiveError]
 importArchive cfg@ArchiveConfig {archivePath} =
  withTempDir cfg "simplex-chat." $ \dir -> do
    Z.withArchive archivePath $ Z.unpackInto dir
-    StorageFiles {chatDb, agentDb, filesPath} <- storageFiles
-    backup chatDb
-    backup agentDb
-    copyFile (dir </> archiveChatDbFile) chatDb
-    copyFile (dir </> archiveAgentDbFile) agentDb
+    fs@StorageFiles {chatStore, agentStore, filesPath} <- storageFiles
+    liftIO $ closeSQLiteStore `withStores` fs
+    backup `withDBs` fs
+    copyFile (dir </> archiveChatDbFile) $ dbFilePath chatStore
+    copyFile (dir </> archiveAgentDbFile) $ dbFilePath agentStore
    copyFiles dir filesPath
      `E.catch` \(e :: E.SomeException) -> pure [AEImport . ChatError . CEException $ show e]
  where
@@ -94,52 +93,69 @@ copyDirectoryFiles fromDir toDir = do

 deleteStorage :: ChatMonad m => m ()
 deleteStorage = do
-  StorageFiles {chatDb, agentDb, filesPath} <- storageFiles
-  removeFile chatDb
-  removeFile agentDb
-  mapM_ removePathForcibly filesPath
-  tmpPath <- readTVarIO =<< asks tempDirectory
-  mapM_ removePathForcibly tmpPath
+  fs <- storageFiles
+  liftIO $ closeSQLiteStore `withStores` fs
+  remove `withDBs` fs
+  mapM_ removeDir $ filesPath fs
+  mapM_ removeDir =<< chatReadVar tempDirectory
+  where
+    remove f = whenM (doesFileExist f) $ removeFile f
+    removeDir d = whenM (doesDirectoryExist d) $ removePathForcibly d

 data StorageFiles = StorageFiles
-  { chatDb :: FilePath,
-    chatEncrypted :: TVar Bool,
-    agentDb :: FilePath,
-    agentEncrypted :: TVar Bool,
+  { chatStore :: SQLiteStore,
+    agentStore :: SQLiteStore,
    filesPath :: Maybe FilePath
  }

 storageFiles :: ChatMonad m => m StorageFiles
 storageFiles = do
  ChatController {chatStore, filesFolder, smpAgent} <- ask
-  let SQLiteStore {dbFilePath = chatDb, dbEncrypted = chatEncrypted} = chatStore
-      SQLiteStore {dbFilePath = agentDb, dbEncrypted = agentEncrypted} = agentClientStore smpAgent
+  let agentStore = agentClientStore smpAgent
  filesPath <- readTVarIO filesFolder
-  pure StorageFiles {chatDb, chatEncrypted, agentDb, agentEncrypted, filesPath}
+  pure StorageFiles {chatStore, agentStore, filesPath}

 sqlCipherExport :: forall m. ChatMonad m => DBEncryptionConfig -> m ()
 sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = DBEncryptionKey key'} =
  when (key /= key') $ do
-    fs@StorageFiles {chatDb, chatEncrypted, agentDb, agentEncrypted} <- storageFiles
-    checkFile `with` fs
-    backup `with` fs
-    (export chatDb chatEncrypted >> export agentDb agentEncrypted)
-      `catchChatError` \e -> tryChatError (restore `with` fs) >> throwError e
+    fs <- storageFiles
+    checkFile `withDBs` fs
+    backup `withDBs` fs
+    checkEncryption `withStores` fs
+    removeExported `withDBs` fs
+    export `withDBs` fs
+    -- closing after encryption prevents closing in case wrong encryption key was passed
+    liftIO $ closeSQLiteStore `withStores` fs
+    (moveExported `withStores` fs)
+      `catchChatError` \e -> (restore `withDBs` fs) >> throwError e
  where
-    action `with` StorageFiles {chatDb, agentDb} = action chatDb >> action agentDb
    backup f = copyFile f (f <> ".bak")
    restore f = copyFile (f <> ".bak") f
    checkFile f = unlessM (doesFileExist f) $ throwDBError $ DBErrorNoFile f
-    export f dbEnc = do
-      enc <- readTVarIO dbEnc
+    checkEncryption SQLiteStore {dbEncrypted} = do
+      enc <- readTVarIO dbEncrypted
      when (enc && null key) $ throwDBError DBErrorEncrypted
      when (not enc && not (null key)) $ throwDBError DBErrorPlaintext
-      withDB (`SQL.exec` exportSQL) DBErrorExport
-      renameFile (f <> ".exported") f
-      withDB (`SQL.exec` testSQL) DBErrorOpen
-      atomically $ writeTVar dbEnc $ not (null key')
+    exported = (<> ".exported")
+    removeExported f = whenM (doesFileExist $ exported f) $ removeFile (exported f)
+    moveExported SQLiteStore {dbFilePath = f, dbEncrypted} = do
+      renameFile (exported f) f
+      atomically $ writeTVar dbEncrypted $ not (null key')
+    export f = do
+      withDB f (`SQL.exec` exportSQL) DBErrorExport
+      withDB (exported f) (`SQL.exec` testSQL) DBErrorOpen
      where
-        withDB a err = handleDBError err $ bracket (SQL.open $ T.pack f) SQL.close a
+        withDB f' a err =
+          liftIO (bracket (SQL.open $ T.pack f') SQL.close a $> Nothing)
+            `catch` checkSQLError
+            `catch` (\(e :: SomeException) -> sqliteError' e)
+            >>= mapM_ (throwDBError . err)
+          where
+            checkSQLError e = case SQL.sqlError e of
+              SQL.ErrorNotADatabase -> pure $ Just SQLiteErrorNotADatabase
+              _ -> sqliteError' e
+            sqliteError' :: Show e => e -> m (Maybe SQLiteError)
+            sqliteError' = pure . Just . SQLiteError . show
        exportSQL =
          T.unlines $
            keySQL key
@@ -152,20 +168,12 @@ sqlCipherExport DBEncryptionConfig {currentKey = DBEncryptionKey key, newKey = D
            keySQL key'
              <> [ "PRAGMA foreign_keys = ON;",
                   "PRAGMA secure_delete = ON;",
-                   "PRAGMA auto_vacuum = FULL;",
                   "SELECT count(*) FROM sqlite_master;"
                 ]
        keySQL k = ["PRAGMA key = " <> sqlString k <> ";" | not (null k)]

-handleDBError :: forall m. ChatMonad m => (SQLiteError -> DatabaseError) -> IO () -> m ()
-handleDBError err a =
-  (liftIO a $> Nothing)
-    `catch` checkSQLError
-    `catch` (\(e :: SomeException) -> sqliteError' e)
-    >>= mapM_ (throwDBError . err)
-  where
-    checkSQLError e = case SQL.sqlError e of
-      SQL.ErrorNotADatabase -> pure $ Just SQLiteErrorNotADatabase
-      _ -> sqliteError' e
-    sqliteError' :: Show e => e -> m (Maybe SQLiteError)
-    sqliteError' = pure . Just . SQLiteError . show
+withDBs :: Monad m => (FilePath -> m b) -> StorageFiles -> m b
+action `withDBs` StorageFiles {chatStore, agentStore} = action (dbFilePath chatStore) >> action (dbFilePath agentStore)
+
+withStores :: Monad m => (SQLiteStore -> m b) -> StorageFiles -> m b
+action `withStores` StorageFiles {chatStore, agentStore} = action chatStore >> action agentStore
@@ -221,8 +221,8 @@ data ChatCommand
  | UnmuteUser
  | APIDeleteUser UserId Bool (Maybe UserPwd)
  | DeleteUser UserName Bool (Maybe UserPwd)
-  | APIStartChat ChatCtrlCfg
-  | APIStopChat {closeStore :: Bool}
+  | StartChat {subscribeConnections :: Bool, enableExpireChatItems :: Bool, startXFTPWorkers :: Bool}
+  | APIStopChat
  | APIActivateChat
  | APISuspendChat {suspendTimeout :: Int}
  | ResubscribeAllConnections
@@ -621,14 +621,6 @@ instance ToJSON ChatResponse where
  toJSON = J.genericToJSON . sumTypeJSON $ dropPrefix "CR"
  toEncoding = J.genericToEncoding . sumTypeJSON $ dropPrefix "CR"

-data ChatCtrlCfg = ChatCtrlCfg
-  { subConns :: Bool,
-    enableExpireCIs :: Bool,
-    startXFTPWorkers :: Bool,
-    openDBWithKey :: Maybe DBEncryptionKey
-  }
-  deriving (Show, Generic, FromJSON)
-
 newtype UserPwd = UserPwd {unUserPwd :: Text}
  deriving (Eq, Show)

@@ -1642,7 +1642,7 @@ viewChatError logLevel = \case
    DBErrorEncrypted -> ["error: chat database is already encrypted"]
    DBErrorPlaintext -> ["error: chat database is not encrypted"]
    DBErrorExport e -> ["error encrypting database: " <> sqliteError' e]
-    DBErrorOpen e -> ["error opening database: " <> sqliteError' e]
+    DBErrorOpen e -> ["error opening database after encryption: " <> sqliteError' e]
    e -> ["chat database error: " <> sShow e]
  ChatErrorAgent err entity_ -> case err of
    CMD PROHIBITED -> [withConnEntity <> "error: command is prohibited"]