core: support down migrations to allow reverting to the previous version (#2072)

* core: support down migrations to allow reverting to the previous version

* update schema

* update simplexmq

* rename errors

* remove unused functions

* migration UI, test migration

* update migration UI

* return current migrations in CRVersionInfo

* update simplexmq

* test down migrations

* cleanup ios

* show migrations in log
This commit is contained in:
Evgeny Poberezkin
2023-03-27 18:34:48 +01:00
committed by GitHub
parent f5c11b8faf
commit c96ba30018
26 changed files with 365 additions and 222 deletions
+3 -3
View File
@@ -49,7 +49,7 @@ import Simplex.Messaging.Agent.Client (AgentLocks, SMPTestFailure)
import Simplex.Messaging.Agent.Env.SQLite (AgentConfig, NetworkConfig)
import Simplex.Messaging.Agent.Lock
import Simplex.Messaging.Agent.Protocol
import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore)
import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation, SQLiteStore, UpMigration)
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Notifications.Protocol (DeviceToken (..), NtfTknStatus)
@@ -101,7 +101,7 @@ coreVersionInfo buildTimestamp simplexmqCommit =
data ChatConfig = ChatConfig
{ agentConfig :: AgentConfig,
yesToMigrations :: Bool,
confirmMigrations :: MigrationConfirmation,
defaultServers :: DefaultAgentServers,
tbqSize :: Natural,
fileChunkSize :: Integer,
@@ -415,7 +415,7 @@ data ChatResponse
| CRUserProfile {user :: User, profile :: Profile}
| CRUserProfileNoChange {user :: User}
| CRUserPrivacy {user :: User}
| CRVersionInfo {versionInfo :: CoreVersionInfo}
| CRVersionInfo {versionInfo :: CoreVersionInfo, chatMigrations :: [UpMigration], agentMigrations :: [UpMigration]}
| CRInvitation {user :: User, connReqInvitation :: ConnReqInvitation}
| CRSentConfirmation {user :: User}
| CRSentInvitation {user :: User, customUserProfile :: Maybe Profile}
+7 -3
View File
@@ -11,18 +11,22 @@ import Simplex.Chat
import Simplex.Chat.Controller
import Simplex.Chat.Options (ChatOpts (..), CoreChatOpts (..))
import Simplex.Chat.Types
import System.Exit (exitFailure)
import UnliftIO.Async
simplexChatCore :: ChatConfig -> ChatOpts -> Maybe (Notification -> IO ()) -> (User -> ChatController -> IO ()) -> IO ()
simplexChatCore cfg@ChatConfig {yesToMigrations} opts@ChatOpts {coreOptions = CoreChatOpts {dbFilePrefix, dbKey, logAgent}} sendToast chat =
simplexChatCore cfg@ChatConfig {confirmMigrations} opts@ChatOpts {coreOptions = CoreChatOpts {dbFilePrefix, dbKey, logAgent}} sendToast chat =
case logAgent of
Just level -> do
setLogLevel level
withGlobalLogging logCfg initRun
_ -> initRun
where
initRun = do
db@ChatDatabase {chatStore} <- createChatDatabase dbFilePrefix dbKey yesToMigrations
initRun = createChatDatabase dbFilePrefix dbKey confirmMigrations >>= either exit run
exit e = do
putStrLn $ "Error opening database: " <> show e
exitFailure
run db@ChatDatabase {chatStore} = do
u <- getCreateActiveUser chatStore
cc <- newChatController db (Just u) cfg opts sendToast
runSimplexChat opts u cc chat
@@ -12,3 +12,11 @@ ALTER TABLE users ADD COLUMN view_pwd_hash BLOB;
ALTER TABLE users ADD COLUMN view_pwd_salt BLOB;
ALTER TABLE users ADD COLUMN show_ntfs INTEGER NOT NULL DEFAULT 1;
|]
down_m20230317_hidden_profiles :: Query
down_m20230317_hidden_profiles =
[sql|
ALTER TABLE users DROP COLUMN view_pwd_hash;
ALTER TABLE users DROP COLUMN view_pwd_salt;
ALTER TABLE users DROP COLUMN show_ntfs;
|]
@@ -1,6 +1,7 @@
CREATE TABLE migrations(
name TEXT NOT NULL,
ts TEXT NOT NULL,
down TEXT,
PRIMARY KEY(name)
);
CREATE TABLE contact_profiles(
+21 -74
View File
@@ -12,6 +12,7 @@ import Control.Monad.Except
import Control.Monad.Reader
import Data.Aeson (ToJSON (..))
import qualified Data.Aeson as J
import Data.Bifunctor (first)
import qualified Data.ByteString.Base64.URL as U
import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
@@ -37,26 +38,17 @@ import Simplex.Chat.Mobile.WebRTC
import Simplex.Chat.Options
import Simplex.Chat.Store
import Simplex.Chat.Types
import Simplex.Messaging.Agent.Env.SQLite (AgentConfig (yesToMigrations), createAgentStore)
import Simplex.Messaging.Agent.Store.SQLite (closeSQLiteStore)
import Simplex.Messaging.Agent.Env.SQLite (createAgentStore)
import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..), MigrationError)
import Simplex.Messaging.Client (defaultNetworkConfig)
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON)
import Simplex.Messaging.Protocol (BasicAuth (..), CorrId (..), ProtoServerWithAuth (..), ProtocolServer (..), SMPServerWithAuth)
import Simplex.Messaging.Util (catchAll, safeDecodeUtf8)
import Simplex.Messaging.Util (catchAll, liftEitherWith, safeDecodeUtf8)
import System.Timeout (timeout)
foreign export ccall "chat_migrate_init" cChatMigrateInit :: CString -> CString -> Ptr (StablePtr ChatController) -> IO CJSONString
-- TODO remove
foreign export ccall "chat_migrate_db" cChatMigrateDB :: CString -> CString -> IO CJSONString
-- chat_init is deprecated
foreign export ccall "chat_init" cChatInit :: CString -> IO (StablePtr ChatController)
-- TODO remove
foreign export ccall "chat_init_key" cChatInitKey :: CString -> CString -> IO (StablePtr ChatController)
foreign export ccall "chat_migrate_init" cChatMigrateInit :: CString -> CString -> CString -> Ptr (StablePtr ChatController) -> IO CJSONString
foreign export ccall "chat_send_cmd" cChatSendCmd :: StablePtr ChatController -> CString -> IO CJSONString
@@ -75,35 +67,17 @@ foreign export ccall "chat_encrypt_media" cChatEncryptMedia :: CString -> Ptr Wo
foreign export ccall "chat_decrypt_media" cChatDecryptMedia :: CString -> Ptr Word8 -> CInt -> IO CString
-- | check / migrate database and initialize chat controller on success
cChatMigrateInit :: CString -> CString -> Ptr (StablePtr ChatController) -> IO CJSONString
cChatMigrateInit fp key ctrl = do
cChatMigrateInit :: CString -> CString -> CString -> Ptr (StablePtr ChatController) -> IO CJSONString
cChatMigrateInit fp key conf ctrl = do
dbPath <- peekCAString fp
dbKey <- peekCAString key
confirm <- peekCAString conf
r <-
chatMigrateInit dbPath dbKey >>= \case
chatMigrateInit dbPath dbKey confirm >>= \case
Right cc -> (newStablePtr cc >>= poke ctrl) $> DBMOk
Left e -> pure e
newCAString . LB.unpack $ J.encode r
-- | check and migrate the database
-- This function validates that the encryption is correct and runs migrations - it should be called before cChatInitKey
-- TODO remove
cChatMigrateDB :: CString -> CString -> IO CJSONString
cChatMigrateDB fp key =
((,) <$> peekCAString fp <*> peekCAString key) >>= uncurry chatMigrateDB >>= newCAString . LB.unpack . J.encode
-- | initialize chat controller (deprecated)
-- The active user has to be created and the chat has to be started before most commands can be used.
cChatInit :: CString -> IO (StablePtr ChatController)
cChatInit fp = peekCAString fp >>= chatInit >>= newStablePtr
-- | initialize chat controller with encrypted database
-- The active user has to be created and the chat has to be started before most commands can be used.
-- TODO remove
cChatInitKey :: CString -> CString -> IO (StablePtr ChatController)
cChatInitKey fp key =
((,) <$> peekCAString fp <*> peekCAString key) >>= uncurry chatInitKey >>= newStablePtr
-- | send command to chat (same syntax as in terminal for now)
cChatSendCmd :: StablePtr ChatController -> CString -> IO CJSONString
cChatSendCmd cPtr cCmd = do
@@ -159,10 +133,7 @@ mobileChatOpts dbFilePrefix dbKey =
defaultMobileConfig :: ChatConfig
defaultMobileConfig =
defaultChatConfig
{ yesToMigrations = True,
agentConfig = (agentConfig defaultChatConfig) {yesToMigrations = True}
}
defaultChatConfig {confirmMigrations = MCYesUp}
type CJSONString = CString
@@ -171,60 +142,36 @@ getActiveUser_ st = find activeUser <$> withTransaction st getUsers
data DBMigrationResult
= DBMOk
| DBMInvalidConfirmation
| DBMErrorNotADatabase {dbFile :: String}
| DBMError {dbFile :: String, migrationError :: String}
| DBMErrorMigration {dbFile :: String, migrationError :: MigrationError}
| DBMErrorSQL {dbFile :: String, migrationSQLError :: String}
deriving (Show, Generic)
instance ToJSON DBMigrationResult where
toJSON = J.genericToJSON . sumTypeJSON $ dropPrefix "DBM"
toEncoding = J.genericToEncoding . sumTypeJSON $ dropPrefix "DBM"
chatMigrateInit :: String -> String -> IO (Either DBMigrationResult ChatController)
chatMigrateInit dbFilePrefix dbKey = runExceptT $ do
chatStore <- migrate createChatStore $ chatStoreFile dbFilePrefix
agentStore <- migrate createAgentStore $ agentStoreFile dbFilePrefix
chatMigrateInit :: String -> String -> String -> IO (Either DBMigrationResult ChatController)
chatMigrateInit dbFilePrefix dbKey confirm = runExceptT $ do
confirmMigrations <- liftEitherWith (const DBMInvalidConfirmation) $ strDecode $ B.pack confirm
chatStore <- migrate createChatStore (chatStoreFile dbFilePrefix) confirmMigrations
agentStore <- migrate createAgentStore (agentStoreFile dbFilePrefix) confirmMigrations
liftIO $ initialize chatStore ChatDatabase {chatStore, agentStore}
where
initialize st db = do
user_ <- getActiveUser_ st
newChatController db user_ defaultMobileConfig (mobileChatOpts dbFilePrefix dbKey) Nothing
migrate createStore dbFile =
migrate createStore dbFile confirmMigrations =
ExceptT $
(Right <$> createStore dbFile dbKey True)
(first (DBMErrorMigration dbFile) <$> createStore dbFile dbKey confirmMigrations)
`catch` (pure . checkDBError)
`catchAll` (pure . dbError)
where
checkDBError e = case sqlError e of
DB.ErrorNotADatabase -> Left $ DBMErrorNotADatabase dbFile
_ -> dbError e
dbError e = Left . DBMError dbFile $ show e
-- TODO remove
chatMigrateDB :: String -> String -> IO DBMigrationResult
chatMigrateDB dbFilePrefix dbKey =
migrate createChatStore (chatStoreFile dbFilePrefix) >>= \case
DBMOk -> migrate createAgentStore (agentStoreFile dbFilePrefix)
e -> pure e
where
migrate createStore dbFile =
((createStore dbFile dbKey True >>= closeSQLiteStore) $> DBMOk)
`catch` (pure . checkDBError)
`catchAll` (pure . dbError)
where
checkDBError e = case sqlError e of
DB.ErrorNotADatabase -> DBMErrorNotADatabase dbFile
_ -> dbError e
dbError e = DBMError dbFile $ show e
chatInit :: String -> IO ChatController
chatInit = (`chatInitKey` "")
-- TODO remove
chatInitKey :: String -> String -> IO ChatController
chatInitKey dbFilePrefix dbKey = do
db@ChatDatabase {chatStore} <- createChatDatabase dbFilePrefix dbKey True
user_ <- getActiveUser_ chatStore
newChatController db user_ defaultMobileConfig (mobileChatOpts dbFilePrefix dbKey) Nothing
dbError e = Left . DBMErrorSQL dbFile $ show e
chatSendCmd :: ChatController -> String -> IO JSONString
chatSendCmd cc s = LB.unpack . J.encode . APIResponse Nothing <$> runReaderT (execChatCommand $ B.pack s) cc
+60 -60
View File
@@ -23,6 +23,7 @@ module Simplex.Chat.Store
UserContactLink (..),
AutoAccept (..),
createChatStore,
migrations, -- used in tests
chatStoreFile,
agentStoreFile,
createUserRecord,
@@ -273,10 +274,9 @@ import Data.Bifunctor (first)
import qualified Data.ByteString.Base64 as B64
import Data.ByteString.Char8 (ByteString)
import Data.Either (rights)
import Data.Function (on)
import Data.Functor (($>))
import Data.Int (Int64)
import Data.List (sortBy, sortOn)
import Data.List (sortOn)
import Data.List.NonEmpty (NonEmpty)
import qualified Data.List.NonEmpty as L
import Data.Maybe (fromMaybe, isJust, isNothing, listToMaybe, mapMaybe)
@@ -353,7 +353,7 @@ import Simplex.Chat.Protocol
import Simplex.Chat.Types
import Simplex.Chat.Util (week)
import Simplex.Messaging.Agent.Protocol (ACorrId, AgentMsgId, ConnId, InvitationId, MsgMeta (..), UserId)
import Simplex.Messaging.Agent.Store.SQLite (SQLiteStore (..), createSQLiteStore, firstRow, firstRow', maybeFirstRow, withTransaction)
import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation, MigrationError, SQLiteStore (..), createSQLiteStore, firstRow, firstRow', maybeFirstRow, withTransaction)
import Simplex.Messaging.Agent.Store.SQLite.Migrations (Migration (..))
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON)
@@ -362,71 +362,71 @@ import Simplex.Messaging.Transport.Client (TransportHost)
import Simplex.Messaging.Util (eitherToMaybe, safeDecodeUtf8)
import UnliftIO.STM
schemaMigrations :: [(String, Query)]
schemaMigrations :: [(String, Query, Maybe Query)]
schemaMigrations =
[ ("20220101_initial", m20220101_initial),
("20220122_v1_1", m20220122_v1_1),
("20220205_chat_item_status", m20220205_chat_item_status),
("20220210_deduplicate_contact_requests", m20220210_deduplicate_contact_requests),
("20220224_messages_fks", m20220224_messages_fks),
("20220301_smp_servers", m20220301_smp_servers),
("20220302_profile_images", m20220302_profile_images),
("20220304_msg_quotes", m20220304_msg_quotes),
("20220321_chat_item_edited", m20220321_chat_item_edited),
("20220404_files_status_fields", m20220404_files_status_fields),
("20220514_profiles_user_id", m20220514_profiles_user_id),
("20220626_auto_reply", m20220626_auto_reply),
("20220702_calls", m20220702_calls),
("20220715_groups_chat_item_id", m20220715_groups_chat_item_id),
("20220811_chat_items_indices", m20220811_chat_items_indices),
("20220812_incognito_profiles", m20220812_incognito_profiles),
("20220818_chat_notifications", m20220818_chat_notifications),
("20220822_groups_host_conn_custom_user_profile_id", m20220822_groups_host_conn_custom_user_profile_id),
("20220823_delete_broken_group_event_chat_items", m20220823_delete_broken_group_event_chat_items),
("20220824_profiles_local_alias", m20220824_profiles_local_alias),
("20220909_commands", m20220909_commands),
("20220926_connection_alias", m20220926_connection_alias),
("20220928_settings", m20220928_settings),
("20221001_shared_msg_id_indices", m20221001_shared_msg_id_indices),
("20221003_delete_broken_integrity_error_chat_items", m20221003_delete_broken_integrity_error_chat_items),
("20221004_idx_msg_deliveries_message_id", m20221004_idx_msg_deliveries_message_id),
("20221011_user_contact_links_group_id", m20221011_user_contact_links_group_id),
("20221012_inline_files", m20221012_inline_files),
("20221019_unread_chat", m20221019_unread_chat),
("20221021_auto_accept__group_links", m20221021_auto_accept__group_links),
("20221024_contact_used", m20221024_contact_used),
("20221025_chat_settings", m20221025_chat_settings),
("20221029_group_link_id", m20221029_group_link_id),
("20221112_server_password", m20221112_server_password),
("20221115_server_cfg", m20221115_server_cfg),
("20221129_delete_group_feature_items", m20221129_delete_group_feature_items),
("20221130_delete_item_deleted", m20221130_delete_item_deleted),
("20221209_verified_connection", m20221209_verified_connection),
("20221210_idxs", m20221210_idxs),
("20221211_group_description", m20221211_group_description),
("20221212_chat_items_timed", m20221212_chat_items_timed),
("20221214_live_message", m20221214_live_message),
("20221222_chat_ts", m20221222_chat_ts),
("20221223_idx_chat_items_item_status", m20221223_idx_chat_items_item_status),
("20221230_idxs", m20221230_idxs),
("20230107_connections_auth_err_counter", m20230107_connections_auth_err_counter),
("20230111_users_agent_user_id", m20230111_users_agent_user_id),
("20230117_fkey_indexes", m20230117_fkey_indexes),
("20230118_recreate_smp_servers", m20230118_recreate_smp_servers),
("20230129_drop_chat_items_group_idx", m20230129_drop_chat_items_group_idx),
("20230206_item_deleted_by_group_member_id", m20230206_item_deleted_by_group_member_id),
("20230303_group_link_role", m20230303_group_link_role),
("20230317_hidden_profiles", m20230317_hidden_profiles)
[ ("20220101_initial", m20220101_initial, Nothing),
("20220122_v1_1", m20220122_v1_1, Nothing),
("20220205_chat_item_status", m20220205_chat_item_status, Nothing),
("20220210_deduplicate_contact_requests", m20220210_deduplicate_contact_requests, Nothing),
("20220224_messages_fks", m20220224_messages_fks, Nothing),
("20220301_smp_servers", m20220301_smp_servers, Nothing),
("20220302_profile_images", m20220302_profile_images, Nothing),
("20220304_msg_quotes", m20220304_msg_quotes, Nothing),
("20220321_chat_item_edited", m20220321_chat_item_edited, Nothing),
("20220404_files_status_fields", m20220404_files_status_fields, Nothing),
("20220514_profiles_user_id", m20220514_profiles_user_id, Nothing),
("20220626_auto_reply", m20220626_auto_reply, Nothing),
("20220702_calls", m20220702_calls, Nothing),
("20220715_groups_chat_item_id", m20220715_groups_chat_item_id, Nothing),
("20220811_chat_items_indices", m20220811_chat_items_indices, Nothing),
("20220812_incognito_profiles", m20220812_incognito_profiles, Nothing),
("20220818_chat_notifications", m20220818_chat_notifications, Nothing),
("20220822_groups_host_conn_custom_user_profile_id", m20220822_groups_host_conn_custom_user_profile_id, Nothing),
("20220823_delete_broken_group_event_chat_items", m20220823_delete_broken_group_event_chat_items, Nothing),
("20220824_profiles_local_alias", m20220824_profiles_local_alias, Nothing),
("20220909_commands", m20220909_commands, Nothing),
("20220926_connection_alias", m20220926_connection_alias, Nothing),
("20220928_settings", m20220928_settings, Nothing),
("20221001_shared_msg_id_indices", m20221001_shared_msg_id_indices, Nothing),
("20221003_delete_broken_integrity_error_chat_items", m20221003_delete_broken_integrity_error_chat_items, Nothing),
("20221004_idx_msg_deliveries_message_id", m20221004_idx_msg_deliveries_message_id, Nothing),
("20221011_user_contact_links_group_id", m20221011_user_contact_links_group_id, Nothing),
("20221012_inline_files", m20221012_inline_files, Nothing),
("20221019_unread_chat", m20221019_unread_chat, Nothing),
("20221021_auto_accept__group_links", m20221021_auto_accept__group_links, Nothing),
("20221024_contact_used", m20221024_contact_used, Nothing),
("20221025_chat_settings", m20221025_chat_settings, Nothing),
("20221029_group_link_id", m20221029_group_link_id, Nothing),
("20221112_server_password", m20221112_server_password, Nothing),
("20221115_server_cfg", m20221115_server_cfg, Nothing),
("20221129_delete_group_feature_items", m20221129_delete_group_feature_items, Nothing),
("20221130_delete_item_deleted", m20221130_delete_item_deleted, Nothing),
("20221209_verified_connection", m20221209_verified_connection, Nothing),
("20221210_idxs", m20221210_idxs, Nothing),
("20221211_group_description", m20221211_group_description, Nothing),
("20221212_chat_items_timed", m20221212_chat_items_timed, Nothing),
("20221214_live_message", m20221214_live_message, Nothing),
("20221222_chat_ts", m20221222_chat_ts, Nothing),
("20221223_idx_chat_items_item_status", m20221223_idx_chat_items_item_status, Nothing),
("20221230_idxs", m20221230_idxs, Nothing),
("20230107_connections_auth_err_counter", m20230107_connections_auth_err_counter, Nothing),
("20230111_users_agent_user_id", m20230111_users_agent_user_id, Nothing),
("20230117_fkey_indexes", m20230117_fkey_indexes, Nothing),
("20230118_recreate_smp_servers", m20230118_recreate_smp_servers, Nothing),
("20230129_drop_chat_items_group_idx", m20230129_drop_chat_items_group_idx, Nothing),
("20230206_item_deleted_by_group_member_id", m20230206_item_deleted_by_group_member_id, Nothing),
("20230303_group_link_role", m20230303_group_link_role, Nothing),
("20230317_hidden_profiles", m20230317_hidden_profiles, Just down_m20230317_hidden_profiles)
-- ("20230304_file_description", m20230304_file_description)
]
-- | The list of migrations in ascending order by date
migrations :: [Migration]
migrations = sortBy (compare `on` name) $ map migration schemaMigrations
migrations = sortOn name $ map migration schemaMigrations
where
migration (name, query) = Migration {name = name, up = fromQuery query}
migration (name, up, down) = Migration {name, up = fromQuery up, down = fromQuery <$> down}
createChatStore :: FilePath -> String -> Bool -> IO SQLiteStore
createChatStore :: FilePath -> String -> MigrationConfirmation -> IO (Either MigrationError SQLiteStore)
createChatStore dbFilePath dbKey = createSQLiteStore dbFilePath dbKey migrations
chatStoreFile :: FilePath -> FilePath
+1 -1
View File
@@ -117,7 +117,7 @@ responseToView user_ ChatConfig {logLevel, testView} liveItems ts = \case
CRUserProfile u p -> ttyUser u $ viewUserProfile p
CRUserProfileNoChange u -> ttyUser u ["user profile did not change"]
CRUserPrivacy u -> ttyUserPrefix u $ viewUserPrivacy u
CRVersionInfo info -> viewVersionInfo logLevel info
CRVersionInfo info _ _ -> viewVersionInfo logLevel info
CRInvitation u cReq -> ttyUser u $ viewConnReqInvitation cReq
CRSentConfirmation u -> ttyUser u ["confirmation sent!"]
CRSentInvitation u customUserProfile -> ttyUser u $ viewSentInvitation customUserProfile testView