Merge branch 'master' into master-ios

docs/rfcs/2023-04-28-files-encryption.md

@@ -0,0 +1,64 @@
+# Encrpting local app files
+
+## Problem
+
+Currently, the files are stored in the file storage unencrypted, unlike the database.
+
+There are multiple operations in the app that access files:
+
+1. Sending files via SMP - chat core reads the files chunk by chunk and sends them. The file can be encrypted once sent and the "encrypted" flag added.
+
+2. Sending files via XFTP - simplexmq encrypts the file first and then sends it. Currently, we are deleting the file from chat, once its uploaded, there is no reason to keep unencrypted file (from XFTP point of view) once its encrypted.
+
+3. Viewing images in the mobile apps.
+
+4. Playing voice files in the mobile apps.
+
+5. Playing videos and showing video previews in mobile apps.
+
+6. Saving files from the app storage to the device.
+
+## Possible solutions
+
+### System encryption
+
+A possible approach is to use platform-specific encryption mechanism. The problem with that approach is inconsistency between platforms, and that the files in chat archive will probably be unencrypted in this case.
+
+### App encryption
+
+Files will be encrypted once received, using storage key, and the core would expose C apis to mobile apps:
+
+1. Read the file with decryption - this can be used for image previews, for example, as a replacement for OS file reading.
+
+2. Copy the file with decryption to some permanent destination - this can be used for saving files to the device.
+
+3. Copy the file into a temporary location with decryption - this can be used for playing voice/video files. The app would remove the files once no longer used, and this temporary location can be cleaned on each app start, to clean up the files that the app failed to remove. Alternative to that would be to have both encrypted and decrypted copies available for the file, with paths stored in the database, and clean up process removed decrypted copies once no longer used - there should be some flags to indicate when decrypted copy can be deleted.
+
+For specific use cases:
+
+1. Viewing images in the mobile apps.
+  - iOS: we use `UIImage(contentsOfFile path: String)`. We could use `init?(data: Data)` instead, and decrypt the file in memory before passing it to the image view. Images are small enough for this approach to be ok, and in any case the image is read to memory as a whole.
+  - Android: we use `BitmapFactory.decodeFileDescriptor` (?). We could use ... 
+
+2. Playing voice files in the mobile apps.
+  - iOS: we use `AVAudioPlayer.init(contentsOf: URL)` to play the file. We could either decrypt the file before playing it, or, given that voice files are small (even if we increase allowed duration, they are still likely to be under 1mb), we could use `init(data: Data)` to avoid creating decrypted file.
+  - Android: we use `MediaPlayer.setDataSource(filePath)`. We could use ...
+
+3. Showing video previews.
+  - iOS: ...
+  - Android: ...
+
+  Possibly, we will need to store preview as a separate file, to avoid decrypting the whole video just to show preview.
+
+4. Playing video files.
+  - iOS: we use `AVPlayer(url: URL)`, the file will have to be decrypted for playback.
+  - Android: ...
+
+5. Saving files from the app storage to the device.
+  The file will have to be decrypted, passed to the system, and then decrypted copy deleted once no longer needed.
+
+### Which key to use for encryption
+
+1. Derive file encryption key from database storage key. The downside for this approach is managing key changes - they will be slow. Also, if file encryption is made optional, and in any case, for the existing users all files are not encrypted yet, we will need somehow to track which files are encrypted.
+
+2. Random per-file encryption key stored in the database. Given that the database is already encrypted, it can be a better approach, and it makes it easier to manage file encryption/decryption. File keys will not be sent to the client application, but they will be accessible via the database queries of course.

src/Simplex/Chat/Mobile/File.hs

@@ -19,7 +19,6 @@ import Data.ByteString (ByteString)
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as LB
 import qualified Data.ByteString.Lazy.Char8 as LB'
-import Data.Int (Int64)
 import Data.Word (Word8)
 import Foreign.C
 import Foreign.Marshal.Alloc (mallocBytes)
@@ -53,7 +52,7 @@ chatWriteFile path s = do
     <$> runExceptT (CF.writeFile file $ LB.fromStrict s)
 
 data ReadFileResult
-  = RFResult {fileSize :: Int64}
+  = RFResult {fileSize :: Int}
   | RFError {readError :: String}
   deriving (Generic)
 
@@ -65,7 +64,7 @@ cChatReadFile cPath cKey cNonce = do
   key <- B.packCString cKey
   nonce <- B.packCString cNonce
   (r, s) <- chatReadFile path key nonce
-  let r' = LB.toStrict (J.encode r) <> "\NUL"
+  let r' = LB.toStrict $ J.encode r <> "\NUL"
   ptr <- mallocBytes $ B.length r' + B.length s
   putByteString ptr r'
   unless (B.null s) $ putByteString (ptr `plusPtr` B.length r') s
@@ -73,8 +72,9 @@ cChatReadFile cPath cKey cNonce = do
 
 chatReadFile :: FilePath -> ByteString -> ByteString -> IO (ReadFileResult, ByteString)
 chatReadFile path keyStr nonceStr = do
-  either ((,"") . RFError) (\s -> (RFResult $ LB.length s, LB.toStrict s)) <$> runExceptT readFile_
+  either ((,"") . RFError) result <$> runExceptT readFile_
   where
+    result s = let s' = LB.toStrict s in (RFResult $ B.length s', s')
     readFile_ :: ExceptT String IO LB.ByteString
     readFile_ = do
       key <- liftEither $ strDecode keyStr

tests/MobileTests.hs

@@ -1,19 +1,25 @@
 {-# LANGUAGE CPP #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
 module MobileTests where
 
 import ChatTests.Utils
 import Control.Monad.Except
 import Crypto.Random (getRandomBytes)
+import Data.Aeson (FromJSON (..))
+import qualified Data.Aeson as J
 import Data.ByteString (ByteString)
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as BS
+import qualified Data.ByteString.Lazy.Char8 as LB
 import Data.Word (Word8)
 import Foreign.C
 import Foreign.Marshal.Alloc (mallocBytes)
 import Foreign.Ptr
 import Simplex.Chat.Mobile
+import Simplex.Chat.Mobile.File
 import Simplex.Chat.Mobile.Shared
 import Simplex.Chat.Mobile.WebRTC
 import Simplex.Chat.Store
@@ -21,7 +27,9 @@ import Simplex.Chat.Store.Profiles
 import Simplex.Chat.Types (AgentUserId (..), Profile (..))
 import Simplex.Messaging.Agent.Store.SQLite (MigrationConfirmation (..))
 import qualified Simplex.Messaging.Crypto as C
+import Simplex.Messaging.Crypto.File (CryptoFileArgs (..))
 import Simplex.Messaging.Encoding.String
+import Simplex.Messaging.Parsers (dropPrefix, sumTypeJSON)
 import System.FilePath ((</>))
 import Test.Hspec
 
@@ -30,8 +38,9 @@ mobileTests = do
   describe "mobile API" $ do
     it "start new chat without user" testChatApiNoUser
     it "start new chat with existing user" testChatApi
-    fit "should encrypt/decrypt WebRTC frames" testMediaApi
-    fit "should encrypt/decrypt WebRTC frames via C API" testMediaCApi
+    it "should encrypt/decrypt WebRTC frames" testMediaApi
+    it "should encrypt/decrypt WebRTC frames via C API" testMediaCApi
+    it "should read/write encrypted files via C API" testFileCApi
 
 noActiveUser :: String
 #if defined(darwin_HOST_OS) && defined(swiftJSON)
@@ -158,6 +167,35 @@ testMediaCApi _ = do
           cLen = fromIntegral len
       ptr <- mallocBytes len
       putByteString ptr frame
-      cKeyStr <- newCString $ BS.unpack keyStr
-      (f cKeyStr ptr cLen >>= peekCString) `shouldReturn` ""
+      cKeyStr <- newCAString $ BS.unpack keyStr
+      (f cKeyStr ptr cLen >>= peekCAString) `shouldReturn` ""
       getByteString ptr cLen
+
+instance FromJSON WriteFileResult where parseJSON = J.genericParseJSON . sumTypeJSON $ dropPrefix "WF"
+
+instance FromJSON ReadFileResult where parseJSON = J.genericParseJSON . sumTypeJSON $ dropPrefix "RF"
+
+testFileCApi :: FilePath -> IO ()
+testFileCApi tmp = do
+  src <- B.readFile "./tests/fixtures/test.pdf"
+  cPath <- newCAString $ tmp </> "test.pdf"
+  let len = B.length src
+      cLen = fromIntegral len
+  ptr <- mallocBytes $ B.length src
+  putByteString ptr src
+  r <- peekCAString =<< cChatWriteFile cPath ptr cLen
+  Just (WFResult (CFArgs key nonce)) <- jDecode r
+  cKey <- encodedCString key
+  cNonce <- encodedCString nonce
+  ptr' <- cChatReadFile cPath cKey cNonce
+  -- the returned pointer contains NUL-terminated JSON string of ReadFileResult followed by the file contents
+  r' <- peekCAString $ castPtr ptr'
+  Just (RFResult sz) <- jDecode r'
+  contents <- getByteString (ptr' `plusPtr` (length r' + 1)) $ fromIntegral sz
+  contents `shouldBe` src
+  sz `shouldBe` len
+  where
+    jDecode :: FromJSON a => String -> IO (Maybe a)
+    jDecode = pure . J.decode . LB.pack
+    encodedCString :: StrEncoding a => a -> IO CString
+    encodedCString = newCAString . BS.unpack . strEncode