replace base64-bytestring with base64 (#1065)

* replace base64-bytestring with base64

* minify

* use bytestring-0.10 compatible fork

PR pending...

* bump base64 fork with text compat

* move compat details to modules

* switch repo

* add back module

* cleanup

* minify

* clean imports

* rename

---------

Co-authored-by: Evgeny Poberezkin <evgeny@poberezkin.com>
This commit is contained in:
Alexander Bondarenko
2024-03-28 01:35:09 +02:00
committed by GitHub
parent 5e0123313c
commit ee90ea6a69
19 changed files with 105 additions and 54 deletions

View File

@@ -14,6 +14,12 @@ source-repository-package
location: https://github.com/simplex-chat/aeson.git
tag: aab7b5a14d6c5ea64c64dcaee418de1bb00dcc2b
-- old bs/text compat for 8.10
source-repository-package
type: git
location: https://github.com/simplex-chat/base64.git
tag: 2d77b6dbcaffc00570a70be8694049f3710e7c94
source-repository-package
type: git
location: https://github.com/simplex-chat/hs-socks.git

View File

@@ -31,7 +31,7 @@ dependencies:
- async == 2.2.*
- attoparsec == 0.14.*
- base >= 4.14 && < 5
- base64-bytestring >= 1.0 && < 1.3
- base64 == 1.0.*
- case-insensitive == 1.2.*
- composition == 1.0.*
- constraints >= 0.12 && < 0.14

View File

@@ -119,6 +119,8 @@ library
Simplex.Messaging.Crypto.SNTRUP761.Bindings.FFI
Simplex.Messaging.Crypto.SNTRUP761.Bindings.RNG
Simplex.Messaging.Encoding
Simplex.Messaging.Encoding.Base64
Simplex.Messaging.Encoding.Base64.URL
Simplex.Messaging.Encoding.String
Simplex.Messaging.Notifications.Client
Simplex.Messaging.Notifications.Protocol
@@ -187,7 +189,7 @@ library
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -259,7 +261,7 @@ executable ntf-server
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -332,7 +334,7 @@ executable smp-agent
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -405,7 +407,7 @@ executable smp-server
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -478,7 +480,7 @@ executable xftp
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -551,7 +553,7 @@ executable xftp-server
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14
@@ -658,7 +660,7 @@ test-suite simplexmq-test
, async ==2.2.*
, attoparsec ==0.14.*
, base >=4.14 && <5
, base64-bytestring >=1.0 && <1.3
, base64 ==1.0.*
, case-insensitive ==1.2.*
, composition ==1.0.*
, constraints >=0.12 && <0.14

View File

@@ -17,7 +17,6 @@ import Control.Monad
import Control.Monad.Except
import Control.Monad.Reader
import Data.Bifunctor (first)
import qualified Data.ByteString.Base64.URL as B64
import Data.ByteString.Builder (byteString)
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
@@ -46,6 +45,7 @@ import Simplex.FileTransfer.Server.StoreLog
import Simplex.FileTransfer.Transport
import qualified Simplex.Messaging.Crypto as C
import qualified Simplex.Messaging.Crypto.Lazy as LC
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Protocol (CorrId, RcvPublicDhKey, RcvPublicAuthKey, RecipientId, TransmissionAuth)
import Simplex.Messaging.Server (dummyVerifyCmd, verifyCmdAuthorization)
@@ -382,7 +382,7 @@ processXFTPRequest HTTP2Body {bodyPart} = \case
\used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
receive = do
path <- asks $ filesPath . config
let fPath = path </> B.unpack (B64.encode senderId)
let fPath = path </> B.unpack (U.encode senderId)
receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
Right () -> do
stats <- asks serverStats

View File

@@ -142,7 +142,6 @@ import Crypto.Random (ChaChaDRG)
import qualified Data.Aeson as J
import qualified Data.Aeson.TH as J
import Data.Bifunctor (bimap, first, second)
import Data.ByteString.Base64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Composition ((.:.))
@@ -182,6 +181,7 @@ import Simplex.Messaging.Client
import Simplex.Messaging.Client.Agent ()
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.Base64 (encode)
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Notifications.Client
import Simplex.Messaging.Notifications.Protocol

View File

@@ -163,7 +163,6 @@ import Data.Aeson (FromJSON (..), ToJSON (..))
import qualified Data.Aeson.TH as J
import Data.Attoparsec.ByteString.Char8 (Parser)
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.ByteString.Base64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Functor (($>))
@@ -202,6 +201,7 @@ import Simplex.Messaging.Crypto.Ratchet
SndE2ERatchetParams
)
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.Base64 (base64P, encode)
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers
import Simplex.Messaging.Protocol

View File

@@ -231,7 +231,6 @@ import Data.Bifunctor (first, second)
import Data.ByteArray (ScrubbedBytes)
import qualified Data.ByteArray as BA
import Data.ByteString (ByteString)
import qualified Data.ByteString.Base64.URL as U
import qualified Data.ByteString.Char8 as B
import Data.Char (toLower)
import Data.Functor (($>))
@@ -271,6 +270,7 @@ import Simplex.Messaging.Crypto.File (CryptoFile (..), CryptoFileArgs (..))
import Simplex.Messaging.Crypto.Ratchet (RatchetX448, SkippedMsgDiff (..), SkippedMsgKeys, PQEncryption (..), PQSupport (..))
import qualified Simplex.Messaging.Crypto.Ratchet as CR
import Simplex.Messaging.Encoding
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Notifications.Protocol (DeviceToken (..), NtfSubscriptionId, NtfTknStatus (..), NtfTokenId, SMPQueueNtf (..))
import Simplex.Messaging.Notifications.Types

View File

@@ -211,8 +211,6 @@ import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Bifunctor (bimap, first)
import Data.ByteArray (ByteArrayAccess)
import qualified Data.ByteArray as BA
import Data.ByteString.Base64 (decode, encode)
import qualified Data.ByteString.Base64.URL as U
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.ByteString.Lazy (fromStrict, toStrict)
@@ -230,6 +228,8 @@ import Database.SQLite.Simple.ToField (ToField (..))
import GHC.TypeLits (ErrorMessage (..), KnownNat, Nat, TypeError, natVal, type (+))
import Network.Transport.Internal (decodeWord16, encodeWord16)
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.Base64 (decode, encode)
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers (blobFieldDecoder, parseAll, parseString)
import Simplex.Messaging.Util ((<$?>))

View File

@@ -0,0 +1,29 @@
{-# LANGUAGE OverloadedStrings #-}
-- | Compatibility wrappers for base64 package, Base64 (padded) variant.
module Simplex.Messaging.Encoding.Base64 where
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Base64.Types (extractBase64)
import Data.Bifunctor (first)
import Data.ByteString.Base64 (decodeBase64Untyped, encodeBase64')
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import qualified Data.Text as T
encode :: ByteString -> ByteString
encode = extractBase64 . encodeBase64'
{-# INLINE encode #-}
decode :: ByteString -> Either String ByteString
decode = first T.unpack . decodeBase64Untyped
{-# INLINE decode #-}
base64P :: A.Parser ByteString
base64P = do
str <- A.takeWhile1 (`B.elem` base64Alphabet)
pad <- A.takeWhile (== '=') -- correct amount of padding can be derived from str length
either (fail . T.unpack) pure $ decodeBase64Untyped (str <> pad)
base64Alphabet :: ByteString
base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"

View File

@@ -0,0 +1,33 @@
{-# LANGUAGE OverloadedStrings #-}
-- | Compatibility wrappers for base64 package, Base64URL-padded variant.
module Simplex.Messaging.Encoding.Base64.URL where
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Base64.Types (extractBase64)
import Data.Bifunctor (first)
import Data.ByteString.Base64.URL (decodeBase64Lenient, decodeBase64UnpaddedUntyped, decodeBase64Untyped, encodeBase64')
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import qualified Data.Text as T
encode :: ByteString -> ByteString
encode = extractBase64 . encodeBase64'
{-# INLINE encode #-}
decode :: ByteString -> Either String ByteString
decode = first T.unpack . decodeBase64Untyped
{-# INLINE decode #-}
decodeLenient :: ByteString -> ByteString
decodeLenient = decodeBase64Lenient
{-# INLINE decodeLenient #-}
base64urlP :: A.Parser ByteString
base64urlP = do
str <- A.takeWhile1 (`B.elem` base64AlphabetURL)
_pad <- A.takeWhile (== '=') -- correct amount of padding can be derived from str length
either (fail . T.unpack) pure $ decodeBase64UnpaddedUntyped str
base64AlphabetURL :: ByteString
base64AlphabetURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"

View File

@@ -10,7 +10,6 @@ module Simplex.Messaging.Encoding.String
strToJSON,
strToJEncoding,
strParseJSON,
base64urlP,
strEncodeList,
strListP,
)
@@ -23,10 +22,8 @@ import qualified Data.Aeson.Encoding as JE
import qualified Data.Aeson.Types as JT
import Data.Attoparsec.ByteString.Char8 (Parser)
import qualified Data.Attoparsec.ByteString.Char8 as A
import qualified Data.ByteString.Base64.URL as U
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Char (isAlphaNum)
import Data.Int (Int64)
import qualified Data.List.NonEmpty as L
import Data.Set (Set)
@@ -38,6 +35,7 @@ import Data.Time.Clock.System (SystemTime (..))
import Data.Time.Format.ISO8601
import Data.Word (Word16, Word32)
import Simplex.Messaging.Encoding
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Parsers (parseAll)
import Simplex.Messaging.Util ((<$?>))
@@ -54,19 +52,16 @@ class StrEncoding a where
strDecode :: ByteString -> Either String a
strDecode = parseAll strP
strP :: Parser a
strP = strDecode <$?> base64urlP
strP = strDecode <$?> U.base64urlP
-- base64url encoding/decoding of ByteStrings - the parser only allows non-empty strings
instance StrEncoding ByteString where
strEncode = U.encode
{-# INLINE strEncode #-}
strDecode = U.decode
strP = base64urlP
base64urlP :: Parser ByteString
base64urlP = do
str <- A.takeWhile1 (\c -> isAlphaNum c || c == '-' || c == '_')
pad <- A.takeWhile (== '=')
either fail pure $ U.decode (str <> pad)
{-# INLINE strDecode #-}
strP = U.base64urlP
{-# INLINE strP #-}
newtype Str = Str {unStr :: ByteString}
deriving (Eq, Show)

View File

@@ -27,8 +27,9 @@ import Data.Aeson (ToJSON, (.=))
import qualified Data.Aeson as J
import qualified Data.Aeson.Encoding as JE
import qualified Data.Aeson.TH as JQ
import Data.Base64.Types (extractBase64)
import Data.Bifunctor (first)
import qualified Data.ByteString.Base64.URL as U
import qualified Data.ByteString.Base64.URL as UP
import Data.ByteString.Builder (lazyByteString)
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Lazy.Char8 as LB
@@ -46,6 +47,7 @@ import Network.HTTP2.Client (Request)
import qualified Network.HTTP2.Client as H
import Network.Socket (HostName, ServiceName)
import qualified Simplex.Messaging.Crypto as C
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Notifications.Protocol
import Simplex.Messaging.Notifications.Server.Push.APNS.Internal
@@ -91,8 +93,8 @@ signedJWTToken pk (JWTToken hdr claims) = do
pure $ hc <> "." <> serialize sig
where
jwtEncode :: ToJSON a => a -> ByteString
jwtEncode = U.encodeUnpadded . LB.toStrict . J.encode
serialize sig = U.encodeUnpadded $ encodeASN1' DER [Start Sequence, IntVal (EC.sign_r sig), IntVal (EC.sign_s sig), End Sequence]
jwtEncode = extractBase64 . UP.encodeBase64Unpadded' . LB.toStrict . J.encode
serialize sig = extractBase64 . UP.encodeBase64Unpadded' $ encodeASN1' DER [Start Sequence, IntVal (EC.sign_r sig), IntVal (EC.sign_s sig), End Sequence]
readECPrivateKey :: FilePath -> IO EC.PrivateKey
readECPrivateKey f = do

View File

@@ -10,10 +10,9 @@ import qualified Data.Aeson as J
import Data.Attoparsec.ByteString.Char8 (Parser)
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Bifunctor (first)
import Data.ByteString.Base64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Char (isAlphaNum, toLower)
import Data.Char (toLower)
import Data.String
import Data.Text (Text)
import qualified Data.Text as T
@@ -24,23 +23,8 @@ import Database.SQLite.Simple (ResultError (..), SQLData (..))
import Database.SQLite.Simple.FromField (FieldParser, returnError)
import Database.SQLite.Simple.Internal (Field (..))
import Database.SQLite.Simple.Ok (Ok (Ok))
import Simplex.Messaging.Util ((<$?>))
import Text.Read (readMaybe)
base64P :: Parser ByteString
base64P = decode <$?> paddedBase64 rawBase64P
paddedBase64 :: Parser ByteString -> Parser ByteString
paddedBase64 raw = (<>) <$> raw <*> pad
where
pad = A.takeWhile (== '=')
rawBase64P :: Parser ByteString
rawBase64P = A.takeWhile1 (\c -> isAlphaNum c || c == '+' || c == '/')
-- rawBase64UriP :: Parser ByteString
-- rawBase64UriP = A.takeWhile1 (\c -> isAlphaNum c || c == '-' || c == '_')
tsISO8601P :: Parser UTCTime
tsISO8601P = maybe (fail "timestamp") pure . parseISO8601 . B.unpack =<< A.takeTill wordEnd

View File

@@ -174,7 +174,6 @@ import qualified Data.Aeson.TH as J
import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
import qualified Data.Attoparsec.ByteString.Char8 as A
import Data.Bifunctor (first)
import qualified Data.ByteString.Base64 as B64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Char (isPrint, isSpace)
@@ -192,6 +191,7 @@ import GHC.TypeLits (ErrorMessage (..), TypeError, type (+))
import Network.Socket (ServiceName)
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding
import qualified Simplex.Messaging.Encoding.Base64 as B64
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers
import Simplex.Messaging.ServiceScheme

View File

@@ -45,7 +45,6 @@ import Control.Monad.IO.Unlift
import Control.Monad.Reader
import Crypto.Random
import Data.Bifunctor (first)
import Data.ByteString.Base64 (encode)
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Either (fromRight, partitionEithers)
@@ -68,6 +67,7 @@ import Network.Socket (ServiceName, Socket, socketToHandle)
import Simplex.Messaging.Agent.Lock
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding (Encoding (smpEncode))
import Simplex.Messaging.Encoding.Base64 (encode)
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Protocol
import Simplex.Messaging.Server.Control

View File

@@ -42,7 +42,6 @@ import Control.Monad.Trans.Except
import qualified Data.Aeson as J
import qualified Data.Aeson.Types as JT
import Data.Bifunctor (bimap, first)
import qualified Data.ByteString.Base64.URL as U
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import Data.Text.Encoding (encodeUtf8)
@@ -51,6 +50,7 @@ import SMPAgentClient (agentCfg, initAgentServers, initAgentServers2, testDB, te
import SMPClient (cfg, cfgV7, testPort, testPort2, testStoreLogFile2, withSmpServer, withSmpServerConfigOn, withSmpServerStoreLogOn)
import Simplex.Messaging.Agent hiding (createConnection, joinConnection, sendMessage)
import Simplex.Messaging.Agent.Client (ProtocolTestFailure (..), ProtocolTestStep (..), withStore')
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Agent.Env.SQLite (AgentConfig, Env (..), InitialAgentServers)
import Simplex.Messaging.Agent.Protocol hiding (CON, CONF, INFO)
import Simplex.Messaging.Agent.Store.SQLite (getSavedNtfToken)
@@ -345,7 +345,7 @@ testRunNTFServerTests :: ATransport -> NtfServer -> IO (Maybe ProtocolTestFailur
testRunNTFServerTests t srv =
withNtfServerThreadOn t ntfTestPort $ \ntf -> do
a <- liftIO $ getSMPAgentClient' 1 agentCfg initAgentServers testDB
r <- runRight $ testProtocolServer a 1 $ ProtoServerWithAuth srv Nothing
r <- runRight $ testProtocolServer a 1 $ ProtoServerWithAuth srv Nothing
killThread ntf
pure r

View File

@@ -15,7 +15,6 @@ import Control.Concurrent (threadDelay)
import qualified Data.Aeson as J
import qualified Data.Aeson.Types as JT
import Data.Bifunctor (first)
import qualified Data.ByteString.Base64.URL as U
import Data.ByteString.Char8 (ByteString)
import Data.Text.Encoding (encodeUtf8)
import NtfClient
@@ -35,6 +34,7 @@ import ServerTests
import qualified Simplex.Messaging.Agent.Protocol as AP
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Notifications.Protocol
import Simplex.Messaging.Notifications.Server.Push.APNS

View File

@@ -22,7 +22,6 @@ import Control.Exception (SomeException, try)
import Control.Monad
import Control.Monad.IO.Class
import Data.Bifunctor (first)
import Data.ByteString.Base64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import qualified Data.Set as S
@@ -31,6 +30,7 @@ import GHC.Stack (withFrozenCallStack)
import SMPClient
import qualified Simplex.Messaging.Crypto as C
import Simplex.Messaging.Encoding
import Simplex.Messaging.Encoding.Base64 (encode)
import Simplex.Messaging.Encoding.String
import Simplex.Messaging.Parsers (parseAll)
import Simplex.Messaging.Protocol

View File

@@ -12,7 +12,6 @@ import Control.Exception (SomeException)
import Control.Monad
import Control.Monad.Except
import Control.Monad.IO.Unlift
import qualified Data.ByteString.Base64.URL as B64
import Data.ByteString.Char8 (ByteString)
import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
@@ -26,6 +25,7 @@ import Simplex.FileTransfer.Transport (XFTPRcvChunkSpec (..), XFTPErrorType (..)
import Simplex.Messaging.Client (ProtocolClientError (..))
import qualified Simplex.Messaging.Crypto as C
import qualified Simplex.Messaging.Crypto.Lazy as LC
import qualified Simplex.Messaging.Encoding.Base64.URL as U
import Simplex.Messaging.Protocol (BasicAuth, SenderId)
import Simplex.Messaging.Server.Expiration (ExpirationConfig (..))
import Simplex.Messaging.Util (liftIOEither)
@@ -75,7 +75,7 @@ createTestChunk fp = do
pure bytes
readChunk :: SenderId -> IO ByteString
readChunk sId = B.readFile (xftpServerFiles </> B.unpack (B64.encode sId))
readChunk sId = B.readFile (xftpServerFiles </> B.unpack (U.encode sId))
testFileChunkDelivery :: Expectation
testFileChunkDelivery = xftpTest $ \c -> runRight_ $ runTestFileChunkDelivery c c