{-# LANGUAGE DataKinds #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-ambiguous-fields #-} module Simplex.Chat.Web ( WebChannelPreview (..), WebMessage (..), WebMemberProfile (..), WebFileInfo (..), webPreviewWorker, writeCorsConfig, removeStaleFiles, publicGroupIdFileName, channelContentChanged, channelProfileUpdated, channelRemoved, extractOrigin, ) where import Control.Concurrent.STM (check, flushTQueue) import Control.Exception (SomeException) import Control.Logger.Simple import Control.Monad import Control.Monad.Except (runExceptT) import Data.Either (rights) import Data.Int (Int64) import qualified Data.Aeson as J import qualified Data.Aeson.TH as JQ import qualified Data.ByteString.Char8 as B import qualified Data.ByteString.Lazy as LB import Data.Text.Encoding (encodeUtf8) import qualified Data.Map.Strict as M import qualified Data.Set as S import Data.Maybe (isJust, mapMaybe, maybeToList) import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.IO as TIO import Data.Time.Clock (UTCTime, getCurrentTime) import Simplex.Chat.Controller (ChatController (..), CorsOrigin (..), PublishableGroup (..), WebPreviewConfig (..), WebPreviewState (..), storeCxt) import Simplex.Chat.Markdown (FormattedText (..), MarkdownList, parseMaybeMarkdownList) import Simplex.Chat.Messages ( CChatItem (..), CIDirection (..), CIFile (..), CIMeta (..), CIQDirection (..), CIQuote (..), CIReactionCount, ChatItem (..), ChatType (..), ) import Simplex.Chat.Messages.CIContent (ciMsgContent) import Simplex.Chat.Protocol (MsgContent, MsgRef (..), QuotedMsg (..), isReport) import Simplex.Chat.Store.Groups (getGroupOwners, getRelayPublishableGroups, updatePublicMemberCount) import Simplex.Chat.Store.Messages (getGroupWebPreviewItems) import Simplex.Chat.Store.Shared (getGroupInfo) import Simplex.Chat.Types ( B64UrlByteString, GroupInfo (..), GroupMember (..), GroupProfile (..), GroupSummary (..), ImageData, LocalProfile (..), MemberId, PublicGroupAccess (..), PublicGroupProfile (..), User (..), ) import Simplex.Messaging.Agent.Store.Common (withTransaction) import Simplex.Messaging.Encoding.String (strEncode) import Simplex.Messaging.Util (catchOwn, catchOwn', eitherToMaybe, safeDecodeUtf8, tshow) import Simplex.Messaging.Parsers (defaultJSON) import System.Directory (createDirectoryIfMissing, listDirectory, removeFile, renameFile) import System.FilePath (dropExtension, takeExtension, ()) import qualified URI.ByteString as U import UnliftIO.STM data WebFileInfo = WebFileInfo { fileName :: String, fileSize :: Integer } deriving (Show) data WebMemberProfile = WebMemberProfile { memberId :: MemberId, displayName :: Text, image :: Maybe ImageData } deriving (Show) data WebMessage = WebMessage { sender :: Maybe MemberId, ts :: UTCTime, content :: MsgContent, formattedText :: Maybe MarkdownList, file :: Maybe WebFileInfo, quote :: Maybe QuotedMsg, reactions :: [CIReactionCount], forward :: Maybe Bool, edited :: Bool } deriving (Show) data WebChannelPreview = WebChannelPreview { channel :: GroupProfile, shortDescription :: Maybe MarkdownList, welcomeMessage :: Maybe MarkdownList, members :: [WebMemberProfile], subscribers :: Maybe Int64, messages :: [WebMessage], updatedAt :: UTCTime } deriving (Show) $(JQ.deriveJSON defaultJSON ''WebFileInfo) $(JQ.deriveJSON defaultJSON ''WebMemberProfile) $(JQ.deriveJSON defaultJSON ''WebMessage) $(JQ.deriveJSON defaultJSON ''WebChannelPreview) webPreviewWorker :: WebPreviewConfig -> ChatController -> [User] -> IO () webPreviewWorker cfg@WebPreviewConfig {webJsonDir, webCorsFile, webUpdateInterval} cc users = forM_ (webPreviewState cc) $ \wps -> do createDirectoryIfMissing True webJsonDir initPublishableGroups wps cleanStaleFiles wps regenerateCors wps seedRoutinePending wps forever $ workerLoop wps `catchOwn` \e -> logError ("web preview worker error: " <> tshow e) where cxt = storeCxt cc workerLoop wps@WebPreviewState {priorityRender, filesToRemove, corsNeeded, routinePending, wakeSignal} = do drainRemovals drainPriority handleCors renderRoutine noRoutine <- atomically $ S.null <$> readTVar routinePending when noRoutine waitRefresh where drainRemovals = atomically (tryReadTQueue filesToRemove) >>= \case Nothing -> pure () Just f -> do removeFile (webJsonDir f) `catchOwn'` \(_ :: SomeException) -> pure () drainRemovals -- flush the whole queue and render each group once: a burst of changes in one -- channel enqueues its id many times, but only needs a single render drainPriority = do gIds <- atomically $ flushTQueue priorityRender forM_ (S.fromList gIds) $ renderOneGroup wps handleCors = do needed <- atomically $ swapTVar corsNeeded False when needed $ regenerateCors wps -- render a single routine item; the main loop calls this once per iteration renderRoutine = do mGId <- atomically $ do pending <- readTVar routinePending case S.minView pending of Nothing -> pure Nothing Just (gId, rest) -> writeTVar routinePending rest >> pure (Just gId) forM_ mGId $ renderOneGroup wps -- routine list drained: wait for the refresh timer or a change signal; only the timer -- seeds the next full sweep, a change just returns to let the main loop service it waitRefresh = do delay <- registerDelay (webUpdateInterval * 1000000) timerFired <- atomically $ (True <$ (readTVar delay >>= check)) `orElse` (False <$ takeTMVar wakeSignal) when timerFired $ seedRoutinePending wps initPublishableGroups WebPreviewState {publishableGroupIds} = do rows <- withTransaction (chatStore cc) $ \db -> concat <$> mapM (getRelayPublishableGroups db) users let gIds = M.fromList [(gId, toPublishableGroup pgId access) | (gId, pgId, access) <- rows] atomically $ writeTVar publishableGroupIds gIds cleanStaleFiles WebPreviewState {publishableGroupIds} = do ids <- readTVarIO publishableGroupIds let activeFiles = S.fromList $ map pgFileName $ M.elems ids removeStaleFiles webJsonDir activeFiles regenerateCors WebPreviewState {publishableGroupIds} = do ids <- readTVarIO publishableGroupIds let entries = mapMaybe pgCorsEntry $ M.elems ids forM_ webCorsFile $ writeCorsConfig entries seedRoutinePending WebPreviewState {publishableGroupIds, routinePending} = atomically $ M.keysSet <$> readTVar publishableGroupIds >>= writeTVar routinePending renderOneGroup WebPreviewState {publishableGroupIds} gId = do publishable <- atomically $ M.member gId <$> readTVar publishableGroupIds when publishable $ renderOrRemoveStale `catchOwn'` \(e :: SomeException) -> logError $ "web preview: error rendering group " <> T.pack (show gId) <> ": " <> T.pack (show e) where renderOrRemoveStale = do r <- withTransaction (chatStore cc) $ \db -> findUser $ \u -> fmap (\g -> (u, g)) <$> runExceptT (getGroupInfo db cxt u gId) case r of Just (u, gInfo) | hasPublicGroup gInfo -> void $ renderGroupPreview cfg cc u gInfo _ -> do fName <- atomically $ do pg <- M.lookup gId <$> readTVar publishableGroupIds modifyTVar' publishableGroupIds (M.delete gId) pure $ pgFileName <$> pg forM_ fName $ \f -> removeFile (webJsonDir f) `catchOwn'` \(_ :: SomeException) -> pure () logInfo $ "web preview: group " <> T.pack (show gId) <> " no longer publishable" findUser f = go users where go [] = pure Nothing go (u : us) = f u >>= \case Right a -> pure (Just a) Left _ -> go us renderGroupPreview :: WebPreviewConfig -> ChatController -> User -> GroupInfo -> IO (Maybe (Text, CorsOrigin)) renderGroupPreview WebPreviewConfig {webJsonDir, webPreviewItemCount} cc user gInfo@GroupInfo {groupProfile = gp@GroupProfile {shortDescr = sd, description = wd, publicGroup}, groupSummary = GroupSummary {publicMemberCount}} = case publicGroup of Just PublicGroupProfile {publicGroupId, publicGroupAccess} -> do let fName = publicGroupIdFileName publicGroupId <> ".json" -- backfill the subscriber count for channels created before it was tracked subscribers <- case publicMemberCount of Just _ -> pure publicMemberCount Nothing -> do g_ <- withTransaction (chatStore cc) (\db -> runExceptT $ updatePublicMemberCount db cxt user gInfo) pure $ eitherToMaybe g_ >>= \GroupInfo {groupSummary = GroupSummary {publicMemberCount = pmc}} -> pmc (items, owners) <- withTransaction (chatStore cc) $ \db -> do is <- getGroupWebPreviewItems db user gInfo webPreviewItemCount os <- getGroupOwners db cxt user gInfo pure (is, os) ts <- getCurrentTime let rendered = mapMaybe toRenderedItem $ rights items msgs = map fst rendered senders = collectSenders $ map memberToProfile owners <> concatMap snd rendered preview = WebChannelPreview { channel = gp, shortDescription = toFormattedText =<< sd, welcomeMessage = toFormattedText =<< wd, members = senders, subscribers, messages = msgs, updatedAt = ts } let destPath = webJsonDir fName tmpPath = destPath <> ".tmp" LB.writeFile tmpPath (J.encode preview) renameFile tmpPath destPath pure $ corsEntry publicGroupId <$> publicGroupAccess Nothing -> pure Nothing where cxt = storeCxt cc channelContentChanged :: ChatController -> Int64 -> STM () channelContentChanged cc gId = forM_ (webPreviewState cc) $ \WebPreviewState {publishableGroupIds, priorityRender, routinePending, wakeSignal} -> do ids <- readTVar publishableGroupIds when (M.member gId ids) $ do writeTQueue priorityRender gId modifyTVar' routinePending (S.delete gId) void $ tryPutTMVar wakeSignal () channelProfileUpdated :: ChatController -> Int64 -> GroupProfile -> STM () channelProfileUpdated cc gId GroupProfile {publicGroup} = forM_ (webPreviewState cc) $ \WebPreviewState {publishableGroupIds, priorityRender, filesToRemove, corsNeeded, routinePending, wakeSignal} -> case publicGroup of Just PublicGroupProfile {publicGroupId, publicGroupAccess} -> do let pg = PublishableGroup { pgFileName = publicGroupIdFileName publicGroupId <> ".json", pgCorsEntry = corsEntry publicGroupId <$> publicGroupAccess } modifyTVar' publishableGroupIds (M.insert gId pg) writeTQueue priorityRender gId modifyTVar' routinePending (S.delete gId) writeTVar corsNeeded True void $ tryPutTMVar wakeSignal () Nothing -> do ids <- readTVar publishableGroupIds forM_ (pgFileName <$> M.lookup gId ids) $ writeTQueue filesToRemove modifyTVar' publishableGroupIds (M.delete gId) modifyTVar' routinePending (S.delete gId) writeTVar corsNeeded True void $ tryPutTMVar wakeSignal () channelRemoved :: ChatController -> Int64 -> STM () channelRemoved cc gId = forM_ (webPreviewState cc) $ \WebPreviewState {publishableGroupIds, filesToRemove, corsNeeded, routinePending, wakeSignal} -> do ids <- readTVar publishableGroupIds forM_ (pgFileName <$> M.lookup gId ids) $ writeTQueue filesToRemove modifyTVar' publishableGroupIds (M.delete gId) modifyTVar' routinePending (S.delete gId) writeTVar corsNeeded True void $ tryPutTMVar wakeSignal () toRenderedItem :: CChatItem 'CTGroup -> Maybe (WebMessage, [WebMemberProfile]) toRenderedItem (CChatItem _ ChatItem {chatDir, meta = CIMeta {itemTs, itemTimed, itemForwarded, itemEdited}, content, formattedText, quotedItem, reactions, file}) | isJust itemTimed = Nothing | otherwise = case ciMsgContent content of Just mc | not (isReport mc) -> let (sender, senderProfile) = case chatDir of CIGroupRcv m@GroupMember {memberId} -> (Just memberId, [memberToProfile m]) _ -> (Nothing, []) quotedProfile = case quotedItem of Just CIQuote {chatDir = CIQGroupRcv (Just m)} -> [memberToProfile m] _ -> [] in Just ( WebMessage { sender, ts = itemTs, content = mc, formattedText, file = webFileInfo <$> file, quote = quotedItem >>= ciQuoteToQuotedMsg, reactions, forward = if isJust itemForwarded then Just True else Nothing, edited = itemEdited }, senderProfile <> quotedProfile ) _ -> Nothing ciQuoteToQuotedMsg :: CIQuote c -> Maybe QuotedMsg ciQuoteToQuotedMsg CIQuote {chatDir = qDir, sharedMsgId, sentAt, content = qContent} = Just QuotedMsg { msgRef = MsgRef { msgId = sharedMsgId, sentAt, sent = case qDir of CIQDirectSnd -> True CIQGroupSnd -> True _ -> False, memberId = case qDir of CIQGroupRcv (Just GroupMember {memberId}) -> Just memberId _ -> Nothing }, content = qContent } webFileInfo :: CIFile d -> WebFileInfo webFileInfo CIFile {fileName, fileSize} = WebFileInfo {fileName, fileSize} collectSenders :: [WebMemberProfile] -> [WebMemberProfile] collectSenders = M.elems . M.fromList . map (\p@WebMemberProfile {memberId} -> (memberId, p)) memberToProfile :: GroupMember -> WebMemberProfile memberToProfile GroupMember {memberId, memberProfile = LocalProfile {displayName, image}} = WebMemberProfile {memberId, displayName, image} toPublishableGroup :: B64UrlByteString -> Maybe PublicGroupAccess -> PublishableGroup toPublishableGroup pgId access = PublishableGroup { pgFileName = publicGroupIdFileName pgId <> ".json", pgCorsEntry = corsEntry pgId <$> access } corsEntry :: B64UrlByteString -> PublicGroupAccess -> (Text, CorsOrigin) corsEntry publicGroupId PublicGroupAccess {groupWebPage, allowEmbedding} = let fName = T.pack $ publicGroupIdFileName publicGroupId <> ".json" origin | allowEmbedding = CorsAny | otherwise = CorsOrigins $ mapMaybe extractOrigin $ maybeToList groupWebPage in (fName, origin) extractOrigin :: Text -> Maybe Text extractOrigin url = case U.parseURI U.laxURIParserOptions (encodeUtf8 url) of Right uri@U.URI {uriScheme = U.Scheme sch, uriAuthority = Just _} | sch == "https" || sch == "http" -> let originUri = uri {U.uriPath = "", U.uriQuery = U.Query [], U.uriFragment = Nothing} origin = safeDecodeUtf8 $ U.serializeURIRef' originUri in if T.all safeOriginChar origin then Just origin else Nothing _ -> Nothing where -- percent-encoded bytes in the host (e.g. %22, %0a) are decoded by serializeURIRef', -- so reject any origin with characters that could break out of the Caddy CORS config or header safeOriginChar c = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c `elem` (".-:/[]" :: [Char]) channelPath :: Text channelPath = "/channel/" writeCorsConfig :: [(Text, CorsOrigin)] -> FilePath -> IO () writeCorsConfig entries path = TIO.writeFile path $ T.unlines $ ["map {path} {cors_origin} {"] <> map corsLine entries <> [ " default \"\"", "}", "header " <> channelPath <> "*.json Access-Control-Allow-Origin {cors_origin}", "header " <> channelPath <> "*.json Access-Control-Allow-Methods \"GET, OPTIONS\"" ] where corsLine (fName, origin) = case origin of CorsAny -> " " <> channelPath <> fName <> " \"*\"" CorsOrigins origins -> case origins of [] -> " # " <> fName <> " (no origin configured)" (o : _) -> " " <> channelPath <> fName <> " \"" <> o <> "\"" removeStaleFiles :: FilePath -> S.Set FilePath -> IO () removeStaleFiles dir activeFiles = do let -- matches ".json" and leftover ".json.tmp" from an interrupted write isPreviewFile f = let f' = if takeExtension f == ".tmp" then dropExtension f else f base = dropExtension f' in takeExtension f' == ".json" && not (null base) && all isBase64Url base isBase64Url c = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '=' allFiles <- S.filter isPreviewFile . S.fromList <$> listDirectory dir forM_ (S.difference allFiles activeFiles) $ \f -> removeFile (dir f) `catchOwn'` \(e :: SomeException) -> logError $ "web preview: error removing stale file " <> T.pack f <> ": " <> tshow e toFormattedText :: Text -> Maybe MarkdownList toFormattedText t = case parseMaybeMarkdownList t of Just fts | any hasFormat fts -> Just fts _ -> Nothing where hasFormat (FormattedText fmt _) = isJust fmt publicGroupIdFileName :: B64UrlByteString -> String publicGroupIdFileName = B.unpack . strEncode hasPublicGroup :: GroupInfo -> Bool hasPublicGroup GroupInfo {groupProfile = GroupProfile {publicGroup}} = isJust publicGroup