diff --git a/apps/ios/Shared/Views/Chat/ChatItem/MsgContentView.swift b/apps/ios/Shared/Views/Chat/ChatItem/MsgContentView.swift index 8e32b51856..a69150a985 100644 --- a/apps/ios/Shared/Views/Chat/ChatItem/MsgContentView.swift +++ b/apps/ios/Shared/Views/Chat/ChatItem/MsgContentView.swift @@ -663,10 +663,10 @@ func messageText( t = viaHost(smpHosts) } case let .command(cmdStr): - snippet = snippet ?? UIFont.monospacedSystemFont(ofSize: descr.pointSize, weight: .regular) - attrs[.font] = snippet - t = "/" + cmdStr if !preview && commands { + snippet = snippet ?? UIFont.monospacedSystemFont(ofSize: descr.pointSize, weight: .regular) + attrs[.font] = snippet + t = "/" + cmdStr attrs[.foregroundColor] = uiLinkColor attrs[commandAttrKey] = t handleTaps = true diff --git a/plans/2026-10-29-markdown-edge-cases.md b/plans/2026-10-29-markdown-edge-cases.md new file mode 100644 index 0000000000..c2401aec97 --- /dev/null +++ b/plans/2026-10-29-markdown-edge-cases.md @@ -0,0 +1,64 @@ +# Markdown: edge cases + +Date: 2026-10-29. Follows `2026-10-29-markdown-links-and-line-breaks.md`. + +Haskell paths without a directory are in `src/Simplex/Chat/`. + +## Decisions + +``` +E1 phones a phone starts with "+" or "0"; another number of 7 to 22 characters is text +E2 domains a link without a scheme needs a lowercase top-level domain +E3 "_" and "__" a closing "_" followed by a letter or a digit is text (CommonMark) +E4 "#" names digits only after "#" are text +E5 commands, iOS plain text when the chat does not use commands, as Android and desktop +E6 quotes ">" needs a space or a tab after it, or the line end +E7 numbered lines after a paragraph line, only "1." or "1)" starts a list (CommonMark example 304) +E8 "---" under text a rule; "===" under text stays a heading +E9 delimiter runs a closing delimiter followed by the same character is text +E11 code spans code spans inside a format are content of the format +E12 headings the spaces after "#" are part of the heading mark +``` + +Unchanged: E10 nested formats, E13 links without a scheme, E14 links after quotes (GFM), E15 non-Latin links, E16 task lists. + +## Inline + +`formattedP c f`: + +``` +content pieces to a closing c on the same line: + text without c, "`" and line breaks + a code span, or its backticks when it is not closed in the line + a run of c longer than 1 + a single c followed by a letter or a digit, for "_" +otherwise as before: across lines, or the delimiter and the rest of the line as text +``` + +`doubleP c f`: + +``` +content text without c and "`", and code spans, to cc +closing cc not followed by c, nor by a letter or a digit for "_" +``` + +Other: + +``` +phoneP Phone when it starts with "+" or "0"; text when 7 to 22 characters otherwise +isDomain the top-level domain in lowercase ASCII letters +nameRefP "#" with digits only: not a name +``` + +## Blocks + +``` +quoteMarkP ">", then a space or a tab, or the line end +headingMarkP "#" run, a space, then spaces and tabs +paragraphP continuation: a paragraph line, or a numbered list line with a number other than 1 +underlineLevel "=" only; HeadingUnderline 2 is not produced, kept for decoding +``` + +## iOS + +`MsgContentView.swift`, `messageText`: a command is drawn as code with "/" and its name only when it can be sent; otherwise its text as typed. diff --git a/src/Simplex/Chat/Markdown.hs b/src/Simplex/Chat/Markdown.hs index 3fb6705cc9..3250594c5d 100644 --- a/src/Simplex/Chat/Markdown.hs +++ b/src/Simplex/Chat/Markdown.hs @@ -25,7 +25,7 @@ import qualified Data.Attoparsec.Text as A import Data.Bifunctor (first) import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as B -import Data.Char (isAlpha, isAscii, isDigit, isPunctuation, isSpace, isSymbol) +import Data.Char (isAlpha, isAlphaNum, isAscii, isAsciiLower, isDigit, isPunctuation, isSpace, isSymbol) import Data.Either (fromRight, isLeft, lefts) import Data.Functor (($>)) import Data.List (intercalate) @@ -366,10 +366,13 @@ markdownP = joinMarkdown <$> A.many' fragmentP | otherwise -> wordP Nothing -> fail "" formattedP :: Char -> Format -> Parser Markdown - formattedP c f = do - s <- A.char c *> A.takeTill (\x -> x == c || x == '\n') - (A.char c $> md c f s) <|> (markdown f <$> acrossLinesP s) <|> noFormat (c `T.cons` s) + formattedP c f = A.char c *> ((md c f . T.concat <$> A.many' pieceP <* A.char c) <|> openedP) where + pieceP = A.takeWhile1 (\x -> x /= c && x /= '`' && x /= '\n') <|> codeTextP <|> delimitersP + delimitersP = A.takeWhile1 (== c) >>= \r -> r <$ (guard (T.length r > 1) <|> lookAhead (void $ A.satisfy $ notClosingBefore c)) + openedP = do + s <- A.takeTill (\x -> x == c || x == '\n') + (markdown f <$> acrossLinesP s) <|> noFormat (c `T.cons` s) acrossLinesP s = do s' <- ((s <> "\n") <>) <$> (A.char '\n' *> A.takeTill (== c) <* A.char c) if T.null s || isSpace (T.head s) || isSpace (T.last s') then fail "not formatted" else pure s' @@ -380,12 +383,19 @@ markdownP = joinMarkdown <$> A.many' fragmentP | otherwise = markdown f s doubleP :: Char -> Format -> Parser Markdown doubleP c f = do - s <- A.string cc *> A.takeTill (== c) <* A.string cc + s <- A.string cc *> (T.concat <$> A.many' (A.takeWhile1 (\x -> x /= c && x /= '`') <|> codeTextP)) <* A.string cc <* notP (A.satisfy $ notClosingBefore c) if T.null s || isSpace (T.head s) || isSpace (T.last s) then fail "not formatted" else pure $ markdown f s where cc = T.pack [c, c] + notClosingBefore :: Char -> Char -> Bool + notClosingBefore c x = x == c || (c == '_' && isAlphaNum x) + codeTextP :: Parser Text + codeTextP = do + ticks <- A.takeWhile1 (== '`') + let pieceP = A.takeWhile1 (\x -> x /= '`' && x /= '\n') <|> (A.takeWhile1 (== '`') >>= \r -> r <$ guard (r /= ticks)) + ((\s -> ticks <> s <> ticks) . T.concat <$> A.many' pieceP <* A.string ticks) <|> pure ticks codeSpanP :: Parser Markdown codeSpanP = do ticks <- A.takeWhile1 (== '`') @@ -414,6 +424,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP where nameRef word | pfx == '@' && T.all (/= '.') name = Left "not a name" + | pfx == '#' && T.all isDigit name = Left "not a name" | otherwise = mkMd <$> strDecode (encodeUtf8 full) where (name, punct) = splitPunctuation word @@ -477,7 +488,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP code <- optional $ conc4 <$> phoneSep <*> "(" <*> A.takeWhile1 isDigit <*> ")" s <- segmentsP $ fromMaybe "" country <> fromMaybe "" code let len = T.length s - if 7 <= len && len <= 22 then pure $ markdown Phone s else fail "not phone" + if 7 <= len && len <= 22 then pure $ if isJust country || "0" `T.isPrefixOf` s then markdown Phone s else unmarked s else fail "not phone" segmentsP s | T.length s > 22 = fail "not phone" | otherwise = optional ((<>) <$> phoneSep <*> A.takeWhile1 isDigit) >>= maybe (pure s) (segmentsP . (s <>)) @@ -531,7 +542,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP isDomain_ name tld = (let n = T.length name in n >= 1 && n <= 24) && (let n = T.length tld in n >= 2 && n <= 8) - && (let p c = isAscii c && isAlpha c in T.all p name && T.all p tld) + && (let p c = isAscii c && isAlpha c in T.all p name && T.all isAsciiLower tld) isEmail s = T.any (== '@') s && Email.isValid (encodeUtf8 s) noFormat = pure . unmarked simplexUriFormat :: Maybe Text -> AConnectionLink -> Format @@ -658,7 +669,7 @@ headingMarkP = do ind <- A.takeWhile isIndentChar hashes <- A.takeWhile1 (== '#') guard $ T.length hashes <= 6 - sp <- A.string " " + sp <- (<>) <$> A.string " " <*> A.takeWhile isIndentChar hasTextP pure (T.length hashes, ind <> hashes <> sp) @@ -704,7 +715,7 @@ quoteP = do quoteMarkP :: Parser Text quoteMarkP = do ind <- A.takeWhile isIndentChar - sp <- A.char '>' *> A.option "" (T.singleton <$> A.satisfy isIndentChar) + sp <- A.char '>' *> ((T.singleton <$> A.satisfy isIndentChar) <|> ("" <$ lookAhead (void lineEndP <|> A.endOfInput))) pure $ ind <> ">" <> sp blankQuoteLineP :: Parser Text @@ -716,10 +727,13 @@ quoteStartP = optional (blankQuoteLineP *> lineEndP) *> quoteMarkP *> hasTextP paragraphP :: Parser MarkdownList paragraphP = do l <- lineTextP - ls <- A.many' $ (,[],) <$> lineEndP <* notP underlineP <* paragraphLineP <*> lineTextP + ls <- A.many' $ (,[],) <$> lineEndP <* notP underlineP <* (paragraphLineP <|> numberedTextLineP) <*> lineTextP underline <- A.option [] $ (\br (lvl, u) -> [markupText HardBreak br, markupText (HeadingUnderline lvl) u]) <$> lineEndP <*> underlineP pure $ inlineLines l ls <> underline +numberedTextLineP :: Parser () +numberedTextLineP = lookAhead $ listMarkP >>= \(num, _) -> guard $ maybe False (`notElem` ["1.", "1)"]) num + underlineP :: Parser (Int, Text) underlineP = lineTextP >>= \l -> maybe mzero (pure . (,l)) (underlineLevel l) @@ -811,7 +825,6 @@ headingClosing c = underlineLevel :: Text -> Maybe Int underlineLevel t = case T.uncons s of Just ('=', _) | T.all (== '=') s -> Just 1 - Just ('-', _) | T.all (== '-') s -> Just 2 _ -> Nothing where s = T.dropWhileEnd isSpace $ T.dropWhile isIndentChar t diff --git a/tests/MarkdownTests.hs b/tests/MarkdownTests.hs index d5176ce39b..b26ae8921a 100644 --- a/tests/MarkdownTests.hs +++ b/tests/MarkdownTests.hs @@ -312,6 +312,8 @@ textWithUri = describe "text with Uri" do ".com" <==> ".com" "example.academytoolong" <==> "example.academytoolong" "simplex:/example" <==> "simplex:/example" + "end of sentence.Next sentence" <==> "end of sentence.Next sentence" + "Visit Example.Com today" <==> "Visit Example.Com today" it "SimpleX links" do let inv = "/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-2%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D2%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D" ("https://simplex.chat" <> inv) <==> simplexLink XLInvitation ("simplex:" <> inv) ["smp.simplex.im"] ("https://simplex.chat" <> inv) @@ -406,6 +408,12 @@ textWithPhone = describe "text with Phone" do it "ignored as markdown (double spaces)" $ do "test 07777 777 777 test" <==> "test 07777 777 777 test" "*test* 07777 777 777 test" <==> bold "test" <> " 07777 777 777 test" + it "ignored as markdown (not starting with + or 0)" $ do + "order 12345678 shipped" <==> "order 12345678 shipped" + "on 2024-10-29" <==> "on 2024-10-29" + "server 192.168.1.1" <==> "server 192.168.1.1" + "version 7.1.0.14" <==> "version 7.1.0.14" + "total 1 000 000" <==> "total 1 000 000" mention :: Text -> Text -> Markdown mention = Markdown . Just . Mention @@ -480,7 +488,6 @@ textWithSimplexNames = describe "text with SimpleX names" do "#my-channel.simplex" <==> sname NTPublicGroup TLDSimplex "my-channel" [] "my-channel.simplex" "hello #privacy!" <==> "hello " <> sname NTPublicGroup TLDSimplex "privacy" [] "privacy" <> "!" "see #privacy.simplex now" <==> "see " <> sname NTPublicGroup TLDSimplex "privacy" [] "privacy.simplex" <> " now" - "#123" <==> sname NTPublicGroup TLDSimplex "123" [] "123" it "channel names - subdomains" do "#support.acme.simplex" <==> sname NTPublicGroup TLDSimplex "acme" ["support"] "support.acme.simplex" "#a.b.acme.simplex" <==> sname NTPublicGroup TLDSimplex "acme" ["b", "a"] "a.b.acme.simplex" @@ -496,6 +503,9 @@ textWithSimplexNames = describe "text with SimpleX names" do "@my-name.simplex" <==> sname NTContact TLDSimplex "my-name" [] "my-name.simplex" "@alice.example.com" <==> sname NTContact TLDWeb "alice.example.com" [] "alice.example.com" it "not parsed as names" do + "#123" <==> "#123" + "issue #42 is fixed" <==> "issue #42 is fixed" + "I'm #1!" <==> "I'm #1!" "#secret#" <==> markdown Secret "secret" "##double secret##" <==> markdown Secret "#double secret#" "#" <==> "#" @@ -532,6 +542,20 @@ inlineMarkdownGFM = describe "inline markdown of GFM" do "a__b__" ==> "a__b__" markdownText (fmt boldF "a") `shouldBe` "*a*" markdownText (fmt bold2F "a") `shouldBe` "**a**" + it "underscores inside words, as CommonMark" do + "_private_var = 1" <==> "_private_var = 1" + "use _snake_case_ here" <==> "use " <> markdown Italic "snake_case" <> " here" + "_foo_bar_baz_" <==> markdown Italic "foo_bar_baz" + "__foo__bar" <==> "__foo__bar" + "__init__.py" ==> bold2 "init" <> ".py" + it "a closing delimiter followed by the same character is text" do + "*a**b*" <==> bold "a**b" + "*foo**bar**baz*" <==> bold "foo**bar**baz" + "**a****b**" <==> "**a****b**" + "*a**" <==> "*a**" + it "code spans inside a format" do + "*a `*`*" <==> bold "a `*`" + "**a `**` b**" <==> bold2 "a `**` b" it "code spans" do "`` x ` y ``" ==> markdown Snippet "x ` y" "``x ` y``" <== markdown Snippet "x ` y" @@ -754,6 +778,7 @@ headings = describe "headings" do "#\n- b" <===> (["#", lineBr, bullet "- ", "b"], [Paragraph (rng 0 1), List [entry 3 4 []]]) it "indentation and inline content" do " # a" <===> ([mk (HeadingMark 1) " # ", "a"], [Heading 1 (rng 1 2)]) + "# a" <===> ([mk (HeadingMark 1) "# ", "a"], [Heading 1 (rng 1 2)]) "# *a* b" <===> ([heading1, fmt boldF "a", " b"], [Heading 1 (rng 1 3)]) it "closing sequences" do "# a #" <===> ([heading1, "a", mk HeadingClose " #"], [Heading 1 (rng 1 2)]) @@ -766,12 +791,13 @@ headings = describe "headings" do "# a\n# b" <===> ([heading1, "a", lineBr, heading1, "b"], [Heading 1 (rng 1 2), Heading 1 (rng 4 5)]) it "setext headings" do "a\n===" <===> (["a", hardBr, mk (HeadingUnderline 1) "==="], [Heading 1 (rng 0 1)]) - "a\n---" <===> (["a", hardBr, mk (HeadingUnderline 2) "---"], [Heading 2 (rng 0 1)]) - "a\n-" <===> (["a", hardBr, mk (HeadingUnderline 2) "-"], [Heading 2 (rng 0 1)]) "a\nb\n=" <===> (["a", hardBr, "b", hardBr, mk (HeadingUnderline 1) "="], [Heading 1 (rng 0 3)]) - "a\nb\n---" <===> (["a", hardBr, "b", hardBr, mk (HeadingUnderline 2) "---"], [Heading 2 (rng 0 3)]) "a\n === " <===> (["a", hardBr, mk (HeadingUnderline 1) " === "], [Heading 1 (rng 0 1)]) - "a\n---\nb" <===> (["a", hardBr, mk (HeadingUnderline 2) "---", lineBr, "b"], [Heading 2 (rng 0 1), Paragraph (rng 4 5)]) + it "dashes under text are a rule, not an underline" do + "a\n---" <===> (["a", lineBr, mk RuleMark "---"], [Paragraph (rng 0 1), Rule (rng 2 3)]) + "a\nb\n---" <===> (["a", hardBr, "b", lineBr, mk RuleMark "---"], [Paragraph (rng 0 3), Rule (rng 4 5)]) + "a\n---\nb" <===> (["a", lineBr, mk RuleMark "---", lineBr, "b"], [Paragraph (rng 0 1), Rule (rng 2 3), Paragraph (rng 4 5)]) + noBlocks "a\n-" it "underlines only under paragraphs" do "a\n- - -" <===> (["a", lineBr, mk RuleMark "- - -"], [Paragraph (rng 0 1), Rule (rng 2 3)]) "a\n***" <===> (["a", lineBr, mk RuleMark "***"], [Paragraph (rng 0 1), Rule (rng 2 3)]) @@ -813,6 +839,11 @@ lists = describe "lists" do noBlocks "1. " noBlocks "1234567890. a" noBlocks "a. b" + it "after a paragraph line, only a list numbered 1 starts" do + "a\n1. b" <===> (["a", lineBr, numbered "1.", "b"], [Paragraph (rng 0 1), List [ListEntry (Just "1.") (rng 3 4) []]]) + "a\n1) b" <===> (["a", lineBr, numbered "1)", "b"], [Paragraph (rng 0 1), List [ListEntry (Just "1)") (rng 3 4) []]]) + noBlocks "a\n14. b" + "a\n14. b\n- c" <===> (["a", hardBr, "14. b", lineBr, bullet "- ", "c"], [Paragraph (rng 0 3), List [entry 5 6 []]]) it "nesting by the column of the marker" do "- a\n - b" <===> ([bullet "- ", "a", lineBr, bullet " - ", "b"], [List [entry 1 2 [entry 4 5 []]]]) "1. a\n - b" <===> ([numbered "1.", "a", lineBr, bullet " - ", "b"], [List [ListEntry (Just "1.") (rng 1 2) [entry 4 5 []]]]) @@ -859,11 +890,13 @@ quotes :: Spec quotes = describe "quotes" do it "quote marks" do "> a" <===> ([quoteMark "> ", "a"], [Quote (rng 1 2)]) - ">a" <===> ([quoteMark ">", "a"], [Quote (rng 1 2)]) ">\ta" <===> ([quoteMark ">\t", "a"], [Quote (rng 1 2)]) "> a" <===> ([quoteMark "> ", " a"], [Quote (rng 1 2)]) " > a" <===> ([quoteMark " > ", "a"], [Quote (rng 1 2)]) - ">a\n> b" <===> ([quoteMark ">", "a", hardBr, quoteMark "> ", "b"], [Quote (rng 1 5)]) + it "a quote mark without a space or a tab after it is text" do + noBlocks ">a" + noBlocks ">_< sorry" + ">a\n> b" <===> ([">a", lineBr, quoteMark "> ", "b"], [Paragraph (rng 0 1), Quote (rng 3 4)]) it "quote marks without text are text" do noBlocks ">" noBlocks "> " @@ -1048,7 +1081,7 @@ liveMessages = describe "incomplete syntax of live messages" do linearTimeParsing :: Spec linearTimeParsing = describe "bounded scans for linear-time parsing" do it "phone numbers: at most 22 characters, as on master" do - "1 1 1 1 1 1 1 1 1 1 1 1" <==> "1 " <> phone "1 1 1 1 1 1 1 1 1 1 1" + "0 0 0 0 0 0 0 0 0 0 0 0" <==> "0 " <> phone "0 0 0 0 0 0 0 0 0 0 0" it "link texts stop at \"[\", link targets and autolinks at white space" do "[a [b](https://simplex.chat)" <==> "[a " <> web "b" "https://simplex.chat" "[b](https://simplex.chat)" "[a](https://simplex.chat x)" <==> "[a](" <> uri "https://simplex.chat" <> " x)"