core, ios: markdown phones, domains, underscores, quotes, rules and delimiters (#7943)

2026-11-04 08:37:30 +00:00 · 2026-10-29 22:03:06 +00:00
1 parent b7bae8c47e
commit 094b1837d8
@@ -663,10 +663,10 @@ func messageText(
                    t = viaHost(smpHosts)
                }
            case let .command(cmdStr):
-                snippet = snippet ?? UIFont.monospacedSystemFont(ofSize: descr.pointSize, weight: .regular)
-                attrs[.font] = snippet
-                t = "/" + cmdStr
                if !preview && commands {
+                    snippet = snippet ?? UIFont.monospacedSystemFont(ofSize: descr.pointSize, weight: .regular)
+                    attrs[.font] = snippet
+                    t = "/" + cmdStr
                    attrs[.foregroundColor] = uiLinkColor
                    attrs[commandAttrKey] = t
                    handleTaps = true
@@ -0,0 +1,64 @@
+# Markdown: edge cases
+
+Date: 2026-10-29. Follows `2026-10-29-markdown-links-and-line-breaks.md`.
+
+Haskell paths without a directory are in `src/Simplex/Chat/`.
+
+## Decisions
+
+```
+E1   phones               a phone starts with "+" or "0"; another number of 7 to 22 characters is text
+E2   domains              a link without a scheme needs a lowercase top-level domain
+E3   "_" and "__"         a closing "_" followed by a letter or a digit is text (CommonMark)
+E4   "#" names            digits only after "#" are text
+E5   commands, iOS        plain text when the chat does not use commands, as Android and desktop
+E6   quotes               ">" needs a space or a tab after it, or the line end
+E7   numbered lines       after a paragraph line, only "1." or "1)" starts a list (CommonMark example 304)
+E8   "---" under text     a rule; "===" under text stays a heading
+E9   delimiter runs       a closing delimiter followed by the same character is text
+E11  code spans           code spans inside a format are content of the format
+E12  headings             the spaces after "#" are part of the heading mark
+```
+
+Unchanged: E10 nested formats, E13 links without a scheme, E14 links after quotes (GFM), E15 non-Latin links, E16 task lists.
+
+## Inline
+
+`formattedP c f`:
+
+```
+content    pieces to a closing c on the same line:
+             text without c, "`" and line breaks
+             a code span, or its backticks when it is not closed in the line
+             a run of c longer than 1
+             a single c followed by a letter or a digit, for "_"
+otherwise  as before: across lines, or the delimiter and the rest of the line as text
+```
+
+`doubleP c f`:
+
+```
+content    text without c and "`", and code spans, to cc
+closing    cc not followed by c, nor by a letter or a digit for "_"
+```
+
+Other:
+
+```
+phoneP       Phone when it starts with "+" or "0"; text when 7 to 22 characters otherwise
+isDomain     the top-level domain in lowercase ASCII letters
+nameRefP     "#" with digits only: not a name
+```
+
+## Blocks
+
+```
+quoteMarkP         ">", then a space or a tab, or the line end
+headingMarkP       "#" run, a space, then spaces and tabs
+paragraphP         continuation: a paragraph line, or a numbered list line with a number other than 1
+underlineLevel     "=" only; HeadingUnderline 2 is not produced, kept for decoding
+```
+
+## iOS
+
+`MsgContentView.swift`, `messageText`: a command is drawn as code with "/" and its name only when it can be sent; otherwise its text as typed.
@@ -25,7 +25,7 @@ import qualified Data.Attoparsec.Text as A
 import Data.Bifunctor (first)
 import Data.ByteString.Char8 (ByteString)
 import qualified Data.ByteString.Char8 as B
-import Data.Char (isAlpha, isAscii, isDigit, isPunctuation, isSpace, isSymbol)
+import Data.Char (isAlpha, isAlphaNum, isAscii, isAsciiLower, isDigit, isPunctuation, isSpace, isSymbol)
 import Data.Either (fromRight, isLeft, lefts)
 import Data.Functor (($>))
 import Data.List (intercalate)
@@ -366,10 +366,13 @@ markdownP = joinMarkdown <$> A.many' fragmentP
            | otherwise -> wordP
        Nothing -> fail ""
    formattedP :: Char -> Format -> Parser Markdown
-    formattedP c f = do
-      s <- A.char c *> A.takeTill (\x -> x == c || x == '\n')
-      (A.char c $> md c f s) <|> (markdown f <$> acrossLinesP s) <|> noFormat (c `T.cons` s)
+    formattedP c f = A.char c *> ((md c f . T.concat <$> A.many' pieceP <* A.char c) <|> openedP)
      where
+        pieceP = A.takeWhile1 (\x -> x /= c && x /= '`' && x /= '\n') <|> codeTextP <|> delimitersP
+        delimitersP = A.takeWhile1 (== c) >>= \r -> r <$ (guard (T.length r > 1) <|> lookAhead (void $ A.satisfy $ notClosingBefore c))
+        openedP = do
+          s <- A.takeTill (\x -> x == c || x == '\n')
+          (markdown f <$> acrossLinesP s) <|> noFormat (c `T.cons` s)
        acrossLinesP s = do
          s' <- ((s <> "\n") <>) <$> (A.char '\n' *> A.takeTill (== c) <* A.char c)
          if T.null s || isSpace (T.head s) || isSpace (T.last s') then fail "not formatted" else pure s'
@@ -380,12 +383,19 @@ markdownP = joinMarkdown <$> A.many' fragmentP
      | otherwise = markdown f s
    doubleP :: Char -> Format -> Parser Markdown
    doubleP c f = do
-      s <- A.string cc *> A.takeTill (== c) <* A.string cc
+      s <- A.string cc *> (T.concat <$> A.many' (A.takeWhile1 (\x -> x /= c && x /= '`') <|> codeTextP)) <* A.string cc <* notP (A.satisfy $ notClosingBefore c)
      if T.null s || isSpace (T.head s) || isSpace (T.last s)
        then fail "not formatted"
        else pure $ markdown f s
      where
        cc = T.pack [c, c]
+    notClosingBefore :: Char -> Char -> Bool
+    notClosingBefore c x = x == c || (c == '_' && isAlphaNum x)
+    codeTextP :: Parser Text
+    codeTextP = do
+      ticks <- A.takeWhile1 (== '`')
+      let pieceP = A.takeWhile1 (\x -> x /= '`' && x /= '\n') <|> (A.takeWhile1 (== '`') >>= \r -> r <$ guard (r /= ticks))
+      ((\s -> ticks <> s <> ticks) . T.concat <$> A.many' pieceP <* A.string ticks) <|> pure ticks
    codeSpanP :: Parser Markdown
    codeSpanP = do
      ticks <- A.takeWhile1 (== '`')
@@ -414,6 +424,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP
      where
        nameRef word
          | pfx == '@' && T.all (/= '.') name = Left "not a name"
+          | pfx == '#' && T.all isDigit name = Left "not a name"
          | otherwise = mkMd <$> strDecode (encodeUtf8 full)
          where
            (name, punct) = splitPunctuation word
@@ -477,7 +488,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP
      code <- optional $ conc4 <$> phoneSep <*> "(" <*> A.takeWhile1 isDigit <*> ")"
      s <- segmentsP $ fromMaybe "" country <> fromMaybe "" code
      let len = T.length s
-      if 7 <= len && len <= 22 then pure $ markdown Phone s else fail "not phone"
+      if 7 <= len && len <= 22 then pure $ if isJust country || "0" `T.isPrefixOf` s then markdown Phone s else unmarked s else fail "not phone"
    segmentsP s
      | T.length s > 22 = fail "not phone"
      | otherwise = optional ((<>) <$> phoneSep <*> A.takeWhile1 isDigit) >>= maybe (pure s) (segmentsP . (s <>))
@@ -531,7 +542,7 @@ markdownP = joinMarkdown <$> A.many' fragmentP
        isDomain_ name tld =
          (let n = T.length name in n >= 1 && n <= 24)
            && (let n = T.length tld in n >= 2 && n <= 8)
-            && (let p c = isAscii c && isAlpha c in T.all p name && T.all p tld)
+            && (let p c = isAscii c && isAlpha c in T.all p name && T.all isAsciiLower tld)
    isEmail s = T.any (== '@') s && Email.isValid (encodeUtf8 s)
    noFormat = pure . unmarked
    simplexUriFormat :: Maybe Text -> AConnectionLink -> Format
@@ -658,7 +669,7 @@ headingMarkP = do
  ind <- A.takeWhile isIndentChar
  hashes <- A.takeWhile1 (== '#')
  guard $ T.length hashes <= 6
-  sp <- A.string " "
+  sp <- (<>) <$> A.string " " <*> A.takeWhile isIndentChar
  hasTextP
  pure (T.length hashes, ind <> hashes <> sp)

@@ -704,7 +715,7 @@ quoteP = do
 quoteMarkP :: Parser Text
 quoteMarkP = do
  ind <- A.takeWhile isIndentChar
-  sp <- A.char '>' *> A.option "" (T.singleton <$> A.satisfy isIndentChar)
+  sp <- A.char '>' *> ((T.singleton <$> A.satisfy isIndentChar) <|> ("" <$ lookAhead (void lineEndP <|> A.endOfInput)))
  pure $ ind <> ">" <> sp

 blankQuoteLineP :: Parser Text
@@ -716,10 +727,13 @@ quoteStartP = optional (blankQuoteLineP *> lineEndP) *> quoteMarkP *> hasTextP
 paragraphP :: Parser MarkdownList
 paragraphP = do
  l <- lineTextP
-  ls <- A.many' $ (,[],) <$> lineEndP <* notP underlineP <* paragraphLineP <*> lineTextP
+  ls <- A.many' $ (,[],) <$> lineEndP <* notP underlineP <* (paragraphLineP <|> numberedTextLineP) <*> lineTextP
  underline <- A.option [] $ (\br (lvl, u) -> [markupText HardBreak br, markupText (HeadingUnderline lvl) u]) <$> lineEndP <*> underlineP
  pure $ inlineLines l ls <> underline

+numberedTextLineP :: Parser ()
+numberedTextLineP = lookAhead $ listMarkP >>= \(num, _) -> guard $ maybe False (`notElem` ["1.", "1)"]) num
+
 underlineP :: Parser (Int, Text)
 underlineP = lineTextP >>= \l -> maybe mzero (pure . (,l)) (underlineLevel l)

@@ -811,7 +825,6 @@ headingClosing c =
 underlineLevel :: Text -> Maybe Int
 underlineLevel t = case T.uncons s of
  Just ('=', _) | T.all (== '=') s -> Just 1
-  Just ('-', _) | T.all (== '-') s -> Just 2
  _ -> Nothing
  where
    s = T.dropWhileEnd isSpace $ T.dropWhile isIndentChar t
@@ -312,6 +312,8 @@ textWithUri = describe "text with Uri" do
    ".com" <==> ".com"
    "example.academytoolong" <==> "example.academytoolong"
    "simplex:/example" <==> "simplex:/example"
+    "end of sentence.Next sentence" <==> "end of sentence.Next sentence"
+    "Visit Example.Com today" <==> "Visit Example.Com today"
  it "SimpleX links" do
    let inv = "/invitation#/?v=1&smp=smp%3A%2F%2F1234-w%3D%3D%40smp.simplex.im%3A5223%2F3456-w%3D%3D%23%2F%3Fv%3D1-2%26dh%3DMCowBQYDK2VuAyEAjiswwI3O_NlS8Fk3HJUW870EY2bAwmttMBsvRB9eV3o%253D&e2e=v%3D2%26x3dh%3DMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D%2CMEIwBQYDK2VvAzkAmKuSYeQ_m0SixPDS8Wq8VBaTS1cW-Lp0n0h4Diu-kUpR-qXx4SDJ32YGEFoGFGSbGPry5Ychr6U%3D"
    ("https://simplex.chat" <> inv) <==> simplexLink XLInvitation ("simplex:" <> inv) ["smp.simplex.im"] ("https://simplex.chat" <> inv)
@@ -406,6 +408,12 @@ textWithPhone = describe "text with Phone" do
  it "ignored as markdown (double spaces)" $ do
    "test 07777  777  777 test" <==> "test 07777  777  777 test"
    "*test* 07777  777  777 test" <==> bold "test" <> " 07777  777  777 test"
+  it "ignored as markdown (not starting with + or 0)" $ do
+    "order 12345678 shipped" <==> "order 12345678 shipped"
+    "on 2024-10-29" <==> "on 2024-10-29"
+    "server 192.168.1.1" <==> "server 192.168.1.1"
+    "version 7.1.0.14" <==> "version 7.1.0.14"
+    "total 1 000 000" <==> "total 1 000 000"

 mention :: Text -> Text -> Markdown
 mention = Markdown . Just . Mention
@@ -480,7 +488,6 @@ textWithSimplexNames = describe "text with SimpleX names" do
    "#my-channel.simplex" <==> sname NTPublicGroup TLDSimplex "my-channel" [] "my-channel.simplex"
    "hello #privacy!" <==> "hello " <> sname NTPublicGroup TLDSimplex "privacy" [] "privacy" <> "!"
    "see #privacy.simplex now" <==> "see " <> sname NTPublicGroup TLDSimplex "privacy" [] "privacy.simplex" <> " now"
-    "#123" <==> sname NTPublicGroup TLDSimplex "123" [] "123"
  it "channel names - subdomains" do
    "#support.acme.simplex" <==> sname NTPublicGroup TLDSimplex "acme" ["support"] "support.acme.simplex"
    "#a.b.acme.simplex" <==> sname NTPublicGroup TLDSimplex "acme" ["b", "a"] "a.b.acme.simplex"
@@ -496,6 +503,9 @@ textWithSimplexNames = describe "text with SimpleX names" do
    "@my-name.simplex" <==> sname NTContact TLDSimplex "my-name" [] "my-name.simplex"
    "@alice.example.com" <==> sname NTContact TLDWeb "alice.example.com" [] "alice.example.com"
  it "not parsed as names" do
+    "#123" <==> "#123"
+    "issue #42 is fixed" <==> "issue #42 is fixed"
+    "I'm #1!" <==> "I'm #1!"
    "#secret#" <==> markdown Secret "secret"
    "##double secret##" <==> markdown Secret "#double secret#"
    "#" <==> "#"
@@ -532,6 +542,20 @@ inlineMarkdownGFM = describe "inline markdown of GFM" do
    "a__b__" ==> "a__b__"
    markdownText (fmt boldF "a") `shouldBe` "*a*"
    markdownText (fmt bold2F "a") `shouldBe` "**a**"
+  it "underscores inside words, as CommonMark" do
+    "_private_var = 1" <==> "_private_var = 1"
+    "use _snake_case_ here" <==> "use " <> markdown Italic "snake_case" <> " here"
+    "_foo_bar_baz_" <==> markdown Italic "foo_bar_baz"
+    "__foo__bar" <==> "__foo__bar"
+    "__init__.py" ==> bold2 "init" <> ".py"
+  it "a closing delimiter followed by the same character is text" do
+    "*a**b*" <==> bold "a**b"
+    "*foo**bar**baz*" <==> bold "foo**bar**baz"
+    "**a****b**" <==> "**a****b**"
+    "*a**" <==> "*a**"
+  it "code spans inside a format" do
+    "*a `*`*" <==> bold "a `*`"
+    "**a `**` b**" <==> bold2 "a `**` b"
  it "code spans" do
    "`` x ` y ``" ==> markdown Snippet "x ` y"
    "``x ` y``" <== markdown Snippet "x ` y"
@@ -754,6 +778,7 @@ headings = describe "headings" do
    "#\n- b" <===> (["#", lineBr, bullet "- ", "b"], [Paragraph (rng 0 1), List [entry 3 4 []]])
  it "indentation and inline content" do
    "  # a" <===> ([mk (HeadingMark 1) "  # ", "a"], [Heading 1 (rng 1 2)])
+    "#   a" <===> ([mk (HeadingMark 1) "#   ", "a"], [Heading 1 (rng 1 2)])
    "# *a* b" <===> ([heading1, fmt boldF "a", " b"], [Heading 1 (rng 1 3)])
  it "closing sequences" do
    "# a #" <===> ([heading1, "a", mk HeadingClose " #"], [Heading 1 (rng 1 2)])
@@ -766,12 +791,13 @@ headings = describe "headings" do
    "# a\n# b" <===> ([heading1, "a", lineBr, heading1, "b"], [Heading 1 (rng 1 2), Heading 1 (rng 4 5)])
  it "setext headings" do
    "a\n===" <===> (["a", hardBr, mk (HeadingUnderline 1) "==="], [Heading 1 (rng 0 1)])
-    "a\n---" <===> (["a", hardBr, mk (HeadingUnderline 2) "---"], [Heading 2 (rng 0 1)])
-    "a\n-" <===> (["a", hardBr, mk (HeadingUnderline 2) "-"], [Heading 2 (rng 0 1)])
    "a\nb\n=" <===> (["a", hardBr, "b", hardBr, mk (HeadingUnderline 1) "="], [Heading 1 (rng 0 3)])
-    "a\nb\n---" <===> (["a", hardBr, "b", hardBr, mk (HeadingUnderline 2) "---"], [Heading 2 (rng 0 3)])
    "a\n  ===  " <===> (["a", hardBr, mk (HeadingUnderline 1) "  ===  "], [Heading 1 (rng 0 1)])
-    "a\n---\nb" <===> (["a", hardBr, mk (HeadingUnderline 2) "---", lineBr, "b"], [Heading 2 (rng 0 1), Paragraph (rng 4 5)])
+  it "dashes under text are a rule, not an underline" do
+    "a\n---" <===> (["a", lineBr, mk RuleMark "---"], [Paragraph (rng 0 1), Rule (rng 2 3)])
+    "a\nb\n---" <===> (["a", hardBr, "b", lineBr, mk RuleMark "---"], [Paragraph (rng 0 3), Rule (rng 4 5)])
+    "a\n---\nb" <===> (["a", lineBr, mk RuleMark "---", lineBr, "b"], [Paragraph (rng 0 1), Rule (rng 2 3), Paragraph (rng 4 5)])
+    noBlocks "a\n-"
  it "underlines only under paragraphs" do
    "a\n- - -" <===> (["a", lineBr, mk RuleMark "- - -"], [Paragraph (rng 0 1), Rule (rng 2 3)])
    "a\n***" <===> (["a", lineBr, mk RuleMark "***"], [Paragraph (rng 0 1), Rule (rng 2 3)])
@@ -813,6 +839,11 @@ lists = describe "lists" do
    noBlocks "1. "
    noBlocks "1234567890. a"
    noBlocks "a. b"
+  it "after a paragraph line, only a list numbered 1 starts" do
+    "a\n1. b" <===> (["a", lineBr, numbered "1.", "b"], [Paragraph (rng 0 1), List [ListEntry (Just "1.") (rng 3 4) []]])
+    "a\n1) b" <===> (["a", lineBr, numbered "1)", "b"], [Paragraph (rng 0 1), List [ListEntry (Just "1)") (rng 3 4) []]])
+    noBlocks "a\n14. b"
+    "a\n14. b\n- c" <===> (["a", hardBr, "14. b", lineBr, bullet "- ", "c"], [Paragraph (rng 0 3), List [entry 5 6 []]])
  it "nesting by the column of the marker" do
    "- a\n  - b" <===> ([bullet "- ", "a", lineBr, bullet "  - ", "b"], [List [entry 1 2 [entry 4 5 []]]])
    "1. a\n   - b" <===> ([numbered "1.", "a", lineBr, bullet "   - ", "b"], [List [ListEntry (Just "1.") (rng 1 2) [entry 4 5 []]]])
@@ -859,11 +890,13 @@ quotes :: Spec
 quotes = describe "quotes" do
  it "quote marks" do
    "> a" <===> ([quoteMark "> ", "a"], [Quote (rng 1 2)])
-    ">a" <===> ([quoteMark ">", "a"], [Quote (rng 1 2)])
    ">\ta" <===> ([quoteMark ">\t", "a"], [Quote (rng 1 2)])
    ">  a" <===> ([quoteMark "> ", " a"], [Quote (rng 1 2)])
    "  > a" <===> ([quoteMark "  > ", "a"], [Quote (rng 1 2)])
-    ">a\n> b" <===> ([quoteMark ">", "a", hardBr, quoteMark "> ", "b"], [Quote (rng 1 5)])
+  it "a quote mark without a space or a tab after it is text" do
+    noBlocks ">a"
+    noBlocks ">_< sorry"
+    ">a\n> b" <===> ([">a", lineBr, quoteMark "> ", "b"], [Paragraph (rng 0 1), Quote (rng 3 4)])
  it "quote marks without text are text" do
    noBlocks ">"
    noBlocks "> "
@@ -1048,7 +1081,7 @@ liveMessages = describe "incomplete syntax of live messages" do
 linearTimeParsing :: Spec
 linearTimeParsing = describe "bounded scans for linear-time parsing" do
  it "phone numbers: at most 22 characters, as on master" do
-    "1 1 1 1 1 1 1 1 1 1 1 1" <==> "1 " <> phone "1 1 1 1 1 1 1 1 1 1 1"
+    "0 0 0 0 0 0 0 0 0 0 0 0" <==> "0 " <> phone "0 0 0 0 0 0 0 0 0 0 0"
  it "link texts stop at \"[\", link targets and autolinks at white space" do
    "[a [b](https://simplex.chat)" <==> "[a " <> web "b" "https://simplex.chat" "[b](https://simplex.chat)"
    "[a](https://simplex.chat x)" <==> "[a](" <> uri "https://simplex.chat" <> " x)"