website: channel pages show every empty line of a message (#8028)

.github/workflows/build.yml

@@ -38,6 +38,9 @@ on:
       - "apps/simplex-badge-service/web/**"
       - "scripts/ci/packages_release.py"
       - "scripts/ci/test_packages_release.py"
+      - "website/src/js/simplex-lib.jsc"
+      - "website/test/**"
+      - "website/package.json"
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
     inputs:

scripts/ci/README.md

@@ -115,13 +115,27 @@ The reduced build runs one Linux job, build-linux (ubuntu-22.04 x86_64), for eve
 - CI, in build-linux: `scope-tests.sh ci-scripts` (step "Test changed scopes").
 - Local: `python3 scripts/ci/test_packages_release.py`; Python 3.11 or newer is required for tomllib (packages_release.py).
 
+#### website
+
+- Globs:
+  - website/src/js/simplex-lib.jsc
+  - website/test/**
+  - website/package.json
+- CI:
+  - in build-linux: `scope-tests.sh website` (step "Test changed scopes");
+  - on master pushes: build and deploy of website/**, images/**, blog/**, docs/** and PRIVACY.md (web.yml).
+- Local:
+  ```sh
+  cd website
+  npm test
+  ```
+
 #### Scopes lacking CI tests
 
 | Scope | Paths | CI |
 |---|---|---|
 | ios | apps/ios/** | compile only |
 | types package | packages/simplex-chat-client/types/typescript/** | compile only |
-| website | website/**<br>images/**<br>blog/**<br>docs/**<br>PRIVACY.md | build and deploy on master pushes (web.yml) |
 | ts-client | packages/simplex-chat-client/typescript/** | - |
 | webrtc | packages/simplex-chat-webrtc/** | - |
 | mcp | packages/simplex-chat-mcp/** | - |

scripts/ci/packages_release.py

@@ -82,6 +82,7 @@ BUILD_CHANGES = {
 SCOPE_CHANGES = {
     "badge-web": ["apps/simplex-badge-service/web/**"],
     "ci-scripts": ["scripts/ci/packages_release.py", "scripts/ci/test_packages_release.py"],
+    "website": ["website/src/js/simplex-lib.jsc", "website/test/**", "website/package.json"],
 }
 CABAL_FILE = "simplex-chat.cabal"
 CABAL_VERSION = re.compile(r"^(version:\s*)(\S+)", re.M)

scripts/ci/scope-tests.sh

@@ -11,8 +11,11 @@ case "${1:-}" in
   ci-scripts)
     python3 scripts/ci/test_packages_release.py
     ;;
+  website)
+    (cd website && npm test)
+    ;;
   *)
-    echo "usage: $0 badge-web | ci-scripts" >&2
+    echo "usage: $0 badge-web | ci-scripts | website" >&2
     exit 1
     ;;
 esac

scripts/ci/test_packages_release.py

@@ -514,6 +514,10 @@ class ChangesTest(GitRepoTest):
         self.assertEqual(self.builds("scripts/ci/packages_release.py"), set())
         self.assertEqual(self.values["scopes"], "ci-scripts")
 
+    def test_website_library_change_tests_its_scope(self):
+        self.assertEqual(self.builds("website/src/js/simplex-lib.jsc"), set())
+        self.assertEqual(self.values["scopes"], "website")
+
 
 class PackageStateTest(GitRepoTest):
     def setUp(self):

website/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "build": "npm run build:js && npm run build:eleventy && npm run build:tailwind",
     "start": "npx eleventy --serve",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "node --test test/simplex-lib.test.js",
     "build:js": "cp ./node_modules/qrcode/build/qrcode.js ./src/js/ && cp ./node_modules/@plausible-analytics/tracker/plausible.js ./src/js/page.js && ./copy_call.sh",
     "build:eleventy": "eleventy",
     "build:tailwind": "npx tailwindcss -i tailwind.css -o _site/css/tailwind.css",

website/src/js/channel-preview.jsc

@@ -507,7 +507,6 @@ const STYLE = `
   min-width: 0;
 }
 
-.simplex-preview-text .md-gap { height: 0.7em; }
 .simplex-preview-text .md-h1 { font-size: 1.5em; line-height: 1.3; font-weight: 700; }
 .simplex-preview-text .md-h2 { font-size: 1.3em; line-height: 1.3; font-weight: 700; }
 .simplex-preview-text .md-h3 { font-size: 1.15em; line-height: 1.3; font-weight: 700; }
@@ -927,7 +926,7 @@ function render(container, data, channelLink, showAppBadges) {
     var welcomeMsg = {
       sender: null,
       ts: messages.length > 0 ? messages[0].ts : new Date().toISOString(),
-      content: { type: 'text', text: typeof welcome === 'string' ? welcome : '' },
+      content: { type: 'text', text: typeof welcome === 'string' ? welcome : channel.description },
       formattedText: Array.isArray(welcome) ? welcome : null,
       markdownBlocks: data.welcomeMessageBlocks,
       reactions: []
@@ -1547,7 +1546,7 @@ function appendTextBlock(bubble, msg) {
   textDiv.className = 'simplex-preview-text';
   const meta = renderMetaHTML(msg);
   if (msg.formattedText && msg.formattedText.length > 0) {
-    textDiv.innerHTML = renderMarkdown(msg.formattedText, msg.markdownBlocks) + meta;
+    textDiv.innerHTML = renderMarkdown(msg.formattedText, msg.markdownBlocks, msg.content.text) + meta;
   } else {
     textDiv.innerHTML = escapeHtml(msg.content.text || '') + meta;
   }

website/src/js/simplex-lib.jsc

@@ -58,6 +58,8 @@ const MAX_LIST_DEPTH = 7;
 
 const MAX_TABLE_CELLS = 1000;
 
+const MAX_EMPTY_LINES = 100;
+
 const COPY_BUTTON =
   '<button type="button" class="md-copy" aria-label="Copy">' +
   '<svg class="md-icon-copy" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><rect x="9" y="9" width="11" height="11" rx="2.5"></rect><path d="M15 9V6.5A2.5 2.5 0 0 0 12.5 4h-6A2.5 2.5 0 0 0 4 6.5v6A2.5 2.5 0 0 0 6.5 15H9"></path></svg>' +
@@ -128,13 +130,26 @@ function validBlocks(fts, blocks) {
   });
 }
 
-function renderBlocks(fts, blocks) {
+function emptyLines(fts, from, to, between) {
+  let lineBreaks = 0;
+  for (const { format } of fts.slice(from, to)) {
+    const token = (format && format.type === 'markup' && format.token) || {};
+    lineBreaks += token.type === 'paragraphBreak' ? token.lineBreaks : token.type === 'lineBreak' ? 1 : 0;
+  }
+  return Math.min(Math.max(lineBreaks - (between ? 1 : 0), 0), MAX_EMPTY_LINES);
+}
+
+function renderBlocks(fts, blocks, messageText) {
+  const isMessage = typeof messageText === 'string';
   let html = '';
   let pos = 0;
   const segments = (r) => renderSegments(fts.slice(r.from, r.to), true);
   const isParagraphBreak = (ft) => ft.format && ft.format.type === 'markup' && ft.format.token && ft.format.token.type === 'paragraphBreak';
+  const trailingLines = (from) => emptyLines(fts, from, fts.length, !messageText.endsWith('\n'));
+  const inlineLast = (b) => isMessage && b === blocks[blocks.length - 1] && trailingLines(b.content.to) === 0;
   const gap = (to) => {
-    if (html && fts.slice(pos, to).some(isParagraphBreak)) html += '<div class="md-gap"></div>';
+    if (isMessage) html += '<br>'.repeat(emptyLines(fts, pos, to, html !== ''));
+    else if (html && fts.slice(pos, to).some(isParagraphBreak)) html += '<div class="md-gap"></div>';
     pos = Math.max(pos, to);
   };
   const block = (r, open, close) => {
@@ -143,6 +158,7 @@ function renderBlocks(fts, blocks) {
     pos = r.to;
   };
   const entry = (e, level) => {
+    if (isMessage) gap(e.content.from);
     html += `<div class="md-entry" style="padding-left: ${1.25 * Math.min(level, 6)}em;"><span class="md-mark">${escapeHtml(String(e.number ?? '•'))}</span><span class="md-text">${segments(e.content)}</span></div>`;
     pos = e.content.to;
     e.items.forEach((item) => entry(item, level + 1));
@@ -162,7 +178,8 @@ function renderBlocks(fts, blocks) {
   for (const b of blocks) {
     switch (b.type) {
       case 'paragraph':
-        block(b.content, '<div>', '</div>');
+        if (inlineLast(b)) block(b.content, '', '');
+        else block(b.content, '<div>', '</div>');
         break;
       case 'heading':
         block(b.content, `<div class="md-h${b.level === 1 || b.level === 2 || b.level === 3 ? b.level : 4}">`, '</div>');
@@ -185,6 +202,7 @@ function renderBlocks(fts, blocks) {
         break;
     }
   }
+  if (isMessage) html += '<br>'.repeat(trailingLines(pos));
   return html;
 }
 
@@ -200,8 +218,8 @@ function markupHtml(token) {
   }
 }
 
-function renderMarkdown(fts, blocks) {
-  return validBlocks(fts, blocks) ? renderBlocks(fts, blocks) : renderSegments(fts, false);
+function renderMarkdown(fts, blocks, messageText) {
+  return validBlocks(fts, blocks) ? renderBlocks(fts, blocks, messageText) : renderSegments(fts, false);
 }
 
 function renderSegments(fts, withBlocks) {

website/test/simplex-lib.test.js

@@ -0,0 +1,159 @@
+const assert = require('node:assert');
+const fs = require('node:fs');
+const path = require('node:path');
+const test = require('node:test');
+const vm = require('node:vm');
+
+const lib = {};
+vm.runInNewContext(fs.readFileSync(path.join(__dirname, '../src/js/simplex-lib.jsc'), 'utf8'), lib);
+const { escapeHtml, renderMarkdown } = lib;
+
+const plain = (text) => ({ text });
+const markup = (type, text, fields) => ({ format: { type: 'markup', token: { type, ...fields } }, text });
+const lineBreak = markup('lineBreak', '\n');
+const hardBreak = markup('hardBreak', '\n');
+const paragraphBreak = (lineBreaks, text = '\n'.repeat(lineBreaks)) => markup('paragraphBreak', text, { lineBreaks });
+const headingMark = markup('headingMark', '# ', { level: 1 });
+const listMark = (text) => markup('listMark', text);
+const quoteMark = (text) => markup('quoteMark', text);
+const fence = markup('fenceMark', '```');
+
+const range = (from, to) => ({ from, to });
+const paragraph = (from, to) => ({ type: 'paragraph', content: range(from, to) });
+const heading = (from, to) => ({ type: 'heading', level: 1, content: range(from, to) });
+const entry = (from, to, items = []) => ({ content: range(from, to), items });
+const list = (...items) => ({ type: 'list', items });
+
+const br = (lines) => '<br>'.repeat(lines);
+const div = (html) => `<div>${html}</div>`;
+const h1 = (html) => `<div class="md-h1">${html}</div>`;
+const li = (html, level = 0) => `<div class="md-entry" style="padding-left: ${1.25 * level}em;"><span class="md-mark">•</span><span class="md-text">${html}</span></div>`;
+
+const check = (fts, blocks, html, text = '') => assert.strictEqual(renderMarkdown(fts, blocks, text), html);
+
+test('each empty line between blocks is one line', () => {
+  const blocks = [heading(1, 2), paragraph(3, 4)];
+  check([headingMark, plain('a'), lineBreak, plain('b')], blocks, h1('a') + 'b');
+  check([headingMark, plain('a'), paragraphBreak(2), plain('b')], blocks, h1('a') + br(1) + 'b');
+  check([headingMark, plain('a'), paragraphBreak(3), plain('b')], blocks, h1('a') + br(2) + 'b');
+  check([headingMark, plain('a'), paragraphBreak(3, '\n  \n\t\n'), plain('b')], blocks, h1('a') + br(2) + 'b');
+});
+
+test('empty lines before the first block', () => {
+  check([paragraphBreak(2), headingMark, plain('a')], [heading(2, 3)], br(2) + h1('a'));
+  check([paragraphBreak(2, '\n \n'), listMark('- '), plain('a')], [list(entry(2, 3))], br(2) + li('a'));
+});
+
+test('empty lines after the last block', () => {
+  const fts = [headingMark, plain('a')];
+  const blocks = [heading(1, 2)];
+  check(fts, blocks, h1('a'), '# a');
+  check(fts, blocks, h1('a'), '# a\n');
+  check([...fts, lineBreak], blocks, h1('a') + br(1), '# a\n\n');
+  check([...fts, paragraphBreak(2)], blocks, h1('a') + br(2), '# a\n\n\n');
+  check([...fts, paragraphBreak(1, '\n  ')], blocks, h1('a'), '# a\n  ');
+  check([...fts, paragraphBreak(1, '\n  ')], blocks, h1('a') + br(1), '# a\n  \n');
+  check([...fts, paragraphBreak(2, '\n\n  ')], blocks, h1('a') + br(1), '# a\n\n  ');
+  check([...fts, paragraphBreak(2, '\n\n  ')], blocks, h1('a') + br(2), '# a\n\n  \n');
+});
+
+test('a message ending in a paragraph ends in inline text', () => {
+  const twoParagraphs = [paragraph(0, 1), paragraph(2, 3)];
+  const oneParagraph = [paragraph(0, 1)];
+  check([plain('a'), paragraphBreak(2), plain('b')], twoParagraphs, div('a') + br(1) + 'b', 'a\n\nb');
+  check([plain('a'), paragraphBreak(3), plain('b')], twoParagraphs, div('a') + br(2) + 'b', 'a\n\n\nb');
+  check([paragraphBreak(2), plain('a')], [paragraph(1, 2)], br(2) + 'a', '\n\na');
+  check([plain('a'), paragraphBreak(1, '\n  ')], oneParagraph, 'a', 'a\n  ');
+});
+
+test('a message ending in a paragraph with empty lines at the end', () => {
+  const oneParagraph = [paragraph(0, 1)];
+  check([plain('a'), lineBreak], oneParagraph, div('a') + br(1), 'a\n\n');
+  check([plain('a'), paragraphBreak(2)], oneParagraph, div('a') + br(2), 'a\n\n\n');
+  check([plain('a'), paragraphBreak(1, '\n  ')], oneParagraph, div('a') + br(1), 'a\n  \n');
+});
+
+test('empty lines between list entries', () => {
+  check(
+    [listMark('- '), plain('a'), paragraphBreak(2), listMark('- '), plain('b'), paragraphBreak(3), listMark('- '), plain('c')],
+    [list(entry(1, 2), entry(4, 5), entry(7, 8))],
+    li('a') + br(1) + li('b') + br(2) + li('c')
+  );
+  check(
+    [listMark('- '), plain('a'), lineBreak, listMark('  - '), plain('b'), paragraphBreak(2), listMark('- '), plain('c')],
+    [list(entry(1, 2, [entry(4, 5)]), entry(7, 8))],
+    li('a') + li('b', 1) + br(1) + li('c')
+  );
+  check(
+    [listMark('- '), plain('a'), paragraphBreak(2), listMark('  - '), plain('b')],
+    [list(entry(1, 2, [entry(4, 5)]))],
+    li('a') + br(1) + li('b', 1)
+  );
+});
+
+test('empty lines next to lists', () => {
+  check(
+    [plain('a'), paragraphBreak(3), listMark('- '), plain('b')],
+    [paragraph(0, 1), list(entry(3, 4))],
+    div('a') + br(2) + li('b')
+  );
+  check(
+    [listMark('- '), plain('a'), paragraphBreak(2), plain('b')],
+    [list(entry(1, 2)), paragraph(3, 4)],
+    li('a') + br(1) + 'b'
+  );
+});
+
+test('empty lines next to code blocks, tables and rules', () => {
+  const html = renderMarkdown(
+    [fence, hardBreak, plain('x'), hardBreak, fence, paragraphBreak(2), markup('ruleMark', '---')],
+    [{ type: 'codeBlock', content: range(2, 3) }, { type: 'rule', mark: range(6, 7) }],
+    ''
+  );
+  assert.ok(html.startsWith('<div class="md-code"><pre>x</pre>'));
+  assert.ok(html.endsWith('</button></div><br><hr class="md-rule">'));
+  check(
+    [plain('a'), paragraphBreak(2), markup('rowMark', '| '), plain('x'), markup('cellMark', ' |'), hardBreak, markup('delimiterRow', '|---|', { columns: ['none'] }), paragraphBreak(3), plain('b')],
+    [paragraph(0, 1), { type: 'table', columns: ['none'], header: [range(3, 4)], rows: [] }, paragraph(8, 9)],
+    div('a') + br(1) + '<div class="md-table"><div class="md-frame"><table><tr><th style="text-align: left;">x</th></tr></table></div></div>' + br(2) + 'b'
+  );
+});
+
+test('empty lines in a quote are lines of its text', () => {
+  check(
+    [quoteMark('> '), plain('a'), hardBreak, quoteMark('>'), hardBreak, quoteMark('> '), plain('b')],
+    [{ type: 'quote', content: range(1, 7) }],
+    '<div class="md-quote"><span class="md-bar"></span><div class="md-text">a<br><br>b</div></div>'
+  );
+});
+
+test('at most 100 empty lines in a row', () => {
+  check([plain('a'), paragraphBreak(5000), plain('b')], [paragraph(0, 1), paragraph(2, 3)], div('a') + br(100) + 'b');
+  check([paragraphBreak(5000), plain('a')], [paragraph(1, 2)], br(100) + 'a');
+  check([plain('a'), paragraphBreak(5000)], [paragraph(0, 1)], div('a') + br(100));
+  check(
+    [listMark('- '), plain('a'), paragraphBreak(5000), listMark('- '), plain('b')],
+    [list(entry(1, 2), entry(4, 5))],
+    li('a') + br(100) + li('b')
+  );
+});
+
+test('plain text and inline formatting without blocks', () => {
+  assert.strictEqual(escapeHtml('a\n\n\nb'), 'a<br><br><br>b');
+  check([{ format: { type: 'bold', singleStar: true }, text: 'a' }, plain('\nb')], undefined, '<strong>a</strong><br>b');
+});
+
+test('invalid blocks are drawn as text', () => {
+  check([plain('a'), paragraphBreak(2), plain('b')], [{ type: 'unknown' }], 'a<br><br>b');
+});
+
+test('text outside messages keeps one gap between blocks', () => {
+  const page = (fts, blocks, html) => assert.strictEqual(renderMarkdown(fts, blocks), html);
+  const gap = '<div class="md-gap"></div>';
+  const blocks = [heading(1, 2), paragraph(3, 4)];
+  page([headingMark, plain('a'), lineBreak, plain('b')], blocks, h1('a') + div('b'));
+  page([headingMark, plain('a'), paragraphBreak(3), plain('b')], blocks, h1('a') + gap + div('b'));
+  page([plain('a'), paragraphBreak(2), listMark('- '), plain('b')], [paragraph(0, 1), list(entry(3, 4))], div('a') + gap + li('b'));
+  page([listMark('- '), plain('a'), paragraphBreak(2), listMark('- '), plain('b')], [list(entry(1, 2), entry(4, 5))], li('a') + li('b'));
+  page([paragraphBreak(2), plain('a'), paragraphBreak(2)], [paragraph(1, 2)], div('a'));
+});