From 888ce25d4f15c10aea121907f4fd47169004d60f Mon Sep 17 00:00:00 2001
From: shum <shum@liber.li>
Date: Thu, 28 Aug 2025 12:03:23 +0000
Subject: [PATCH] apps/simplex-llm-bot: reponse buffering

---
 apps/simplex-llm-bot/bot.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/apps/simplex-llm-bot/bot.py b/apps/simplex-llm-bot/bot.py
index 7135826f5c..fa09b962bd 100644
--- a/apps/simplex-llm-bot/bot.py
+++ b/apps/simplex-llm-bot/bot.py
@@ -5,6 +5,7 @@ from simpx.extension import ChatWrapper
 from collections import defaultdict
 import traceback
 import os
+import time
 
 chat_histories = defaultdict(list)
 
@@ -72,6 +73,7 @@ if __name__ == "__main__":
               chat_info: Chat information dictionary.
               args: User's prompt.
           """
+
           # Create a ChatWrapper instance using the provided chat_info and the bot's client.
           chat = ChatWrapper(chat_info, bot.client)
 
@@ -101,6 +103,23 @@ if __name__ == "__main__":
                   stream=True  # Enable streaming of the response.
               )
 
+              # Buffer to accumulate tokens until a sentence is complete
+              buffer = ""
+              last_update = time.time()
+              UPDATE_INTERVAL = 2  # seconds
+
+              async def flush_buffer(force: bool = False):
+                  """Push complete sentences to the live message at UPDATE_INTERVAL-second cadence."""
+                  nonlocal buffer, current_response, last_update
+                  now = time.time()
+
+                  if force or (buffer.rstrip().endswith(('.', '?', '!'))
+                              and now - last_update >= UPDATE_INTERVAL):
+                      current_response += buffer
+                      await live_msg.update_live(current_response.strip())
+                      buffer = ""
+                      last_update = now
+
               # Process each streaming chunk.
               for chunk in response:
                   # Extract text content from the current chunk.
@@ -109,13 +128,13 @@ if __name__ == "__main__":
                   if "<|eot_id|>" in chunk_text:
                       chunk_text = chunk_text.replace("<|eot_id|>", "")
                       if chunk_text:                       # keep any leftover text
-                        current_response += chunk_text
-                        await live_msg.update_live(current_response)
+                        buffer += chunk_text
                       break                               # no more data expected
-                  if chunk_text:
-                      current_response += chunk_text
-                      # Update the live message with the accumulated response.
-                      await live_msg.update_live(f"{current_response}")
+                  buffer += chunk_text
+                  await flush_buffer()
+
+              # Final flush for any remaining text
+              await flush_buffer(force=True)
               
               chat_histories[chat_id].append({"role": "assistant", "content": current_response})
               # Finalize the live message.