feat streaming the inference in realtime

gui.py

@@ -362,6 +362,7 @@ class ModelLoaderThread(QThread):
 class AskThread(QThread):
     tool_call = pyqtSignal(str)
     usage = pyqtSignal(dict)
+    token = pyqtSignal(str)
     finished_ok = pyqtSignal(str)
     failed = pyqtSignal(str)
 
@@ -377,6 +378,7 @@ class AskThread(QThread):
                 self._messages,
                 on_tool_call=self.tool_call.emit,
                 on_usage=self.usage.emit,
+                on_token=self.token.emit,
             )
         except Exception as e:
             self.failed.emit(str(e))
@@ -394,6 +396,11 @@ class ChatWindow(QMainWindow):
         self._history: list[tuple[str, str]] = []
         self._pending_question = ""
         self._ask_thread: AskThread | None = None
+        # Position in chat_view's document where the in-progress streamed
+        # answer card starts, and a cursor into its body cell for appending
+        # incoming tokens; both None when no answer is currently streaming.
+        self._stream_start_pos: int | None = None
+        self._stream_body_cursor: QTextCursor | None = None
 
         central = QWidget()
         self.setCentralWidget(central)
@@ -526,6 +533,8 @@ class ChatWindow(QMainWindow):
         self._append_user(question)
         self._set_input_enabled(False)
         self.statusBar().showMessage("Thinking...")
+        self._stream_start_pos = None
+        self._stream_body_cursor = None
 
         messages = chatbot.build_messages(self._history, question)
         self._pending_question = question
@@ -533,11 +542,43 @@ class ChatWindow(QMainWindow):
         self._ask_thread = AskThread(self._llm, messages)
         self._ask_thread.tool_call.connect(self._on_tool_call)
         self._ask_thread.usage.connect(self._on_usage)
+        self._ask_thread.token.connect(self._on_token)
         self._ask_thread.finished_ok.connect(self._on_answer)
         self._ask_thread.failed.connect(self._on_ask_failed)
         self._ask_thread.start()
 
+    def _on_token(self, piece: str) -> None:
+        if self._stream_body_cursor is None:
+            self.statusBar().showMessage("Generating...")
+            cursor = self.chat_view.textCursor()
+            cursor.movePosition(QTextCursor.MoveOperation.End)
+            self._stream_start_pos = cursor.position()
+            accent, bg = ROLE_STYLES["assistant"]
+            body = _insert_message_card(cursor, accent, bg)
+            body.insertHtml(f'<span style="color:{accent}; font-weight:600; font-size:8.5pt;">ASSISTANT</span>')
+            body.insertBlock(QTextBlockFormat(), QTextCharFormat())
+            self._stream_body_cursor = body
+
+        char_format = QTextCharFormat()
+        char_format.setForeground(QColor(TEXT_PRIMARY))
+        self._stream_body_cursor.insertText(piece, char_format)
+        self.chat_view.moveCursor(QTextCursor.MoveOperation.End)
+
+    def _discard_stream_card(self) -> None:
+        """Remove the in-progress streamed answer card, if any, so a
+        tool-call round's raw output or a failed generation doesn't linger
+        in the chat view."""
+        if self._stream_start_pos is None:
+            return
+        cursor = self.chat_view.textCursor()
+        cursor.setPosition(self._stream_start_pos)
+        cursor.movePosition(QTextCursor.MoveOperation.End, QTextCursor.MoveMode.KeepAnchor)
+        cursor.removeSelectedText()
+        self._stream_start_pos = None
+        self._stream_body_cursor = None
+
     def _on_tool_call(self, query: str) -> None:
+        self._discard_stream_card()
         self.statusBar().showMessage(f"Searching: {query}")
         self._append_system(f"Searching the web for: {query}")
 
@@ -547,6 +588,10 @@ class ChatWindow(QMainWindow):
         self.context_status_label.setText(f"Context: {total} / {self._n_ctx} ({pct:.0f}%)")
 
     def _on_answer(self, answer: str) -> None:
+        # The streamed card holds the raw, plain-text answer; swap it out for
+        # a fully markdown-rendered and syntax-highlighted one now that the
+        # complete text is available.
+        self._discard_stream_card()
         self._append_assistant(answer)
         self._history.append((self._pending_question, answer.strip()))
         self.statusBar().showMessage("Ready.")
@@ -554,6 +599,7 @@ class ChatWindow(QMainWindow):
         self._set_input_enabled(True)
 
     def _on_ask_failed(self, message: str) -> None:
+        self._discard_stream_card()
         self._append_system(f"Error: {message}")
         self.statusBar().showMessage("Ready.")
         self._ask_thread = None