From 484eb670b291afa07f2f7b976fafe02bdc9ac71d Mon Sep 17 00:00:00 2001
From: "A.J. Shulman" <Shulman.aj@gmail.com>
Date: Wed, 21 Aug 2024 16:06:31 -0400
Subject: added answer streaming parsing so it provides realtime parsing and
 then follow-up questions and citations are added when its finished

---
 .../views/nodes/ChatBox/StreamedAnswerParser.ts    | 73 ++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 src/client/views/nodes/ChatBox/StreamedAnswerParser.ts

(limited to 'src/client/views/nodes/ChatBox/StreamedAnswerParser.ts')

diff --git a/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts b/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts
new file mode 100644
index 000000000..3585cab4a
--- /dev/null
+++ b/src/client/views/nodes/ChatBox/StreamedAnswerParser.ts
@@ -0,0 +1,73 @@
+import { threadId } from 'worker_threads';
+
+enum ParserState {
+    Outside,
+    InGroundedText,
+    InNormalText,
+}
+
+export class StreamedAnswerParser {
+    private state: ParserState = ParserState.Outside;
+    private buffer: string = '';
+    private result: string = '';
+    private isStartOfLine: boolean = true;
+
+    public parse(char: string): string {
+        switch (this.state) {
+            case ParserState.Outside:
+                if (char === '<') {
+                    this.buffer = '<';
+                } else if (char === '>') {
+                    if (this.buffer.startsWith('<grounded_text')) {
+                        this.state = ParserState.InGroundedText;
+                    } else if (this.buffer.startsWith('<normal_text')) {
+                        this.state = ParserState.InNormalText;
+                    }
+                    this.buffer = '';
+                } else {
+                    this.buffer += char;
+                }
+                break;
+
+            case ParserState.InGroundedText:
+            case ParserState.InNormalText:
+                if (char === '<') {
+                    this.buffer = '<';
+                } else if (this.buffer.startsWith('</grounded_text') && char === '>') {
+                    this.state = ParserState.Outside;
+                    this.buffer = '';
+                } else if (this.buffer.startsWith('</normal_text') && char === '>') {
+                    this.state = ParserState.Outside;
+                    this.buffer = '';
+                } else if (this.buffer.startsWith('<')) {
+                    this.buffer += char;
+                } else {
+                    this.processChar(char);
+                }
+                break;
+        }
+
+        return this.result.trim();
+    }
+
+    private processChar(char: string): void {
+        if (this.isStartOfLine && char === ' ') {
+            // Skip leading spaces
+            return;
+        }
+        if (char === '\n') {
+            this.result += char;
+            this.isStartOfLine = true;
+        } else {
+            this.result += char;
+            this.isStartOfLine = false;
+        }
+    }
+
+    public reset(): void {
+        this.state = ParserState.Outside;
+        this.buffer = '';
+        this.result = '';
+        this.isStartOfLine = true;
+    }
+}
-- 
cgit v1.2.3-70-g09d2