diff --git a/src/core/message-parsing/CodeBlockStateMachine.ts b/src/core/message-parsing/CodeBlockStateMachine.ts
new file mode 100644
index 0000000000..4fab47c65c
--- /dev/null
+++ b/src/core/message-parsing/CodeBlockStateMachine.ts
@@ -0,0 +1,61 @@
+import { ParseContext, CodeBlockState } from "./ParseContext"
+
+export interface ProcessedTextResult {
+ processedText: string
+ suppressXmlParsing: boolean
+ stateChanged: boolean
+ nextIndex: number
+}
+
+export interface CodeBlockBoundary {
+ found: boolean
+ endIndex: number
+ isComplete: boolean
+}
+
+export class CodeBlockStateMachine {
+ /**
+ * Process incoming text and manage code block state transitions
+ */
+ processText(text: string, context: ParseContext): ProcessedTextResult {
+ // Simple approach: scan for ``` patterns and track state
+ let result = ""
+ let i = 0
+ let stateChanged = false
+
+ while (i < text.length) {
+ // Check for ``` pattern at current position
+ if (this.isCodeBlockBoundary(text, i)) {
+ // Found ``` - toggle state
+ if (context.codeBlockState === CodeBlockState.OUTSIDE) {
+ context.codeBlockState = CodeBlockState.INSIDE
+ stateChanged = true
+ } else if (context.codeBlockState === CodeBlockState.INSIDE) {
+ context.codeBlockState = CodeBlockState.OUTSIDE
+ stateChanged = true
+ }
+ // Include the ``` in the result
+ result += "```"
+ i += 3
+ } else {
+ // Regular character
+ result += text[i]
+ i++
+ }
+ }
+
+ return {
+ processedText: result,
+ suppressXmlParsing: context.codeBlockState === CodeBlockState.INSIDE,
+ stateChanged,
+ nextIndex: i,
+ }
+ }
+
+ /**
+ * Check if there's a ``` pattern at the given position
+ */
+ private isCodeBlockBoundary(text: string, pos: number): boolean {
+ return pos <= text.length - 3 && text[pos] === "`" && text[pos + 1] === "`" && text[pos + 2] === "`"
+ }
+}
diff --git a/src/core/message-parsing/DirectiveStreamingParser.ts b/src/core/message-parsing/DirectiveStreamingParser.ts
index 1a2b2eb18d..0e64d64749 100644
--- a/src/core/message-parsing/DirectiveStreamingParser.ts
+++ b/src/core/message-parsing/DirectiveStreamingParser.ts
@@ -1,6 +1,6 @@
import * as sax from "sax"
import { Directive } from "./directives"
-import { ParseContext } from "./ParseContext"
+import { ParseContext, CodeBlockState } from "./ParseContext"
import { DirectiveRegistryFactory } from "./DirectiveRegistryFactory"
import { FallbackParser } from "./FallbackParser"
import { XmlUtils } from "./XmlUtils"
@@ -14,6 +14,10 @@ export class DirectiveStreamingParser {
contentBlocks: [],
hasXmlTags: false,
hasIncompleteXml: XmlUtils.hasIncompleteXml(assistantMessage),
+ codeBlockState: CodeBlockState.OUTSIDE,
+ pendingBackticks: "",
+ codeBlockContent: "",
+ codeBlockStartIndex: -1,
}
const parser = sax.parser(false, { lowercase: true })
@@ -22,28 +26,41 @@ export class DirectiveStreamingParser {
let activeHandler: any = null
parser.onopentag = (node: sax.Tag) => {
- context.hasXmlTags = true
- tagStack.push(node.name)
- const handler = this.registry.getHandler(node.name)
+ // Only process XML tags if NOT inside code block
+ if (context.codeBlockState !== CodeBlockState.INSIDE) {
+ context.hasXmlTags = true
+ tagStack.push(node.name)
+ const handler = this.registry.getHandler(node.name)
- if (handler) {
- activeHandler = handler
- this.registry.getTextHandler().setState("none")
- }
- if (activeHandler) {
- activeHandler.onOpenTag(node, context)
+ if (handler) {
+ activeHandler = handler
+ this.registry.getTextHandler().setState("none")
+ }
+ if (activeHandler) {
+ activeHandler.onOpenTag(node, context)
+ }
+ } else {
+ // Inside code block - treat as plain text
+ const tagText = `<${node.name}${this.attributesToString(node.attributes)}>`
+ this.registry.getTextHandler().onText(tagText, context)
}
}
parser.onclosetag = (tagName: string) => {
- if (activeHandler) {
- activeHandler.onCloseTag(tagName, context)
- if (tagName === activeHandler.tagName) {
- activeHandler = null
- this.registry.getTextHandler().setState("text")
+ if (context.codeBlockState !== CodeBlockState.INSIDE) {
+ // Normal XML processing
+ if (activeHandler) {
+ activeHandler.onCloseTag(tagName, context)
+ if (tagName === activeHandler.tagName) {
+ activeHandler = null
+ this.registry.getTextHandler().setState("text")
+ }
}
+ tagStack.pop()
+ } else {
+ // Inside code block - treat as plain text
+ this.registry.getTextHandler().onText(``, context)
}
- tagStack.pop()
}
parser.ontext = (text: string) => {
@@ -77,4 +94,16 @@ export class DirectiveStreamingParser {
return context.contentBlocks
}
+
+ /**
+ * Convert SAX node attributes to string representation
+ */
+ private static attributesToString(attributes: { [key: string]: string }): string {
+ if (!attributes || Object.keys(attributes).length === 0) {
+ return ""
+ }
+ return Object.entries(attributes)
+ .map(([key, value]) => ` ${key}="${value}"`)
+ .join("")
+ }
}
diff --git a/src/core/message-parsing/ParseContext.ts b/src/core/message-parsing/ParseContext.ts
index 7868d88445..bc615b5198 100644
--- a/src/core/message-parsing/ParseContext.ts
+++ b/src/core/message-parsing/ParseContext.ts
@@ -1,8 +1,21 @@
import { Directive } from "./directives"
+export enum CodeBlockState {
+ OUTSIDE = "outside", // Normal parsing mode
+ INSIDE = "inside", // Inside code block - suppress XML
+ PARTIAL_START = "partial_start", // Detected partial ``` at start
+ PARTIAL_END = "partial_end", // Detected partial ``` at end
+}
+
export interface ParseContext {
currentText: string
contentBlocks: Directive[]
hasXmlTags: boolean
hasIncompleteXml: boolean
+
+ // Code block state tracking
+ codeBlockState: CodeBlockState
+ pendingBackticks: string // For partial ``` detection
+ codeBlockContent: string // Accumulated content inside code blocks
+ codeBlockStartIndex: number // Track where code block started
}
diff --git a/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts b/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts
new file mode 100644
index 0000000000..311c2055ac
--- /dev/null
+++ b/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts
@@ -0,0 +1,65 @@
+import { suite, test, expect } from "vitest"
+import { CodeBlockStateMachine } from "../CodeBlockStateMachine"
+import { ParseContext, CodeBlockState } from "../ParseContext"
+
+suite("CodeBlockStateMachine", () => {
+ function createContext(): ParseContext {
+ return {
+ currentText: "",
+ contentBlocks: [],
+ hasXmlTags: false,
+ hasIncompleteXml: false,
+ codeBlockState: CodeBlockState.OUTSIDE,
+ pendingBackticks: "",
+ codeBlockContent: "",
+ codeBlockStartIndex: -1,
+ }
+ }
+
+ test("should detect complete code block boundary", () => {
+ const stateMachine = new CodeBlockStateMachine()
+ const context = createContext()
+ const input = "```\ncode content\n```"
+
+ const result = stateMachine.processText(input, context)
+
+ expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+ expect(result.processedText).toBe("```\ncode content\n```")
+ })
+
+ test("should handle false positive backticks", () => {
+ const stateMachine = new CodeBlockStateMachine()
+ const context = createContext()
+
+ // Single backtick should not trigger code block
+ const result1 = stateMachine.processText("text `single` more", context)
+ expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+
+ // Two backticks should not trigger code block
+ const result2 = stateMachine.processText("text ``double`` more", context)
+ expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+ })
+
+ test("should toggle state correctly for code blocks", () => {
+ const stateMachine = new CodeBlockStateMachine()
+ const context = createContext()
+
+ // Start outside
+ expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+
+ // Process opening ```
+ const result1 = stateMachine.processText("```", context)
+ expect(context.codeBlockState).toBe(CodeBlockState.INSIDE)
+ expect(result1.suppressXmlParsing).toBe(true)
+
+ // Process content inside
+ const result2 = stateMachine.processText("content", context)
+ expect(context.codeBlockState).toBe(CodeBlockState.INSIDE)
+ expect(result2.suppressXmlParsing).toBe(true)
+
+ // Process closing ```
+ const result3 = stateMachine.processText("```", context)
+ expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+ expect(result3.suppressXmlParsing).toBe(false)
+ })
+})
diff --git a/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts b/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts
index 66012c24a0..32f17e12ab 100644
--- a/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts
+++ b/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts
@@ -78,4 +78,49 @@ suite("DirectiveStreamingParser", () => {
} as TextDirective,
])
})
+
+ test("should handle mixed content with code blocks and directives", () => {
+ const input =
+ "Some text ```\n\nThis is code\n\n```\nMore text\n\nThis is a real directive\ninfo\n"
+ const result = DirectiveStreamingParser.parse(input)
+ expect(result).toHaveLength(2)
+ expect(result[0].type).toBe("text")
+ expect((result[0] as TextDirective).content).toContain("")
+ expect(result[1].type).toBe("log_message")
+ })
+
+ test("should handle multiple code blocks in single message", () => {
+ const input =
+ "Text ```\ncontent1\n``` middle ```\ncontent2\n``` end"
+ const result = DirectiveStreamingParser.parse(input)
+ expect(result).toEqual([
+ {
+ type: "text",
+ content:
+ "Text ```\ncontent1\n``` middle ```\ncontent2\n``` end",
+ partial: true,
+ } as TextDirective,
+ ])
+ })
+
+ test("should handle nested backticks inside code blocks", () => {
+ const input = "Text ```\nSome `nested` backticks here\n``` end"
+ const result = DirectiveStreamingParser.parse(input)
+ expect(result).toEqual([
+ {
+ type: "text",
+ content: "Text ```\nSome `nested` backticks here\n``` end",
+ partial: true,
+ } as TextDirective,
+ ])
+ })
+
+ test("should not treat incomplete backticks as code blocks", () => {
+ const input =
+ "Text with `single` and ``double`` backticks Should be parsedinfo"
+ const result = DirectiveStreamingParser.parse(input)
+ expect(result).toHaveLength(2)
+ expect(result[0].type).toBe("text")
+ expect(result[1].type).toBe("log_message")
+ })
})
diff --git a/src/core/message-parsing/handlers/TextDirectiveHandler.ts b/src/core/message-parsing/handlers/TextDirectiveHandler.ts
index 7162a76438..8cd1eb7485 100644
--- a/src/core/message-parsing/handlers/TextDirectiveHandler.ts
+++ b/src/core/message-parsing/handlers/TextDirectiveHandler.ts
@@ -1,10 +1,12 @@
import { BaseDirectiveHandler } from "./BaseDirectiveHandler"
-import { ParseContext } from "../ParseContext"
+import { ParseContext, CodeBlockState } from "../ParseContext"
import { TextDirective } from "../directives"
+import { CodeBlockStateMachine } from "../CodeBlockStateMachine"
export class TextDirectiveHandler extends BaseDirectiveHandler {
readonly tagName = "text"
private currentState: "text" | "none" = "text"
+ private stateMachine = new CodeBlockStateMachine()
override canHandle(tagName: string): boolean {
return false // Text handler is fallback
@@ -12,7 +14,12 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
override onText(text: string, context: ParseContext): void {
if (this.currentState === "text") {
- context.currentText += text
+ // Process text through the code block state machine
+ const result = this.stateMachine.processText(text, context)
+
+ // Always add processed text to current text
+ // The suppressXmlParsing flag is used by the parser to decide whether to process XML tags
+ context.currentText += result.processedText
}
}
@@ -21,6 +28,19 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
}
override onEnd(context: ParseContext): void {
+ // Handle any remaining code block content
+ if (context.codeBlockContent) {
+ context.currentText += context.codeBlockContent
+ context.codeBlockContent = ""
+ }
+
+ // Handle any pending backticks that weren't completed
+ if (context.pendingBackticks) {
+ context.currentText += context.pendingBackticks
+ context.pendingBackticks = ""
+ }
+
+ // Create text directive if we have content
if (context.currentText.trim()) {
context.contentBlocks.push({
type: "text",
@@ -29,4 +49,11 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
} as TextDirective)
}
}
+
+ /**
+ * Check if we're currently inside a code block
+ */
+ isInsideCodeBlock(context: ParseContext): boolean {
+ return context.codeBlockState === CodeBlockState.INSIDE
+ }
}