WIP: Implement code block detection for DirectiveStreamingParser

- Add CodeBlockState enum and extend ParseContext with code block tracking
- Create CodeBlockStateMachine to detect ` boundaries and manage state
- Update TextDirectiveHandler to integrate with state machine
- Modify DirectiveStreamingParser to suppress XML parsing inside code blocks
- Add comprehensive tests for code block handling

NOTE: Tests pass but implementation does NOT work in runtime.
The log messages still appear in Output window instead of being treated
as plain text within code blocks. The real parsing flow differs from
test scenarios and needs further investigation.

src/core/message-parsing/CodeBlockStateMachine.ts

@@ -0,0 +1,61 @@
+import { ParseContext, CodeBlockState } from "./ParseContext"
+
+export interface ProcessedTextResult {
+	processedText: string
+	suppressXmlParsing: boolean
+	stateChanged: boolean
+	nextIndex: number
+}
+
+export interface CodeBlockBoundary {
+	found: boolean
+	endIndex: number
+	isComplete: boolean
+}
+
+export class CodeBlockStateMachine {
+	/**
+	 * Process incoming text and manage code block state transitions
+	 */
+	processText(text: string, context: ParseContext): ProcessedTextResult {
+		// Simple approach: scan for ``` patterns and track state
+		let result = ""
+		let i = 0
+		let stateChanged = false
+
+		while (i < text.length) {
+			// Check for ``` pattern at current position
+			if (this.isCodeBlockBoundary(text, i)) {
+				// Found ``` - toggle state
+				if (context.codeBlockState === CodeBlockState.OUTSIDE) {
+					context.codeBlockState = CodeBlockState.INSIDE
+					stateChanged = true
+				} else if (context.codeBlockState === CodeBlockState.INSIDE) {
+					context.codeBlockState = CodeBlockState.OUTSIDE
+					stateChanged = true
+				}
+				// Include the ``` in the result
+				result += "```"
+				i += 3
+			} else {
+				// Regular character
+				result += text[i]
+				i++
+			}
+		}
+
+		return {
+			processedText: result,
+			suppressXmlParsing: context.codeBlockState === CodeBlockState.INSIDE,
+			stateChanged,
+			nextIndex: i,
+		}
+	}
+
+	/**
+	 * Check if there's a ``` pattern at the given position
+	 */
+	private isCodeBlockBoundary(text: string, pos: number): boolean {
+		return pos <= text.length - 3 && text[pos] === "`" && text[pos + 1] === "`" && text[pos + 2] === "`"
+	}
+}

src/core/message-parsing/DirectiveStreamingParser.ts

@@ -1,6 +1,6 @@
 import * as sax from "sax"
 import { Directive } from "./directives"
-import { ParseContext } from "./ParseContext"
+import { ParseContext, CodeBlockState } from "./ParseContext"
 import { DirectiveRegistryFactory } from "./DirectiveRegistryFactory"
 import { FallbackParser } from "./FallbackParser"
 import { XmlUtils } from "./XmlUtils"
@@ -14,6 +14,10 @@ export class DirectiveStreamingParser {
 			contentBlocks: [],
 			hasXmlTags: false,
 			hasIncompleteXml: XmlUtils.hasIncompleteXml(assistantMessage),
+			codeBlockState: CodeBlockState.OUTSIDE,
+			pendingBackticks: "",
+			codeBlockContent: "",
+			codeBlockStartIndex: -1,
 		}
 
 		const parser = sax.parser(false, { lowercase: true })
@@ -22,28 +26,41 @@ export class DirectiveStreamingParser {
 		let activeHandler: any = null
 
 		parser.onopentag = (node: sax.Tag) => {
-			context.hasXmlTags = true
-			tagStack.push(node.name)
-			const handler = this.registry.getHandler(node.name)
+			// Only process XML tags if NOT inside code block
+			if (context.codeBlockState !== CodeBlockState.INSIDE) {
+				context.hasXmlTags = true
+				tagStack.push(node.name)
+				const handler = this.registry.getHandler(node.name)
 
-			if (handler) {
-				activeHandler = handler
-				this.registry.getTextHandler().setState("none")
-			}
-			if (activeHandler) {
-				activeHandler.onOpenTag(node, context)
+				if (handler) {
+					activeHandler = handler
+					this.registry.getTextHandler().setState("none")
+				}
+				if (activeHandler) {
+					activeHandler.onOpenTag(node, context)
+				}
+			} else {
+				// Inside code block - treat as plain text
+				const tagText = `<${node.name}${this.attributesToString(node.attributes)}>`
+				this.registry.getTextHandler().onText(tagText, context)
 			}
 		}
 
 		parser.onclosetag = (tagName: string) => {
-			if (activeHandler) {
-				activeHandler.onCloseTag(tagName, context)
-				if (tagName === activeHandler.tagName) {
-					activeHandler = null
-					this.registry.getTextHandler().setState("text")
+			if (context.codeBlockState !== CodeBlockState.INSIDE) {
+				// Normal XML processing
+				if (activeHandler) {
+					activeHandler.onCloseTag(tagName, context)
+					if (tagName === activeHandler.tagName) {
+						activeHandler = null
+						this.registry.getTextHandler().setState("text")
+					}
 				}
+				tagStack.pop()
+			} else {
+				// Inside code block - treat as plain text
+				this.registry.getTextHandler().onText(`</${tagName}>`, context)
 			}
-			tagStack.pop()
 		}
 
 		parser.ontext = (text: string) => {
@@ -77,4 +94,16 @@ export class DirectiveStreamingParser {
 
 		return context.contentBlocks
 	}
+
+	/**
+	 * Convert SAX node attributes to string representation
+	 */
+	private static attributesToString(attributes: { [key: string]: string }): string {
+		if (!attributes || Object.keys(attributes).length === 0) {
+			return ""
+		}
+		return Object.entries(attributes)
+			.map(([key, value]) => ` ${key}="${value}"`)
+			.join("")
+	}
 }

src/core/message-parsing/ParseContext.ts

@@ -1,8 +1,21 @@
 import { Directive } from "./directives"
 
+export enum CodeBlockState {
+	OUTSIDE = "outside", // Normal parsing mode
+	INSIDE = "inside", // Inside code block - suppress XML
+	PARTIAL_START = "partial_start", // Detected partial ``` at start
+	PARTIAL_END = "partial_end", // Detected partial ``` at end
+}
+
 export interface ParseContext {
 	currentText: string
 	contentBlocks: Directive[]
 	hasXmlTags: boolean
 	hasIncompleteXml: boolean
+
+	// Code block state tracking
+	codeBlockState: CodeBlockState
+	pendingBackticks: string // For partial ``` detection
+	codeBlockContent: string // Accumulated content inside code blocks
+	codeBlockStartIndex: number // Track where code block started
 }

src/core/message-parsing/__tests__/code-block-state-machine.spec.ts

@@ -0,0 +1,65 @@
+import { suite, test, expect } from "vitest"
+import { CodeBlockStateMachine } from "../CodeBlockStateMachine"
+import { ParseContext, CodeBlockState } from "../ParseContext"
+
+suite("CodeBlockStateMachine", () => {
+	function createContext(): ParseContext {
+		return {
+			currentText: "",
+			contentBlocks: [],
+			hasXmlTags: false,
+			hasIncompleteXml: false,
+			codeBlockState: CodeBlockState.OUTSIDE,
+			pendingBackticks: "",
+			codeBlockContent: "",
+			codeBlockStartIndex: -1,
+		}
+	}
+
+	test("should detect complete code block boundary", () => {
+		const stateMachine = new CodeBlockStateMachine()
+		const context = createContext()
+		const input = "```\ncode content\n```"
+
+		const result = stateMachine.processText(input, context)
+
+		expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+		expect(result.processedText).toBe("```\ncode content\n```")
+	})
+
+	test("should handle false positive backticks", () => {
+		const stateMachine = new CodeBlockStateMachine()
+		const context = createContext()
+
+		// Single backtick should not trigger code block
+		const result1 = stateMachine.processText("text `single` more", context)
+		expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+
+		// Two backticks should not trigger code block
+		const result2 = stateMachine.processText("text ``double`` more", context)
+		expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+	})
+
+	test("should toggle state correctly for code blocks", () => {
+		const stateMachine = new CodeBlockStateMachine()
+		const context = createContext()
+
+		// Start outside
+		expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+
+		// Process opening ```
+		const result1 = stateMachine.processText("```", context)
+		expect(context.codeBlockState).toBe(CodeBlockState.INSIDE)
+		expect(result1.suppressXmlParsing).toBe(true)
+
+		// Process content inside
+		const result2 = stateMachine.processText("content", context)
+		expect(context.codeBlockState).toBe(CodeBlockState.INSIDE)
+		expect(result2.suppressXmlParsing).toBe(true)
+
+		// Process closing ```
+		const result3 = stateMachine.processText("```", context)
+		expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE)
+		expect(result3.suppressXmlParsing).toBe(false)
+	})
+})

src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts

@@ -78,4 +78,49 @@ suite("DirectiveStreamingParser", () => {
 			} as TextDirective,
 		])
 	})
+
+	test("should handle mixed content with code blocks and directives", () => {
+		const input =
+			"Some text ```\n<log_message>\n<message>This is code</message>\n</log_message>\n```\nMore text\n<log_message>\n<message>This is a real directive</message>\n<level>info</level>\n</log_message>"
+		const result = DirectiveStreamingParser.parse(input)
+		expect(result).toHaveLength(2)
+		expect(result[0].type).toBe("text")
+		expect((result[0] as TextDirective).content).toContain("<log_message>")
+		expect(result[1].type).toBe("log_message")
+	})
+
+	test("should handle multiple code blocks in single message", () => {
+		const input =
+			"Text ```\n<directive1>content1</directive1>\n``` middle ```\n<directive2>content2</directive2>\n``` end"
+		const result = DirectiveStreamingParser.parse(input)
+		expect(result).toEqual([
+			{
+				type: "text",
+				content:
+					"Text ```\n<directive1>content1</directive1>\n``` middle ```\n<directive2>content2</directive2>\n``` end",
+				partial: true,
+			} as TextDirective,
+		])
+	})
+
+	test("should handle nested backticks inside code blocks", () => {
+		const input = "Text ```\nSome `nested` backticks here\n``` end"
+		const result = DirectiveStreamingParser.parse(input)
+		expect(result).toEqual([
+			{
+				type: "text",
+				content: "Text ```\nSome `nested` backticks here\n``` end",
+				partial: true,
+			} as TextDirective,
+		])
+	})
+
+	test("should not treat incomplete backticks as code blocks", () => {
+		const input =
+			"Text with `single` and ``double`` backticks <log_message><message>Should be parsed</message><level>info</level></log_message>"
+		const result = DirectiveStreamingParser.parse(input)
+		expect(result).toHaveLength(2)
+		expect(result[0].type).toBe("text")
+		expect(result[1].type).toBe("log_message")
+	})
 })

src/core/message-parsing/handlers/TextDirectiveHandler.ts

@@ -1,10 +1,12 @@
 import { BaseDirectiveHandler } from "./BaseDirectiveHandler"
-import { ParseContext } from "../ParseContext"
+import { ParseContext, CodeBlockState } from "../ParseContext"
 import { TextDirective } from "../directives"
+import { CodeBlockStateMachine } from "../CodeBlockStateMachine"
 
 export class TextDirectiveHandler extends BaseDirectiveHandler {
 	readonly tagName = "text"
 	private currentState: "text" | "none" = "text"
+	private stateMachine = new CodeBlockStateMachine()
 
 	override canHandle(tagName: string): boolean {
 		return false // Text handler is fallback
@@ -12,7 +14,12 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
 
 	override onText(text: string, context: ParseContext): void {
 		if (this.currentState === "text") {
-			context.currentText += text
+			// Process text through the code block state machine
+			const result = this.stateMachine.processText(text, context)
+
+			// Always add processed text to current text
+			// The suppressXmlParsing flag is used by the parser to decide whether to process XML tags
+			context.currentText += result.processedText
 		}
 	}
 
@@ -21,6 +28,19 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
 	}
 
 	override onEnd(context: ParseContext): void {
+		// Handle any remaining code block content
+		if (context.codeBlockContent) {
+			context.currentText += context.codeBlockContent
+			context.codeBlockContent = ""
+		}
+
+		// Handle any pending backticks that weren't completed
+		if (context.pendingBackticks) {
+			context.currentText += context.pendingBackticks
+			context.pendingBackticks = ""
+		}
+
+		// Create text directive if we have content
 		if (context.currentText.trim()) {
 			context.contentBlocks.push({
 				type: "text",
@@ -29,4 +49,11 @@ export class TextDirectiveHandler extends BaseDirectiveHandler {
 			} as TextDirective)
 		}
 	}
+
+	/**
+	 * Check if we're currently inside a code block
+	 */
+	isInsideCodeBlock(context: ParseContext): boolean {
+		return context.codeBlockState === CodeBlockState.INSIDE
+	}
 }