From 51440392bee24c2e241befc17a17524158946614 Mon Sep 17 00:00:00 2001 From: "Steven T. Cramer" Date: Mon, 16 Jun 2025 20:23:26 +0700 Subject: [PATCH] WIP: Implement code block detection for DirectiveStreamingParser - Add CodeBlockState enum and extend ParseContext with code block tracking - Create CodeBlockStateMachine to detect ` boundaries and manage state - Update TextDirectiveHandler to integrate with state machine - Modify DirectiveStreamingParser to suppress XML parsing inside code blocks - Add comprehensive tests for code block handling NOTE: Tests pass but implementation does NOT work in runtime. The log messages still appear in Output window instead of being treated as plain text within code blocks. The real parsing flow differs from test scenarios and needs further investigation. --- .../message-parsing/CodeBlockStateMachine.ts | 61 +++++++++++++++++ .../DirectiveStreamingParser.ts | 61 ++++++++++++----- src/core/message-parsing/ParseContext.ts | 13 ++++ .../code-block-state-machine.spec.ts | 65 +++++++++++++++++++ .../directive-streaming-parser.spec.ts | 45 +++++++++++++ .../handlers/TextDirectiveHandler.ts | 31 ++++++++- 6 files changed, 258 insertions(+), 18 deletions(-) create mode 100644 src/core/message-parsing/CodeBlockStateMachine.ts create mode 100644 src/core/message-parsing/__tests__/code-block-state-machine.spec.ts diff --git a/src/core/message-parsing/CodeBlockStateMachine.ts b/src/core/message-parsing/CodeBlockStateMachine.ts new file mode 100644 index 0000000000..4fab47c65c --- /dev/null +++ b/src/core/message-parsing/CodeBlockStateMachine.ts @@ -0,0 +1,61 @@ +import { ParseContext, CodeBlockState } from "./ParseContext" + +export interface ProcessedTextResult { + processedText: string + suppressXmlParsing: boolean + stateChanged: boolean + nextIndex: number +} + +export interface CodeBlockBoundary { + found: boolean + endIndex: number + isComplete: boolean +} + +export class CodeBlockStateMachine { + /** + * Process incoming text and manage code block state transitions + */ + processText(text: string, context: ParseContext): ProcessedTextResult { + // Simple approach: scan for ``` patterns and track state + let result = "" + let i = 0 + let stateChanged = false + + while (i < text.length) { + // Check for ``` pattern at current position + if (this.isCodeBlockBoundary(text, i)) { + // Found ``` - toggle state + if (context.codeBlockState === CodeBlockState.OUTSIDE) { + context.codeBlockState = CodeBlockState.INSIDE + stateChanged = true + } else if (context.codeBlockState === CodeBlockState.INSIDE) { + context.codeBlockState = CodeBlockState.OUTSIDE + stateChanged = true + } + // Include the ``` in the result + result += "```" + i += 3 + } else { + // Regular character + result += text[i] + i++ + } + } + + return { + processedText: result, + suppressXmlParsing: context.codeBlockState === CodeBlockState.INSIDE, + stateChanged, + nextIndex: i, + } + } + + /** + * Check if there's a ``` pattern at the given position + */ + private isCodeBlockBoundary(text: string, pos: number): boolean { + return pos <= text.length - 3 && text[pos] === "`" && text[pos + 1] === "`" && text[pos + 2] === "`" + } +} diff --git a/src/core/message-parsing/DirectiveStreamingParser.ts b/src/core/message-parsing/DirectiveStreamingParser.ts index 1a2b2eb18d..0e64d64749 100644 --- a/src/core/message-parsing/DirectiveStreamingParser.ts +++ b/src/core/message-parsing/DirectiveStreamingParser.ts @@ -1,6 +1,6 @@ import * as sax from "sax" import { Directive } from "./directives" -import { ParseContext } from "./ParseContext" +import { ParseContext, CodeBlockState } from "./ParseContext" import { DirectiveRegistryFactory } from "./DirectiveRegistryFactory" import { FallbackParser } from "./FallbackParser" import { XmlUtils } from "./XmlUtils" @@ -14,6 +14,10 @@ export class DirectiveStreamingParser { contentBlocks: [], hasXmlTags: false, hasIncompleteXml: XmlUtils.hasIncompleteXml(assistantMessage), + codeBlockState: CodeBlockState.OUTSIDE, + pendingBackticks: "", + codeBlockContent: "", + codeBlockStartIndex: -1, } const parser = sax.parser(false, { lowercase: true }) @@ -22,28 +26,41 @@ export class DirectiveStreamingParser { let activeHandler: any = null parser.onopentag = (node: sax.Tag) => { - context.hasXmlTags = true - tagStack.push(node.name) - const handler = this.registry.getHandler(node.name) + // Only process XML tags if NOT inside code block + if (context.codeBlockState !== CodeBlockState.INSIDE) { + context.hasXmlTags = true + tagStack.push(node.name) + const handler = this.registry.getHandler(node.name) - if (handler) { - activeHandler = handler - this.registry.getTextHandler().setState("none") - } - if (activeHandler) { - activeHandler.onOpenTag(node, context) + if (handler) { + activeHandler = handler + this.registry.getTextHandler().setState("none") + } + if (activeHandler) { + activeHandler.onOpenTag(node, context) + } + } else { + // Inside code block - treat as plain text + const tagText = `<${node.name}${this.attributesToString(node.attributes)}>` + this.registry.getTextHandler().onText(tagText, context) } } parser.onclosetag = (tagName: string) => { - if (activeHandler) { - activeHandler.onCloseTag(tagName, context) - if (tagName === activeHandler.tagName) { - activeHandler = null - this.registry.getTextHandler().setState("text") + if (context.codeBlockState !== CodeBlockState.INSIDE) { + // Normal XML processing + if (activeHandler) { + activeHandler.onCloseTag(tagName, context) + if (tagName === activeHandler.tagName) { + activeHandler = null + this.registry.getTextHandler().setState("text") + } } + tagStack.pop() + } else { + // Inside code block - treat as plain text + this.registry.getTextHandler().onText(``, context) } - tagStack.pop() } parser.ontext = (text: string) => { @@ -77,4 +94,16 @@ export class DirectiveStreamingParser { return context.contentBlocks } + + /** + * Convert SAX node attributes to string representation + */ + private static attributesToString(attributes: { [key: string]: string }): string { + if (!attributes || Object.keys(attributes).length === 0) { + return "" + } + return Object.entries(attributes) + .map(([key, value]) => ` ${key}="${value}"`) + .join("") + } } diff --git a/src/core/message-parsing/ParseContext.ts b/src/core/message-parsing/ParseContext.ts index 7868d88445..bc615b5198 100644 --- a/src/core/message-parsing/ParseContext.ts +++ b/src/core/message-parsing/ParseContext.ts @@ -1,8 +1,21 @@ import { Directive } from "./directives" +export enum CodeBlockState { + OUTSIDE = "outside", // Normal parsing mode + INSIDE = "inside", // Inside code block - suppress XML + PARTIAL_START = "partial_start", // Detected partial ``` at start + PARTIAL_END = "partial_end", // Detected partial ``` at end +} + export interface ParseContext { currentText: string contentBlocks: Directive[] hasXmlTags: boolean hasIncompleteXml: boolean + + // Code block state tracking + codeBlockState: CodeBlockState + pendingBackticks: string // For partial ``` detection + codeBlockContent: string // Accumulated content inside code blocks + codeBlockStartIndex: number // Track where code block started } diff --git a/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts b/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts new file mode 100644 index 0000000000..311c2055ac --- /dev/null +++ b/src/core/message-parsing/__tests__/code-block-state-machine.spec.ts @@ -0,0 +1,65 @@ +import { suite, test, expect } from "vitest" +import { CodeBlockStateMachine } from "../CodeBlockStateMachine" +import { ParseContext, CodeBlockState } from "../ParseContext" + +suite("CodeBlockStateMachine", () => { + function createContext(): ParseContext { + return { + currentText: "", + contentBlocks: [], + hasXmlTags: false, + hasIncompleteXml: false, + codeBlockState: CodeBlockState.OUTSIDE, + pendingBackticks: "", + codeBlockContent: "", + codeBlockStartIndex: -1, + } + } + + test("should detect complete code block boundary", () => { + const stateMachine = new CodeBlockStateMachine() + const context = createContext() + const input = "```\ncode content\n```" + + const result = stateMachine.processText(input, context) + + expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE) + expect(result.processedText).toBe("```\ncode content\n```") + }) + + test("should handle false positive backticks", () => { + const stateMachine = new CodeBlockStateMachine() + const context = createContext() + + // Single backtick should not trigger code block + const result1 = stateMachine.processText("text `single` more", context) + expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE) + + // Two backticks should not trigger code block + const result2 = stateMachine.processText("text ``double`` more", context) + expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE) + }) + + test("should toggle state correctly for code blocks", () => { + const stateMachine = new CodeBlockStateMachine() + const context = createContext() + + // Start outside + expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE) + + // Process opening ``` + const result1 = stateMachine.processText("```", context) + expect(context.codeBlockState).toBe(CodeBlockState.INSIDE) + expect(result1.suppressXmlParsing).toBe(true) + + // Process content inside + const result2 = stateMachine.processText("content", context) + expect(context.codeBlockState).toBe(CodeBlockState.INSIDE) + expect(result2.suppressXmlParsing).toBe(true) + + // Process closing ``` + const result3 = stateMachine.processText("```", context) + expect(context.codeBlockState).toBe(CodeBlockState.OUTSIDE) + expect(result3.suppressXmlParsing).toBe(false) + }) +}) diff --git a/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts b/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts index 66012c24a0..32f17e12ab 100644 --- a/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts +++ b/src/core/message-parsing/__tests__/directive-streaming-parser.spec.ts @@ -78,4 +78,49 @@ suite("DirectiveStreamingParser", () => { } as TextDirective, ]) }) + + test("should handle mixed content with code blocks and directives", () => { + const input = + "Some text ```\n\nThis is code\n\n```\nMore text\n\nThis is a real directive\ninfo\n" + const result = DirectiveStreamingParser.parse(input) + expect(result).toHaveLength(2) + expect(result[0].type).toBe("text") + expect((result[0] as TextDirective).content).toContain("") + expect(result[1].type).toBe("log_message") + }) + + test("should handle multiple code blocks in single message", () => { + const input = + "Text ```\ncontent1\n``` middle ```\ncontent2\n``` end" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: + "Text ```\ncontent1\n``` middle ```\ncontent2\n``` end", + partial: true, + } as TextDirective, + ]) + }) + + test("should handle nested backticks inside code blocks", () => { + const input = "Text ```\nSome `nested` backticks here\n``` end" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: "Text ```\nSome `nested` backticks here\n``` end", + partial: true, + } as TextDirective, + ]) + }) + + test("should not treat incomplete backticks as code blocks", () => { + const input = + "Text with `single` and ``double`` backticks Should be parsedinfo" + const result = DirectiveStreamingParser.parse(input) + expect(result).toHaveLength(2) + expect(result[0].type).toBe("text") + expect(result[1].type).toBe("log_message") + }) }) diff --git a/src/core/message-parsing/handlers/TextDirectiveHandler.ts b/src/core/message-parsing/handlers/TextDirectiveHandler.ts index 7162a76438..8cd1eb7485 100644 --- a/src/core/message-parsing/handlers/TextDirectiveHandler.ts +++ b/src/core/message-parsing/handlers/TextDirectiveHandler.ts @@ -1,10 +1,12 @@ import { BaseDirectiveHandler } from "./BaseDirectiveHandler" -import { ParseContext } from "../ParseContext" +import { ParseContext, CodeBlockState } from "../ParseContext" import { TextDirective } from "../directives" +import { CodeBlockStateMachine } from "../CodeBlockStateMachine" export class TextDirectiveHandler extends BaseDirectiveHandler { readonly tagName = "text" private currentState: "text" | "none" = "text" + private stateMachine = new CodeBlockStateMachine() override canHandle(tagName: string): boolean { return false // Text handler is fallback @@ -12,7 +14,12 @@ export class TextDirectiveHandler extends BaseDirectiveHandler { override onText(text: string, context: ParseContext): void { if (this.currentState === "text") { - context.currentText += text + // Process text through the code block state machine + const result = this.stateMachine.processText(text, context) + + // Always add processed text to current text + // The suppressXmlParsing flag is used by the parser to decide whether to process XML tags + context.currentText += result.processedText } } @@ -21,6 +28,19 @@ export class TextDirectiveHandler extends BaseDirectiveHandler { } override onEnd(context: ParseContext): void { + // Handle any remaining code block content + if (context.codeBlockContent) { + context.currentText += context.codeBlockContent + context.codeBlockContent = "" + } + + // Handle any pending backticks that weren't completed + if (context.pendingBackticks) { + context.currentText += context.pendingBackticks + context.pendingBackticks = "" + } + + // Create text directive if we have content if (context.currentText.trim()) { context.contentBlocks.push({ type: "text", @@ -29,4 +49,11 @@ export class TextDirectiveHandler extends BaseDirectiveHandler { } as TextDirective) } } + + /** + * Check if we're currently inside a code block + */ + isInsideCodeBlock(context: ParseContext): boolean { + return context.codeBlockState === CodeBlockState.INSIDE + } }