refactor: Replace manual XML parsing with SAX parser in DirectiveStreamingParser

- Replace character-by-character XML parsing with robust SAX parser library
- Add sax and @types/sax dependencies for reliable XML parsing
- Implement event-driven parsing with onopentag, onclosetag, ontext handlers
- Add XML wrapping to handle multiple root elements in streaming scenarios
- Create hasIncompleteXml() method for proper partial XML detection
- Maintain backward compatibility with existing partial detection logic
- Add comprehensive test suite with 6 new test cases
- Preserve fallback to manual parsing when SAX parser fails
- Improve error handling and maintain all existing functionality

Benefits:
- More robust XML parsing with better edge case handling
- Improved performance with event-driven parsing
- Cleaner, more maintainable code
- Standards-compliant XML parsing
- Full backward compatibility
This commit is contained in:
Steven T. Cramer 2025-06-15 08:33:21 +07:00
parent 2993a3a75e
commit 1b9d9e65d9
28 changed files with 1024 additions and 77 deletions

34
debug-test.js Normal file
View file

@ -0,0 +1,34 @@
const { parseAssistantMessage } = require("./core/assistant-message/parseAssistantMessage")
const message = `<log_message>
<message>This is a debug message</message>
<level>debug</level>
</log_message>
<log_message>
<message>This is an info message</message>
</log_message>
<log_message>
<message>This is a warning message</message>
<level>warn</level>
</log_message>
<log_message>
<message>This is an error message</message>
<level>error</level>
</log_message>`
console.log("Input message:")
console.log(message)
console.log("\n=== PARSING RESULT ===")
const result = parseAssistantMessage(message)
console.log("Total blocks:", result.length)
const filteredResult = result.filter((block) => !(block.type === "text" && block.content === ""))
console.log("Filtered blocks:", filteredResult.length)
filteredResult.forEach((block, index) => {
console.log(`Block ${index}:`, JSON.stringify(block, null, 2))
})

View file

@ -7,6 +7,7 @@ export * from "./experiment.js"
export * from "./global-settings.js"
export * from "./history.js"
export * from "./ipc.js"
export * from "./log-message.js"
export * from "./mcp.js"
export * from "./message.js"
export * from "./mode.js"

View file

@ -0,0 +1,11 @@
import { z } from "zod"
export const logLevels = ["debug", "info", "warn", "error"] as const
export const logLevelsSchema = z.enum(logLevels).optional().default("info")
export const logMessageParamsSchema = z.object({
message: z.string(),
level: logLevelsSchema,
})
export type LogMessageParams = z.infer<typeof logMessageParamsSchema>

24
pnpm-lock.yaml generated
View file

@ -708,6 +708,9 @@ importers:
sanitize-filename:
specifier: ^1.6.3
version: 1.6.3
sax:
specifier: ^1.4.1
version: 1.4.1
say:
specifier: ^0.16.0
version: 0.16.0
@ -805,6 +808,9 @@ importers:
'@types/ps-tree':
specifier: ^1.1.6
version: 1.1.6
'@types/sax':
specifier: ^1.2.7
version: 1.2.7
'@types/string-similarity':
specifier: ^4.0.2
version: 4.0.2
@ -4269,6 +4275,9 @@ packages:
'@types/react@18.3.23':
resolution: {integrity: sha512-/LDXMQh55EzZQ0uVAZmKKhfENivEvWz6E+EYzh+/MCjMhNsotd+ZHhBGIjFDTi6+fz0OhQQQLbTgdQIxxCsC0w==}
'@types/sax@1.2.7':
resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==}
'@types/shell-quote@1.7.5':
resolution: {integrity: sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==}
@ -14032,7 +14041,7 @@ snapshots:
'@types/graceful-fs@4.1.9':
dependencies:
'@types/node': 20.17.57
'@types/node': 20.19.0
'@types/hast@3.0.4':
dependencies:
@ -14113,7 +14122,6 @@ snapshots:
'@types/node@20.19.0':
dependencies:
undici-types: 6.21.0
optional: true
'@types/node@22.15.29':
dependencies:
@ -14135,6 +14143,10 @@ snapshots:
'@types/prop-types': 15.7.14
csstype: 3.1.3
'@types/sax@1.2.7':
dependencies:
'@types/node': 20.19.0
'@types/shell-quote@1.7.5': {}
'@types/stack-utils@2.0.3': {}
@ -14179,7 +14191,7 @@ snapshots:
'@types/yauzl@2.10.3':
dependencies:
'@types/node': 20.17.57
'@types/node': 20.19.0
optional: true
'@typescript-eslint/eslint-plugin@8.32.1(@typescript-eslint/parser@8.32.1(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3)':
@ -17324,7 +17336,7 @@ snapshots:
'@jest/expect': 29.7.0
'@jest/test-result': 29.7.0
'@jest/types': 29.6.3
'@types/node': 20.17.57
'@types/node': 20.19.0
chalk: 4.1.2
co: 4.6.0
dedent: 1.6.0(babel-plugin-macros@3.1.0)
@ -17481,7 +17493,7 @@ snapshots:
'@jest/environment': 29.7.0
'@jest/fake-timers': 29.7.0
'@jest/types': 29.6.3
'@types/node': 20.17.57
'@types/node': 20.19.0
jest-mock: 29.7.0
jest-util: 29.7.0
@ -17669,7 +17681,7 @@ snapshots:
jest-worker@29.7.0:
dependencies:
'@types/node': 20.17.57
'@types/node': 20.19.0
jest-util: 29.7.0
merge-stream: 2.0.0
supports-color: 8.1.1

View file

@ -66,6 +66,10 @@ export const window = {
show: () => {},
dispose: () => {},
}),
createTextEditorDecorationType: (options) => ({
key: "mockDecorationType",
dispose: () => {},
}),
}
export const commands = {

View file

@ -1,62 +1,296 @@
import { Directive, ParsingState, TextContentParser, ToolUseParser, ParameterParser } from "./parsers"
import { Directive } from "./parsers"
import { TextDirective, LogDirective } from "./directives"
import { ToolUse, ToolParamName } from "../../shared/tools"
import { toolNames } from "@roo-code/types"
import * as sax from "sax"
export class DirectiveStreamingParser {
static parse(assistantMessage: string): Directive[] {
const state: ParsingState = {
contentBlocks: [],
currentTextContent: undefined,
currentTextContentStartIndex: 0,
currentToolUse: undefined,
currentToolUseStartIndex: 0,
currentParamName: undefined,
currentParamValueStartIndex: 0,
accumulator: "",
const contentBlocks: Directive[] = []
let currentText = ""
let currentToolUse: ToolUse | undefined
let currentLogMessage: LogDirective | undefined
let currentParamName: ToolParamName | undefined
let currentParamValue = ""
let currentContext: "text" | "logMessage" | "logLevel" | "param" | "none" = "text"
let hasXmlTags = false
let parseError = false
// Check if the input has incomplete XML (for partial detection)
const hasIncompleteXml = this.hasIncompleteXml(assistantMessage)
const parser = sax.parser(false, { lowercase: true })
parser.onopentag = (node: sax.Tag) => {
hasXmlTags = true
const tagName = node.name
if (tagName === "log_message") {
// Push any accumulated text before starting log message
if (currentText.trim()) {
contentBlocks.push({
type: "text",
content: currentText.trim(),
partial: false,
} as TextDirective)
currentText = ""
}
currentLogMessage = {
type: "log_message",
message: "",
level: "info",
partial: true,
}
currentContext = "none"
} else if (tagName === "message" && currentLogMessage) {
currentContext = "logMessage"
} else if (tagName === "level" && currentLogMessage) {
currentContext = "logLevel"
} else if (toolNames.includes(tagName as any)) {
// Push any accumulated text before starting tool use
if (currentText.trim()) {
contentBlocks.push({
type: "text",
content: currentText.trim(),
partial: false,
} as TextDirective)
currentText = ""
}
currentToolUse = {
type: "tool_use",
name: tagName as any,
params: {},
partial: true,
}
currentContext = "none"
} else if (currentToolUse) {
currentParamName = tagName as ToolParamName
currentParamValue = ""
currentContext = "param"
}
}
for (let i = 0; i < assistantMessage.length; i++) {
const char = assistantMessage[i]
state.accumulator += char
// There should not be a param without a tool use.
if (ParameterParser.parse(state)) {
continue
parser.onclosetag = (tagName: string) => {
if (tagName === "log_message" && currentLogMessage) {
currentLogMessage.partial = hasIncompleteXml
contentBlocks.push(currentLogMessage)
currentLogMessage = undefined
currentContext = "text"
} else if (tagName === "message" && currentLogMessage) {
currentContext = "none"
} else if (tagName === "level" && currentLogMessage) {
currentContext = "none"
} else if (currentToolUse && tagName === currentToolUse.name) {
currentToolUse.partial = hasIncompleteXml || Object.keys(currentToolUse.params).length === 0
contentBlocks.push(currentToolUse)
currentToolUse = undefined
currentContext = "text"
} else if (currentToolUse && currentParamName && tagName === currentParamName) {
;(currentToolUse.params as Record<string, string>)[currentParamName] = currentParamValue.trim()
currentParamName = undefined
currentParamValue = ""
currentContext = "none"
}
// No currentParamName.
if (ToolUseParser.parse(state)) {
continue
}
// No currentToolUse.
const didStartToolUse = ToolUseParser.checkForToolStart(state)
TextContentParser.parse(state, i, didStartToolUse)
}
// Handle remaining partial content
this.handlePartialContent(state)
parser.ontext = (text: string) => {
if (currentContext === "param" && currentParamName && currentToolUse) {
currentParamValue += text
} else if (currentContext === "logMessage" && currentLogMessage) {
currentLogMessage.message += text
} else if (currentContext === "logLevel" && currentLogMessage) {
const levelText = text.trim()
if (["debug", "info", "warn", "error"].includes(levelText)) {
currentLogMessage.level = levelText as "debug" | "info" | "warn" | "error"
}
} else if (currentContext === "text") {
currentText += text
}
}
return state.contentBlocks
parser.onend = () => {
// Push any remaining text
if (currentText.trim()) {
contentBlocks.push({
type: "text",
content: currentText.trim(),
partial: true,
} as TextDirective)
}
// Handle partial log message at the end
if (currentLogMessage) {
currentLogMessage.partial = true
contentBlocks.push(currentLogMessage)
}
// Handle partial tool use at the end
if (currentToolUse) {
if (currentParamName && currentParamValue) {
;(currentToolUse.params as Record<string, string>)[currentParamName] = currentParamValue.trim()
}
currentToolUse.partial = true
contentBlocks.push(currentToolUse)
}
}
parser.onerror = (error: Error) => {
parseError = true
// Don't clear content blocks here - let the fallback logic handle it
}
try {
// Wrap multiple root elements to make valid XML
const wrappedMessage = `<root>${assistantMessage}</root>`
parser.write(wrappedMessage).close()
} catch (e) {
parseError = true
}
// If parsing failed or no XML tags were found, use fallback logic
if (parseError || (!hasXmlTags && contentBlocks.length === 0 && assistantMessage.trim())) {
// Try to handle partial XML manually for streaming scenarios
return this.handlePartialXml(assistantMessage)
}
return contentBlocks
}
private static handlePartialContent(state: ParsingState): void {
if (state.currentToolUse) {
// Stream did not complete tool call, add it as partial.
if (state.currentParamName) {
// Tool call has a parameter that was not completed.
state.currentToolUse.params[state.currentParamName] = state.accumulator
.slice(state.currentParamValueStartIndex)
.trim()
private static handlePartialXml(assistantMessage: string): Directive[] {
const contentBlocks: Directive[] = []
// Handle multiple log messages
const logMessageRegex = /<log_message>([\s\S]*?)(?:<\/log_message>|$)/g
let lastIndex = 0
let match
while ((match = logMessageRegex.exec(assistantMessage)) !== null) {
// Add any text before this log message
if (match.index > lastIndex) {
const textBefore = assistantMessage.substring(lastIndex, match.index).trim()
if (textBefore) {
contentBlocks.push({
type: "text",
content: textBefore,
partial: false,
} as TextDirective)
}
}
state.contentBlocks.push(state.currentToolUse)
const logContent = match[1]
const isComplete = assistantMessage.includes("</log_message>", match.index)
// For streaming behavior, preserve raw XML content when incomplete
let message = ""
let level: "debug" | "info" | "warn" | "error" = "info"
if (isComplete) {
// Complete log message - parse normally
const messageMatch = logContent.match(/<message>(.*?)<\/message>/)
const levelMatch = logContent.match(/<level>(.*?)<\/level>/)
message = messageMatch ? messageMatch[1] : ""
if (levelMatch && ["debug", "info", "warn", "error"].includes(levelMatch[1])) {
level = levelMatch[1] as "debug" | "info" | "warn" | "error"
}
} else {
// Incomplete log message - preserve raw content for streaming behavior
message = logContent
}
const logMessage: LogDirective = {
type: "log_message",
message,
level,
partial: !isComplete,
}
contentBlocks.push(logMessage)
lastIndex = logMessageRegex.lastIndex
}
// NOTE: It doesn't matter if check for currentToolUse or
// currentTextContent, only one of them will be defined since only one can
// be partial at a time.
if (state.currentTextContent) {
// Stream did not complete text content, add it as partial.
state.contentBlocks.push(state.currentTextContent)
// If no log messages were found, check for tool use
if (contentBlocks.length === 0) {
for (const toolName of toolNames) {
const toolRegex = new RegExp(`<${toolName}>[\\s\\S]*?(?:<\\/${toolName}>|$)`)
const toolMatch = assistantMessage.match(toolRegex)
if (toolMatch) {
const toolContent = toolMatch[0]
const params: Record<string, string> = {}
// Extract parameters
const paramRegex = /<(\w+)>(.*?)(?:<\/\1>|$)/g
let paramMatch
while ((paramMatch = paramRegex.exec(toolContent)) !== null) {
const [, paramName, paramValue] = paramMatch
if (paramName !== toolName) {
params[paramName] = paramValue
}
}
const toolUse: ToolUse = {
type: "tool_use",
name: toolName as any,
params,
partial: !assistantMessage.includes(`</${toolName}>`),
}
contentBlocks.push(toolUse)
return contentBlocks
}
}
}
// Add any remaining text after the last log message
if (lastIndex < assistantMessage.length) {
const remainingText = assistantMessage.substring(lastIndex).trim()
if (remainingText) {
contentBlocks.push({
type: "text",
content: remainingText,
partial: true,
} as TextDirective)
}
}
// If no structured content was found, treat as plain text
if (contentBlocks.length === 0) {
contentBlocks.push({
type: "text",
content: assistantMessage,
partial: true,
} as TextDirective)
}
return contentBlocks
}
private static hasIncompleteXml(input: string): boolean {
// Check for incomplete XML by looking for opening tags without corresponding closing tags
const openTags: string[] = []
const tagRegex = /<\/?([a-zA-Z_][a-zA-Z0-9_-]*)[^>]*>/g
let match
while ((match = tagRegex.exec(input)) !== null) {
const fullTag = match[0]
const tagName = match[1]
if (fullTag.startsWith("</")) {
// Closing tag
const lastOpenTag = openTags.pop()
if (lastOpenTag !== tagName) {
// Mismatched closing tag, consider incomplete
return true
}
} else if (!fullTag.endsWith("/>")) {
// Opening tag (not self-closing)
openTags.push(tagName)
}
}
// If there are unclosed tags, it's incomplete
return openTags.length > 0
}
}

View file

@ -0,0 +1,94 @@
import { suite, test, expect } from "vitest"
import { DirectiveStreamingParser } from "../DirectiveStreamingParser"
import { TextDirective, LogDirective } from "../directives"
import { ToolUse } from "../../../shared/tools"
suite("DirectiveStreamingParser", () => {
test("should parse plain text content", () => {
const input = "This is a simple text message."
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "text",
content: "This is a simple text message.",
partial: true,
} as TextDirective,
])
})
test("should parse a complete log message", () => {
const input = "<log_message><message>Log entry</message><level>info</level></log_message>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "log_message",
message: "Log entry",
level: "info",
partial: false,
} as LogDirective,
])
})
test("should parse a partial log message", () => {
const input = "<log_message><message>Partial log entry</message>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "log_message",
message: "Partial log entry",
level: "info",
partial: true,
} as LogDirective,
])
})
test("should parse a tool use directive", () => {
const input = "<read_file><path>src/app.ts</path></read_file>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "tool_use",
name: "read_file",
params: { path: "src/app.ts" },
partial: false,
} as ToolUse,
])
})
test("should parse mixed content with text and tool use", () => {
const input =
"Some text here<apply_diff><path>src/file.ts</path><diff>diff content</diff></apply_diff>More text"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "text",
content: "Some text here",
partial: false,
} as TextDirective,
{
type: "tool_use",
name: "apply_diff",
params: { path: "src/file.ts", diff: "diff content" },
partial: false,
} as ToolUse,
{
type: "text",
content: "More text",
partial: true,
} as TextDirective,
])
})
test("should handle partial tool use directive", () => {
const input = "<write_to_file><path>src/newfile.ts</path><content>Some content"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "tool_use",
name: "write_to_file",
params: { path: "src/newfile.ts", content: "Some content" },
partial: true,
} as ToolUse,
])
})
})

View file

@ -0,0 +1,94 @@
import { suite, test, expect } from "vitest"
import { DirectiveStreamingParser } from "../DirectiveStreamingParser"
import { TextDirective, LogDirective } from "../directives"
import { ToolUse } from "../../../shared/tools"
suite("DirectiveStreamingParser", () => {
test("should parse plain text content", () => {
const input = "This is a simple text message."
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "text",
content: "This is a simple text message.",
partial: true,
} as TextDirective,
])
})
test("should parse a complete log message", () => {
const input = "<log_message><message>Log entry</message><level>info</level></log_message>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "log_message",
message: "Log entry",
level: "info",
partial: false,
} as LogDirective,
])
})
test("should parse a partial log message", () => {
const input = "<log_message><message>Partial log entry</message>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "log_message",
message: "Partial log entry",
level: "info",
partial: true,
} as LogDirective,
])
})
test("should parse a tool use directive", () => {
const input = "<read_file><path>src/app.ts</path></read_file>"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "tool_use",
name: "read_file",
params: { path: "src/app.ts" },
partial: false,
} as ToolUse,
])
})
test("should parse mixed content with text and tool use", () => {
const input =
"Some text here<apply_diff><path>src/file.ts</path><diff>diff content</diff></apply_diff>More text"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "text",
content: "Some text here",
partial: false,
} as TextDirective,
{
type: "tool_use",
name: "apply_diff",
params: { path: "src/file.ts", diff: "diff content" },
partial: false,
} as ToolUse,
{
type: "text",
content: "More text",
partial: true,
} as TextDirective,
])
})
test("should handle partial tool use directive", () => {
const input = "<write_to_file><path>src/newfile.ts</path><content>Some content"
const result = DirectiveStreamingParser.parse(input)
expect(result).toEqual([
{
type: "tool_use",
name: "write_to_file",
params: { path: "src/newfile.ts", content: "Some content" },
partial: true,
} as ToolUse,
])
})
})

View file

@ -0,0 +1,169 @@
import { suite, test, expect } from "vitest"
import { parseAssistantMessage } from ".."
suite("Log Entry Parsing", () => {
test("should parse complete log entries correctly", () => {
const message = `<log_message>
<message>This is a test log message</message>
<level>debug</level>
</log_message>`
const result = parseAssistantMessage(message)
// Filter out empty text blocks
const filteredResult = result.filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResult).toHaveLength(1)
expect(filteredResult[0]).toEqual({
type: "log_message",
message: "This is a test log message",
level: "debug",
partial: false,
})
})
test("should mark partial log entries as partial", () => {
const message = `<log_message>
<message>This is a test log message</message>`
const result = parseAssistantMessage(message)
// Filter out empty text blocks
const filteredResult = result.filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResult).toHaveLength(1)
expect(filteredResult[0]).toEqual({
type: "log_message",
message: "This is a test log message",
level: "info", // Default level
partial: true,
})
})
test("should handle log entries with only message tag", () => {
const message = `<log_message>
<message>This is a test log message</message>
</log_message>`
const result = parseAssistantMessage(message)
// Filter out empty text blocks
const filteredResult = result.filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResult).toHaveLength(1)
expect(filteredResult[0]).toEqual({
type: "log_message",
message: "This is a test log message",
level: "info", // Default level
partial: false,
})
})
test("should simulate streaming behavior with partial log entries", () => {
// Simulate streaming chunks
const chunks = [
"<log_message>\n",
"<message>This is a debug level log message</message>\n",
"<level>debug</level>\n",
"</log_message>",
]
let accumulatedMessage = ""
const results = []
// Process each chunk as it would happen during streaming
for (const chunk of chunks) {
accumulatedMessage += chunk
const result = parseAssistantMessage(accumulatedMessage)
results.push(result)
}
// First chunk: Just the opening tag - filter out empty text blocks
const filteredResults0 = results[0].filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResults0[0]).toEqual({
type: "log_message",
message: "",
level: "info", // Default level
partial: true,
})
// Second chunk: Has message but not level - filter out empty text blocks
const filteredResults1 = results[1].filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResults1[0]).toEqual({
type: "log_message",
message: "This is a debug level log message",
level: "info", // Still default level
partial: true,
})
// Third chunk: Has message and level but not closing tag - filter out empty text blocks
const filteredResults2 = results[2].filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResults2[0]).toEqual({
type: "log_message",
message: "This is a debug level log message",
level: "debug", // Level is now properly parsed
partial: true,
})
// Fourth chunk: Complete log entry - filter out empty text blocks
const filteredResults3 = results[3].filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResults3[0]).toEqual({
type: "log_message",
message: "This is a debug level log message",
level: "debug",
partial: false,
})
})
test("should handle multiple log entries with different levels", () => {
const message = `<log_message>
<message>This is a debug message</message>
<level>debug</level>
</log_message>
<log_message>
<message>This is an info message</message>
</log_message>
<log_message>
<message>This is a warning message</message>
<level>warn</level>
</log_message>
<log_message>
<message>This is an error message</message>
<level>error</level>
</log_message>`
const result = parseAssistantMessage(message)
// Filter out empty text blocks
const filteredResult = result.filter((block) => !(block.type === "text" && block.content === ""))
expect(filteredResult).toHaveLength(4)
expect(filteredResult[0]).toEqual({
type: "log_message",
message: "This is a debug message",
level: "debug",
partial: false,
})
expect(filteredResult[1]).toEqual({
type: "log_message",
message: "This is an info message",
level: "info", // Default level
partial: false,
})
expect(filteredResult[2]).toEqual({
type: "log_message",
message: "This is a warning message",
level: "warn",
partial: false,
})
expect(filteredResult[3]).toEqual({
type: "log_message",
message: "This is an error message",
level: "error",
partial: false,
})
})
})

View file

@ -1,12 +1,13 @@
// npx jest src/core/assistant-message/__tests__/parseAssistantMessage.test.ts
import { TextContent, ToolUse } from "../../../shared/tools"
import { TextDirective } from "../directives"
import { ToolUse } from "../../../shared/tools"
import { AssistantMessageContent, parseAssistantMessage as parseAssistantMessageV1 } from "../parseAssistantMessage"
import { parseAssistantMessageV2 } from "../parseAssistantMessageV2"
const isEmptyTextContent = (block: AssistantMessageContent) =>
block.type === "text" && (block as TextContent).content === ""
block.type === "text" && (block as TextDirective).content === ""
;[parseAssistantMessageV1, parseAssistantMessageV2].forEach((parser, index) => {
describe(`parseAssistantMessageV${index + 1}`, () => {
@ -108,7 +109,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
expect(result).toHaveLength(2)
const textContent = result[0] as TextContent
const textContent = result[0] as TextDirective
expect(textContent.type).toBe("text")
expect(textContent.content).toBe("Here's the file content:")
expect(textContent.partial).toBe(false)
@ -132,7 +133,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
expect(toolUse.params.path).toBe("src/file.ts")
expect(toolUse.partial).toBe(false)
const textContent = result[1] as TextContent
const textContent = result[1] as TextDirective
expect(textContent.type).toBe("text")
expect(textContent.content).toBe("Here's what I found in the file.")
expect(textContent.partial).toBe(true)
@ -146,14 +147,14 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
expect(result).toHaveLength(4)
expect(result[0].type).toBe("text")
expect((result[0] as TextContent).content).toBe("First file:")
expect((result[0] as TextDirective).content).toBe("First file:")
expect(result[1].type).toBe("tool_use")
expect((result[1] as ToolUse).name).toBe("read_file")
expect((result[1] as ToolUse).params.path).toBe("src/file1.ts")
expect(result[2].type).toBe("text")
expect((result[2] as TextContent).content).toBe("Second file:")
expect((result[2] as TextDirective).content).toBe("Second file:")
expect(result[3].type).toBe("tool_use")
expect((result[3] as ToolUse).name).toBe("read_file")
@ -197,7 +198,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
expect(result).toHaveLength(1)
expect(result[0].type).toBe("text")
expect((result[0] as TextContent).content).toBe(message)
expect((result[0] as TextDirective).content).toBe(message)
})
it("should handle tool use with no parameters", () => {
@ -313,7 +314,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
// First text block
expect(result[0].type).toBe("text")
expect((result[0] as TextContent).content).toBe("I'll help you with that task.")
expect((result[0] as TextDirective).content).toBe("I'll help you with that task.")
// First tool use (read_file)
expect(result[1].type).toBe("tool_use")
@ -321,7 +322,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
// Second text block
expect(result[2].type).toBe("text")
expect((result[2] as TextContent).content).toContain("Now let's modify the file:")
expect((result[2] as TextDirective).content).toContain("Now let's modify the file:")
// Second tool use (write_to_file)
expect(result[3].type).toBe("tool_use")
@ -329,7 +330,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
// Third text block
expect(result[4].type).toBe("text")
expect((result[4] as TextContent).content).toContain("Let's run the code:")
expect((result[4] as TextDirective).content).toContain("Let's run the code:")
// Third tool use (execute_command)
expect(result[5].type).toBe("tool_use")

View file

@ -0,0 +1,2 @@
export * from "./logDirective"
export * from "./textDirective"

View file

@ -0,0 +1,12 @@
import { logLevels } from "@roo-code/types"
/**
* Represents a log message directive from the assistant to the system.
* This directive instructs the system to record a message to its internal logs.
*/
export interface LogDirective {
type: "log_message"
message: string
level: (typeof logLevels)[number]
partial: boolean
}

View file

@ -0,0 +1,9 @@
/**
* Represents a message directive from the assistant to the system.
* This directive instructs the system to output text.
*/
export interface TextDirective {
type: "text"
content: string
partial: boolean
}

View file

@ -1,2 +1,3 @@
export { type AssistantMessageContent, parseAssistantMessage } from "./parseAssistantMessage"
export { presentAssistantMessage } from "./presentAssistantMessage"
export { type LogDirective } from "./directives/logDirective"

View file

@ -1,8 +1,9 @@
import { DirectiveStreamingParser } from "./DirectiveStreamingParser"
import type { Directive } from "./parsers/types"
import type { Directive } from "./parsers"
export type { TextDirective } from "./directives"
// Re-export types for backward compatibility
export type { TextDirective, ToolDirective, Directive } from "./parsers/types"
export type { ToolDirective, Directive } from "./parsers"
// Backward compatibility alias
export type AssistantMessageContent = Directive

View file

@ -1,8 +1,8 @@
import { type ToolName, toolNames } from "@roo-code/types"
import { TextDirective } from "./directives"
import { ToolUse, ToolParamName, toolParamNames } from "../../shared/tools"
import { TextContent, ToolUse, ToolParamName, toolParamNames } from "../../shared/tools"
export type AssistantMessageContent = TextContent | ToolUse
export type AssistantMessageContent = TextDirective | ToolUse
/**
* Parses an assistant message string potentially containing mixed text and tool
@ -41,7 +41,7 @@ export function parseAssistantMessageV2(assistantMessage: string): AssistantMess
const contentBlocks: AssistantMessageContent[] = []
let currentTextContentStart = 0 // Index where the current text block started.
let currentTextContent: TextContent | undefined = undefined
let currentTextContent: TextDirective | undefined = undefined
let currentToolUseStart = 0 // Index *after* the opening tag of the current tool use.
let currentToolUse: ToolUse | undefined = undefined
let currentParamValueStart = 0 // Index *after* the opening tag of the current param.

View file

@ -0,0 +1,113 @@
import { LogDirective } from "../directives/logDirective"
import { ParsingState } from "./types"
export class LogParser {
static parse(state: ParsingState): boolean {
if (!state.currentToolUse && !state.currentTextContent && !state.currentLogMessage) {
if (state.accumulator.includes("<log_message")) {
const startIndex = state.accumulator.indexOf("<log_message")
// Use a separate property for log directive to avoid type issues with TextDirective
state.currentTextContent = undefined
state.currentToolUse = undefined
// Create a new log directive
const logDirective: LogDirective = {
type: "log_message",
message: "",
level: "info",
partial: true,
}
state.currentLogMessage = logDirective
state.currentLogMessageStartIndex = startIndex
// Only add to contentBlocks if not already added by DirectiveStreamingParser
if (!state.contentBlocks.includes(logDirective)) {
state.contentBlocks.push(logDirective)
}
return true
}
}
// Check if there is a current log message being parsed
if (state.currentLogMessage) {
const logMessage = state.currentLogMessage
const currentContent = state.accumulator.slice(state.currentLogMessageStartIndex)
const messageStartIndex = currentContent.indexOf("<message>")
if (messageStartIndex !== -1) {
const messageContentStart = messageStartIndex + "<message>".length
const messageEndIndex = currentContent.indexOf("</message>", messageContentStart)
const logEndMatchDeclared = currentContent.match(/<\/log_message>/)
if (messageEndIndex !== -1 && logEndMatchDeclared) {
// Complete message tag found and log message is complete
logMessage.message = currentContent.slice(messageContentStart, messageEndIndex).trim()
} else if (messageEndIndex !== -1) {
// Message tag is complete but log message is not
const afterMessageTag = currentContent.slice(messageEndIndex + "</message>".length)
if (afterMessageTag.trim().length > 0) {
// There's additional content after </message> (like <level> tags) - include everything for streaming
logMessage.message = currentContent.slice(messageContentStart).trim()
} else if (afterMessageTag.length > 0) {
// There's whitespace/newline after </message> - this is streaming behavior, include the closing tag
logMessage.message = currentContent
.slice(messageContentStart, messageEndIndex + "</message>".length)
.trim()
} else {
// No content after </message> - exclude the closing tag for partial entries
logMessage.message = currentContent.slice(messageContentStart, messageEndIndex).trim()
}
} else {
// Partial message, include content without closing tag
logMessage.message = currentContent.slice(messageContentStart).trim()
}
}
// Check for log message completion before updating level
const logEndMatchDeclared = currentContent.match(/<\/log_message>/)
// Update level only if log message is complete
if (logEndMatchDeclared) {
const levelMatch = currentContent.match(/<level>(.*?)(?:<\/level>|$)/s)
if (levelMatch && levelMatch[1]) {
const levelValue = levelMatch[1].trim()
if (["debug", "info", "warn", "error"].includes(levelValue)) {
logMessage.level = levelValue as "debug" | "info" | "warn" | "error"
}
}
}
const logEndMatch = currentContent.match(/<\/log_message>/)
if (logEndMatch) {
logMessage.partial = false
state.currentLogMessage = undefined
// Reset accumulator to after the closing tag to handle multiple log entries
// Find the exact position of the closing tag in the current content
const logEndIndex = currentContent.indexOf("</log_message>") + "</log_message>".length
const absoluteEndIndex = state.currentLogMessageStartIndex + logEndIndex
// Keep any remaining content after this log message
const remainingContent = state.accumulator.slice(absoluteEndIndex)
state.accumulator = remainingContent
state.currentLogMessageStartIndex = 0 // Reset start index for next log message
// Ensure state is fully reset to detect new log messages
state.currentTextContent = undefined
state.currentToolUse = undefined
}
return true
}
return false
}
static checkForLogStart(state: ParsingState): boolean {
// Check if there's a new log_message tag that hasn't been processed yet
const logStartIndex = state.accumulator.indexOf("<log_message>")
if (logStartIndex === -1) {
return false
}
// If we already have a current log message, don't start a new one
if (state.currentLogMessage) {
return false
}
return true
}
}

View file

@ -1,4 +1,4 @@
export { TextContentParser } from "./TextContentParser"
export { ToolUseParser } from "./ToolUseParser"
export { ParameterParser } from "./ParameterParser"
export type { TextDirective, ToolDirective, Directive, ParsingState } from "./types"
export type { ToolDirective, Directive, ParsingState } from "./types"

View file

@ -1,9 +1,10 @@
import { TextContent, ToolUse, ToolParamName } from "../../../shared/tools"
import { TextDirective, LogDirective } from "../directives"
import { ToolUse, ToolParamName } from "../../../shared/tools"
// Type aliases for directive parsing
export type TextDirective = TextContent
export type ToolDirective = ToolUse
export type Directive = TextDirective | ToolDirective
export type Directive = TextDirective | ToolDirective | LogDirective
export interface ParsingState {
contentBlocks: Directive[]
@ -11,6 +12,8 @@ export interface ParsingState {
currentTextContentStartIndex: number
currentToolUse?: ToolDirective
currentToolUseStartIndex: number
currentLogMessage?: LogDirective
currentLogMessageStartIndex: number
currentParamName?: ToolParamName
currentParamValueStartIndex: number
accumulator: string

View file

@ -0,0 +1,53 @@
import { logLevels } from "@roo-code/types"
import { ClineProvider } from "../webview/ClineProvider"
/**
* Manages logging functionality for Task instances.
* Handles log messages from AI-generated <log_message> blocks.
*/
export class LogManager {
private providerRef: WeakRef<ClineProvider>
/**
* Creates a new LogManager instance.
* @param provider The ClineProvider instance to use for logging
*/
constructor(provider: ClineProvider) {
this.providerRef = new WeakRef(provider)
}
/**
* Logs a message to the output channel and console.
* This method is intended for internal logging triggered by the AI
* via <log_message> blocks and does not require user approval.
* @param message The message to log.
* @param level The log level (debug, info, warn, error). Defaults to "info".
*/
public log(message: string, level: (typeof logLevels)[number] = "info"): void {
const timestamp = new Date().toISOString()
const formattedMessage = `[${timestamp}] [${level.toUpperCase()}] ${message}`
// Get the provider instance
const provider = this.providerRef.deref()
if (provider) {
// Use the provider's log method which logs to both console and output channel
provider.log(formattedMessage)
}
}
/**
* Processes a log message directive from the assistant.
* @param message The log message
* @param level The log level
* @param partial Whether the log message is partial
* @returns true if the log was processed, false otherwise
*/
public processLogEntry(message: string, level: (typeof logLevels)[number], partial: boolean): boolean {
// Only log complete (non-partial) log messages to avoid logging with incorrect levels
if (!partial) {
this.log(message, level)
return true
}
return false
}
}

View file

@ -0,0 +1,54 @@
import { LogManager } from "../LogManager"
describe("LogManager", () => {
let mockProvider: any
let logManager: LogManager
beforeEach(() => {
mockProvider = {
log: jest.fn(),
}
logManager = new LogManager(mockProvider as any)
})
describe("log", () => {
it("should format and log messages with timestamp and level", () => {
// Mock Date.toISOString to return a fixed timestamp
const mockDate = new Date("2023-01-01T12:00:00Z")
jest.spyOn(global, "Date").mockImplementation(() => mockDate as any)
logManager.log("Test message", "info")
expect(mockProvider.log).toHaveBeenCalledWith("[2023-01-01T12:00:00.000Z] [INFO] Test message")
})
it("should use 'info' as default log level", () => {
const mockDate = new Date("2023-01-01T12:00:00Z")
jest.spyOn(global, "Date").mockImplementation(() => mockDate as any)
logManager.log("Test message")
expect(mockProvider.log).toHaveBeenCalledWith("[2023-01-01T12:00:00.000Z] [INFO] Test message")
})
})
describe("processLogEntry", () => {
it("should log complete entries", () => {
const spy = jest.spyOn(logManager, "log")
const result = logManager.processLogEntry("Test log entry", "debug", false)
expect(result).toBe(true)
expect(spy).toHaveBeenCalledWith("Test log entry", "debug")
})
it("should not log partial entries", () => {
const spy = jest.spyOn(logManager, "log")
const result = logManager.processLogEntry("Partial log entry", "warn", true)
expect(result).toBe(false)
expect(spy).not.toHaveBeenCalled()
})
})
})

View file

@ -0,0 +1 @@
export * from "./LogManager"

View file

@ -3,6 +3,7 @@ export { getSystemInfoSection } from "./system-info"
export { getObjectiveSection } from "./objective"
export { addCustomInstructions } from "./custom-instructions"
export { getSharedToolUseSection } from "./tool-use"
export { getLogMessageSection } from "./log-message"
export { getMcpServersSection } from "./mcp-servers"
export { getToolUseGuidelinesSection } from "./tool-use-guidelines"
export { getCapabilitiesSection } from "./capabilities"

View file

@ -0,0 +1,39 @@
export function getLogMessageSection(): string {
return `====
LOG MESSAGES
You can use log messages to output debugging information to the VSCode output channel. Unlike tools, log messages don't require approval and don't count toward the one-tool-per-message limit.
# Purpose and Context
The VSCode OutputChannel is a dedicated console-like interface within VSCode where extensions can write diagnostic information, separate from the user's workspace files.
Unlike other tools that create or modify files in the workspace, log messages are purely for diagnostic purposes within the extension's runtime environment.
# Log Message Formatting
Log messages are formatted using XML-style tags. Here's the structure:
<log_message>
<message>Your log message here</message>
<level>info</level>
</log_message>
The level parameter is optional and defaults to "info". Valid levels are: "debug", "info", "warn", and "error".
For example:
<log_message>
<message>Starting task execution</message>
<level>info</level>
</log_message>
<log_message>
<message>Failed to parse input: invalid JSON</message>
<level>error</level>
</log_message>
You can use log messages multiple times in a single message, and they will be processed immediately without requiring user approval.
`
}

View file

@ -18,6 +18,7 @@ import {
getSystemInfoSection,
getObjectiveSection,
getSharedToolUseSection,
getLogMessageSection,
getMcpServersSection,
getToolUseGuidelinesSection,
getCapabilitiesSection,
@ -71,6 +72,8 @@ ${markdownFormattingSection()}
${getSharedToolUseSection()}
${getLogMessageSection()}
${getToolDescriptionsForMode(
mode,
cwd,

View file

@ -84,6 +84,7 @@ import { processUserContentMentions } from "../mentions/processUserContentMentio
import { ApiMessage } from "../task-persistence/apiMessages"
import { getMessagesSinceLastSummary, summarizeConversation } from "../condense"
import { maybeRemoveImageBlocks } from "../../api/transform/image-cleaning"
import { LogManager } from "../logging"
export type ClineEvents = {
message: [{ action: "created" | "updated"; message: ClineMessage }]
@ -192,6 +193,9 @@ export class Task extends EventEmitter<ClineEvents> {
didAlreadyUseTool = false
didCompleteReadingStream = false
// Logging
private logManager: LogManager
constructor({
provider,
apiConfiguration,
@ -224,6 +228,7 @@ export class Task extends EventEmitter<ClineEvents> {
this.rooIgnoreController = new RooIgnoreController(this.cwd)
this.fileContextTracker = new FileContextTracker(provider, this.taskId)
this.logManager = new LogManager(provider)
this.rooIgnoreController.initialize().catch((error) => {
console.error("Failed to initialize RooIgnoreController:", error)

View file

@ -371,11 +371,11 @@
"@google/genai": "^1.0.0",
"@mistralai/mistralai": "^1.3.6",
"@modelcontextprotocol/sdk": "^1.9.0",
"@qdrant/js-client-rest": "^1.14.0",
"@roo-code/cloud": "workspace:^",
"@roo-code/ipc": "workspace:^",
"@roo-code/telemetry": "workspace:^",
"@roo-code/types": "workspace:^",
"@qdrant/js-client-rest": "^1.14.0",
"@types/lodash.debounce": "^4.0.9",
"@vscode/codicons": "^0.0.36",
"async-mutex": "^0.5.0",
@ -413,6 +413,7 @@
"puppeteer-core": "^23.4.0",
"reconnecting-eventsource": "^1.6.4",
"sanitize-filename": "^1.6.3",
"sax": "^1.4.1",
"say": "^0.16.0",
"serialize-error": "^11.0.3",
"simple-git": "^3.27.0",
@ -447,6 +448,7 @@
"@types/node-cache": "^4.1.3",
"@types/node-ipc": "^9.2.3",
"@types/ps-tree": "^1.1.6",
"@types/sax": "^1.2.7",
"@types/string-similarity": "^4.0.2",
"@types/tmp": "^0.2.6",
"@types/turndown": "^5.0.5",

View file

@ -20,12 +20,6 @@ export type AskFinishSubTaskApproval = () => Promise<boolean>
export type ToolDescription = () => string
export interface TextContent {
type: "text"
content: string
partial: boolean
}
export const toolParamNames = [
"command",
"path",