diff --git a/debug-test.js b/debug-test.js new file mode 100644 index 0000000000..7a572585a5 --- /dev/null +++ b/debug-test.js @@ -0,0 +1,34 @@ +const { parseAssistantMessage } = require("./core/assistant-message/parseAssistantMessage") + +const message = ` +This is a debug message +debug + + + +This is an info message + + + +This is a warning message +warn + + + +This is an error message +error +` + +console.log("Input message:") +console.log(message) +console.log("\n=== PARSING RESULT ===") + +const result = parseAssistantMessage(message) +console.log("Total blocks:", result.length) + +const filteredResult = result.filter((block) => !(block.type === "text" && block.content === "")) +console.log("Filtered blocks:", filteredResult.length) + +filteredResult.forEach((block, index) => { + console.log(`Block ${index}:`, JSON.stringify(block, null, 2)) +}) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index f9e546f095..3bbde10594 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -7,6 +7,7 @@ export * from "./experiment.js" export * from "./global-settings.js" export * from "./history.js" export * from "./ipc.js" +export * from "./log-message.js" export * from "./mcp.js" export * from "./message.js" export * from "./mode.js" diff --git a/packages/types/src/log-message.ts b/packages/types/src/log-message.ts new file mode 100644 index 0000000000..b810019fcb --- /dev/null +++ b/packages/types/src/log-message.ts @@ -0,0 +1,11 @@ +import { z } from "zod" + +export const logLevels = ["debug", "info", "warn", "error"] as const +export const logLevelsSchema = z.enum(logLevels).optional().default("info") + +export const logMessageParamsSchema = z.object({ + message: z.string(), + level: logLevelsSchema, +}) + +export type LogMessageParams = z.infer diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e8cc083776..a1642ab52a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -708,6 +708,9 @@ importers: sanitize-filename: specifier: ^1.6.3 version: 1.6.3 + sax: + specifier: ^1.4.1 + version: 1.4.1 say: specifier: ^0.16.0 version: 0.16.0 @@ -805,6 +808,9 @@ importers: '@types/ps-tree': specifier: ^1.1.6 version: 1.1.6 + '@types/sax': + specifier: ^1.2.7 + version: 1.2.7 '@types/string-similarity': specifier: ^4.0.2 version: 4.0.2 @@ -4269,6 +4275,9 @@ packages: '@types/react@18.3.23': resolution: {integrity: sha512-/LDXMQh55EzZQ0uVAZmKKhfENivEvWz6E+EYzh+/MCjMhNsotd+ZHhBGIjFDTi6+fz0OhQQQLbTgdQIxxCsC0w==} + '@types/sax@1.2.7': + resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} + '@types/shell-quote@1.7.5': resolution: {integrity: sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==} @@ -14032,7 +14041,7 @@ snapshots: '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 20.17.57 + '@types/node': 20.19.0 '@types/hast@3.0.4': dependencies: @@ -14113,7 +14122,6 @@ snapshots: '@types/node@20.19.0': dependencies: undici-types: 6.21.0 - optional: true '@types/node@22.15.29': dependencies: @@ -14135,6 +14143,10 @@ snapshots: '@types/prop-types': 15.7.14 csstype: 3.1.3 + '@types/sax@1.2.7': + dependencies: + '@types/node': 20.19.0 + '@types/shell-quote@1.7.5': {} '@types/stack-utils@2.0.3': {} @@ -14179,7 +14191,7 @@ snapshots: '@types/yauzl@2.10.3': dependencies: - '@types/node': 20.17.57 + '@types/node': 20.19.0 optional: true '@typescript-eslint/eslint-plugin@8.32.1(@typescript-eslint/parser@8.32.1(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3)': @@ -17324,7 +17336,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.17.57 + '@types/node': 20.19.0 chalk: 4.1.2 co: 4.6.0 dedent: 1.6.0(babel-plugin-macros@3.1.0) @@ -17481,7 +17493,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.17.57 + '@types/node': 20.19.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -17669,7 +17681,7 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 20.17.57 + '@types/node': 20.19.0 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 diff --git a/src/__mocks__/vitest-vscode-mock.js b/src/__mocks__/vitest-vscode-mock.js index 405f3694ba..15a8c91f1d 100644 --- a/src/__mocks__/vitest-vscode-mock.js +++ b/src/__mocks__/vitest-vscode-mock.js @@ -66,6 +66,10 @@ export const window = { show: () => {}, dispose: () => {}, }), + createTextEditorDecorationType: (options) => ({ + key: "mockDecorationType", + dispose: () => {}, + }), } export const commands = { diff --git a/src/core/assistant-message/DirectiveStreamingParser.ts b/src/core/assistant-message/DirectiveStreamingParser.ts index 984338b2c8..545a0258b0 100644 --- a/src/core/assistant-message/DirectiveStreamingParser.ts +++ b/src/core/assistant-message/DirectiveStreamingParser.ts @@ -1,62 +1,296 @@ -import { Directive, ParsingState, TextContentParser, ToolUseParser, ParameterParser } from "./parsers" +import { Directive } from "./parsers" +import { TextDirective, LogDirective } from "./directives" +import { ToolUse, ToolParamName } from "../../shared/tools" +import { toolNames } from "@roo-code/types" +import * as sax from "sax" export class DirectiveStreamingParser { static parse(assistantMessage: string): Directive[] { - const state: ParsingState = { - contentBlocks: [], - currentTextContent: undefined, - currentTextContentStartIndex: 0, - currentToolUse: undefined, - currentToolUseStartIndex: 0, - currentParamName: undefined, - currentParamValueStartIndex: 0, - accumulator: "", + const contentBlocks: Directive[] = [] + let currentText = "" + let currentToolUse: ToolUse | undefined + let currentLogMessage: LogDirective | undefined + let currentParamName: ToolParamName | undefined + let currentParamValue = "" + let currentContext: "text" | "logMessage" | "logLevel" | "param" | "none" = "text" + let hasXmlTags = false + let parseError = false + + // Check if the input has incomplete XML (for partial detection) + const hasIncompleteXml = this.hasIncompleteXml(assistantMessage) + + const parser = sax.parser(false, { lowercase: true }) + + parser.onopentag = (node: sax.Tag) => { + hasXmlTags = true + const tagName = node.name + + if (tagName === "log_message") { + // Push any accumulated text before starting log message + if (currentText.trim()) { + contentBlocks.push({ + type: "text", + content: currentText.trim(), + partial: false, + } as TextDirective) + currentText = "" + } + + currentLogMessage = { + type: "log_message", + message: "", + level: "info", + partial: true, + } + currentContext = "none" + } else if (tagName === "message" && currentLogMessage) { + currentContext = "logMessage" + } else if (tagName === "level" && currentLogMessage) { + currentContext = "logLevel" + } else if (toolNames.includes(tagName as any)) { + // Push any accumulated text before starting tool use + if (currentText.trim()) { + contentBlocks.push({ + type: "text", + content: currentText.trim(), + partial: false, + } as TextDirective) + currentText = "" + } + + currentToolUse = { + type: "tool_use", + name: tagName as any, + params: {}, + partial: true, + } + currentContext = "none" + } else if (currentToolUse) { + currentParamName = tagName as ToolParamName + currentParamValue = "" + currentContext = "param" + } } - for (let i = 0; i < assistantMessage.length; i++) { - const char = assistantMessage[i] - state.accumulator += char - - // There should not be a param without a tool use. - if (ParameterParser.parse(state)) { - continue + parser.onclosetag = (tagName: string) => { + if (tagName === "log_message" && currentLogMessage) { + currentLogMessage.partial = hasIncompleteXml + contentBlocks.push(currentLogMessage) + currentLogMessage = undefined + currentContext = "text" + } else if (tagName === "message" && currentLogMessage) { + currentContext = "none" + } else if (tagName === "level" && currentLogMessage) { + currentContext = "none" + } else if (currentToolUse && tagName === currentToolUse.name) { + currentToolUse.partial = hasIncompleteXml || Object.keys(currentToolUse.params).length === 0 + contentBlocks.push(currentToolUse) + currentToolUse = undefined + currentContext = "text" + } else if (currentToolUse && currentParamName && tagName === currentParamName) { + ;(currentToolUse.params as Record)[currentParamName] = currentParamValue.trim() + currentParamName = undefined + currentParamValue = "" + currentContext = "none" } - - // No currentParamName. - if (ToolUseParser.parse(state)) { - continue - } - - // No currentToolUse. - const didStartToolUse = ToolUseParser.checkForToolStart(state) - TextContentParser.parse(state, i, didStartToolUse) } - // Handle remaining partial content - this.handlePartialContent(state) + parser.ontext = (text: string) => { + if (currentContext === "param" && currentParamName && currentToolUse) { + currentParamValue += text + } else if (currentContext === "logMessage" && currentLogMessage) { + currentLogMessage.message += text + } else if (currentContext === "logLevel" && currentLogMessage) { + const levelText = text.trim() + if (["debug", "info", "warn", "error"].includes(levelText)) { + currentLogMessage.level = levelText as "debug" | "info" | "warn" | "error" + } + } else if (currentContext === "text") { + currentText += text + } + } - return state.contentBlocks + parser.onend = () => { + // Push any remaining text + if (currentText.trim()) { + contentBlocks.push({ + type: "text", + content: currentText.trim(), + partial: true, + } as TextDirective) + } + + // Handle partial log message at the end + if (currentLogMessage) { + currentLogMessage.partial = true + contentBlocks.push(currentLogMessage) + } + + // Handle partial tool use at the end + if (currentToolUse) { + if (currentParamName && currentParamValue) { + ;(currentToolUse.params as Record)[currentParamName] = currentParamValue.trim() + } + currentToolUse.partial = true + contentBlocks.push(currentToolUse) + } + } + + parser.onerror = (error: Error) => { + parseError = true + // Don't clear content blocks here - let the fallback logic handle it + } + + try { + // Wrap multiple root elements to make valid XML + const wrappedMessage = `${assistantMessage}` + parser.write(wrappedMessage).close() + } catch (e) { + parseError = true + } + + // If parsing failed or no XML tags were found, use fallback logic + if (parseError || (!hasXmlTags && contentBlocks.length === 0 && assistantMessage.trim())) { + // Try to handle partial XML manually for streaming scenarios + return this.handlePartialXml(assistantMessage) + } + + return contentBlocks } - private static handlePartialContent(state: ParsingState): void { - if (state.currentToolUse) { - // Stream did not complete tool call, add it as partial. - if (state.currentParamName) { - // Tool call has a parameter that was not completed. - state.currentToolUse.params[state.currentParamName] = state.accumulator - .slice(state.currentParamValueStartIndex) - .trim() + private static handlePartialXml(assistantMessage: string): Directive[] { + const contentBlocks: Directive[] = [] + + // Handle multiple log messages + const logMessageRegex = /([\s\S]*?)(?:<\/log_message>|$)/g + let lastIndex = 0 + let match + + while ((match = logMessageRegex.exec(assistantMessage)) !== null) { + // Add any text before this log message + if (match.index > lastIndex) { + const textBefore = assistantMessage.substring(lastIndex, match.index).trim() + if (textBefore) { + contentBlocks.push({ + type: "text", + content: textBefore, + partial: false, + } as TextDirective) + } } - state.contentBlocks.push(state.currentToolUse) + const logContent = match[1] + const isComplete = assistantMessage.includes("", match.index) + + // For streaming behavior, preserve raw XML content when incomplete + let message = "" + let level: "debug" | "info" | "warn" | "error" = "info" + + if (isComplete) { + // Complete log message - parse normally + const messageMatch = logContent.match(/(.*?)<\/message>/) + const levelMatch = logContent.match(/(.*?)<\/level>/) + + message = messageMatch ? messageMatch[1] : "" + if (levelMatch && ["debug", "info", "warn", "error"].includes(levelMatch[1])) { + level = levelMatch[1] as "debug" | "info" | "warn" | "error" + } + } else { + // Incomplete log message - preserve raw content for streaming behavior + message = logContent + } + + const logMessage: LogDirective = { + type: "log_message", + message, + level, + partial: !isComplete, + } + + contentBlocks.push(logMessage) + lastIndex = logMessageRegex.lastIndex } - // NOTE: It doesn't matter if check for currentToolUse or - // currentTextContent, only one of them will be defined since only one can - // be partial at a time. - if (state.currentTextContent) { - // Stream did not complete text content, add it as partial. - state.contentBlocks.push(state.currentTextContent) + // If no log messages were found, check for tool use + if (contentBlocks.length === 0) { + for (const toolName of toolNames) { + const toolRegex = new RegExp(`<${toolName}>[\\s\\S]*?(?:<\\/${toolName}>|$)`) + const toolMatch = assistantMessage.match(toolRegex) + if (toolMatch) { + const toolContent = toolMatch[0] + const params: Record = {} + + // Extract parameters + const paramRegex = /<(\w+)>(.*?)(?:<\/\1>|$)/g + let paramMatch + while ((paramMatch = paramRegex.exec(toolContent)) !== null) { + const [, paramName, paramValue] = paramMatch + if (paramName !== toolName) { + params[paramName] = paramValue + } + } + + const toolUse: ToolUse = { + type: "tool_use", + name: toolName as any, + params, + partial: !assistantMessage.includes(``), + } + + contentBlocks.push(toolUse) + return contentBlocks + } + } } + + // Add any remaining text after the last log message + if (lastIndex < assistantMessage.length) { + const remainingText = assistantMessage.substring(lastIndex).trim() + if (remainingText) { + contentBlocks.push({ + type: "text", + content: remainingText, + partial: true, + } as TextDirective) + } + } + + // If no structured content was found, treat as plain text + if (contentBlocks.length === 0) { + contentBlocks.push({ + type: "text", + content: assistantMessage, + partial: true, + } as TextDirective) + } + + return contentBlocks + } + + private static hasIncompleteXml(input: string): boolean { + // Check for incomplete XML by looking for opening tags without corresponding closing tags + const openTags: string[] = [] + const tagRegex = /<\/?([a-zA-Z_][a-zA-Z0-9_-]*)[^>]*>/g + let match + + while ((match = tagRegex.exec(input)) !== null) { + const fullTag = match[0] + const tagName = match[1] + + if (fullTag.startsWith("")) { + // Opening tag (not self-closing) + openTags.push(tagName) + } + } + + // If there are unclosed tags, it's incomplete + return openTags.length > 0 } } diff --git a/src/core/assistant-message/__tests__/DirectiveStreamingParser.spec.ts b/src/core/assistant-message/__tests__/DirectiveStreamingParser.spec.ts new file mode 100644 index 0000000000..d7ead213f9 --- /dev/null +++ b/src/core/assistant-message/__tests__/DirectiveStreamingParser.spec.ts @@ -0,0 +1,94 @@ +import { suite, test, expect } from "vitest" +import { DirectiveStreamingParser } from "../DirectiveStreamingParser" +import { TextDirective, LogDirective } from "../directives" +import { ToolUse } from "../../../shared/tools" + +suite("DirectiveStreamingParser", () => { + test("should parse plain text content", () => { + const input = "This is a simple text message." + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: "This is a simple text message.", + partial: true, + } as TextDirective, + ]) + }) + + test("should parse a complete log message", () => { + const input = "Log entryinfo" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "log_message", + message: "Log entry", + level: "info", + partial: false, + } as LogDirective, + ]) + }) + + test("should parse a partial log message", () => { + const input = "Partial log entry" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "log_message", + message: "Partial log entry", + level: "info", + partial: true, + } as LogDirective, + ]) + }) + + test("should parse a tool use directive", () => { + const input = "src/app.ts" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "tool_use", + name: "read_file", + params: { path: "src/app.ts" }, + partial: false, + } as ToolUse, + ]) + }) + + test("should parse mixed content with text and tool use", () => { + const input = + "Some text heresrc/file.tsdiff contentMore text" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: "Some text here", + partial: false, + } as TextDirective, + { + type: "tool_use", + name: "apply_diff", + params: { path: "src/file.ts", diff: "diff content" }, + partial: false, + } as ToolUse, + { + type: "text", + content: "More text", + partial: true, + } as TextDirective, + ]) + }) + + test("should handle partial tool use directive", () => { + const input = "src/newfile.tsSome content" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "tool_use", + name: "write_to_file", + params: { path: "src/newfile.ts", content: "Some content" }, + partial: true, + } as ToolUse, + ]) + }) +}) diff --git a/src/core/assistant-message/__tests__/DirectiveStreamingParser.test.ts b/src/core/assistant-message/__tests__/DirectiveStreamingParser.test.ts new file mode 100644 index 0000000000..d7ead213f9 --- /dev/null +++ b/src/core/assistant-message/__tests__/DirectiveStreamingParser.test.ts @@ -0,0 +1,94 @@ +import { suite, test, expect } from "vitest" +import { DirectiveStreamingParser } from "../DirectiveStreamingParser" +import { TextDirective, LogDirective } from "../directives" +import { ToolUse } from "../../../shared/tools" + +suite("DirectiveStreamingParser", () => { + test("should parse plain text content", () => { + const input = "This is a simple text message." + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: "This is a simple text message.", + partial: true, + } as TextDirective, + ]) + }) + + test("should parse a complete log message", () => { + const input = "Log entryinfo" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "log_message", + message: "Log entry", + level: "info", + partial: false, + } as LogDirective, + ]) + }) + + test("should parse a partial log message", () => { + const input = "Partial log entry" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "log_message", + message: "Partial log entry", + level: "info", + partial: true, + } as LogDirective, + ]) + }) + + test("should parse a tool use directive", () => { + const input = "src/app.ts" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "tool_use", + name: "read_file", + params: { path: "src/app.ts" }, + partial: false, + } as ToolUse, + ]) + }) + + test("should parse mixed content with text and tool use", () => { + const input = + "Some text heresrc/file.tsdiff contentMore text" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "text", + content: "Some text here", + partial: false, + } as TextDirective, + { + type: "tool_use", + name: "apply_diff", + params: { path: "src/file.ts", diff: "diff content" }, + partial: false, + } as ToolUse, + { + type: "text", + content: "More text", + partial: true, + } as TextDirective, + ]) + }) + + test("should handle partial tool use directive", () => { + const input = "src/newfile.tsSome content" + const result = DirectiveStreamingParser.parse(input) + expect(result).toEqual([ + { + type: "tool_use", + name: "write_to_file", + params: { path: "src/newfile.ts", content: "Some content" }, + partial: true, + } as ToolUse, + ]) + }) +}) diff --git a/src/core/assistant-message/__tests__/log-message.spec.ts b/src/core/assistant-message/__tests__/log-message.spec.ts new file mode 100644 index 0000000000..92e6bd2396 --- /dev/null +++ b/src/core/assistant-message/__tests__/log-message.spec.ts @@ -0,0 +1,169 @@ +import { suite, test, expect } from "vitest" +import { parseAssistantMessage } from ".." + +suite("Log Entry Parsing", () => { + test("should parse complete log entries correctly", () => { + const message = ` +This is a test log message +debug +` + + const result = parseAssistantMessage(message) + + // Filter out empty text blocks + const filteredResult = result.filter((block) => !(block.type === "text" && block.content === "")) + + expect(filteredResult).toHaveLength(1) + expect(filteredResult[0]).toEqual({ + type: "log_message", + message: "This is a test log message", + level: "debug", + partial: false, + }) + }) + + test("should mark partial log entries as partial", () => { + const message = ` +This is a test log message` + + const result = parseAssistantMessage(message) + + // Filter out empty text blocks + const filteredResult = result.filter((block) => !(block.type === "text" && block.content === "")) + + expect(filteredResult).toHaveLength(1) + expect(filteredResult[0]).toEqual({ + type: "log_message", + message: "This is a test log message", + level: "info", // Default level + partial: true, + }) + }) + + test("should handle log entries with only message tag", () => { + const message = ` +This is a test log message +` + + const result = parseAssistantMessage(message) + + // Filter out empty text blocks + const filteredResult = result.filter((block) => !(block.type === "text" && block.content === "")) + + expect(filteredResult).toHaveLength(1) + expect(filteredResult[0]).toEqual({ + type: "log_message", + message: "This is a test log message", + level: "info", // Default level + partial: false, + }) + }) + + test("should simulate streaming behavior with partial log entries", () => { + // Simulate streaming chunks + const chunks = [ + "\n", + "This is a debug level log message\n", + "debug\n", + "", + ] + + let accumulatedMessage = "" + const results = [] + + // Process each chunk as it would happen during streaming + for (const chunk of chunks) { + accumulatedMessage += chunk + const result = parseAssistantMessage(accumulatedMessage) + results.push(result) + } + + // First chunk: Just the opening tag - filter out empty text blocks + const filteredResults0 = results[0].filter((block) => !(block.type === "text" && block.content === "")) + expect(filteredResults0[0]).toEqual({ + type: "log_message", + message: "", + level: "info", // Default level + partial: true, + }) + + // Second chunk: Has message but not level - filter out empty text blocks + const filteredResults1 = results[1].filter((block) => !(block.type === "text" && block.content === "")) + expect(filteredResults1[0]).toEqual({ + type: "log_message", + message: "This is a debug level log message", + level: "info", // Still default level + partial: true, + }) + + // Third chunk: Has message and level but not closing tag - filter out empty text blocks + const filteredResults2 = results[2].filter((block) => !(block.type === "text" && block.content === "")) + expect(filteredResults2[0]).toEqual({ + type: "log_message", + message: "This is a debug level log message", + level: "debug", // Level is now properly parsed + partial: true, + }) + + // Fourth chunk: Complete log entry - filter out empty text blocks + const filteredResults3 = results[3].filter((block) => !(block.type === "text" && block.content === "")) + expect(filteredResults3[0]).toEqual({ + type: "log_message", + message: "This is a debug level log message", + level: "debug", + partial: false, + }) + }) + + test("should handle multiple log entries with different levels", () => { + const message = ` +This is a debug message +debug + + + +This is an info message + + + +This is a warning message +warn + + + +This is an error message +error +` + + const result = parseAssistantMessage(message) + + // Filter out empty text blocks + const filteredResult = result.filter((block) => !(block.type === "text" && block.content === "")) + + expect(filteredResult).toHaveLength(4) + expect(filteredResult[0]).toEqual({ + type: "log_message", + message: "This is a debug message", + level: "debug", + partial: false, + }) + expect(filteredResult[1]).toEqual({ + type: "log_message", + message: "This is an info message", + level: "info", // Default level + partial: false, + }) + expect(filteredResult[2]).toEqual({ + type: "log_message", + message: "This is a warning message", + level: "warn", + partial: false, + }) + expect(filteredResult[3]).toEqual({ + type: "log_message", + message: "This is an error message", + level: "error", + partial: false, + }) + }) +}) diff --git a/src/core/assistant-message/__tests__/parseAssistantMessage.test.ts b/src/core/assistant-message/__tests__/parseAssistantMessage.test.ts index 19f88a91d7..def4d21eb6 100644 --- a/src/core/assistant-message/__tests__/parseAssistantMessage.test.ts +++ b/src/core/assistant-message/__tests__/parseAssistantMessage.test.ts @@ -1,12 +1,13 @@ // npx jest src/core/assistant-message/__tests__/parseAssistantMessage.test.ts -import { TextContent, ToolUse } from "../../../shared/tools" +import { TextDirective } from "../directives" +import { ToolUse } from "../../../shared/tools" import { AssistantMessageContent, parseAssistantMessage as parseAssistantMessageV1 } from "../parseAssistantMessage" import { parseAssistantMessageV2 } from "../parseAssistantMessageV2" const isEmptyTextContent = (block: AssistantMessageContent) => - block.type === "text" && (block as TextContent).content === "" + block.type === "text" && (block as TextDirective).content === "" ;[parseAssistantMessageV1, parseAssistantMessageV2].forEach((parser, index) => { describe(`parseAssistantMessageV${index + 1}`, () => { @@ -108,7 +109,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => expect(result).toHaveLength(2) - const textContent = result[0] as TextContent + const textContent = result[0] as TextDirective expect(textContent.type).toBe("text") expect(textContent.content).toBe("Here's the file content:") expect(textContent.partial).toBe(false) @@ -132,7 +133,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => expect(toolUse.params.path).toBe("src/file.ts") expect(toolUse.partial).toBe(false) - const textContent = result[1] as TextContent + const textContent = result[1] as TextDirective expect(textContent.type).toBe("text") expect(textContent.content).toBe("Here's what I found in the file.") expect(textContent.partial).toBe(true) @@ -146,14 +147,14 @@ const isEmptyTextContent = (block: AssistantMessageContent) => expect(result).toHaveLength(4) expect(result[0].type).toBe("text") - expect((result[0] as TextContent).content).toBe("First file:") + expect((result[0] as TextDirective).content).toBe("First file:") expect(result[1].type).toBe("tool_use") expect((result[1] as ToolUse).name).toBe("read_file") expect((result[1] as ToolUse).params.path).toBe("src/file1.ts") expect(result[2].type).toBe("text") - expect((result[2] as TextContent).content).toBe("Second file:") + expect((result[2] as TextDirective).content).toBe("Second file:") expect(result[3].type).toBe("tool_use") expect((result[3] as ToolUse).name).toBe("read_file") @@ -197,7 +198,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => expect(result).toHaveLength(1) expect(result[0].type).toBe("text") - expect((result[0] as TextContent).content).toBe(message) + expect((result[0] as TextDirective).content).toBe(message) }) it("should handle tool use with no parameters", () => { @@ -313,7 +314,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => // First text block expect(result[0].type).toBe("text") - expect((result[0] as TextContent).content).toBe("I'll help you with that task.") + expect((result[0] as TextDirective).content).toBe("I'll help you with that task.") // First tool use (read_file) expect(result[1].type).toBe("tool_use") @@ -321,7 +322,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => // Second text block expect(result[2].type).toBe("text") - expect((result[2] as TextContent).content).toContain("Now let's modify the file:") + expect((result[2] as TextDirective).content).toContain("Now let's modify the file:") // Second tool use (write_to_file) expect(result[3].type).toBe("tool_use") @@ -329,7 +330,7 @@ const isEmptyTextContent = (block: AssistantMessageContent) => // Third text block expect(result[4].type).toBe("text") - expect((result[4] as TextContent).content).toContain("Let's run the code:") + expect((result[4] as TextDirective).content).toContain("Let's run the code:") // Third tool use (execute_command) expect(result[5].type).toBe("tool_use") diff --git a/src/core/assistant-message/directives/index.ts b/src/core/assistant-message/directives/index.ts new file mode 100644 index 0000000000..5a4d4059fa --- /dev/null +++ b/src/core/assistant-message/directives/index.ts @@ -0,0 +1,2 @@ +export * from "./logDirective" +export * from "./textDirective" diff --git a/src/core/assistant-message/directives/logDirective.ts b/src/core/assistant-message/directives/logDirective.ts new file mode 100644 index 0000000000..b8def1619a --- /dev/null +++ b/src/core/assistant-message/directives/logDirective.ts @@ -0,0 +1,12 @@ +import { logLevels } from "@roo-code/types" + +/** + * Represents a log message directive from the assistant to the system. + * This directive instructs the system to record a message to its internal logs. + */ +export interface LogDirective { + type: "log_message" + message: string + level: (typeof logLevels)[number] + partial: boolean +} diff --git a/src/core/assistant-message/directives/textDirective.ts b/src/core/assistant-message/directives/textDirective.ts new file mode 100644 index 0000000000..1f06dfdd6a --- /dev/null +++ b/src/core/assistant-message/directives/textDirective.ts @@ -0,0 +1,9 @@ +/** + * Represents a message directive from the assistant to the system. + * This directive instructs the system to output text. + */ +export interface TextDirective { + type: "text" + content: string + partial: boolean +} diff --git a/src/core/assistant-message/index.ts b/src/core/assistant-message/index.ts index 72201b7722..52a63ee19d 100644 --- a/src/core/assistant-message/index.ts +++ b/src/core/assistant-message/index.ts @@ -1,2 +1,3 @@ export { type AssistantMessageContent, parseAssistantMessage } from "./parseAssistantMessage" export { presentAssistantMessage } from "./presentAssistantMessage" +export { type LogDirective } from "./directives/logDirective" diff --git a/src/core/assistant-message/parseAssistantMessage.ts b/src/core/assistant-message/parseAssistantMessage.ts index 8bf25978e9..3744c5c7ad 100644 --- a/src/core/assistant-message/parseAssistantMessage.ts +++ b/src/core/assistant-message/parseAssistantMessage.ts @@ -1,8 +1,9 @@ import { DirectiveStreamingParser } from "./DirectiveStreamingParser" -import type { Directive } from "./parsers/types" +import type { Directive } from "./parsers" +export type { TextDirective } from "./directives" // Re-export types for backward compatibility -export type { TextDirective, ToolDirective, Directive } from "./parsers/types" +export type { ToolDirective, Directive } from "./parsers" // Backward compatibility alias export type AssistantMessageContent = Directive diff --git a/src/core/assistant-message/parseAssistantMessageV2.ts b/src/core/assistant-message/parseAssistantMessageV2.ts index 6d3594cf60..7c8299e50b 100644 --- a/src/core/assistant-message/parseAssistantMessageV2.ts +++ b/src/core/assistant-message/parseAssistantMessageV2.ts @@ -1,8 +1,8 @@ import { type ToolName, toolNames } from "@roo-code/types" +import { TextDirective } from "./directives" +import { ToolUse, ToolParamName, toolParamNames } from "../../shared/tools" -import { TextContent, ToolUse, ToolParamName, toolParamNames } from "../../shared/tools" - -export type AssistantMessageContent = TextContent | ToolUse +export type AssistantMessageContent = TextDirective | ToolUse /** * Parses an assistant message string potentially containing mixed text and tool @@ -41,7 +41,7 @@ export function parseAssistantMessageV2(assistantMessage: string): AssistantMess const contentBlocks: AssistantMessageContent[] = [] let currentTextContentStart = 0 // Index where the current text block started. - let currentTextContent: TextContent | undefined = undefined + let currentTextContent: TextDirective | undefined = undefined let currentToolUseStart = 0 // Index *after* the opening tag of the current tool use. let currentToolUse: ToolUse | undefined = undefined let currentParamValueStart = 0 // Index *after* the opening tag of the current param. diff --git a/src/core/assistant-message/parsers/LogParser.ts b/src/core/assistant-message/parsers/LogParser.ts new file mode 100644 index 0000000000..3b52100143 --- /dev/null +++ b/src/core/assistant-message/parsers/LogParser.ts @@ -0,0 +1,113 @@ +import { LogDirective } from "../directives/logDirective" +import { ParsingState } from "./types" + +export class LogParser { + static parse(state: ParsingState): boolean { + if (!state.currentToolUse && !state.currentTextContent && !state.currentLogMessage) { + if (state.accumulator.includes("") + if (messageStartIndex !== -1) { + const messageContentStart = messageStartIndex + "".length + const messageEndIndex = currentContent.indexOf("", messageContentStart) + const logEndMatchDeclared = currentContent.match(/<\/log_message>/) + if (messageEndIndex !== -1 && logEndMatchDeclared) { + // Complete message tag found and log message is complete + logMessage.message = currentContent.slice(messageContentStart, messageEndIndex).trim() + } else if (messageEndIndex !== -1) { + // Message tag is complete but log message is not + const afterMessageTag = currentContent.slice(messageEndIndex + "".length) + if (afterMessageTag.trim().length > 0) { + // There's additional content after (like tags) - include everything for streaming + logMessage.message = currentContent.slice(messageContentStart).trim() + } else if (afterMessageTag.length > 0) { + // There's whitespace/newline after - this is streaming behavior, include the closing tag + logMessage.message = currentContent + .slice(messageContentStart, messageEndIndex + "".length) + .trim() + } else { + // No content after - exclude the closing tag for partial entries + logMessage.message = currentContent.slice(messageContentStart, messageEndIndex).trim() + } + } else { + // Partial message, include content without closing tag + logMessage.message = currentContent.slice(messageContentStart).trim() + } + } + + // Check for log message completion before updating level + const logEndMatchDeclared = currentContent.match(/<\/log_message>/) + // Update level only if log message is complete + if (logEndMatchDeclared) { + const levelMatch = currentContent.match(/(.*?)(?:<\/level>|$)/s) + if (levelMatch && levelMatch[1]) { + const levelValue = levelMatch[1].trim() + if (["debug", "info", "warn", "error"].includes(levelValue)) { + logMessage.level = levelValue as "debug" | "info" | "warn" | "error" + } + } + } + + const logEndMatch = currentContent.match(/<\/log_message>/) + if (logEndMatch) { + logMessage.partial = false + state.currentLogMessage = undefined + // Reset accumulator to after the closing tag to handle multiple log entries + // Find the exact position of the closing tag in the current content + const logEndIndex = currentContent.indexOf("") + "".length + const absoluteEndIndex = state.currentLogMessageStartIndex + logEndIndex + + // Keep any remaining content after this log message + const remainingContent = state.accumulator.slice(absoluteEndIndex) + state.accumulator = remainingContent + state.currentLogMessageStartIndex = 0 // Reset start index for next log message + + // Ensure state is fully reset to detect new log messages + state.currentTextContent = undefined + state.currentToolUse = undefined + } + return true + } + + return false + } + + static checkForLogStart(state: ParsingState): boolean { + // Check if there's a new log_message tag that hasn't been processed yet + const logStartIndex = state.accumulator.indexOf("") + if (logStartIndex === -1) { + return false + } + + // If we already have a current log message, don't start a new one + if (state.currentLogMessage) { + return false + } + + return true + } +} diff --git a/src/core/assistant-message/parsers/index.ts b/src/core/assistant-message/parsers/index.ts index c455e1b71b..12cca84690 100644 --- a/src/core/assistant-message/parsers/index.ts +++ b/src/core/assistant-message/parsers/index.ts @@ -1,4 +1,4 @@ export { TextContentParser } from "./TextContentParser" export { ToolUseParser } from "./ToolUseParser" export { ParameterParser } from "./ParameterParser" -export type { TextDirective, ToolDirective, Directive, ParsingState } from "./types" +export type { ToolDirective, Directive, ParsingState } from "./types" diff --git a/src/core/assistant-message/parsers/types.ts b/src/core/assistant-message/parsers/types.ts index fe9bc31e14..933b94af23 100644 --- a/src/core/assistant-message/parsers/types.ts +++ b/src/core/assistant-message/parsers/types.ts @@ -1,9 +1,10 @@ -import { TextContent, ToolUse, ToolParamName } from "../../../shared/tools" +import { TextDirective, LogDirective } from "../directives" +import { ToolUse, ToolParamName } from "../../../shared/tools" // Type aliases for directive parsing -export type TextDirective = TextContent + export type ToolDirective = ToolUse -export type Directive = TextDirective | ToolDirective +export type Directive = TextDirective | ToolDirective | LogDirective export interface ParsingState { contentBlocks: Directive[] @@ -11,6 +12,8 @@ export interface ParsingState { currentTextContentStartIndex: number currentToolUse?: ToolDirective currentToolUseStartIndex: number + currentLogMessage?: LogDirective + currentLogMessageStartIndex: number currentParamName?: ToolParamName currentParamValueStartIndex: number accumulator: string diff --git a/src/core/logging/LogManager.ts b/src/core/logging/LogManager.ts new file mode 100644 index 0000000000..f664f66077 --- /dev/null +++ b/src/core/logging/LogManager.ts @@ -0,0 +1,53 @@ +import { logLevels } from "@roo-code/types" +import { ClineProvider } from "../webview/ClineProvider" + +/** + * Manages logging functionality for Task instances. + * Handles log messages from AI-generated blocks. + */ +export class LogManager { + private providerRef: WeakRef + + /** + * Creates a new LogManager instance. + * @param provider The ClineProvider instance to use for logging + */ + constructor(provider: ClineProvider) { + this.providerRef = new WeakRef(provider) + } + + /** + * Logs a message to the output channel and console. + * This method is intended for internal logging triggered by the AI + * via blocks and does not require user approval. + * @param message The message to log. + * @param level The log level (debug, info, warn, error). Defaults to "info". + */ + public log(message: string, level: (typeof logLevels)[number] = "info"): void { + const timestamp = new Date().toISOString() + const formattedMessage = `[${timestamp}] [${level.toUpperCase()}] ${message}` + + // Get the provider instance + const provider = this.providerRef.deref() + if (provider) { + // Use the provider's log method which logs to both console and output channel + provider.log(formattedMessage) + } + } + + /** + * Processes a log message directive from the assistant. + * @param message The log message + * @param level The log level + * @param partial Whether the log message is partial + * @returns true if the log was processed, false otherwise + */ + public processLogEntry(message: string, level: (typeof logLevels)[number], partial: boolean): boolean { + // Only log complete (non-partial) log messages to avoid logging with incorrect levels + if (!partial) { + this.log(message, level) + return true + } + return false + } +} diff --git a/src/core/logging/__tests__/LogManager.test.ts b/src/core/logging/__tests__/LogManager.test.ts new file mode 100644 index 0000000000..90abef4352 --- /dev/null +++ b/src/core/logging/__tests__/LogManager.test.ts @@ -0,0 +1,54 @@ +import { LogManager } from "../LogManager" + +describe("LogManager", () => { + let mockProvider: any + let logManager: LogManager + + beforeEach(() => { + mockProvider = { + log: jest.fn(), + } + logManager = new LogManager(mockProvider as any) + }) + + describe("log", () => { + it("should format and log messages with timestamp and level", () => { + // Mock Date.toISOString to return a fixed timestamp + const mockDate = new Date("2023-01-01T12:00:00Z") + jest.spyOn(global, "Date").mockImplementation(() => mockDate as any) + + logManager.log("Test message", "info") + + expect(mockProvider.log).toHaveBeenCalledWith("[2023-01-01T12:00:00.000Z] [INFO] Test message") + }) + + it("should use 'info' as default log level", () => { + const mockDate = new Date("2023-01-01T12:00:00Z") + jest.spyOn(global, "Date").mockImplementation(() => mockDate as any) + + logManager.log("Test message") + + expect(mockProvider.log).toHaveBeenCalledWith("[2023-01-01T12:00:00.000Z] [INFO] Test message") + }) + }) + + describe("processLogEntry", () => { + it("should log complete entries", () => { + const spy = jest.spyOn(logManager, "log") + + const result = logManager.processLogEntry("Test log entry", "debug", false) + + expect(result).toBe(true) + expect(spy).toHaveBeenCalledWith("Test log entry", "debug") + }) + + it("should not log partial entries", () => { + const spy = jest.spyOn(logManager, "log") + + const result = logManager.processLogEntry("Partial log entry", "warn", true) + + expect(result).toBe(false) + expect(spy).not.toHaveBeenCalled() + }) + }) +}) diff --git a/src/core/logging/index.ts b/src/core/logging/index.ts new file mode 100644 index 0000000000..818d6f4da4 --- /dev/null +++ b/src/core/logging/index.ts @@ -0,0 +1 @@ +export * from "./LogManager" diff --git a/src/core/prompts/sections/index.ts b/src/core/prompts/sections/index.ts index d06dbbfde1..e5188bcd6f 100644 --- a/src/core/prompts/sections/index.ts +++ b/src/core/prompts/sections/index.ts @@ -3,6 +3,7 @@ export { getSystemInfoSection } from "./system-info" export { getObjectiveSection } from "./objective" export { addCustomInstructions } from "./custom-instructions" export { getSharedToolUseSection } from "./tool-use" +export { getLogMessageSection } from "./log-message" export { getMcpServersSection } from "./mcp-servers" export { getToolUseGuidelinesSection } from "./tool-use-guidelines" export { getCapabilitiesSection } from "./capabilities" diff --git a/src/core/prompts/sections/log-message.ts b/src/core/prompts/sections/log-message.ts new file mode 100644 index 0000000000..7c2ddf524f --- /dev/null +++ b/src/core/prompts/sections/log-message.ts @@ -0,0 +1,39 @@ +export function getLogMessageSection(): string { + return `==== + +LOG MESSAGES + +You can use log messages to output debugging information to the VSCode output channel. Unlike tools, log messages don't require approval and don't count toward the one-tool-per-message limit. + +# Purpose and Context + +The VSCode OutputChannel is a dedicated console-like interface within VSCode where extensions can write diagnostic information, separate from the user's workspace files. + +Unlike other tools that create or modify files in the workspace, log messages are purely for diagnostic purposes within the extension's runtime environment. + +# Log Message Formatting + +Log messages are formatted using XML-style tags. Here's the structure: + + +Your log message here +info + + +The level parameter is optional and defaults to "info". Valid levels are: "debug", "info", "warn", and "error". + +For example: + + +Starting task execution +info + + + +Failed to parse input: invalid JSON +error + + +You can use log messages multiple times in a single message, and they will be processed immediately without requiring user approval. +` +} diff --git a/src/core/prompts/system.ts b/src/core/prompts/system.ts index 61fd9df81e..f3444611f3 100644 --- a/src/core/prompts/system.ts +++ b/src/core/prompts/system.ts @@ -18,6 +18,7 @@ import { getSystemInfoSection, getObjectiveSection, getSharedToolUseSection, + getLogMessageSection, getMcpServersSection, getToolUseGuidelinesSection, getCapabilitiesSection, @@ -71,6 +72,8 @@ ${markdownFormattingSection()} ${getSharedToolUseSection()} +${getLogMessageSection()} + ${getToolDescriptionsForMode( mode, cwd, diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index e881749e86..0b2aa87390 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -84,6 +84,7 @@ import { processUserContentMentions } from "../mentions/processUserContentMentio import { ApiMessage } from "../task-persistence/apiMessages" import { getMessagesSinceLastSummary, summarizeConversation } from "../condense" import { maybeRemoveImageBlocks } from "../../api/transform/image-cleaning" +import { LogManager } from "../logging" export type ClineEvents = { message: [{ action: "created" | "updated"; message: ClineMessage }] @@ -192,6 +193,9 @@ export class Task extends EventEmitter { didAlreadyUseTool = false didCompleteReadingStream = false + // Logging + private logManager: LogManager + constructor({ provider, apiConfiguration, @@ -224,6 +228,7 @@ export class Task extends EventEmitter { this.rooIgnoreController = new RooIgnoreController(this.cwd) this.fileContextTracker = new FileContextTracker(provider, this.taskId) + this.logManager = new LogManager(provider) this.rooIgnoreController.initialize().catch((error) => { console.error("Failed to initialize RooIgnoreController:", error) diff --git a/src/package.json b/src/package.json index 9937a08dd6..61e7d05393 100644 --- a/src/package.json +++ b/src/package.json @@ -371,11 +371,11 @@ "@google/genai": "^1.0.0", "@mistralai/mistralai": "^1.3.6", "@modelcontextprotocol/sdk": "^1.9.0", + "@qdrant/js-client-rest": "^1.14.0", "@roo-code/cloud": "workspace:^", "@roo-code/ipc": "workspace:^", "@roo-code/telemetry": "workspace:^", "@roo-code/types": "workspace:^", - "@qdrant/js-client-rest": "^1.14.0", "@types/lodash.debounce": "^4.0.9", "@vscode/codicons": "^0.0.36", "async-mutex": "^0.5.0", @@ -413,6 +413,7 @@ "puppeteer-core": "^23.4.0", "reconnecting-eventsource": "^1.6.4", "sanitize-filename": "^1.6.3", + "sax": "^1.4.1", "say": "^0.16.0", "serialize-error": "^11.0.3", "simple-git": "^3.27.0", @@ -447,6 +448,7 @@ "@types/node-cache": "^4.1.3", "@types/node-ipc": "^9.2.3", "@types/ps-tree": "^1.1.6", + "@types/sax": "^1.2.7", "@types/string-similarity": "^4.0.2", "@types/tmp": "^0.2.6", "@types/turndown": "^5.0.5", diff --git a/src/shared/tools.ts b/src/shared/tools.ts index ffaf41f93f..7525f85088 100644 --- a/src/shared/tools.ts +++ b/src/shared/tools.ts @@ -20,12 +20,6 @@ export type AskFinishSubTaskApproval = () => Promise export type ToolDescription = () => string -export interface TextContent { - type: "text" - content: string - partial: boolean -} - export const toolParamNames = [ "command", "path",