fix: improve HTML entity unescaping for Gemini and other models

- Add HTML entity unescaping to applyDiffTool for non-Claude models
- Enhance unescapeHtmlEntities function to handle more entity types
- Add support for alternative encodings like ', /, \, `
- Add comprehensive tests for new entity types
- Improve comments to clarify the purpose of unescaping

Fixes #7890
This commit is contained in:
Roo Code 2025-09-11 07:35:13 +00:00
parent 8fee3127ff
commit 6477cad443
4 changed files with 71 additions and 3 deletions

View file

@ -25,6 +25,8 @@ export async function applyDiffToolLegacy(
const relPath: string | undefined = block.params.path
let diffContent: string | undefined = block.params.diff
// Unescape HTML entities for non-Claude models (e.g., Gemini, DeepSeek, Llama)
// These models may return content with escaped characters that need to be unescaped
if (diffContent && !cline.api.getModel().id.includes("claude")) {
diffContent = unescapeHtmlEntities(diffContent)
}

View file

@ -73,9 +73,11 @@ export async function writeToFileTool(
cline.diffViewProvider.editType = fileExists ? "modify" : "create"
}
// pre-processing newContent for cases where weaker models might add artifacts like markdown codeblock markers (deepseek/llama) or extra escape characters (gemini)
// Pre-processing newContent for cases where models might add artifacts
// Some models (DeepSeek/Llama) add markdown codeblock markers
// Others (Gemini) return content with HTML-escaped characters
if (newContent.startsWith("```")) {
// cline handles cases where it includes language specifiers like ```python ```js
// Handle cases where it includes language specifiers like ```python ```js
newContent = newContent.split("\n").slice(1).join("\n")
}
@ -83,6 +85,8 @@ export async function writeToFileTool(
newContent = newContent.split("\n").slice(0, -1).join("\n")
}
// Unescape HTML entities for non-Claude models (e.g., Gemini, DeepSeek, Llama)
// These models may return content with escaped characters that need to be unescaped
if (!cline.api.getModel().id.includes("claude")) {
newContent = unescapeHtmlEntities(newContent)
}

View file

@ -0,0 +1,57 @@
import { describe, it, expect } from "vitest"
import { unescapeHtmlEntities } from "../text-normalization"
describe("Extended HTML entity unescaping", () => {
describe("unescapeHtmlEntities", () => {
it("unescapes alternative apostrophe encoding", () => {
const input = "It's working"
const expected = "It's working"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("unescapes forward slash", () => {
const input = "path/to/file"
const expected = "path/to/file"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("unescapes backslash", () => {
const input = "C:\Users\file"
const expected = "C:\\Users\\file"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("unescapes backtick", () => {
const input = "`code`"
const expected = "`code`"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("unescapes non-breaking space", () => {
const input = "Hello World"
const expected = "Hello World"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("handles complex mixed content with all entity types", () => {
const input =
"<div class="test">It's a  test/path\file with `code` & more</div>"
const expected = '<div class="test">It\'s a test/path\\file with `code` & more</div>'
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("handles Gemini-style escaped markdown content", () => {
const input =
"```python\n&lt;search&gt;\ndef old_function():\n return &#x27;old&#x27;\n&lt;/search&gt;\n&lt;replace&gt;\ndef new_function():\n return &#x27;new&#x27;\n&lt;/replace&gt;\n```"
const expected =
"```python\n<search>\ndef old_function():\n return 'old'\n</search>\n<replace>\ndef new_function():\n return 'new'\n</replace>\n```"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
it("correctly orders ampersand unescaping to avoid double-unescaping", () => {
const input = "&amp;lt;&amp;gt;&amp;amp;"
const expected = "&lt;&gt;&amp;"
expect(unescapeHtmlEntities(input)).toBe(expected)
})
})
})

View file

@ -91,5 +91,10 @@ export function unescapeHtmlEntities(text: string): string {
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&apos;/g, "'")
.replace(/&amp;/g, "&")
.replace(/&#x27;/g, "'") // Alternative apostrophe encoding
.replace(/&#x2F;/g, "/") // Forward slash
.replace(/&#x5C;/g, "\\") // Backslash
.replace(/&#x60;/g, "`") // Backtick
.replace(/&nbsp;/g, " ") // Non-breaking space
.replace(/&amp;/g, "&") // Must be last to avoid double-unescaping
}