From d6713767b532f9cd8d8a697b62c942c255de8799 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Thu, 18 Sep 2025 00:04:52 +0000 Subject: [PATCH] fix: unescape HTML entities in Gemini provider streaming responses - Apply unescapeHtmlEntities to all text yielded from Gemini API - Fixes issue where square brackets were missing in Python/other code - Add comprehensive tests for HTML entity unescaping - Ensures consistency with other providers Fixes #8107 --- src/api/providers/__tests__/gemini.spec.ts | 147 +++++++++++++++++++++ src/api/providers/gemini.ts | 9 +- 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/api/providers/__tests__/gemini.spec.ts b/src/api/providers/__tests__/gemini.spec.ts index 812c1ae1a6..098dbdcf08 100644 --- a/src/api/providers/__tests__/gemini.spec.ts +++ b/src/api/providers/__tests__/gemini.spec.ts @@ -90,6 +90,133 @@ describe("GeminiHandler", () => { ) }) + it("should unescape HTML entities in text messages", async () => { + // Setup the mock implementation to return text with HTML entities + ;(handler["client"].models.generateContentStream as any).mockResolvedValue({ + [Symbol.asyncIterator]: async function* () { + yield { text: "array[0]" } + yield { text: " and <div>" } + yield { text: " with & symbol" } + yield { usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 } } + }, + }) + + const stream = handler.createMessage(systemPrompt, mockMessages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have unescaped HTML entities + expect(chunks[0]).toEqual({ type: "text", text: "array[0]" }) + expect(chunks[1]).toEqual({ type: "text", text: " and
" }) + expect(chunks[2]).toEqual({ type: "text", text: " with & symbol" }) + }) + + it("should unescape square bracket entities in Python code", async () => { + // Test the exact scenario from the issue + ;(handler["client"].models.generateContentStream as any).mockResolvedValue({ + [Symbol.asyncIterator]: async function* () { + yield { + candidates: [ + { + content: { + parts: [ + { + text: 'print(f"The first character is: {populated_variable[0]}")', + }, + ], + }, + }, + ], + } + yield { usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 } } + }, + }) + + const stream = handler.createMessage(systemPrompt, mockMessages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have properly unescaped square brackets + expect(chunks[0]).toEqual({ + type: "text", + text: 'print(f"The first character is: {populated_variable[0]}")', + }) + }) + + it("should unescape named square bracket entities", async () => { + // Test named entities for square brackets + ;(handler["client"].models.generateContentStream as any).mockResolvedValue({ + [Symbol.asyncIterator]: async function* () { + yield { + candidates: [ + { + content: { + parts: [ + { + text: "matrix[i][j]", + }, + ], + }, + }, + ], + } + yield { usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 } } + }, + }) + + const stream = handler.createMessage(systemPrompt, mockMessages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have properly unescaped named entities + expect(chunks[0]).toEqual({ type: "text", text: "matrix[i][j]" }) + }) + + it("should unescape HTML entities in thinking/reasoning parts", async () => { + // Test that entities are unescaped in thinking parts too + ;(handler["client"].models.generateContentStream as any).mockResolvedValue({ + [Symbol.asyncIterator]: async function* () { + yield { + candidates: [ + { + content: { + parts: [ + { + thought: true, + text: "Need to access array[0] element", + }, + ], + }, + }, + ], + } + yield { usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5 } } + }, + }) + + const stream = handler.createMessage(systemPrompt, mockMessages) + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk) + } + + // Should have properly unescaped in reasoning text + expect(chunks[0]).toEqual({ + type: "reasoning", + text: "Need to access array[0] element", + }) + }) + it("should handle API errors", async () => { const mockError = new Error("Gemini API error") ;(handler["client"].models.generateContentStream as any).mockRejectedValue(mockError) @@ -143,6 +270,26 @@ describe("GeminiHandler", () => { const result = await handler.completePrompt("Test prompt") expect(result).toBe("") }) + + it("should unescape HTML entities in completePrompt response", async () => { + // Mock the response with HTML entities + ;(handler["client"].models.generateContent as any).mockResolvedValue({ + text: "array[0] and <div>", + }) + + const result = await handler.completePrompt("Test prompt") + expect(result).toBe("array[0] and
") + }) + + it("should handle square brackets in completePrompt for Python code", async () => { + // Test the exact scenario from the issue + ;(handler["client"].models.generateContent as any).mockResolvedValue({ + text: 'print(f"The first character is: {populated_variable[0]}")', + }) + + const result = await handler.completePrompt("Update Python script") + expect(result).toBe('print(f"The first character is: {populated_variable[0]}")') + }) }) describe("getModel", () => { diff --git a/src/api/providers/gemini.ts b/src/api/providers/gemini.ts index 775d763a05..50a4f0875f 100644 --- a/src/api/providers/gemini.ts +++ b/src/api/providers/gemini.ts @@ -17,6 +17,7 @@ import { convertAnthropicContentToGemini, convertAnthropicMessageToGemini } from import { t } from "i18next" import type { ApiStream, GroundingSource } from "../transform/stream" import { getModelParams } from "../transform/model-params" +import { unescapeHtmlEntities } from "../../utils/text-normalization" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" import { BaseProvider } from "./base-provider" @@ -109,12 +110,12 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl if (part.thought) { // This is a thinking/reasoning part if (part.text) { - yield { type: "reasoning", text: part.text } + yield { type: "reasoning", text: unescapeHtmlEntities(part.text) } } } else { // This is regular content if (part.text) { - yield { type: "text", text: part.text } + yield { type: "text", text: unescapeHtmlEntities(part.text) } } } } @@ -123,7 +124,7 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl // Fallback to the original text property if no candidates structure else if (chunk.text) { - yield { type: "text", text: chunk.text } + yield { type: "text", text: unescapeHtmlEntities(chunk.text) } } if (chunk.usageMetadata) { @@ -234,7 +235,7 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl config: promptConfig, }) - let text = result.text ?? "" + let text = unescapeHtmlEntities(result.text ?? "") const candidate = result.candidates?.[0] if (candidate?.groundingMetadata) {