scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-08-28 05:27:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: special tokens should not break task processing (#7540)

Browse source
This commit is contained in:
Piotr Wilkin (ilintar) 2025-08-30 16:40:40 +02:00 • committed by GitHub
parent c7d7ad8197
commit 47f594f27b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 10 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/utils/__tests__/tiktoken.spec.ts
View file

@ -25,6 +25,15 @@ describe("tiktoken", () => {
expect(result).toBe(0)
})
it("should not throw on text content with special tokens", async () => {
const content: Anthropic.Messages.ContentBlockParam[] = [
{ type: "text", text: "something<|endoftext|>something" },
]
const result = await tiktoken(content)
expect(result).toBeGreaterThan(0)
})
it("should handle missing text content", async () => {
// Using 'as any' to bypass TypeScript's type checking for this test case
// since we're specifically testing how the function handles undefined text

2
src/utils/tiktoken.ts
View file

@ -24,7 +24,7 @@ export async function tiktoken(content: Anthropic.Messages.ContentBlockParam[]):
const text = block.text || ""
if (text.length > 0) {
const tokens = encoder.encode(text)
const tokens = encoder.encode(text, undefined, [])
totalTokens += tokens.length
}
} else if (block.type === "image") {