From 2badff4776bd074f40916408e64ccf947541103b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:32:51 +0000 Subject: [PATCH] fix(ui): render reasoning in chat instead of raw think tags The chat UI streams via the Responses API but only called onReasoningContent for response.reasoning.delta, an event only its own non-streaming helper synthesizes; the proxy streams response.reasoning_summary_text.delta, so streamed reasoning was dropped and the reasoning panel never appeared. It also never parsed inline blocks, so deployments using merge_reasoning_content_in_choices rendered the literal tags in the answer. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../components/llm_calls/responses_api.tsx | 44 +++++++++--- .../components/llm_calls/thinkTags.test.ts | 67 +++++++++++++++++++ .../src/components/llm_calls/thinkTags.ts | 67 +++++++++++++++++++ 3 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/llm_calls/thinkTags.test.ts create mode 100644 ui/litellm-dashboard/src/components/llm_calls/thinkTags.ts diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index f356b2cb2c3..dc2fbb4e940 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -11,6 +11,7 @@ import { handleCodeInterpreterCall, handleCodeInterpreterOutput, } from "./code_interpreter_handler"; +import { flushThinkTags, initialThinkTagState, splitThinkTags, ThinkTagState } from "./thinkTags"; export type { CodeInterpreterResult } from "./code_interpreter_handler"; @@ -29,6 +30,12 @@ interface NonStreamedResponse { output?: ResponseOutputItem[]; } +const REASONING_DELTA_EVENT_TYPES = new Set([ + "response.reasoning.delta", + "response.reasoning_text.delta", + "response.reasoning_summary_text.delta", +]); + type SynthesizedResponseEvent = | { type: "response.output_item.done"; item: ResponseOutputItem } | { type: "response.reasoning.delta"; delta: string } @@ -206,6 +213,7 @@ export async function makeOpenAIResponsesRequest( let mcpToolUsed = ""; let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" }; + let thinkTagState: ThinkTagState = initialThinkTagState; for await (const event of events) { // Use a type-safe approach to handle events @@ -250,24 +258,30 @@ export async function makeOpenAIResponsesRequest( // 2) only handle actual text deltas if (event.type === "response.output_text.delta" && typeof event.delta === "string") { - const delta = event.delta; - if (delta.length > 0) { - updateTextUI("assistant", delta, selectedModel); + const split = splitThinkTags(thinkTagState, event.delta); + thinkTagState = split.state; - // Calculate time to first token - if (!firstTokenReceived) { - firstTokenReceived = true; - const timeToFirstToken = Date.now() - startTime; + if (split.reasoning.length > 0 && onReasoningContent) { + onReasoningContent(split.reasoning); + } - if (onTimingData && streamingEnabled) { - onTimingData(timeToFirstToken); - } + if (split.text.length > 0) { + updateTextUI("assistant", split.text, selectedModel); + } + + // Calculate time to first token + if ((split.text.length > 0 || split.reasoning.length > 0) && !firstTokenReceived) { + firstTokenReceived = true; + const timeToFirstToken = Date.now() - startTime; + + if (onTimingData && streamingEnabled) { + onTimingData(timeToFirstToken); } } } // Handle reasoning content - if (event.type === "response.reasoning.delta" && "delta" in event) { + if (REASONING_DELTA_EVENT_TYPES.has(event.type) && "delta" in event) { const delta = event.delta; if (typeof delta === "string" && onReasoningContent) { onReasoningContent(delta); @@ -303,6 +317,14 @@ export async function makeOpenAIResponsesRequest( } } + const trailing = flushThinkTags(thinkTagState); + if (trailing.reasoning.length > 0 && onReasoningContent) { + onReasoningContent(trailing.reasoning); + } + if (trailing.text.length > 0) { + updateTextUI("assistant", trailing.text, selectedModel); + } + if (onTotalLatency) { onTotalLatency(Date.now() - startTime); } diff --git a/ui/litellm-dashboard/src/components/llm_calls/thinkTags.test.ts b/ui/litellm-dashboard/src/components/llm_calls/thinkTags.test.ts new file mode 100644 index 00000000000..9d25b7b38c1 --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/thinkTags.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { flushThinkTags, initialThinkTagState, splitThinkTags, ThinkTagState } from "./thinkTags"; + +function run(deltas: string[]): { text: string; reasoning: string } { + const final = deltas.reduce<{ state: ThinkTagState; text: string; reasoning: string }>( + (acc, delta) => { + const split = splitThinkTags(acc.state, delta); + return { + state: split.state, + text: acc.text + split.text, + reasoning: acc.reasoning + split.reasoning, + }; + }, + { state: initialThinkTagState, text: "", reasoning: "" }, + ); + const trailing = flushThinkTags(final.state); + return { text: final.text + trailing.text, reasoning: final.reasoning + trailing.reasoning }; +} + +describe("splitThinkTags", () => { + it("leaves plain text untouched", () => { + expect(run(["Hello! ", "How can I help?"])).toEqual({ text: "Hello! How can I help?", reasoning: "" }); + }); + + it("routes a whole think block to reasoning and keeps the answer as text", () => { + expect(run(["the user said hiHello!"])).toEqual({ + text: "Hello!", + reasoning: "the user said hi", + }); + }); + + it("handles a think block spread across many deltas", () => { + expect(run(["", "the user ", "said hi", "", "Hello!"])).toEqual({ + text: "Hello!", + reasoning: "the user said hi", + }); + }); + + it("handles tags split mid-token across delta boundaries", () => { + expect(run(["reasoninganswer"])).toEqual({ + text: "answer", + reasoning: "reasoning", + }); + }); + + it("never emits a partial tag as text before it is resolved", () => { + const first = splitThinkTags(initialThinkTagState, "answerhiddendone"); + expect(second.text).toBe("done"); + expect(second.reasoning).toBe("hidden"); + }); + + it("emits held-back text that turns out not to be a tag", () => { + expect(run(["3 <", " 4"])).toEqual({ text: "3 < 4", reasoning: "" }); + }); + + it("supports multiple think blocks in one response", () => { + expect(run(["aonebtwo"])).toEqual({ text: "onetwo", reasoning: "ab" }); + }); + + it("flushes an unterminated think block as reasoning", () => { + expect(run(["truncated reasoning"])).toEqual({ text: "", reasoning: "truncated reasoning" }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/thinkTags.ts b/ui/litellm-dashboard/src/components/llm_calls/thinkTags.ts new file mode 100644 index 00000000000..3c58a1e5e9a --- /dev/null +++ b/ui/litellm-dashboard/src/components/llm_calls/thinkTags.ts @@ -0,0 +1,67 @@ +const OPEN_TAG = ""; +const CLOSE_TAG = ""; + +export interface ThinkTagState { + readonly insideThink: boolean; + readonly pending: string; +} + +export interface ThinkTagSplit { + readonly state: ThinkTagState; + readonly text: string; + readonly reasoning: string; +} + +export const initialThinkTagState: ThinkTagState = { insideThink: false, pending: "" }; + +function heldBackLength(buffer: string, tag: string): number { + const max = Math.min(buffer.length, tag.length - 1); + for (let len = max; len > 0; len--) { + if (tag.startsWith(buffer.slice(buffer.length - len))) return len; + } + return 0; +} + +export function splitThinkTags(state: ThinkTagState, delta: string): ThinkTagSplit { + const consume = ( + buffer: string, + insideThink: boolean, + text: string, + reasoning: string, + ): { pending: string; insideThink: boolean; text: string; reasoning: string } => { + const tag = insideThink ? CLOSE_TAG : OPEN_TAG; + const index = buffer.indexOf(tag); + if (index !== -1) { + const before = buffer.slice(0, index); + return consume( + buffer.slice(index + tag.length), + !insideThink, + insideThink ? text : text + before, + insideThink ? reasoning + before : reasoning, + ); + } + const held = heldBackLength(buffer, tag); + const emitted = buffer.slice(0, buffer.length - held); + return { + pending: buffer.slice(buffer.length - held), + insideThink, + text: insideThink ? text : text + emitted, + reasoning: insideThink ? reasoning + emitted : reasoning, + }; + }; + + const result = consume(state.pending + delta, state.insideThink, "", ""); + return { + state: { insideThink: result.insideThink, pending: result.pending }, + text: result.text, + reasoning: result.reasoning, + }; +} + +export function flushThinkTags(state: ThinkTagState): ThinkTagSplit { + return { + state: initialThinkTagState, + text: state.insideThink ? "" : state.pending, + reasoning: state.insideThink ? state.pending : "", + }; +}