From d7d9fa7530cd0dffacfb69821024917544fb7110 Mon Sep 17 00:00:00 2001 From: Houyun Sun <18763177732@139.com> Date: Wed, 28 Oct 2026 19:34:47 +0800 Subject: [PATCH] fix(token_counter): count assistant refusal content parts An assistant message replayed in the history can hold a `refusal` content part - it's part of OpenAI's assistant message param shape, and a model that refused a turn sends it back on the next request. token_counter had no branch for it, so it fell through to the catch-all and raised UnsupportedContentTypeError, which also turned /utils/token_counter into a 500 on the proxy. A refusal part is just text, so count it in the text branch: read `text`, fall back to `refusal`, nothing to add when both are empty. Folding it into the existing branch keeps the walker under the complexity ceiling. Also added ChatCompletionRefusalObject to the content block union, the schema snapshot and schema.d.ts, and to the supported types listed in the error message. Tests: refusal history now counts exactly like the equivalent text history, and a text part that carries a stray refusal key still counts its text. --- litellm/litellm_core_utils/token_counter.py | 11 +++- litellm/proxy/_lazy_openapi_snapshot.json | 26 ++++++++ litellm/types/llms/openai.py | 14 ++++ .../litellm_core_utils/test_token_counter.py | 65 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 16 ++++- 5 files changed, 128 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 6d50e4b065e..28ecf0a3be5 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -61,6 +61,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionDocumentObject, ChatCompletionRedactedThinkingBlock, + ChatCompletionRefusalObject, ChatCompletionThinkingBlock, ChatCompletionToolParam, OpenAIMessageContentListBlock, @@ -1039,6 +1040,7 @@ def _count_content_list( OpenAIMessageContentListBlock | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock + | ChatCompletionRefusalObject | AnthropicMessagesTextParam | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam @@ -1053,8 +1055,11 @@ def _count_content_list( for c in content_list: if isinstance(c, str): num_tokens += count_function(c) - elif c["type"] == "text": - num_tokens += count_function(str(c.get("text", ""))) + elif c["type"] in ("text", "refusal"): + # `refusal` is OpenAI's content part for a turn the model declined. + # It is plain text, so it counts like a text part - without this, + # replaying a refused assistant message in the history blows up here. + num_tokens += count_function(str(c.get("text") or c.get("refusal") or "")) elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) @@ -1107,7 +1112,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"tool_reference, search_result)." + f"tool_reference, search_result, refusal)." ) return num_tokens except Exception as e: diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 2aaa987ba6a..667c8b1e64c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -48156,6 +48156,29 @@ "title": "ChatCompletionRedactedThinkingBlock", "type": "object" }, + "ChatCompletionRefusalObject": { + "description": "Assistant content part holding the refusal text of a turn the model declined.", + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "refusal": { + "title": "Refusal", + "type": "string" + }, + "type": { + "const": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "refusal" + ], + "title": "ChatCompletionRefusalObject", + "type": "object" + }, "ChatCompletionSystemMessage": { "properties": { "cache_control": { @@ -48501,6 +48524,9 @@ }, { "$ref": "#/components/schemas/ChatCompletionFileObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionRefusalObject" } ] }, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4912f9f1375..88705b245be 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -703,6 +703,19 @@ class ChatCompletionRedactedThinkingBlock(TypedDict, total=False): index: ReadOnly[int] +class OpenAIChatCompletionRefusalObject(TypedDict): + """Assistant content part holding the refusal text of a turn the model declined.""" + + type: Literal["refusal"] + refusal: str + + +class ChatCompletionRefusalObject( + OpenAIChatCompletionRefusalObject, total=False +): # litellm wrapper on top of openai object for handling cached content + cache_control: ChatCompletionCachedContent + + class ChatCompletionReasoningSummaryTextBlock(TypedDict, total=False): type: Required[Literal["summary_text"]] text: str @@ -869,6 +882,7 @@ OpenAIMessageContentListBlock = ( | ChatCompletionDocumentObject | ChatCompletionVideoObject | ChatCompletionFileObject + | ChatCompletionRefusalObject ) OpenAIMessageContent = str | Iterable[OpenAIMessageContentListBlock] diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 92f971f64e9..dca5a37b47c 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2469,3 +2469,68 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk ) == 200_000 ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_counter_counts_assistant_refusal_content_part() -> None: + """ + An assistant history message can carry a `refusal` content part - it is part of + OpenAI's `chat_completion_assistant_message_param` shape and models that refuse a + turn replay it in the next request. token_counter used to reject it with + `Invalid content item type: refusal`, which also turned /utils/token_counter into + a 500 on the proxy. + + A refusal part is plain text, so it should count exactly like the same text sent + as a `text` part. + """ + refusal_text = "I cannot help with that request." + refusal_messages = [ + {"role": "user", "content": "A request"}, + {"role": "assistant", "content": [{"type": "refusal", "refusal": refusal_text}]}, + {"role": "user", "content": "Please suggest a safe alternative."}, + ] + text_messages = [ + {"role": "user", "content": "A request"}, + {"role": "assistant", "content": [{"type": "text", "text": refusal_text}]}, + {"role": "user", "content": "Please suggest a safe alternative."}, + ] + + assert token_counter_new(model="gpt-4o", messages=refusal_messages) == token_counter_new( + model="gpt-4o", messages=text_messages + ) + + +def test_count_content_list_accepts_refusal_block() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "refusal", "refusal": "no"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 2 + ) + + # a part without refusal text still parses, it just contributes nothing + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "refusal"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 0 + ) + + # a text part keeps using `text` and is not matched by the refusal key + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "text", "text": "keep", "refusal": "ignored"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 4 + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8af76e4d16f..6b65dbe3495 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28953,6 +28953,20 @@ export interface components { } & { [key: string]: unknown; }; + /** + * ChatCompletionRefusalObject + * @description Assistant content part holding the refusal text of a turn the model declined. + */ + ChatCompletionRefusalObject: { + cache_control?: components["schemas"]["ChatCompletionCachedContent"]; + /** Refusal */ + refusal: string; + /** + * Type + * @constant + */ + type: "refusal"; + }; /** ChatCompletionSystemMessage */ ChatCompletionSystemMessage: { cache_control?: components["schemas"]["ChatCompletionCachedContent"]; @@ -29091,7 +29105,7 @@ export interface components { ChatCompletionUserMessage: { cache_control?: components["schemas"]["ChatCompletionCachedContent"]; /** Content */ - content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"])[]; + content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"] | components["schemas"]["ChatCompletionRefusalObject"])[]; /** * Role * @constant