fix(token_counter): count assistant refusal content parts

An assistant message replayed in the history can hold a `refusal` content
part - it's part of OpenAI's assistant message param shape, and a model that
refused a turn sends it back on the next request. token_counter had no branch
for it, so it fell through to the catch-all and raised
UnsupportedContentTypeError, which also turned /utils/token_counter into a
500 on the proxy.

A refusal part is just text, so count it in the text branch: read `text`,
fall back to `refusal`, nothing to add when both are empty. Folding it into
the existing branch keeps the walker under the complexity ceiling. Also added
ChatCompletionRefusalObject to the content block union, the schema snapshot
and schema.d.ts, and to the supported types listed in the error message.

Tests: refusal history now counts exactly like the equivalent text history,
and a text part that carries a stray refusal key still counts its text.
This commit is contained in:
Houyun Sun 2026-10-28 19:34:47 +08:00
parent e2d78b69cd
commit d7d9fa7530
5 changed files with 128 additions and 4 deletions

View file

@ -61,6 +61,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionDocumentObject,
ChatCompletionRedactedThinkingBlock,
ChatCompletionRefusalObject,
ChatCompletionThinkingBlock,
ChatCompletionToolParam,
OpenAIMessageContentListBlock,
@ -1039,6 +1040,7 @@ def _count_content_list(
OpenAIMessageContentListBlock
| ChatCompletionThinkingBlock
| ChatCompletionRedactedThinkingBlock
| ChatCompletionRefusalObject
| AnthropicMessagesTextParam
| AnthropicMessagesImageParam
| AnthropicMessagesDocumentParam
@ -1053,8 +1055,11 @@ def _count_content_list(
for c in content_list:
if isinstance(c, str):
num_tokens += count_function(c)
elif c["type"] == "text":
num_tokens += count_function(str(c.get("text", "")))
elif c["type"] in ("text", "refusal"):
# `refusal` is OpenAI's content part for a turn the model declined.
# It is plain text, so it counts like a text part - without this,
# replaying a refused assistant message in the history blows up here.
num_tokens += count_function(str(c.get("text") or c.get("refusal") or ""))
elif c["type"] == "image_url":
image_url = c.get("image_url")
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
@ -1107,7 +1112,7 @@ def _count_content_list(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference, search_result)."
f"tool_reference, search_result, refusal)."
)
return num_tokens
except Exception as e:

View file

@ -48156,6 +48156,29 @@
"title": "ChatCompletionRedactedThinkingBlock",
"type": "object"
},
"ChatCompletionRefusalObject": {
"description": "Assistant content part holding the refusal text of a turn the model declined.",
"properties": {
"cache_control": {
"$ref": "#/components/schemas/ChatCompletionCachedContent"
},
"refusal": {
"title": "Refusal",
"type": "string"
},
"type": {
"const": "refusal",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"refusal"
],
"title": "ChatCompletionRefusalObject",
"type": "object"
},
"ChatCompletionSystemMessage": {
"properties": {
"cache_control": {
@ -48501,6 +48524,9 @@
},
{
"$ref": "#/components/schemas/ChatCompletionFileObject"
},
{
"$ref": "#/components/schemas/ChatCompletionRefusalObject"
}
]
},

View file

@ -703,6 +703,19 @@ class ChatCompletionRedactedThinkingBlock(TypedDict, total=False):
index: ReadOnly[int]
class OpenAIChatCompletionRefusalObject(TypedDict):
"""Assistant content part holding the refusal text of a turn the model declined."""
type: Literal["refusal"]
refusal: str
class ChatCompletionRefusalObject(
OpenAIChatCompletionRefusalObject, total=False
): # litellm wrapper on top of openai object for handling cached content
cache_control: ChatCompletionCachedContent
class ChatCompletionReasoningSummaryTextBlock(TypedDict, total=False):
type: Required[Literal["summary_text"]]
text: str
@ -869,6 +882,7 @@ OpenAIMessageContentListBlock = (
| ChatCompletionDocumentObject
| ChatCompletionVideoObject
| ChatCompletionFileObject
| ChatCompletionRefusalObject
)
OpenAIMessageContent = str | Iterable[OpenAIMessageContentListBlock]

View file

@ -2469,3 +2469,68 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
)
== 200_000
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_token_counter_counts_assistant_refusal_content_part() -> None:
"""
An assistant history message can carry a `refusal` content part - it is part of
OpenAI's `chat_completion_assistant_message_param` shape and models that refuse a
turn replay it in the next request. token_counter used to reject it with
`Invalid content item type: refusal`, which also turned /utils/token_counter into
a 500 on the proxy.
A refusal part is plain text, so it should count exactly like the same text sent
as a `text` part.
"""
refusal_text = "I cannot help with that request."
refusal_messages = [
{"role": "user", "content": "A request"},
{"role": "assistant", "content": [{"type": "refusal", "refusal": refusal_text}]},
{"role": "user", "content": "Please suggest a safe alternative."},
]
text_messages = [
{"role": "user", "content": "A request"},
{"role": "assistant", "content": [{"type": "text", "text": refusal_text}]},
{"role": "user", "content": "Please suggest a safe alternative."},
]
assert token_counter_new(model="gpt-4o", messages=refusal_messages) == token_counter_new(
model="gpt-4o", messages=text_messages
)
def test_count_content_list_accepts_refusal_block() -> None:
from litellm.litellm_core_utils.token_counter import _count_content_list
assert (
_count_content_list(
count_function=len,
content_list=[{"type": "refusal", "refusal": "no"}],
use_default_image_token_count=False,
default_token_count=None,
)
== 2
)
# a part without refusal text still parses, it just contributes nothing
assert (
_count_content_list(
count_function=len,
content_list=[{"type": "refusal"}],
use_default_image_token_count=False,
default_token_count=None,
)
== 0
)
# a text part keeps using `text` and is not matched by the refusal key
assert (
_count_content_list(
count_function=len,
content_list=[{"type": "text", "text": "keep", "refusal": "ignored"}],
use_default_image_token_count=False,
default_token_count=None,
)
== 4
)

View file

@ -28953,6 +28953,20 @@ export interface components {
} & {
[key: string]: unknown;
};
/**
* ChatCompletionRefusalObject
* @description Assistant content part holding the refusal text of a turn the model declined.
*/
ChatCompletionRefusalObject: {
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
/** Refusal */
refusal: string;
/**
* Type
* @constant
*/
type: "refusal";
};
/** ChatCompletionSystemMessage */
ChatCompletionSystemMessage: {
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
@ -29091,7 +29105,7 @@ export interface components {
ChatCompletionUserMessage: {
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
/** Content */
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"])[];
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"] | components["schemas"]["ChatCompletionRefusalObject"])[];
/**
* Role
* @constant