mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(token_counter): count assistant refusal content parts
An assistant message replayed in the history can hold a `refusal` content part - it's part of OpenAI's assistant message param shape, and a model that refused a turn sends it back on the next request. token_counter had no branch for it, so it fell through to the catch-all and raised UnsupportedContentTypeError, which also turned /utils/token_counter into a 500 on the proxy. A refusal part is just text, so count it in the text branch: read `text`, fall back to `refusal`, nothing to add when both are empty. Folding it into the existing branch keeps the walker under the complexity ceiling. Also added ChatCompletionRefusalObject to the content block union, the schema snapshot and schema.d.ts, and to the supported types listed in the error message. Tests: refusal history now counts exactly like the equivalent text history, and a text part that carries a stray refusal key still counts its text.
This commit is contained in:
parent
e2d78b69cd
commit
d7d9fa7530
5 changed files with 128 additions and 4 deletions
|
|
@ -61,6 +61,7 @@ from litellm.types.llms.openai import (
|
|||
AllMessageValues,
|
||||
ChatCompletionDocumentObject,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionRefusalObject,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
OpenAIMessageContentListBlock,
|
||||
|
|
@ -1039,6 +1040,7 @@ def _count_content_list(
|
|||
OpenAIMessageContentListBlock
|
||||
| ChatCompletionThinkingBlock
|
||||
| ChatCompletionRedactedThinkingBlock
|
||||
| ChatCompletionRefusalObject
|
||||
| AnthropicMessagesTextParam
|
||||
| AnthropicMessagesImageParam
|
||||
| AnthropicMessagesDocumentParam
|
||||
|
|
@ -1053,8 +1055,11 @@ def _count_content_list(
|
|||
for c in content_list:
|
||||
if isinstance(c, str):
|
||||
num_tokens += count_function(c)
|
||||
elif c["type"] == "text":
|
||||
num_tokens += count_function(str(c.get("text", "")))
|
||||
elif c["type"] in ("text", "refusal"):
|
||||
# `refusal` is OpenAI's content part for a turn the model declined.
|
||||
# It is plain text, so it counts like a text part - without this,
|
||||
# replaying a refused assistant message in the history blows up here.
|
||||
num_tokens += count_function(str(c.get("text") or c.get("refusal") or ""))
|
||||
elif c["type"] == "image_url":
|
||||
image_url = c.get("image_url")
|
||||
num_tokens += _count_image_tokens(image_url, use_default_image_token_count)
|
||||
|
|
@ -1107,7 +1112,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"tool_reference, search_result)."
|
||||
f"tool_reference, search_result, refusal)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -48156,6 +48156,29 @@
|
|||
"title": "ChatCompletionRedactedThinkingBlock",
|
||||
"type": "object"
|
||||
},
|
||||
"ChatCompletionRefusalObject": {
|
||||
"description": "Assistant content part holding the refusal text of a turn the model declined.",
|
||||
"properties": {
|
||||
"cache_control": {
|
||||
"$ref": "#/components/schemas/ChatCompletionCachedContent"
|
||||
},
|
||||
"refusal": {
|
||||
"title": "Refusal",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"const": "refusal",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"refusal"
|
||||
],
|
||||
"title": "ChatCompletionRefusalObject",
|
||||
"type": "object"
|
||||
},
|
||||
"ChatCompletionSystemMessage": {
|
||||
"properties": {
|
||||
"cache_control": {
|
||||
|
|
@ -48501,6 +48524,9 @@
|
|||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ChatCompletionFileObject"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ChatCompletionRefusalObject"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -703,6 +703,19 @@ class ChatCompletionRedactedThinkingBlock(TypedDict, total=False):
|
|||
index: ReadOnly[int]
|
||||
|
||||
|
||||
class OpenAIChatCompletionRefusalObject(TypedDict):
|
||||
"""Assistant content part holding the refusal text of a turn the model declined."""
|
||||
|
||||
type: Literal["refusal"]
|
||||
refusal: str
|
||||
|
||||
|
||||
class ChatCompletionRefusalObject(
|
||||
OpenAIChatCompletionRefusalObject, total=False
|
||||
): # litellm wrapper on top of openai object for handling cached content
|
||||
cache_control: ChatCompletionCachedContent
|
||||
|
||||
|
||||
class ChatCompletionReasoningSummaryTextBlock(TypedDict, total=False):
|
||||
type: Required[Literal["summary_text"]]
|
||||
text: str
|
||||
|
|
@ -869,6 +882,7 @@ OpenAIMessageContentListBlock = (
|
|||
| ChatCompletionDocumentObject
|
||||
| ChatCompletionVideoObject
|
||||
| ChatCompletionFileObject
|
||||
| ChatCompletionRefusalObject
|
||||
)
|
||||
|
||||
OpenAIMessageContent = str | Iterable[OpenAIMessageContentListBlock]
|
||||
|
|
|
|||
|
|
@ -2469,3 +2469,68 @@ def test_get_modified_max_tokens_leaves_a_gemini_4_long_decode_budget_alone(monk
|
|||
)
|
||||
== 200_000
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_token_counter_counts_assistant_refusal_content_part() -> None:
|
||||
"""
|
||||
An assistant history message can carry a `refusal` content part - it is part of
|
||||
OpenAI's `chat_completion_assistant_message_param` shape and models that refuse a
|
||||
turn replay it in the next request. token_counter used to reject it with
|
||||
`Invalid content item type: refusal`, which also turned /utils/token_counter into
|
||||
a 500 on the proxy.
|
||||
|
||||
A refusal part is plain text, so it should count exactly like the same text sent
|
||||
as a `text` part.
|
||||
"""
|
||||
refusal_text = "I cannot help with that request."
|
||||
refusal_messages = [
|
||||
{"role": "user", "content": "A request"},
|
||||
{"role": "assistant", "content": [{"type": "refusal", "refusal": refusal_text}]},
|
||||
{"role": "user", "content": "Please suggest a safe alternative."},
|
||||
]
|
||||
text_messages = [
|
||||
{"role": "user", "content": "A request"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": refusal_text}]},
|
||||
{"role": "user", "content": "Please suggest a safe alternative."},
|
||||
]
|
||||
|
||||
assert token_counter_new(model="gpt-4o", messages=refusal_messages) == token_counter_new(
|
||||
model="gpt-4o", messages=text_messages
|
||||
)
|
||||
|
||||
|
||||
def test_count_content_list_accepts_refusal_block() -> None:
|
||||
from litellm.litellm_core_utils.token_counter import _count_content_list
|
||||
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "refusal", "refusal": "no"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 2
|
||||
)
|
||||
|
||||
# a part without refusal text still parses, it just contributes nothing
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "refusal"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
# a text part keeps using `text` and is not matched by the refusal key
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "text", "text": "keep", "refusal": "ignored"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 4
|
||||
)
|
||||
|
|
|
|||
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
16
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -28953,6 +28953,20 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/**
|
||||
* ChatCompletionRefusalObject
|
||||
* @description Assistant content part holding the refusal text of a turn the model declined.
|
||||
*/
|
||||
ChatCompletionRefusalObject: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Refusal */
|
||||
refusal: string;
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "refusal";
|
||||
};
|
||||
/** ChatCompletionSystemMessage */
|
||||
ChatCompletionSystemMessage: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
|
|
@ -29091,7 +29105,7 @@ export interface components {
|
|||
ChatCompletionUserMessage: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Content */
|
||||
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"])[];
|
||||
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"] | components["schemas"]["ChatCompletionRefusalObject"])[];
|
||||
/**
|
||||
* Role
|
||||
* @constant
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue