diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 323af72940a..f46563de15b 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -76,6 +76,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionDocumentObject, ChatCompletionRedactedThinkingBlock, + ChatCompletionRefusalObject, ChatCompletionThinkingBlock, ChatCompletionToolParam, OpenAIMessageContentListBlock, @@ -1672,7 +1673,16 @@ def _count_anthropic_content( _TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( - {"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"} + { + "text": "text", + "thinking": "thinking", + "redacted_thinking": "thinking", + "compaction": "content", + # `refusal` is OpenAI's content part for a turn the model declined. It is + # plain text, so it counts like a text part - without this, replaying a + # refused assistant message in the history blows up here. + "refusal": "refusal", + } ) @@ -1683,6 +1693,7 @@ def _count_content_list( OpenAIMessageContentListBlock | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock + | ChatCompletionRefusalObject | AnthropicMessagesTextParam | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam @@ -1752,7 +1763,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"compaction, tool_reference, search_result)." + f"compaction, tool_reference, search_result, refusal)." ) return num_tokens except Exception as e: diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 546022bc0c4..860104671c5 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -48815,6 +48815,28 @@ "title": "ChatCompletionRedactedThinkingBlock", "type": "object" }, + "ChatCompletionRefusalObject": { + "properties": { + "cache_control": { + "$ref": "#/components/schemas/ChatCompletionCachedContent" + }, + "refusal": { + "title": "Refusal", + "type": "string" + }, + "type": { + "const": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "refusal" + ], + "title": "ChatCompletionRefusalObject", + "type": "object" + }, "ChatCompletionSystemMessage": { "properties": { "cache_control": { @@ -49160,6 +49182,9 @@ }, { "$ref": "#/components/schemas/ChatCompletionFileObject" + }, + { + "$ref": "#/components/schemas/ChatCompletionRefusalObject" } ] }, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index d7e55b57a07..2f304aec45d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -712,6 +712,19 @@ class ChatCompletionRedactedThinkingBlock(TypedDict, total=False): index: ReadOnly[int] +class OpenAIChatCompletionRefusalObject(TypedDict): + """Assistant content part holding the refusal text of a turn the model declined.""" + + type: ReadOnly[Literal["refusal"]] + refusal: ReadOnly[str] + + +class ChatCompletionRefusalObject( + OpenAIChatCompletionRefusalObject, total=False +): # litellm wrapper on top of openai object for handling cached content + cache_control: ReadOnly[ChatCompletionCachedContent] + + class ChatCompletionReasoningSummaryTextBlock(TypedDict, total=False): type: Required[Literal["summary_text"]] text: str @@ -878,6 +891,7 @@ OpenAIMessageContentListBlock = ( | ChatCompletionDocumentObject | ChatCompletionVideoObject | ChatCompletionFileObject + | ChatCompletionRefusalObject ) OpenAIMessageContent = str | Iterable[OpenAIMessageContentListBlock] diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 697ac76b4e5..2028e755b32 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -3570,3 +3570,68 @@ def test_openai_model_check_holds_while_openai_models_are_registered_concurrentl registering.result() finally: litellm.open_ai_chat_completion_models.difference_update(registered) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_counter_counts_assistant_refusal_content_part() -> None: + """ + An assistant history message can carry a `refusal` content part - it is part of + OpenAI's `chat_completion_assistant_message_param` shape and models that refuse a + turn replay it in the next request. token_counter used to reject it with + `Invalid content item type: refusal`, which also turned /utils/token_counter into + a 500 on the proxy. + + A refusal part is plain text, so it should count exactly like the same text sent + as a `text` part. + """ + refusal_text = "I cannot help with that request." + refusal_messages = [ + {"role": "user", "content": "A request"}, + {"role": "assistant", "content": [{"type": "refusal", "refusal": refusal_text}]}, + {"role": "user", "content": "Please suggest a safe alternative."}, + ] + text_messages = [ + {"role": "user", "content": "A request"}, + {"role": "assistant", "content": [{"type": "text", "text": refusal_text}]}, + {"role": "user", "content": "Please suggest a safe alternative."}, + ] + + assert token_counter_new(model="gpt-4o", messages=refusal_messages) == token_counter_new( + model="gpt-4o", messages=text_messages + ) + + +def test_count_content_list_accepts_refusal_block() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "refusal", "refusal": "no"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 2 + ) + + # a part without refusal text still parses, it just contributes nothing + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "refusal"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 0 + ) + + # a text part keeps using `text` and is not matched by the refusal key + assert ( + _count_content_list( + count_function=len, + content_list=[{"type": "text", "text": "keep", "refusal": "ignored"}], + use_default_image_token_count=False, + default_token_count=None, + ) + == 4 + ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2082dac6ade..5b4917eb3b6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29111,6 +29111,17 @@ export interface components { } & { [key: string]: unknown; }; + /** ChatCompletionRefusalObject */ + ChatCompletionRefusalObject: { + cache_control?: components["schemas"]["ChatCompletionCachedContent"]; + /** Refusal */ + refusal: string; + /** + * Type + * @constant + */ + type: "refusal"; + }; /** ChatCompletionSystemMessage */ ChatCompletionSystemMessage: { cache_control?: components["schemas"]["ChatCompletionCachedContent"]; @@ -29249,7 +29260,7 @@ export interface components { ChatCompletionUserMessage: { cache_control?: components["schemas"]["ChatCompletionCachedContent"]; /** Content */ - content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"])[]; + content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"] | components["schemas"]["ChatCompletionRefusalObject"])[]; /** * Role * @constant