mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 74ad1d1d37 into 08f2a98250
This commit is contained in:
commit
d8add8aafe
5 changed files with 129 additions and 3 deletions
|
|
@ -76,6 +76,7 @@ from litellm.types.llms.openai import (
|
|||
AllMessageValues,
|
||||
ChatCompletionDocumentObject,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionRefusalObject,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
OpenAIMessageContentListBlock,
|
||||
|
|
@ -1812,7 +1813,16 @@ def _count_anthropic_content(
|
|||
|
||||
|
||||
_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType(
|
||||
{"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"}
|
||||
{
|
||||
"text": "text",
|
||||
"thinking": "thinking",
|
||||
"redacted_thinking": "thinking",
|
||||
"compaction": "content",
|
||||
# `refusal` is OpenAI's content part for a turn the model declined. It is
|
||||
# plain text, so it counts like a text part - without this, replaying a
|
||||
# refused assistant message in the history blows up here.
|
||||
"refusal": "refusal",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1823,6 +1833,7 @@ def _count_content_list(
|
|||
OpenAIMessageContentListBlock
|
||||
| ChatCompletionThinkingBlock
|
||||
| ChatCompletionRedactedThinkingBlock
|
||||
| ChatCompletionRefusalObject
|
||||
| AnthropicMessagesTextParam
|
||||
| AnthropicMessagesImageParam
|
||||
| AnthropicMessagesDocumentParam
|
||||
|
|
@ -1892,7 +1903,7 @@ def _count_content_list(
|
|||
f"Invalid content item type: {content_type}. "
|
||||
f"Expected str or dict with 'type' field "
|
||||
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
|
||||
f"compaction, tool_reference, search_result)."
|
||||
f"compaction, tool_reference, search_result, refusal)."
|
||||
)
|
||||
return num_tokens
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -49746,6 +49746,28 @@
|
|||
"title": "ChatCompletionRedactedThinkingBlock",
|
||||
"type": "object"
|
||||
},
|
||||
"ChatCompletionRefusalObject": {
|
||||
"properties": {
|
||||
"cache_control": {
|
||||
"$ref": "#/components/schemas/ChatCompletionCachedContent"
|
||||
},
|
||||
"refusal": {
|
||||
"title": "Refusal",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"const": "refusal",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"refusal"
|
||||
],
|
||||
"title": "ChatCompletionRefusalObject",
|
||||
"type": "object"
|
||||
},
|
||||
"ChatCompletionSystemMessage": {
|
||||
"properties": {
|
||||
"cache_control": {
|
||||
|
|
@ -50091,6 +50113,9 @@
|
|||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ChatCompletionFileObject"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ChatCompletionRefusalObject"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -723,6 +723,19 @@ class ChatCompletionRedactedThinkingBlock(TypedDict, total=False):
|
|||
index: ReadOnly[int]
|
||||
|
||||
|
||||
class OpenAIChatCompletionRefusalObject(TypedDict):
|
||||
"""Assistant content part holding the refusal text of a turn the model declined."""
|
||||
|
||||
type: ReadOnly[Literal["refusal"]]
|
||||
refusal: ReadOnly[str]
|
||||
|
||||
|
||||
class ChatCompletionRefusalObject(
|
||||
OpenAIChatCompletionRefusalObject, total=False
|
||||
): # litellm wrapper on top of openai object for handling cached content
|
||||
cache_control: ReadOnly[ChatCompletionCachedContent]
|
||||
|
||||
|
||||
class ChatCompletionReasoningSummaryTextBlock(TypedDict, total=False):
|
||||
type: Required[Literal["summary_text"]]
|
||||
text: str
|
||||
|
|
@ -889,6 +902,7 @@ OpenAIMessageContentListBlock = (
|
|||
| ChatCompletionDocumentObject
|
||||
| ChatCompletionVideoObject
|
||||
| ChatCompletionFileObject
|
||||
| ChatCompletionRefusalObject
|
||||
)
|
||||
|
||||
OpenAIMessageContent = str | Iterable[OpenAIMessageContentListBlock]
|
||||
|
|
|
|||
|
|
@ -3625,3 +3625,68 @@ def test_openai_model_check_holds_while_openai_models_are_registered_concurrentl
|
|||
registering.result()
|
||||
finally:
|
||||
litellm.open_ai_chat_completion_models.difference_update(registered)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_token_counter_counts_assistant_refusal_content_part() -> None:
|
||||
"""
|
||||
An assistant history message can carry a `refusal` content part - it is part of
|
||||
OpenAI's `chat_completion_assistant_message_param` shape and models that refuse a
|
||||
turn replay it in the next request. token_counter used to reject it with
|
||||
`Invalid content item type: refusal`, which also turned /utils/token_counter into
|
||||
a 500 on the proxy.
|
||||
|
||||
A refusal part is plain text, so it should count exactly like the same text sent
|
||||
as a `text` part.
|
||||
"""
|
||||
refusal_text = "I cannot help with that request."
|
||||
refusal_messages = [
|
||||
{"role": "user", "content": "A request"},
|
||||
{"role": "assistant", "content": [{"type": "refusal", "refusal": refusal_text}]},
|
||||
{"role": "user", "content": "Please suggest a safe alternative."},
|
||||
]
|
||||
text_messages = [
|
||||
{"role": "user", "content": "A request"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": refusal_text}]},
|
||||
{"role": "user", "content": "Please suggest a safe alternative."},
|
||||
]
|
||||
|
||||
assert token_counter_new(model="gpt-4o", messages=refusal_messages) == token_counter_new(
|
||||
model="gpt-4o", messages=text_messages
|
||||
)
|
||||
|
||||
|
||||
def test_count_content_list_accepts_refusal_block() -> None:
|
||||
from litellm.litellm_core_utils.token_counter import _count_content_list
|
||||
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "refusal", "refusal": "no"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 2
|
||||
)
|
||||
|
||||
# a part without refusal text still parses, it just contributes nothing
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "refusal"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
# a text part keeps using `text` and is not matched by the refusal key
|
||||
assert (
|
||||
_count_content_list(
|
||||
count_function=len,
|
||||
content_list=[{"type": "text", "text": "keep", "refusal": "ignored"}],
|
||||
use_default_image_token_count=False,
|
||||
default_token_count=None,
|
||||
)
|
||||
== 4
|
||||
)
|
||||
|
|
|
|||
13
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
13
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -29909,6 +29909,17 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** ChatCompletionRefusalObject */
|
||||
ChatCompletionRefusalObject: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Refusal */
|
||||
refusal: string;
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "refusal";
|
||||
};
|
||||
/** ChatCompletionSystemMessage */
|
||||
ChatCompletionSystemMessage: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
|
|
@ -30047,7 +30058,7 @@ export interface components {
|
|||
ChatCompletionUserMessage: {
|
||||
cache_control?: components["schemas"]["ChatCompletionCachedContent"];
|
||||
/** Content */
|
||||
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"])[];
|
||||
content: string | (components["schemas"]["ChatCompletionTextObject"] | components["schemas"]["ChatCompletionImageObject"] | components["schemas"]["ChatCompletionAudioObject"] | components["schemas"]["ChatCompletionDocumentObject"] | components["schemas"]["ChatCompletionVideoObject"] | components["schemas"]["ChatCompletionFileObject"] | components["schemas"]["ChatCompletionRefusalObject"])[];
|
||||
/**
|
||||
* Role
|
||||
* @constant
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue