mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
move the content-part list tests into a region main does not touch so the branch merges cleanly
This commit is contained in:
parent
c33fb55168
commit
7a30307b7e
1 changed files with 118 additions and 120 deletions
|
|
@ -8,19 +8,17 @@ import httpx
|
|||
import pytest
|
||||
import respx
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm import ChatCompletionUsageBlock, stream_chunk_builder
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
ChunkProcessor,
|
||||
stream_chunk_builder_text_completion,
|
||||
)
|
||||
from litellm.types.utils import GenericStreamingChunk
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor, stream_chunk_builder_text_completion
|
||||
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
ChatCompletionMessageToolCall,
|
||||
Delta,
|
||||
Function,
|
||||
GenericStreamingChunk,
|
||||
ModelResponseStream,
|
||||
PromptTokensDetails,
|
||||
ServerToolUse,
|
||||
|
|
@ -198,7 +196,11 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks():
|
|||
}
|
||||
]
|
||||
),
|
||||
make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block1"}]),
|
||||
make_chunk(
|
||||
thinking_blocks=[
|
||||
{"type": "thinking", "thinking": None, "signature": "sig_block1"}
|
||||
]
|
||||
),
|
||||
make_chunk(
|
||||
thinking_blocks=[
|
||||
{
|
||||
|
|
@ -216,10 +218,16 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks():
|
|||
}
|
||||
]
|
||||
),
|
||||
make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block2"}]),
|
||||
make_chunk(
|
||||
thinking_blocks=[
|
||||
{"type": "thinking", "thinking": None, "signature": "sig_block2"}
|
||||
]
|
||||
),
|
||||
]
|
||||
|
||||
thinking_chunks = [chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")]
|
||||
thinking_chunks = [
|
||||
chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")
|
||||
]
|
||||
processor = ChunkProcessor(chunks=chunks)
|
||||
result = processor.get_combined_thinking_content(thinking_chunks)
|
||||
|
||||
|
|
@ -289,7 +297,9 @@ def test_cache_read_input_tokens_retained():
|
|||
prompt_tokens=11779,
|
||||
total_tokens=11784,
|
||||
completion_tokens_details=None,
|
||||
prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=11775),
|
||||
prompt_tokens_details=PromptTokensDetails(
|
||||
audio_tokens=None, cached_tokens=11775
|
||||
),
|
||||
cache_creation_input_tokens=4,
|
||||
cache_read_input_tokens=11775,
|
||||
),
|
||||
|
|
@ -323,7 +333,9 @@ def test_cache_read_input_tokens_retained():
|
|||
prompt_tokens=0,
|
||||
total_tokens=214,
|
||||
completion_tokens_details=None,
|
||||
prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0),
|
||||
prompt_tokens_details=PromptTokensDetails(
|
||||
audio_tokens=None, cached_tokens=0
|
||||
),
|
||||
cache_creation_input_tokens=0,
|
||||
cache_read_input_tokens=0,
|
||||
),
|
||||
|
|
@ -383,7 +395,10 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown():
|
|||
)
|
||||
# Sanity: the delta event genuinely lacks the breakdown - this is the input
|
||||
# condition that used to defeat cost calc.
|
||||
assert getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) is None
|
||||
assert (
|
||||
getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None)
|
||||
is None
|
||||
)
|
||||
|
||||
def _usage_chunk(usage, finish_reason):
|
||||
return ModelResponseStream(
|
||||
|
|
@ -502,7 +517,9 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk():
|
|||
prompt_tokens=1234,
|
||||
total_tokens=1239,
|
||||
completion_tokens_details=None,
|
||||
prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=543).model_dump(),
|
||||
prompt_tokens_details=PromptTokensDetails(
|
||||
audio_tokens=None, cached_tokens=543
|
||||
).model_dump(),
|
||||
),
|
||||
index=2,
|
||||
)
|
||||
|
|
@ -519,7 +536,6 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk():
|
|||
|
||||
assert usage.prompt_tokens_details.cached_tokens == 543
|
||||
|
||||
|
||||
def test_stream_chunk_builder_litellm_usage_chunks():
|
||||
"""
|
||||
Validate ChunkProcessor.calculate_usage uses provided usage fields from streaming chunks
|
||||
|
|
@ -593,7 +609,9 @@ def test_stream_chunk_builder_litellm_usage_chunks():
|
|||
chunks = [chunk1, chunk2]
|
||||
processor = ChunkProcessor(chunks=chunks)
|
||||
|
||||
usage = processor.calculate_usage(chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="")
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output=""
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 50
|
||||
assert usage.completion_tokens == 27
|
||||
|
|
@ -637,11 +655,15 @@ def test_calculate_usage_honors_openai_sdk_completion_usage_chunks():
|
|||
provider_specific_fields=None,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704)
|
||||
usage_chunk.usage = CompletionUsage(
|
||||
prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704
|
||||
)
|
||||
assert type(usage_chunk.usage) is CompletionUsage
|
||||
|
||||
chunks = [content_chunk, usage_chunk]
|
||||
usage = ChunkProcessor(chunks=chunks).calculate_usage(chunks=chunks, model="mantle-claude", completion_output="")
|
||||
usage = ChunkProcessor(chunks=chunks).calculate_usage(
|
||||
chunks=chunks, model="mantle-claude", completion_output=""
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 20
|
||||
assert usage.completion_tokens == 60
|
||||
|
|
@ -664,7 +686,9 @@ def test_get_model_from_chunks_azure_model_router():
|
|||
{"model": "gpt-4.1-nano-2025-04-14", "id": "chatcmpl-123", "choices": []},
|
||||
]
|
||||
|
||||
result = ChunkProcessor._get_model_from_chunks(chunks=chunks, first_chunk_model="azure-model-router")
|
||||
result = ChunkProcessor._get_model_from_chunks(
|
||||
chunks=chunks, first_chunk_model="azure-model-router"
|
||||
)
|
||||
|
||||
# Should return the actual model, not the request model
|
||||
assert result == "gpt-4.1-nano-2025-04-14"
|
||||
|
|
@ -675,7 +699,9 @@ def test_get_model_from_chunks_azure_model_router():
|
|||
{"model": "gpt-4", "id": "chatcmpl-456", "choices": []},
|
||||
]
|
||||
|
||||
result_same = ChunkProcessor._get_model_from_chunks(chunks=chunks_same_model, first_chunk_model="gpt-4")
|
||||
result_same = ChunkProcessor._get_model_from_chunks(
|
||||
chunks=chunks_same_model, first_chunk_model="gpt-4"
|
||||
)
|
||||
|
||||
# Should return the first chunk's model when all are the same
|
||||
assert result_same == "gpt-4"
|
||||
|
|
@ -751,7 +777,9 @@ def test_stream_chunk_builder_anthropic_web_search():
|
|||
chunks = [chunk1, chunk2]
|
||||
processor = ChunkProcessor(chunks=chunks)
|
||||
|
||||
usage = processor.calculate_usage(chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="")
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output=""
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 50
|
||||
assert usage.completion_tokens == 27
|
||||
|
|
@ -903,11 +931,15 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields():
|
|||
],
|
||||
)
|
||||
chunk_dict = chunk.model_dump()
|
||||
chunk_dict["_hidden_params"] = {"provider_specific_fields": {"traffic_type": "default"}}
|
||||
chunk_dict["_hidden_params"] = {
|
||||
"provider_specific_fields": {"traffic_type": "default"}
|
||||
}
|
||||
|
||||
response = stream_chunk_builder(chunks=[chunk_dict])
|
||||
assert response is not None
|
||||
assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default"
|
||||
assert (
|
||||
response._hidden_params["provider_specific_fields"]["traffic_type"] == "default"
|
||||
)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks():
|
||||
|
|
@ -952,7 +984,10 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks():
|
|||
assert getattr(response, "vertex_ai_grounding_metadata") == grounding_metadata
|
||||
assert getattr(response, "vertex_ai_url_context_metadata") == url_context_metadata
|
||||
assert response._hidden_params["vertex_ai_grounding_metadata"] == grounding_metadata
|
||||
assert response._hidden_params["vertex_ai_url_context_metadata"] == url_context_metadata
|
||||
assert (
|
||||
response._hidden_params["vertex_ai_url_context_metadata"]
|
||||
== url_context_metadata
|
||||
)
|
||||
|
||||
dumped = response.model_dump()
|
||||
assert dumped["vertex_ai_grounding_metadata"] == grounding_metadata
|
||||
|
|
@ -999,7 +1034,9 @@ def test_stream_chunk_builder_uses_assembled_model_for_provider_metadata():
|
|||
|
||||
def test_stream_chunk_builder_propagates_vertex_ai_safety_results():
|
||||
"""Assembled response must expose safety data under the non-streaming field name."""
|
||||
safety_ratings = [[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]]
|
||||
safety_ratings = [
|
||||
[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]
|
||||
]
|
||||
|
||||
chunk = ModelResponseStream(
|
||||
id="chatcmpl-vertex-safety",
|
||||
|
|
@ -1041,12 +1078,18 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks():
|
|||
)
|
||||
],
|
||||
).model_dump()
|
||||
chunk_dict["_hidden_params"] = {"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]}
|
||||
chunk_dict["_hidden_params"] = {
|
||||
"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]
|
||||
}
|
||||
|
||||
response = stream_chunk_builder(chunks=[chunk_dict])
|
||||
assert response is not None
|
||||
assert getattr(response, "vertex_ai_grounding_metadata") == [{"webSearchQueries": ["test query"]}]
|
||||
assert response.model_dump()["vertex_ai_grounding_metadata"] == [{"webSearchQueries": ["test query"]}]
|
||||
assert getattr(response, "vertex_ai_grounding_metadata") == [
|
||||
{"webSearchQueries": ["test query"]}
|
||||
]
|
||||
assert response.model_dump()["vertex_ai_grounding_metadata"] == [
|
||||
{"webSearchQueries": ["test query"]}
|
||||
]
|
||||
|
||||
|
||||
def test_cost_field_in_usage_chunks():
|
||||
|
|
@ -1055,21 +1098,29 @@ def test_cost_field_in_usage_chunks():
|
|||
id="chatcmpl-1",
|
||||
created=1745513206,
|
||||
model="openrouter/claude",
|
||||
choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))],
|
||||
choices=[
|
||||
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))
|
||||
],
|
||||
usage=chunk1_usage,
|
||||
)
|
||||
|
||||
chunk2_usage = Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025)
|
||||
chunk2_usage = Usage(
|
||||
completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025
|
||||
)
|
||||
chunk2 = ModelResponseStream(
|
||||
id="chatcmpl-1",
|
||||
created=1745513207,
|
||||
model="openrouter/claude",
|
||||
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))],
|
||||
choices=[
|
||||
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
|
||||
],
|
||||
usage=chunk2_usage,
|
||||
)
|
||||
|
||||
processor = ChunkProcessor(chunks=[chunk1, chunk2])
|
||||
usage = processor.calculate_usage(chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi")
|
||||
usage = processor.calculate_usage(
|
||||
chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi"
|
||||
)
|
||||
|
||||
assert hasattr(usage, "cost")
|
||||
assert usage.cost == 0.00025
|
||||
|
|
@ -1152,19 +1203,25 @@ def test_prompt_tokens_details_survive_later_usage_chunk_without_details():
|
|||
id="chatcmpl-1",
|
||||
created=1745513206,
|
||||
model="openai/gpt-5.6-sol",
|
||||
choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))],
|
||||
choices=[
|
||||
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))
|
||||
],
|
||||
usage=Usage(
|
||||
prompt_tokens=6017,
|
||||
completion_tokens=4,
|
||||
total_tokens=6021,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=6004, cache_write_tokens=10),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=6004, cache_write_tokens=10
|
||||
),
|
||||
),
|
||||
)
|
||||
chunk_without_details = ModelResponseStream(
|
||||
id="chatcmpl-1",
|
||||
created=1745513207,
|
||||
model="openai/gpt-5.6-sol",
|
||||
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))],
|
||||
choices=[
|
||||
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
|
||||
],
|
||||
usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021),
|
||||
)
|
||||
|
||||
|
|
@ -1502,7 +1559,9 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate(
|
|||
assert usage.completion_tokens_details.text_tokens == expected_text_tokens
|
||||
|
||||
|
||||
def _openai_chunk(choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None) -> dict[str, object]:
|
||||
def _openai_chunk(
|
||||
choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None
|
||||
) -> dict[str, object]:
|
||||
base: Final = {
|
||||
"id": "chatcmpl-lit6552",
|
||||
"object": "chat.completion.chunk",
|
||||
|
|
@ -1937,11 +1996,7 @@ def test_stream_chunk_builder_litellm_tool_call(respx_mock: respx.MockRouter) ->
|
|||
)
|
||||
|
||||
assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [
|
||||
{
|
||||
"function": {"arguments": '{"city":"Boston"}', "name": "get_weather"},
|
||||
"id": "call_weather",
|
||||
"type": "function",
|
||||
}
|
||||
{"function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, "id": "call_weather", "type": "function"}
|
||||
]
|
||||
assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (5, 3, 8)
|
||||
|
||||
|
|
@ -1963,9 +2018,7 @@ def test_stream_chunk_builder_litellm_tool_call_regular_message(respx_mock: resp
|
|||
|
||||
|
||||
def test_stream_chunk_builder_multiple_tool_calls() -> None:
|
||||
def tool_call_chunk(
|
||||
*tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None
|
||||
) -> ModelResponseStream:
|
||||
def tool_call_chunk(*tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None) -> ModelResponseStream:
|
||||
return _legacy_stream_chunk("openai/gpt-4o", tool_calls=tool_calls, finish_reason=finish_reason)
|
||||
|
||||
chunks: Final = (
|
||||
|
|
@ -1976,9 +2029,7 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None:
|
|||
),
|
||||
tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='{"base": 3, '))),
|
||||
tool_call_chunk(
|
||||
ChatCompletionDeltaToolCall(
|
||||
id="call_add", index=1, type="function", function=Function(name="add", arguments="")
|
||||
)
|
||||
ChatCompletionDeltaToolCall(id="call_add", index=1, type="function", function=Function(name="add", arguments=""))
|
||||
),
|
||||
tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='"exponent": 5}'))),
|
||||
tool_call_chunk(
|
||||
|
|
@ -1992,11 +2043,7 @@ def test_stream_chunk_builder_multiple_tool_calls() -> None:
|
|||
assert response is not None
|
||||
assert response.choices[0].finish_reason == "tool_calls"
|
||||
assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [
|
||||
{
|
||||
"function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"},
|
||||
"id": "call_exp",
|
||||
"type": "function",
|
||||
},
|
||||
{"function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, "id": "call_exp", "type": "function"},
|
||||
{"function": {"arguments": '{"x": 1, "y": 2}', "name": "add"}, "id": "call_add", "type": "function"},
|
||||
]
|
||||
|
||||
|
|
@ -2030,7 +2077,9 @@ def test_grok_bug() -> None:
|
|||
response: Final = stream_chunk_builder(chunks=chunks)
|
||||
|
||||
assert response is not None
|
||||
assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == {"location": "Boston"}
|
||||
assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == {
|
||||
"location": "Boston"
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_openai_audio_output_usage() -> None:
|
||||
|
|
@ -2147,9 +2196,7 @@ def test_stream_chunk_builder_keeps_usage_dict_set_on_a_typed_text_completion_ch
|
|||
|
||||
|
||||
@pytest.mark.parametrize("build", [litellm.stream_chunk_builder, litellm.stream_chunk_builder_text_completion])
|
||||
def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(
|
||||
build: Callable[..., TextCompletionResponse],
|
||||
) -> None:
|
||||
def test_stream_chunk_builder_rebuilds_dict_text_completion_chunks(build: Callable[..., TextCompletionResponse]) -> None:
|
||||
def chunk(choices: list[dict[str, object]], **extra: object) -> dict[str, object]:
|
||||
return {
|
||||
"id": "cmpl-1",
|
||||
|
|
@ -2286,22 +2333,6 @@ def test_stream_chunk_builder_rebuilds_each_text_completion_choice_by_index() ->
|
|||
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (1, 4)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
_text_completion_chunk(_text_choice(0, "Hello", 5, None)),
|
||||
_text_completion_chunk(_text_choice(0, " world", 10, None, "length")),
|
||||
]
|
||||
)
|
||||
|
||||
assert response.choices[0].logprobs.model_dump() == {
|
||||
"tokens": ["Hello", " world"],
|
||||
"token_logprobs": [-0.5, -0.5],
|
||||
"top_logprobs": None,
|
||||
"text_offset": [5, 10],
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_joins_content_part_list_delta():
|
||||
"""A provider streaming delta.content as a content-part list must aggregate as text."""
|
||||
chunks = [
|
||||
|
|
@ -2367,6 +2398,22 @@ def test_delta_content_as_str_reads_text_from_non_mapping_parts():
|
|||
assert delta_content_as_str([_TextPart()]) == "from attribute"
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_null_text_completion_logprobs_fields_null() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
_text_completion_chunk(_text_choice(0, "Hello", 5, None)),
|
||||
_text_completion_chunk(_text_choice(0, " world", 10, None, "length")),
|
||||
]
|
||||
)
|
||||
|
||||
assert response.choices[0].logprobs.model_dump() == {
|
||||
"tokens": ["Hello", " world"],
|
||||
"token_logprobs": [-0.5, -0.5],
|
||||
"top_logprobs": None,
|
||||
"text_offset": [5, 10],
|
||||
}
|
||||
|
||||
|
||||
def test_stream_chunk_builder_keeps_a_text_completion_stream_that_ends_on_a_chat_chunk() -> None:
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=[
|
||||
|
|
@ -2468,9 +2515,7 @@ def test_stream_chunk_builder_rebuilds_text_completion_choice_indexes_that_are_n
|
|||
]
|
||||
)
|
||||
|
||||
assert (
|
||||
tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected
|
||||
)
|
||||
assert tuple(choice.model_dump(include={"index", "text", "finish_reason"}) for choice in response.choices) == expected
|
||||
|
||||
|
||||
def test_stream_chunk_builder_text_completion_survives_uncountable_prompt() -> None:
|
||||
|
|
@ -2537,50 +2582,3 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
|
|||
assert response is not None
|
||||
assert response.choices[0].message.content == "Hello world"
|
||||
assert response.choices[0].message.reasoning_items == reasoning_items
|
||||
|
||||
|
||||
def test_stream_chunk_builder_carries_the_bedrock_guardrail_trace_and_records_it_on_the_logging_object(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A Bedrock stream reports its guardrail trace on the last chunk only; the built response carries it like the
|
||||
non-streaming response and the call bills it as guardrail cost."""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"bedrock/guardrails": {
|
||||
"litellm_provider": "bedrock",
|
||||
"mode": "guardrails",
|
||||
"guardrail_cost_per_unit": {"topicPolicyUnits": 0.00015},
|
||||
}
|
||||
},
|
||||
)
|
||||
trace = {"guardrail": {"inputAssessment": {"gid": {"invocationMetrics": {"usage": {"topicPolicyUnits": 2}}}}}}
|
||||
chunks = [
|
||||
ModelResponseStream(id="s", model="m", choices=[StreamingChoices(index=0, delta=Delta(content="hi"))]),
|
||||
ModelResponseStream(
|
||||
id="s",
|
||||
model="m",
|
||||
choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")],
|
||||
provider_specific_fields={"trace": trace},
|
||||
),
|
||||
]
|
||||
logging_obj = Logging(
|
||||
model="bedrock/m",
|
||||
messages=[{"role": "user", "content": "q"}],
|
||||
stream=True,
|
||||
call_type="acompletion",
|
||||
start_time=None,
|
||||
litellm_call_id="call",
|
||||
function_id="fn",
|
||||
)
|
||||
logging_obj.model_call_details["litellm_params"] = {"metadata": {}}
|
||||
built = stream_chunk_builder(chunks=chunks, messages=[{"role": "user", "content": "q"}], logging_obj=logging_obj)
|
||||
assert built is not None
|
||||
assert getattr(built, "trace") == trace
|
||||
(entry,) = logging_obj.model_call_details["litellm_params"]["metadata"]["standard_logging_guardrail_information"]
|
||||
assert entry["guardrail_usage"] == {"topicPolicyUnits": 2}
|
||||
assert entry["guardrail_cost"] == pytest.approx(2 * 0.00015)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue