From 5f6afa538fa20f0b9fba1e824daab8d719edf9fb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:13:02 +0000 Subject: [PATCH] test(cost): the stream builder bills into hidden response_cost and leaves usage.cost to the upstream --- .../azure_ai/test_azure_ai_cost_calculator.py | 10 +- tests/unit/test_main.py | 351 +++++++----------- 2 files changed, 149 insertions(+), 212 deletions(-) diff --git a/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py index 4f8ffa25abb..cef64f6284f 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -14,7 +14,6 @@ import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( - calculate_azure_model_router_flat_cost, cost_per_token, is_azure_model_router, ) @@ -263,7 +262,14 @@ class TestAzureModelRouterCostBreakdown: assert route.call_count == 1 assert json.loads(route.calls.last.request.content)["model"] == "model-router" assert (final_usage.prompt_tokens, final_usage.completion_tokens) == (5000, 2000) - assert final_usage.cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + assert getattr(final_usage, "cost", None) is None, "the bill reaches a client through the proxy egress only" + assembled: Final = litellm.stream_chunk_builder( + chunks=list(chunks), messages=messages, logging_obj=stream.logging_obj + ) + assert assembled is not None + assert assembled._hidden_params["response_cost"] == pytest.approx( + routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9 + ) @pytest.mark.respx(assert_all_called=True) def test_model_router_completion_reports_routed_cost_plus_flat_fee(self, respx_mock: respx.MockRouter) -> None: diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 1faf6dc12ad..13663cf4dd5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import TypeAdapter @@ -290,9 +290,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") - with patch.object( - client, "post", MagicMock(return_value=mock_raw_response) - ) as mock_create: + with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create: litellm.completion( model="gpt-4o-mini", messages=[ @@ -446,7 +444,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch): else: response = await acompletion(**args, client=client) print(response) - except Exception as e: + except Exception: pass mock_client.assert_called() @@ -2535,9 +2533,7 @@ def test_parallel_function_call_stream_reassembles_each_arguments_payload(): created=1751934860, model="gpt-4.1-mini", object="chat.completion.chunk", - choices=[ - StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason) - ], + choices=[StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)], ) chunks: Final = [ @@ -2668,9 +2664,7 @@ def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_d }, ) route: Final = respx.post("https://api.openai.com/v1/responses").mock( - return_value=httpx.Response( - 200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events) - ) + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) ) response: Final = litellm.completion( @@ -3284,9 +3278,7 @@ async def test_retrying() -> None: ) -def test_anthropic_disable_url_suffix_env_var( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: +def test_anthropic_disable_url_suffix_env_var(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock( return_value=httpx.Response( @@ -3549,9 +3541,7 @@ async def test_async_mock_streaming_request(): print(chunk) complete_response += chunk["choices"][0]["delta"]["content"] or "" - assert ( - complete_response == "LiteLLM is awesome" - ), f"Unexpected response got {complete_response}" + assert complete_response == "LiteLLM is awesome", f"Unexpected response got {complete_response}" def test_mock_request_n_greater_than_1(): @@ -4077,7 +4067,10 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" -_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} +_TOOL: Final[Mapping[str, object]] = { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}, +} # Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. _OPENAI_CLIENT_SETTINGS: Final = frozenset( @@ -4137,7 +4130,9 @@ _OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] def test_every_openai_body_param_has_a_forwarding_case(): - body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + body_params: Final = ( + frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + ) assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " @@ -4896,7 +4891,7 @@ def _stream_builder_logging_obj(model: str = "gpt-4o", custom_llm_provider: str return logging_obj -def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch): +def test_stream_chunk_builder_bills_a_stream_without_writing_into_its_usage(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), @@ -4908,10 +4903,8 @@ def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch ) assert response is not None - usage_cost: Final = getattr(response.usage, "cost", None) - assert usage_cost is not None - assert usage_cost > 0 - assert response._hidden_params["response_cost"] == pytest.approx(usage_cost) + assert getattr(response.usage, "cost", None) is None, "usage.cost is the upstream's field, the bill rides hidden" + assert response._hidden_params["response_cost"] > 0 def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost(): @@ -4942,7 +4935,8 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost() ) assert response is not None - usage_cost: Final = getattr(response.usage, "cost", None) + assert getattr(response.usage, "cost", None) is None + usage_cost: Final = response._hidden_params.get("response_cost") assert usage_cost is not None assert usage_cost > 0 @@ -4953,7 +4947,6 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost() assert breakdown["output_cost"] > 0 assert breakdown["input_cost"] + breakdown["output_cost"] == pytest.approx(usage_cost) assert spend_row_cost == pytest.approx(usage_cost) - assert response._hidden_params.get("response_cost") == pytest.approx(usage_cost) def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): @@ -5019,7 +5012,7 @@ def test_stream_chunk_builder_prices_alias_response_at_its_deployment_model(): entry: Final = litellm.model_cost[deployment_model] expected_cost: Final = 40 * entry["input_cost_per_token"] + 5 * entry["output_cost_per_token"] assert response is not None - assert getattr(response.usage, "cost", None) == pytest.approx(expected_cost) + assert getattr(response.usage, "cost", None) is None assert response._hidden_params.get("response_cost") == pytest.approx(expected_cost) @@ -5062,7 +5055,9 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): assert response._hidden_params["response_cost"] == pytest.approx(0.000704) -def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reported_cost(monkeypatch: pytest.MonkeyPatch): +def test_stream_chunk_builder_bills_the_margin_over_the_xai_reported_cost_it_leaves_in_usage( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5}) usage_chunk: Final = _stream_builder_text_chunk("grok-4", "") usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42) @@ -5078,7 +5073,7 @@ def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reporte ) assert response is not None - assert getattr(response.usage, "cost", None) == pytest.approx(0.63) + assert getattr(response.usage, "cost", None) == pytest.approx(0.42), "the upstream's figure stays in usage.cost" assert response._hidden_params.get("response_cost") == pytest.approx(0.63) assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.63) assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.63) @@ -5584,17 +5579,11 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None: def test_acompletion_params(): import inspect - from litellm.types.completion import CompletionRequest - acompletion_params_odict = inspect.signature(acompletion).parameters completion_params_dict = inspect.signature(completion).parameters - acompletion_params = { - name: param.annotation for name, param in acompletion_params_odict.items() - } - completion_params = { - name: param.annotation for name, param in completion_params_dict.items() - } + acompletion_params = {name: param.annotation for name, param in acompletion_params_odict.items()} + completion_params = {name: param.annotation for name, param in completion_params_dict.items()} keys_acompletion = set(acompletion_params.keys()) keys_completion = set(completion_params.keys()) @@ -5648,9 +5637,7 @@ def test_null_role_response(): import openai openai_client = openai.OpenAI() - with patch.object( - openai_client, "post", side_effect=_openai_mock_response - ) as mock_response: + with patch.object(openai_client, "post", side_effect=_openai_mock_response) as mock_response: response = litellm.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey! how's it going?"}], @@ -5748,7 +5735,8 @@ def test_completion_perplexity_api(): openai_client = OpenAI() with patch.object( - openai_client, "post", + openai_client, + "post", side_effect=_return_pydantic_obj, ) as mock_client: # litellm.set_verbose= True @@ -5791,12 +5779,10 @@ async def test_openai_compatible_custom_api_base(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( - model="{provider}/my-vllm-model".format(provider=provider), + model=f"{provider}/my-vllm-model", messages=messages, response_format={"type": "json_object"}, client=openai_client, @@ -5808,7 +5794,7 @@ async def test_openai_compatible_custom_api_base(provider): mock_call.assert_called_once() - print("Call KWARGS - {}".format(mock_call.call_args.kwargs)) + print(f"Call KWARGS - {mock_call.call_args.kwargs}") assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"] @@ -5842,12 +5828,10 @@ async def test_openai_compatible_custom_api_video(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( - model="{provider}/my-vllm-model".format(provider=provider), + model=f"{provider}/my-vllm-model", messages=messages, response_format={"type": "json_object"}, client=openai_client, @@ -6037,10 +6021,7 @@ async def test_completion_functions_param(): mock_client.assert_called() print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}") assert "tools" in mock_client.call_args.kwargs["json"] - assert ( - "litellm_param_is_function_call" - not in mock_client.call_args.kwargs["json"] - ) + assert "litellm_param_is_function_call" not in mock_client.call_args.kwargs["json"] assert response.choices[0].message.function_call is not None except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -6103,7 +6084,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): messages=messages, client=client, ) - except Exception as e: + except Exception: pass mock_post.assert_called_once() @@ -6116,9 +6097,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): def test_bedrock_deepseek_known_tokenizer_config(monkeypatch): - model = ( - "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" - ) + model = "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" from unittest.mock import Mock import httpx @@ -6189,9 +6168,7 @@ def test_completion_anthropic_hanging(): {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - converted_messages = anthropic_messages_pt( - messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" - ) + converted_messages = anthropic_messages_pt(messages, model="claude-3-sonnet-20240229", llm_provider="anthropic") assert len(converted_messages) == 3 for i, msg in enumerate(converted_messages): @@ -6498,9 +6475,7 @@ def test_completion_logprobs_parses_top_logprobs(): "top_logprobs": 3, "reasoning_effort": "none", } - assert [ - item.token for item in response.choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response.choices[0].logprobs.content[0].top_logprobs] == ["hello", "hi", "hey"] choice: Final = response.choices[0] assert "logprobs" in choice assert "content" in choice.logprobs @@ -6572,9 +6547,11 @@ def test_completion_logprobs_stream_parses_top_logprobs(): "stream_options": {"include_usage": True}, "stream": True, } - assert [ - item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs] == [ + "hello", + "hi", + "hey", + ] @respx.mock @@ -6645,9 +6622,7 @@ async def test_bedrock_converse_parses_scripted_response( "api_key": "test-bedrock-token", "aws_region_name": "us-west-2", } - response: Final = ( - await litellm.acompletion(**request) if async_mode else litellm.completion(**request) - ) + response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token" assert json.loads(route.calls[0].request.content) == { @@ -6734,9 +6709,7 @@ def test_bedrock_converse_parses_scripted_tool_call_arguments( tool_call: Final = response.choices[0].message.tool_calls[0] assert route.call_count == 1 assert tool_call.function.name == "trade" - assert json.loads(tool_call.function.arguments) == { - "orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}] - } + assert json.loads(tool_call.function.arguments) == {"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]} @pytest.mark.parametrize( @@ -6774,9 +6747,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( expected_request_body: dict[str, object], fake_provider_credentials: None, ): - route: Final = respx.post( - f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke" - ).mock(return_value=httpx.Response(200, json=response_body)) + route: Final = respx.post(f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke").mock( + return_value=httpx.Response(200, json=response_body) + ) response: Final = litellm.completion( model=model, @@ -6787,9 +6760,7 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( ) request: Final = route.calls[0].request - assert request.headers["authorization"].startswith( - "AWS4-HMAC-SHA256 Credential=unit-test/" - ) + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=unit-test/") assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"] assert request.headers["x-amz-date"].endswith("Z") assert json.loads(request.content) == expected_request_body @@ -6799,9 +6770,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( @respx.mock def test_completion_azure_ad_token_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -6816,9 +6787,9 @@ def test_completion_azure_ad_token_is_forwarded(openai_api_response): @respx.mock def test_completion_azure_extra_headers_are_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -6839,9 +6810,9 @@ def test_completion_azure_extra_headers_are_forwarded(openai_api_response): @respx.mock def test_completion_azure_api_key_argument_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) response: Final = litellm.completion( model="azure/gpt-6.1-sol", @@ -7498,9 +7469,7 @@ def test_completion_deep_infra(drop_params): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) if not drop_params: with pytest.raises(litellm.exceptions.UnsupportedParamsError): @@ -7564,9 +7533,7 @@ def test_completion_deep_infra_mistral(): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) response: Final = completion( model=model_name, @@ -7615,19 +7582,13 @@ def test_unified_auth_params(provider, model, project, region_name, token): translated_optional_params = litellm.utils.get_optional_params(**data) if provider == "azure": - special_auth_params = ( - litellm.AzureOpenAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AzureOpenAIConfig().get_mapped_special_auth_params() elif provider == "bedrock": - special_auth_params = ( - litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() elif provider == "vertex_ai": special_auth_params = litellm.VertexAIConfig().get_mapped_special_auth_params() elif provider == "watsonx": - special_auth_params = ( - litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() for param, value in special_auth_params.items(): assert param in data @@ -7659,9 +7620,7 @@ async def test_dynamic_azure_params(stream, sync_mode): mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF CLIENT IS USED (NO PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=mock_client - ) as mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=mock_client) as mock_client: try: # client.chat.completions.with_raw_response.create = mock_client if sync_mode: @@ -7690,9 +7649,7 @@ async def test_dynamic_azure_params(stream, sync_mode): new_mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF NEW CLIENT IS USED (PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=new_mock_client - ) as new_mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=new_mock_client) as new_mock_client: try: if sync_mode: _ = completion( @@ -7903,9 +7860,7 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse) - ) as mock_call: + with patch.object(openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)) as mock_call: mock_call.return_value.headers = {} mock_call.return_value.parse.return_value = litellm.ModelResponse( choices=[{"message": {"role": "assistant", "content": "Hello"}}] @@ -7922,10 +7877,7 @@ def test_completion_novita_ai(): assert response.choices[0].message.content == "Hello" # Verify model is passed correctly - assert ( - mock_call.call_args.kwargs["body"]["model"] - == "meta-llama/llama-3.3-70b-instruct" - ) + assert mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct" # Verify messages are passed correctly assert mock_call.call_args.kwargs["body"]["messages"] == messages @@ -7982,9 +7934,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): litellm.set_verbose = True monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features) - with patch.object( - client, "post", return_value=MagicMock() - ) as mock_completion: + with patch.object(client, "post", return_value=MagicMock()) as mock_completion: try: resp = litellm.completion( model="openai/gpt-3.5-turbo", @@ -7997,9 +7947,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): mock_completion.assert_called_once() if enable_preview_features: - assert mock_completion.call_args.kwargs["body"]["metadata"] == { - "my-test-key": "my-test-value" - } + assert mock_completion.call_args.kwargs["body"]["metadata"] == {"my-test-key": "my-test-value"} else: assert "metadata" not in mock_completion.call_args.kwargs["body"] @@ -8220,17 +8168,30 @@ async def test_async_responses_still_retries_provider_server_errors(monkeypatch) monkeypatch.setattr(litellm, "num_retries", None) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock as upstream: - response = upstream.post("https://openai-test.invalid/v1/responses").mock(side_effect=[ - httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), - httpx.Response(200, json={ - "id": "resp-retry", "object": "response", "created_at": 1, "status": "completed", - "model": "test-model", "output": [], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - }), - ]) + response = upstream.post("https://openai-test.invalid/v1/responses").mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), + httpx.Response( + 200, + json={ + "id": "resp-retry", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "test-model", + "output": [], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + }, + ), + ] + ) result = await litellm.aresponses( - model="openai/test-model", input="ping", api_key="test-key", - api_base="https://openai-test.invalid/v1", num_retries=1, max_retries=0, + model="openai/test-model", + input="ping", + api_key="test-key", + api_base="https://openai-test.invalid/v1", + num_retries=1, + max_retries=0, ) assert result.status == "completed" assert response.call_count == 2 @@ -8416,9 +8377,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "function_call"} - ], + "choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}], }, ) @@ -8442,9 +8401,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ), ) ) - route: Final = respx.post(f"{api_base}/chat/completions").mock( - side_effect=lambda _: next(scripted_responses) - ) + route: Final = respx.post(f"{api_base}/chat/completions").mock(side_effect=lambda _: next(scripted_responses)) stream: Final = litellm.completion( model="gpt-4o-mini", api_key="sk-test", @@ -8455,11 +8412,14 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ) answer_chunks: Final = tuple(stream) - assert "".join( - chunk.choices[0].delta.content - for chunk in answer_chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "The answer is 42." + assert ( + "".join( + chunk.choices[0].delta.content + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].delta.content + ) + == "The answer is 42." + ) assert tuple( chunk.choices[0].finish_reason for chunk in answer_chunks @@ -8476,12 +8436,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( assert answer_chunks[-1].usage.completion_tokens == 2 assert answer_chunks[-1].usage.total_tokens == 6 assert stream._hidden_params["api_base"] == api_base - assert ( - answer_chunks[0]._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ] - == "17" - ) + assert answer_chunks[0]._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "17" function_stream: Final = litellm.completion( model="gpt-4o-mini", @@ -8493,9 +8448,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( stream=True, ) function_chunks: Final = tuple(function_stream) - assembled: Final = litellm.stream_chunk_builder( - chunks=function_chunks, messages=messages - ) + assembled: Final = litellm.stream_chunk_builder(chunks=function_chunks, messages=messages) function_call: Final = assembled.choices[0].message.function_call assert function_call is not None @@ -8507,12 +8460,8 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( if chunk.choices and chunk.choices[0].finish_reason is not None ) == ("function_call",) assert route.call_count == 2 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } - assert json.loads(route.calls[1].request.content)["function_call"] == { - "name": "get_current_weather" - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} + assert json.loads(route.calls[1].request.content)["function_call"] == {"name": "get_current_weather"} assert json.loads(route.calls[1].request.content)["functions"] == list(functions) @@ -8551,15 +8500,9 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock( - return_value=httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) ) stream: Final = await litellm.acompletion( model="gpt-4o-mini", @@ -8570,11 +8513,10 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( ) chunks: Final = tuple([chunk async for chunk in stream]) - assert "".join( - chunk.choices[0].delta.content - for chunk in chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "async reply" + assert ( + "".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices and chunk.choices[0].delta.content) + == "async reply" + ) assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == ( None, None, @@ -8582,9 +8524,7 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( assert chunks[-1].usage.prompt_tokens == 3 assert chunks[-1].usage.completion_tokens == 2 assert chunks[-1].usage.total_tokens == 5 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} @respx.mock @@ -8621,16 +8561,10 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - return httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock(side_effect=respond) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) async def collect(prompt: str) -> tuple[str, tuple[str, ...]]: stream = await litellm.acompletion( @@ -8640,9 +8574,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( stream=True, ) chunks: Final = tuple([chunk async for chunk in stream]) - response_text: Final = "".join( - chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices - ) + response_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) finish_reasons: Final = tuple( chunk.choices[0].finish_reason for chunk in chunks @@ -8650,9 +8582,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( ) return response_text, finish_reasons - responses: Final = tuple( - await asyncio.gather(collect("request one"), collect("request two")) - ) + responses: Final = tuple(await asyncio.gather(collect("request one"), collect("request two"))) assert tuple(response for response, _ in responses) == ( "response one", @@ -8825,7 +8755,7 @@ def test_openai_streaming_tool_call_arguments_are_valid_json( @respx.mock -def test_streaming_openai_usage_includes_test_owned_cost( +def test_streaming_openai_usage_carries_no_cost_the_upstream_did_not_report( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -8846,9 +8776,7 @@ def test_streaming_openai_usage_includes_test_owned_cost( "object": "chat.completion.chunk", "created": 1, "model": model_name, - "choices": [ - {"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None} - ], + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}], }, { "id": "chatcmpl-cost", @@ -8883,12 +8811,10 @@ def test_streaming_openai_usage_includes_test_owned_cost( ) stream_chunks: Final = tuple(stream) usage_chunks: Final = tuple( - getattr(chunk, "usage", None) - for chunk in stream_chunks - if getattr(chunk, "usage", None) is not None + getattr(chunk, "usage", None) for chunk in stream_chunks if getattr(chunk, "usage", None) is not None ) assert len(usage_chunks) == 1 - assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002) + assert getattr(usage_chunks[0], "cost", None) is None, "the bill reaches a client through the proxy egress only" def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: respx.MockRouter) -> None: @@ -8901,9 +8827,7 @@ def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: "object": "chat.completion", "created": 0, "model": "databricks-meta-llama-3-1-70b-instruct", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 40, "completion_tokens": 1, "total_tokens": 41}, }, ) @@ -9113,9 +9037,12 @@ def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels( assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == ( "adaptive" ) - assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ - "reasoning_effort" - ] == "none" + assert ( + _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ + "reasoning_effort" + ] + == "none" + ) def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params( @@ -9250,7 +9177,9 @@ def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declarin thinking={"type": "enabled", "budget_tokens": 2048, **display}, **default_effort, ) - adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}) + adaptive_alone: Final = body_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display} + ) assert "output_config" not in defaulted assert defaulted == adaptive_alone @@ -9755,7 +9684,9 @@ async def test_aspeech_answers_a_non_string_provider_hint_with_a_bad_request( hint: str, respx_mock: respx.MockRouter ) -> None: with pytest.raises(litellm.BadRequestError) as raised: - await litellm.aspeech(model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123}) + await litellm.aspeech( + model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123} + ) assert raised.value.status_code == 400 assert f"Invalid {hint}" in raised.value.message @@ -9914,7 +9845,9 @@ async def test_converse_fallback_prices_the_current_attempt( } if asynchronous: with pytest.raises(litellm.BadRequestError): - await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) + await litellm.acompletion( + model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs + ) else: with pytest.raises(litellm.BadRequestError): litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) @@ -9938,9 +9871,7 @@ async def test_converse_fallback_prices_the_current_attempt( assert result is not None prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock") expected: Final = ( - 0.001 + 0.002 - if custom_price - else prices["input_cost_per_token"] + prices["output_cost_per_token"] + 0.001 + 0.002 if custom_price else prices["input_cost_per_token"] + prices["output_cost_per_token"] ) active_logging: Final = response.logging_obj if stream else logging_obj assert active_logging.response_cost_calculator(result) == pytest.approx(expected)