mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-05 07:37:50 +00:00
test(cost): the stream builder bills into hidden response_cost and leaves usage.cost to the upstream
This commit is contained in:
devin-ai-integration[bot]
parent
b2e1394b0e
commit
5f6afa538f
2 changed files with 149 additions and 212 deletions
|
|
@ -14,7 +14,6 @@ import litellm
|
|||
from litellm.cost_calculator import completion_cost
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.azure_ai.cost_calculator import (
|
||||
calculate_azure_model_router_flat_cost,
|
||||
cost_per_token,
|
||||
is_azure_model_router,
|
||||
)
|
||||
|
|
@ -263,7 +262,14 @@ class TestAzureModelRouterCostBreakdown:
|
|||
assert route.call_count == 1
|
||||
assert json.loads(route.calls.last.request.content)["model"] == "model-router"
|
||||
assert (final_usage.prompt_tokens, final_usage.completion_tokens) == (5000, 2000)
|
||||
assert final_usage.cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9)
|
||||
assert getattr(final_usage, "cost", None) is None, "the bill reaches a client through the proxy egress only"
|
||||
assembled: Final = litellm.stream_chunk_builder(
|
||||
chunks=list(chunks), messages=messages, logging_obj=stream.logging_obj
|
||||
)
|
||||
assert assembled is not None
|
||||
assert assembled._hidden_params["response_cost"] == pytest.approx(
|
||||
routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9
|
||||
)
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_model_router_completion_reports_routed_cost_plus_flat_fee(self, respx_mock: respx.MockRouter) -> None:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import APITimeoutError, AsyncOpenAI, OpenAI
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -290,9 +290,7 @@ def test_completion_missing_role(openai_api_response):
|
|||
|
||||
print(f"openai_api_response: {openai_api_response}")
|
||||
|
||||
with patch.object(
|
||||
client, "post", MagicMock(return_value=mock_raw_response)
|
||||
) as mock_create:
|
||||
with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create:
|
||||
litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[
|
||||
|
|
@ -446,7 +444,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch):
|
|||
else:
|
||||
response = await acompletion(**args, client=client)
|
||||
print(response)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
mock_client.assert_called()
|
||||
|
|
@ -2535,9 +2533,7 @@ def test_parallel_function_call_stream_reassembles_each_arguments_payload():
|
|||
created=1751934860,
|
||||
model="gpt-4.1-mini",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)
|
||||
],
|
||||
choices=[StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)],
|
||||
)
|
||||
|
||||
chunks: Final = [
|
||||
|
|
@ -2668,9 +2664,7 @@ def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_d
|
|||
},
|
||||
)
|
||||
route: Final = respx.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)
|
||||
)
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events))
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
|
|
@ -3284,9 +3278,7 @@ async def test_retrying() -> None:
|
|||
)
|
||||
|
||||
|
||||
def test_anthropic_disable_url_suffix_env_var(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
def test_anthropic_disable_url_suffix_env_var(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com")
|
||||
default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock(
|
||||
return_value=httpx.Response(
|
||||
|
|
@ -3549,9 +3541,7 @@ async def test_async_mock_streaming_request():
|
|||
print(chunk)
|
||||
complete_response += chunk["choices"][0]["delta"]["content"] or ""
|
||||
|
||||
assert (
|
||||
complete_response == "LiteLLM is awesome"
|
||||
), f"Unexpected response got {complete_response}"
|
||||
assert complete_response == "LiteLLM is awesome", f"Unexpected response got {complete_response}"
|
||||
|
||||
|
||||
def test_mock_request_n_greater_than_1():
|
||||
|
|
@ -4077,7 +4067,10 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai():
|
|||
assert request_body["prompt_cache_key"] == "test-cache-key"
|
||||
|
||||
|
||||
_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}
|
||||
_TOOL: Final[Mapping[str, object]] = {
|
||||
"type": "function",
|
||||
"function": {"name": "f", "parameters": {"type": "object", "properties": {}}},
|
||||
}
|
||||
|
||||
# Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body.
|
||||
_OPENAI_CLIENT_SETTINGS: Final = frozenset(
|
||||
|
|
@ -4137,7 +4130,9 @@ _OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]]
|
|||
|
||||
|
||||
def test_every_openai_body_param_has_a_forwarding_case():
|
||||
body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS
|
||||
body_params: Final = (
|
||||
frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS
|
||||
)
|
||||
assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, (
|
||||
"a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving "
|
||||
"completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key "
|
||||
|
|
@ -4896,7 +4891,7 @@ def _stream_builder_logging_obj(model: str = "gpt-4o", custom_llm_provider: str
|
|||
return logging_obj
|
||||
|
||||
|
||||
def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_stream_chunk_builder_bills_a_stream_without_writing_into_its_usage(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("gpt-4o", "Hello "),
|
||||
|
|
@ -4908,10 +4903,8 @@ def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch
|
|||
)
|
||||
|
||||
assert response is not None
|
||||
usage_cost: Final = getattr(response.usage, "cost", None)
|
||||
assert usage_cost is not None
|
||||
assert usage_cost > 0
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(usage_cost)
|
||||
assert getattr(response.usage, "cost", None) is None, "usage.cost is the upstream's field, the bill rides hidden"
|
||||
assert response._hidden_params["response_cost"] > 0
|
||||
|
||||
|
||||
def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost():
|
||||
|
|
@ -4942,7 +4935,8 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost()
|
|||
)
|
||||
|
||||
assert response is not None
|
||||
usage_cost: Final = getattr(response.usage, "cost", None)
|
||||
assert getattr(response.usage, "cost", None) is None
|
||||
usage_cost: Final = response._hidden_params.get("response_cost")
|
||||
assert usage_cost is not None
|
||||
assert usage_cost > 0
|
||||
|
||||
|
|
@ -4953,7 +4947,6 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost()
|
|||
assert breakdown["output_cost"] > 0
|
||||
assert breakdown["input_cost"] + breakdown["output_cost"] == pytest.approx(usage_cost)
|
||||
assert spend_row_cost == pytest.approx(usage_cost)
|
||||
assert response._hidden_params.get("response_cost") == pytest.approx(usage_cost)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable():
|
||||
|
|
@ -5019,7 +5012,7 @@ def test_stream_chunk_builder_prices_alias_response_at_its_deployment_model():
|
|||
entry: Final = litellm.model_cost[deployment_model]
|
||||
expected_cost: Final = 40 * entry["input_cost_per_token"] + 5 * entry["output_cost_per_token"]
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(expected_cost)
|
||||
assert getattr(response.usage, "cost", None) is None
|
||||
assert response._hidden_params.get("response_cost") == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
|
|
@ -5062,7 +5055,9 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk():
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(0.000704)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reported_cost(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_stream_chunk_builder_bills_the_margin_over_the_xai_reported_cost_it_leaves_in_usage(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5})
|
||||
usage_chunk: Final = _stream_builder_text_chunk("grok-4", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42)
|
||||
|
|
@ -5078,7 +5073,7 @@ def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reporte
|
|||
)
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.63)
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.42), "the upstream's figure stays in usage.cost"
|
||||
assert response._hidden_params.get("response_cost") == pytest.approx(0.63)
|
||||
assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.63)
|
||||
assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.63)
|
||||
|
|
@ -5584,17 +5579,11 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None:
|
|||
def test_acompletion_params():
|
||||
import inspect
|
||||
|
||||
from litellm.types.completion import CompletionRequest
|
||||
|
||||
acompletion_params_odict = inspect.signature(acompletion).parameters
|
||||
completion_params_dict = inspect.signature(completion).parameters
|
||||
|
||||
acompletion_params = {
|
||||
name: param.annotation for name, param in acompletion_params_odict.items()
|
||||
}
|
||||
completion_params = {
|
||||
name: param.annotation for name, param in completion_params_dict.items()
|
||||
}
|
||||
acompletion_params = {name: param.annotation for name, param in acompletion_params_odict.items()}
|
||||
completion_params = {name: param.annotation for name, param in completion_params_dict.items()}
|
||||
|
||||
keys_acompletion = set(acompletion_params.keys())
|
||||
keys_completion = set(completion_params.keys())
|
||||
|
|
@ -5648,9 +5637,7 @@ def test_null_role_response():
|
|||
import openai
|
||||
|
||||
openai_client = openai.OpenAI()
|
||||
with patch.object(
|
||||
openai_client, "post", side_effect=_openai_mock_response
|
||||
) as mock_response:
|
||||
with patch.object(openai_client, "post", side_effect=_openai_mock_response) as mock_response:
|
||||
response = litellm.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hey! how's it going?"}],
|
||||
|
|
@ -5748,7 +5735,8 @@ def test_completion_perplexity_api():
|
|||
openai_client = OpenAI()
|
||||
|
||||
with patch.object(
|
||||
openai_client, "post",
|
||||
openai_client,
|
||||
"post",
|
||||
side_effect=_return_pydantic_obj,
|
||||
) as mock_client:
|
||||
# litellm.set_verbose= True
|
||||
|
|
@ -5791,12 +5779,10 @@ async def test_openai_compatible_custom_api_base(provider):
|
|||
|
||||
openai_client = OpenAI(api_key="fake-key")
|
||||
|
||||
with patch.object(
|
||||
openai_client, "post", new=MagicMock()
|
||||
) as mock_call:
|
||||
with patch.object(openai_client, "post", new=MagicMock()) as mock_call:
|
||||
try:
|
||||
completion(
|
||||
model="{provider}/my-vllm-model".format(provider=provider),
|
||||
model=f"{provider}/my-vllm-model",
|
||||
messages=messages,
|
||||
response_format={"type": "json_object"},
|
||||
client=openai_client,
|
||||
|
|
@ -5808,7 +5794,7 @@ async def test_openai_compatible_custom_api_base(provider):
|
|||
|
||||
mock_call.assert_called_once()
|
||||
|
||||
print("Call KWARGS - {}".format(mock_call.call_args.kwargs))
|
||||
print(f"Call KWARGS - {mock_call.call_args.kwargs}")
|
||||
|
||||
assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"]
|
||||
|
||||
|
|
@ -5842,12 +5828,10 @@ async def test_openai_compatible_custom_api_video(provider):
|
|||
|
||||
openai_client = OpenAI(api_key="fake-key")
|
||||
|
||||
with patch.object(
|
||||
openai_client, "post", new=MagicMock()
|
||||
) as mock_call:
|
||||
with patch.object(openai_client, "post", new=MagicMock()) as mock_call:
|
||||
try:
|
||||
completion(
|
||||
model="{provider}/my-vllm-model".format(provider=provider),
|
||||
model=f"{provider}/my-vllm-model",
|
||||
messages=messages,
|
||||
response_format={"type": "json_object"},
|
||||
client=openai_client,
|
||||
|
|
@ -6037,10 +6021,7 @@ async def test_completion_functions_param():
|
|||
mock_client.assert_called()
|
||||
print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}")
|
||||
assert "tools" in mock_client.call_args.kwargs["json"]
|
||||
assert (
|
||||
"litellm_param_is_function_call"
|
||||
not in mock_client.call_args.kwargs["json"]
|
||||
)
|
||||
assert "litellm_param_is_function_call" not in mock_client.call_args.kwargs["json"]
|
||||
assert response.choices[0].message.function_call is not None
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
@ -6103,7 +6084,7 @@ def test_bedrock_deepseek_custom_prompt_dict():
|
|||
messages=messages,
|
||||
client=client,
|
||||
)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
mock_post.assert_called_once()
|
||||
|
|
@ -6116,9 +6097,7 @@ def test_bedrock_deepseek_custom_prompt_dict():
|
|||
|
||||
|
||||
def test_bedrock_deepseek_known_tokenizer_config(monkeypatch):
|
||||
model = (
|
||||
"deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf"
|
||||
)
|
||||
model = "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf"
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
|
|
@ -6189,9 +6168,7 @@ def test_completion_anthropic_hanging():
|
|||
{"role": "function", "name": "get_capital", "content": "Kokoko"},
|
||||
]
|
||||
|
||||
converted_messages = anthropic_messages_pt(
|
||||
messages, model="claude-3-sonnet-20240229", llm_provider="anthropic"
|
||||
)
|
||||
converted_messages = anthropic_messages_pt(messages, model="claude-3-sonnet-20240229", llm_provider="anthropic")
|
||||
|
||||
assert len(converted_messages) == 3
|
||||
for i, msg in enumerate(converted_messages):
|
||||
|
|
@ -6498,9 +6475,7 @@ def test_completion_logprobs_parses_top_logprobs():
|
|||
"top_logprobs": 3,
|
||||
"reasoning_effort": "none",
|
||||
}
|
||||
assert [
|
||||
item.token for item in response.choices[0].logprobs.content[0].top_logprobs
|
||||
] == ["hello", "hi", "hey"]
|
||||
assert [item.token for item in response.choices[0].logprobs.content[0].top_logprobs] == ["hello", "hi", "hey"]
|
||||
choice: Final = response.choices[0]
|
||||
assert "logprobs" in choice
|
||||
assert "content" in choice.logprobs
|
||||
|
|
@ -6572,9 +6547,11 @@ def test_completion_logprobs_stream_parses_top_logprobs():
|
|||
"stream_options": {"include_usage": True},
|
||||
"stream": True,
|
||||
}
|
||||
assert [
|
||||
item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs
|
||||
] == ["hello", "hi", "hey"]
|
||||
assert [item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs] == [
|
||||
"hello",
|
||||
"hi",
|
||||
"hey",
|
||||
]
|
||||
|
||||
|
||||
@respx.mock
|
||||
|
|
@ -6645,9 +6622,7 @@ async def test_bedrock_converse_parses_scripted_response(
|
|||
"api_key": "test-bedrock-token",
|
||||
"aws_region_name": "us-west-2",
|
||||
}
|
||||
response: Final = (
|
||||
await litellm.acompletion(**request) if async_mode else litellm.completion(**request)
|
||||
)
|
||||
response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request)
|
||||
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token"
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
|
|
@ -6734,9 +6709,7 @@ def test_bedrock_converse_parses_scripted_tool_call_arguments(
|
|||
tool_call: Final = response.choices[0].message.tool_calls[0]
|
||||
assert route.call_count == 1
|
||||
assert tool_call.function.name == "trade"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]
|
||||
}
|
||||
assert json.loads(tool_call.function.arguments) == {"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -6774,9 +6747,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response(
|
|||
expected_request_body: dict[str, object],
|
||||
fake_provider_credentials: None,
|
||||
):
|
||||
route: Final = respx.post(
|
||||
f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke"
|
||||
).mock(return_value=httpx.Response(200, json=response_body))
|
||||
route: Final = respx.post(f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke").mock(
|
||||
return_value=httpx.Response(200, json=response_body)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
|
|
@ -6787,9 +6760,7 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response(
|
|||
)
|
||||
|
||||
request: Final = route.calls[0].request
|
||||
assert request.headers["authorization"].startswith(
|
||||
"AWS4-HMAC-SHA256 Credential=unit-test/"
|
||||
)
|
||||
assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=unit-test/")
|
||||
assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"]
|
||||
assert request.headers["x-amz-date"].endswith("Z")
|
||||
assert json.loads(request.content) == expected_request_body
|
||||
|
|
@ -6799,9 +6770,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response(
|
|||
|
||||
@respx.mock
|
||||
def test_completion_azure_ad_token_is_forwarded(openai_api_response):
|
||||
route: Final = respx.post(
|
||||
"https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions"
|
||||
).mock(return_value=httpx.Response(200, json=openai_api_response))
|
||||
route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=openai_api_response)
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="azure/gpt-6.1-sol",
|
||||
|
|
@ -6816,9 +6787,9 @@ def test_completion_azure_ad_token_is_forwarded(openai_api_response):
|
|||
|
||||
@respx.mock
|
||||
def test_completion_azure_extra_headers_are_forwarded(openai_api_response):
|
||||
route: Final = respx.post(
|
||||
"https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions"
|
||||
).mock(return_value=httpx.Response(200, json=openai_api_response))
|
||||
route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=openai_api_response)
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="azure/gpt-6.1-sol",
|
||||
|
|
@ -6839,9 +6810,9 @@ def test_completion_azure_extra_headers_are_forwarded(openai_api_response):
|
|||
|
||||
@respx.mock
|
||||
def test_completion_azure_api_key_argument_is_forwarded(openai_api_response):
|
||||
route: Final = respx.post(
|
||||
"https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions"
|
||||
).mock(return_value=httpx.Response(200, json=openai_api_response))
|
||||
route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=openai_api_response)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="azure/gpt-6.1-sol",
|
||||
|
|
@ -7498,9 +7469,7 @@ def test_completion_deep_infra(drop_params):
|
|||
],
|
||||
"usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25},
|
||||
}
|
||||
route: Final = respx.post(endpoint).mock(
|
||||
return_value=httpx.Response(200, json=response_body)
|
||||
)
|
||||
route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body))
|
||||
|
||||
if not drop_params:
|
||||
with pytest.raises(litellm.exceptions.UnsupportedParamsError):
|
||||
|
|
@ -7564,9 +7533,7 @@ def test_completion_deep_infra_mistral():
|
|||
],
|
||||
"usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25},
|
||||
}
|
||||
route: Final = respx.post(endpoint).mock(
|
||||
return_value=httpx.Response(200, json=response_body)
|
||||
)
|
||||
route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body))
|
||||
|
||||
response: Final = completion(
|
||||
model=model_name,
|
||||
|
|
@ -7615,19 +7582,13 @@ def test_unified_auth_params(provider, model, project, region_name, token):
|
|||
translated_optional_params = litellm.utils.get_optional_params(**data)
|
||||
|
||||
if provider == "azure":
|
||||
special_auth_params = (
|
||||
litellm.AzureOpenAIConfig().get_mapped_special_auth_params()
|
||||
)
|
||||
special_auth_params = litellm.AzureOpenAIConfig().get_mapped_special_auth_params()
|
||||
elif provider == "bedrock":
|
||||
special_auth_params = (
|
||||
litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params()
|
||||
)
|
||||
special_auth_params = litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params()
|
||||
elif provider == "vertex_ai":
|
||||
special_auth_params = litellm.VertexAIConfig().get_mapped_special_auth_params()
|
||||
elif provider == "watsonx":
|
||||
special_auth_params = (
|
||||
litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params()
|
||||
)
|
||||
special_auth_params = litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params()
|
||||
|
||||
for param, value in special_auth_params.items():
|
||||
assert param in data
|
||||
|
|
@ -7659,9 +7620,7 @@ async def test_dynamic_azure_params(stream, sync_mode):
|
|||
mock_client = AsyncMock(return_value="Hello world!")
|
||||
|
||||
## CHECK IF CLIENT IS USED (NO PARAM CHANGE)
|
||||
with patch.object(
|
||||
client.chat.completions.with_raw_response, "create", new=mock_client
|
||||
) as mock_client:
|
||||
with patch.object(client.chat.completions.with_raw_response, "create", new=mock_client) as mock_client:
|
||||
try:
|
||||
# client.chat.completions.with_raw_response.create = mock_client
|
||||
if sync_mode:
|
||||
|
|
@ -7690,9 +7649,7 @@ async def test_dynamic_azure_params(stream, sync_mode):
|
|||
new_mock_client = AsyncMock(return_value="Hello world!")
|
||||
|
||||
## CHECK IF NEW CLIENT IS USED (PARAM CHANGE)
|
||||
with patch.object(
|
||||
client.chat.completions.with_raw_response, "create", new=new_mock_client
|
||||
) as new_mock_client:
|
||||
with patch.object(client.chat.completions.with_raw_response, "create", new=new_mock_client) as new_mock_client:
|
||||
try:
|
||||
if sync_mode:
|
||||
_ = completion(
|
||||
|
|
@ -7903,9 +7860,7 @@ def test_completion_novita_ai():
|
|||
|
||||
openai_client = OpenAI(api_key="fake-key")
|
||||
|
||||
with patch.object(
|
||||
openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)
|
||||
) as mock_call:
|
||||
with patch.object(openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)) as mock_call:
|
||||
mock_call.return_value.headers = {}
|
||||
mock_call.return_value.parse.return_value = litellm.ModelResponse(
|
||||
choices=[{"message": {"role": "assistant", "content": "Hello"}}]
|
||||
|
|
@ -7922,10 +7877,7 @@ def test_completion_novita_ai():
|
|||
assert response.choices[0].message.content == "Hello"
|
||||
|
||||
# Verify model is passed correctly
|
||||
assert (
|
||||
mock_call.call_args.kwargs["body"]["model"]
|
||||
== "meta-llama/llama-3.3-70b-instruct"
|
||||
)
|
||||
assert mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct"
|
||||
# Verify messages are passed correctly
|
||||
assert mock_call.call_args.kwargs["body"]["messages"] == messages
|
||||
|
||||
|
|
@ -7982,9 +7934,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features):
|
|||
litellm.set_verbose = True
|
||||
|
||||
monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features)
|
||||
with patch.object(
|
||||
client, "post", return_value=MagicMock()
|
||||
) as mock_completion:
|
||||
with patch.object(client, "post", return_value=MagicMock()) as mock_completion:
|
||||
try:
|
||||
resp = litellm.completion(
|
||||
model="openai/gpt-3.5-turbo",
|
||||
|
|
@ -7997,9 +7947,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features):
|
|||
|
||||
mock_completion.assert_called_once()
|
||||
if enable_preview_features:
|
||||
assert mock_completion.call_args.kwargs["body"]["metadata"] == {
|
||||
"my-test-key": "my-test-value"
|
||||
}
|
||||
assert mock_completion.call_args.kwargs["body"]["metadata"] == {"my-test-key": "my-test-value"}
|
||||
else:
|
||||
assert "metadata" not in mock_completion.call_args.kwargs["body"]
|
||||
|
||||
|
|
@ -8220,17 +8168,30 @@ async def test_async_responses_still_retries_provider_server_errors(monkeypatch)
|
|||
monkeypatch.setattr(litellm, "num_retries", None)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock as upstream:
|
||||
response = upstream.post("https://openai-test.invalid/v1/responses").mock(side_effect=[
|
||||
httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}),
|
||||
httpx.Response(200, json={
|
||||
"id": "resp-retry", "object": "response", "created_at": 1, "status": "completed",
|
||||
"model": "test-model", "output": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},
|
||||
}),
|
||||
])
|
||||
response = upstream.post("https://openai-test.invalid/v1/responses").mock(
|
||||
side_effect=[
|
||||
httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}),
|
||||
httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp-retry",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "test-model",
|
||||
"output": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
result = await litellm.aresponses(
|
||||
model="openai/test-model", input="ping", api_key="test-key",
|
||||
api_base="https://openai-test.invalid/v1", num_retries=1, max_retries=0,
|
||||
model="openai/test-model",
|
||||
input="ping",
|
||||
api_key="test-key",
|
||||
api_base="https://openai-test.invalid/v1",
|
||||
num_retries=1,
|
||||
max_retries=0,
|
||||
)
|
||||
assert result.status == "completed"
|
||||
assert response.call_count == 2
|
||||
|
|
@ -8416,9 +8377,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "delta": {}, "finish_reason": "function_call"}
|
||||
],
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}],
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -8442,9 +8401,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
),
|
||||
)
|
||||
)
|
||||
route: Final = respx.post(f"{api_base}/chat/completions").mock(
|
||||
side_effect=lambda _: next(scripted_responses)
|
||||
)
|
||||
route: Final = respx.post(f"{api_base}/chat/completions").mock(side_effect=lambda _: next(scripted_responses))
|
||||
stream: Final = litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
api_key="sk-test",
|
||||
|
|
@ -8455,11 +8412,14 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
)
|
||||
answer_chunks: Final = tuple(stream)
|
||||
|
||||
assert "".join(
|
||||
chunk.choices[0].delta.content
|
||||
for chunk in answer_chunks
|
||||
if chunk.choices and chunk.choices[0].delta.content
|
||||
) == "The answer is 42."
|
||||
assert (
|
||||
"".join(
|
||||
chunk.choices[0].delta.content
|
||||
for chunk in answer_chunks
|
||||
if chunk.choices and chunk.choices[0].delta.content
|
||||
)
|
||||
== "The answer is 42."
|
||||
)
|
||||
assert tuple(
|
||||
chunk.choices[0].finish_reason
|
||||
for chunk in answer_chunks
|
||||
|
|
@ -8476,12 +8436,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
assert answer_chunks[-1].usage.completion_tokens == 2
|
||||
assert answer_chunks[-1].usage.total_tokens == 6
|
||||
assert stream._hidden_params["api_base"] == api_base
|
||||
assert (
|
||||
answer_chunks[0]._hidden_params["additional_headers"][
|
||||
"llm_provider-x-ratelimit-remaining-requests"
|
||||
]
|
||||
== "17"
|
||||
)
|
||||
assert answer_chunks[0]._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "17"
|
||||
|
||||
function_stream: Final = litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
|
|
@ -8493,9 +8448,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
stream=True,
|
||||
)
|
||||
function_chunks: Final = tuple(function_stream)
|
||||
assembled: Final = litellm.stream_chunk_builder(
|
||||
chunks=function_chunks, messages=messages
|
||||
)
|
||||
assembled: Final = litellm.stream_chunk_builder(chunks=function_chunks, messages=messages)
|
||||
function_call: Final = assembled.choices[0].message.function_call
|
||||
|
||||
assert function_call is not None
|
||||
|
|
@ -8507,12 +8460,8 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage(
|
|||
if chunk.choices and chunk.choices[0].finish_reason is not None
|
||||
) == ("function_call",)
|
||||
assert route.call_count == 2
|
||||
assert json.loads(route.calls[0].request.content)["stream_options"] == {
|
||||
"include_usage": True
|
||||
}
|
||||
assert json.loads(route.calls[1].request.content)["function_call"] == {
|
||||
"name": "get_current_weather"
|
||||
}
|
||||
assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True}
|
||||
assert json.loads(route.calls[1].request.content)["function_call"] == {"name": "get_current_weather"}
|
||||
assert json.loads(route.calls[1].request.content)["functions"] == list(functions)
|
||||
|
||||
|
||||
|
|
@ -8551,15 +8500,9 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk(
|
|||
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
|
||||
},
|
||||
)
|
||||
body: Final = "".join(
|
||||
f"data: {json.dumps(event)}\n\n" for event in events
|
||||
) + "data: [DONE]\n\n"
|
||||
route: Final = respx.post(
|
||||
"https://api.openai.com/v1/chat/completions"
|
||||
).mock(
|
||||
return_value=httpx.Response(
|
||||
200, text=body, headers={"content-type": "text/event-stream"}
|
||||
)
|
||||
body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n"
|
||||
route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
stream: Final = await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
|
|
@ -8570,11 +8513,10 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk(
|
|||
)
|
||||
chunks: Final = tuple([chunk async for chunk in stream])
|
||||
|
||||
assert "".join(
|
||||
chunk.choices[0].delta.content
|
||||
for chunk in chunks
|
||||
if chunk.choices and chunk.choices[0].delta.content
|
||||
) == "async reply"
|
||||
assert (
|
||||
"".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices and chunk.choices[0].delta.content)
|
||||
== "async reply"
|
||||
)
|
||||
assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == (
|
||||
None,
|
||||
None,
|
||||
|
|
@ -8582,9 +8524,7 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk(
|
|||
assert chunks[-1].usage.prompt_tokens == 3
|
||||
assert chunks[-1].usage.completion_tokens == 2
|
||||
assert chunks[-1].usage.total_tokens == 5
|
||||
assert json.loads(route.calls[0].request.content)["stream_options"] == {
|
||||
"include_usage": True
|
||||
}
|
||||
assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True}
|
||||
|
||||
|
||||
@respx.mock
|
||||
|
|
@ -8621,16 +8561,10 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate(
|
|||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
},
|
||||
)
|
||||
body: Final = "".join(
|
||||
f"data: {json.dumps(event)}\n\n" for event in events
|
||||
) + "data: [DONE]\n\n"
|
||||
return httpx.Response(
|
||||
200, text=body, headers={"content-type": "text/event-stream"}
|
||||
)
|
||||
body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n"
|
||||
return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"})
|
||||
|
||||
route: Final = respx.post(
|
||||
"https://api.openai.com/v1/chat/completions"
|
||||
).mock(side_effect=respond)
|
||||
route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond)
|
||||
|
||||
async def collect(prompt: str) -> tuple[str, tuple[str, ...]]:
|
||||
stream = await litellm.acompletion(
|
||||
|
|
@ -8640,9 +8574,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate(
|
|||
stream=True,
|
||||
)
|
||||
chunks: Final = tuple([chunk async for chunk in stream])
|
||||
response_text: Final = "".join(
|
||||
chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices
|
||||
)
|
||||
response_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices)
|
||||
finish_reasons: Final = tuple(
|
||||
chunk.choices[0].finish_reason
|
||||
for chunk in chunks
|
||||
|
|
@ -8650,9 +8582,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate(
|
|||
)
|
||||
return response_text, finish_reasons
|
||||
|
||||
responses: Final = tuple(
|
||||
await asyncio.gather(collect("request one"), collect("request two"))
|
||||
)
|
||||
responses: Final = tuple(await asyncio.gather(collect("request one"), collect("request two")))
|
||||
|
||||
assert tuple(response for response, _ in responses) == (
|
||||
"response one",
|
||||
|
|
@ -8825,7 +8755,7 @@ def test_openai_streaming_tool_call_arguments_are_valid_json(
|
|||
|
||||
|
||||
@respx.mock
|
||||
def test_streaming_openai_usage_includes_test_owned_cost(
|
||||
def test_streaming_openai_usage_carries_no_cost_the_upstream_did_not_report(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
|
@ -8846,9 +8776,7 @@ def test_streaming_openai_usage_includes_test_owned_cost(
|
|||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": model_name,
|
||||
"choices": [
|
||||
{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}
|
||||
],
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-cost",
|
||||
|
|
@ -8883,12 +8811,10 @@ def test_streaming_openai_usage_includes_test_owned_cost(
|
|||
)
|
||||
stream_chunks: Final = tuple(stream)
|
||||
usage_chunks: Final = tuple(
|
||||
getattr(chunk, "usage", None)
|
||||
for chunk in stream_chunks
|
||||
if getattr(chunk, "usage", None) is not None
|
||||
getattr(chunk, "usage", None) for chunk in stream_chunks if getattr(chunk, "usage", None) is not None
|
||||
)
|
||||
assert len(usage_chunks) == 1
|
||||
assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002)
|
||||
assert getattr(usage_chunks[0], "cost", None) is None, "the bill reaches a client through the proxy egress only"
|
||||
|
||||
|
||||
def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: respx.MockRouter) -> None:
|
||||
|
|
@ -8901,9 +8827,7 @@ def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock:
|
|||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "databricks-meta-llama-3-1-70b-instruct",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}
|
||||
],
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 40, "completion_tokens": 1, "total_tokens": 41},
|
||||
},
|
||||
)
|
||||
|
|
@ -9113,9 +9037,12 @@ def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels(
|
|||
assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == (
|
||||
"adaptive"
|
||||
)
|
||||
assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[
|
||||
"reasoning_effort"
|
||||
] == "none"
|
||||
assert (
|
||||
_chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[
|
||||
"reasoning_effort"
|
||||
]
|
||||
== "none"
|
||||
)
|
||||
|
||||
|
||||
def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params(
|
||||
|
|
@ -9250,7 +9177,9 @@ def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declarin
|
|||
thinking={"type": "enabled", "budget_tokens": 2048, **display},
|
||||
**default_effort,
|
||||
)
|
||||
adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display})
|
||||
adaptive_alone: Final = body_sent(
|
||||
respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}
|
||||
)
|
||||
|
||||
assert "output_config" not in defaulted
|
||||
assert defaulted == adaptive_alone
|
||||
|
|
@ -9755,7 +9684,9 @@ async def test_aspeech_answers_a_non_string_provider_hint_with_a_bad_request(
|
|||
hint: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
await litellm.aspeech(model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123})
|
||||
await litellm.aspeech(
|
||||
model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123}
|
||||
)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert f"Invalid {hint}" in raised.value.message
|
||||
|
|
@ -9914,7 +9845,9 @@ async def test_converse_fallback_prices_the_current_attempt(
|
|||
}
|
||||
if asynchronous:
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs)
|
||||
await litellm.acompletion(
|
||||
model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs
|
||||
)
|
||||
else:
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs)
|
||||
|
|
@ -9938,9 +9871,7 @@ async def test_converse_fallback_prices_the_current_attempt(
|
|||
assert result is not None
|
||||
prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock")
|
||||
expected: Final = (
|
||||
0.001 + 0.002
|
||||
if custom_price
|
||||
else prices["input_cost_per_token"] + prices["output_cost_per_token"]
|
||||
0.001 + 0.002 if custom_price else prices["input_cost_per_token"] + prices["output_cost_per_token"]
|
||||
)
|
||||
active_logging: Final = response.logging_obj if stream else logging_obj
|
||||
assert active_logging.response_cost_calculator(result) == pytest.approx(expected)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue