diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 57ffe28a4b5..a32b5ebb2a8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -213,7 +213,6 @@ jobs: test-path: >- tests/local_testing/test_cache_preset_key.py tests/local_testing/test_caching_handler.py - tests/local_testing/test_prompt_caching.py tests/local_testing/test_responses_stream_cache_keys.py tests/local_testing/test_unit_test_caching.py workers: 2 diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 72713a36831..e8b3862756f 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -34,9 +34,6 @@ from unittest.mock import AsyncMock, MagicMock, patch # Assuming your trim_messages, shorten_message_to_fit_limit, and get_token_count functions are all in a module named 'message_utils' - - -# Test 1: Check trimming of normal message @pytest.fixture(autouse=True) def reset_mock_cache(): from litellm.utils import _model_cache @@ -44,6 +41,7 @@ def reset_mock_cache(): _model_cache.flush_cache() +# Test 1: Check trimming of normal message def test_basic_trimming(): litellm._turn_on_debug() messages = [ @@ -73,7 +71,9 @@ def test_basic_trimming_no_max_tokens_specified(): print("trimmed messages for gpt-4") print(trimmed_messages) # print(get_token_count(messages=trimmed_messages, model="claude-2")) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_tokens"] + assert ( + get_token_count(messages=trimmed_messages, model="gpt-4") + ) <= litellm.model_cost["gpt-4"]["max_tokens"] # test_basic_trimming_no_max_tokens_specified() @@ -90,7 +90,9 @@ def test_multiple_messages_trimming(): "content": "This is another long message that will also exceed the limit.", }, ] - trimmed_messages = trim_messages(messages=messages, model="gpt-3.5-turbo", max_tokens=20) + trimmed_messages = trim_messages( + messages=messages, model="gpt-3.5-turbo", max_tokens=20 + ) # print(get_token_count(messages=trimmed_messages, model="gpt-3.5-turbo")) assert (get_token_count(messages=trimmed_messages, model="gpt-3.5-turbo")) <= 20 @@ -109,7 +111,9 @@ def test_multiple_messages_no_trimming(): "content": "This is another long message that will also exceed the limit.", }, ] - trimmed_messages = trim_messages(messages=messages, model="gpt-3.5-turbo", max_tokens=100) + trimmed_messages = trim_messages( + messages=messages, model="gpt-3.5-turbo", max_tokens=100 + ) print("Trimmed messages") print(trimmed_messages) assert messages == trimmed_messages @@ -136,7 +140,9 @@ def test_large_trimming_multiple_messages(): def test_large_trimming_single_message(): - messages = [{"role": "user", "content": "This is a singlelongwordthatexceedsthelimit."}] + messages = [ + {"role": "user", "content": "This is a singlelongwordthatexceedsthelimit."} + ] trimmed_messages = trim_messages(messages, max_tokens=5, model="gpt-4-0613") assert (get_token_count(messages=trimmed_messages, model="gpt-4-0613")) <= 5 assert (get_token_count(messages=trimmed_messages, model="gpt-4-0613")) > 0 @@ -267,7 +273,10 @@ def test_trimming_with_model_cost_max_input_tokens(model): }, ] trimmed_messages = trim_messages(messages, model=model) - assert get_token_count(trimmed_messages, model=model) < litellm.model_cost[model]["max_input_tokens"] + assert ( + get_token_count(trimmed_messages, model=model) + < litellm.model_cost[model]["max_input_tokens"] + ) def test_trimming_with_untokenizable_field(caplog: pytest.LogCaptureFixture) -> None: @@ -320,7 +329,9 @@ def test_aget_valid_models(): print(valid_models) # list of openai supported llms on litellm - expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models + expected_models = ( + litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models + ) assert set(valid_models) == set(expected_models) @@ -342,7 +353,9 @@ def test_get_valid_models_with_custom_llm_provider(custom_llm_provider): provider=LlmProviders(custom_llm_provider), ) assert provider_config is not None - valid_models = get_valid_models(check_provider_endpoint=True, custom_llm_provider=custom_llm_provider) + valid_models = get_valid_models( + check_provider_endpoint=True, custom_llm_provider=custom_llm_provider + ) print(valid_models) assert len(valid_models) > 0 assert set(provider_config.get_models()) == set(valid_models) @@ -375,7 +388,9 @@ def test_validate_environment_empty_model(): def test_validate_environment_api_key(): response_obj = validate_environment(model="gpt-5-mini", api_key="sk-my-test-key") - assert response_obj["keys_in_environment"] is True, f"Missing keys={response_obj['missing_keys']}" + assert ( + response_obj["keys_in_environment"] is True + ), f"Missing keys={response_obj['missing_keys']}" def test_validate_environment_api_version(): @@ -385,7 +400,9 @@ def test_validate_environment_api_version(): api_base="https://fake.openai.azure.com/", api_version="2024-02-15", ) - assert response_obj["keys_in_environment"] is True, f"Missing keys={response_obj['missing_keys']}" + assert ( + response_obj["keys_in_environment"] is True + ), f"Missing keys={response_obj['missing_keys']}" def test_validate_environment_api_base_dynamic(): @@ -460,14 +477,18 @@ def test_function_to_dict(): assert function_json["description"] == expected_output["description"] assert function_json["parameters"]["type"] == expected_output["parameters"]["type"] assert ( - function_json["parameters"]["properties"]["location"] == expected_output["parameters"]["properties"]["location"] + function_json["parameters"]["properties"]["location"] + == expected_output["parameters"]["properties"]["location"] ) # the enum can change it can be - which is why we don't assert on unit # {'type': 'string', 'description': 'Temperature unit', 'enum': "['fahrenheit', 'celsius']"} # {'type': 'string', 'description': 'Temperature unit', 'enum': "['celsius', 'fahrenheit']"} - assert function_json["parameters"]["required"] == expected_output["parameters"]["required"] + assert ( + function_json["parameters"]["required"] + == expected_output["parameters"]["required"] + ) print("passed") @@ -509,7 +530,9 @@ def test_get_chat_completion_prompt(): prompt_variables=None, ) - assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}] + assert litellm_logging_obj.messages == [ + {"role": "user", "content": updated_message} + ] def test_redact_msgs_from_logs(): @@ -581,7 +604,9 @@ def test_redact_embedding_response(): litellm.turn_off_message_logging = True # Create a test EmbeddingResponse with usage data - original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) + original_usage = litellm.Usage( + prompt_tokens=10, completion_tokens=0, total_tokens=10 + ) original_data = [ {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]}, {"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]}, @@ -617,7 +642,9 @@ def test_redact_embedding_response(): # Assert the redacted response preserves critical metadata assert _redacted_response_obj.usage == original_usage # usage should be preserved - assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved + assert ( + _redacted_response_obj.model == "text-embedding-3-small" + ) # model should be preserved assert _redacted_response_obj.object == "list" # object should be preserved # Assert sensitive data is cleared @@ -671,8 +698,12 @@ def test_redact_msgs_from_logs_with_dynamic_params(): ) # Test Case 1: standard_callback_dynamic_params = False (or not set) - standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params + standard_callback_dynamic_params = StandardCallbackDynamicParams( + turn_off_message_logging=False + ) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + standard_callback_dynamic_params + ) _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -681,8 +712,12 @@ def test_redact_msgs_from_logs_with_dynamic_params(): assert _redacted_response_obj.choices[0].message.content == test_content # Test Case 2: standard_callback_dynamic_params = True - standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params + standard_callback_dynamic_params = StandardCallbackDynamicParams( + turn_off_message_logging=True + ) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + standard_callback_dynamic_params + ) _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -693,7 +728,9 @@ def test_redact_msgs_from_logs_with_dynamic_params(): # Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging # since litellm.turn_off_message_logging is True redaction should occur standard_callback_dynamic_params = StandardCallbackDynamicParams() - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( + standard_callback_dynamic_params + ) _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -798,7 +835,9 @@ def test_get_llm_provider_ft_models(): @pytest.mark.parametrize("langfuse_trace_id", [None, "my-unique-trace-id"]) -@pytest.mark.parametrize("langfuse_existing_trace_id", [None, "my-unique-existing-trace-id"]) +@pytest.mark.parametrize( + "langfuse_existing_trace_id", [None, "my-unique-existing-trace-id"] +) def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): """ - Unit test for `_get_trace_id` function in Logging obj @@ -837,13 +876,22 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): ## if existing_trace_id exists if langfuse_existing_trace_id is not None: - assert litellm_logging_obj._get_trace_id(service_name="langfuse") == langfuse_existing_trace_id + assert ( + litellm_logging_obj._get_trace_id(service_name="langfuse") + == langfuse_existing_trace_id + ) ## if trace_id exists elif langfuse_trace_id is not None: - assert litellm_logging_obj._get_trace_id(service_name="langfuse") == langfuse_trace_id + assert ( + litellm_logging_obj._get_trace_id(service_name="langfuse") + == langfuse_trace_id + ) ## if no trace_id or existing_trace_id is provided, use litellm_trace_id else: - assert litellm_logging_obj._get_trace_id(service_name="langfuse") == litellm_logging_obj.litellm_trace_id + assert ( + litellm_logging_obj._get_trace_id(service_name="langfuse") + == litellm_logging_obj.litellm_trace_id + ) def test_convert_model_response_object(): @@ -966,7 +1014,9 @@ def test_async_http_handler(mock_async_client): concurrent_limit = 2 # Mock the transport creation to return a specific transport - with mock.patch.object(AsyncHTTPHandler, "_create_async_transport") as mock_create_transport: + with mock.patch.object( + AsyncHTTPHandler, "_create_async_transport" + ) as mock_create_transport: mock_transport = mock.MagicMock() mock_create_transport.return_value = mock_transport @@ -1073,7 +1123,9 @@ def test_is_base64_encoded_2(): [ { "role": "user", - "content": [{"type": "image_url", "url": "https://example.com/image.png"}], + "content": [ + {"type": "image_url", "url": "https://example.com/image.png"} + ], } ], True, @@ -1149,7 +1201,10 @@ def test_models_by_provider(): continue elif k == "sample_spec": continue - elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse": + elif ( + v["litellm_provider"] == "sagemaker" + or v["litellm_provider"] == "bedrock_converse" + ): continue elif v.get("mode") in ("search", "evaluation"): continue @@ -1157,7 +1212,9 @@ def test_models_by_provider(): providers.add(v["litellm_provider"]) for provider in providers: - assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider) + assert provider in models_by_provider.keys() or JSONProviderRegistry.exists( + provider + ) @pytest.mark.parametrize( @@ -1168,11 +1225,16 @@ def test_models_by_provider(): ({"user_api_key_end_user_id": "123"}, True, None), ], ) -def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id): +def test_get_end_user_id_for_cost_tracking( + litellm_params, disable_end_user_cost_tracking, expected_end_user_id +): from litellm.utils import get_end_user_id_for_cost_tracking litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking - assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id + assert ( + get_end_user_id_for_cost_tracking(litellm_params=litellm_params) + == expected_end_user_id + ) @pytest.mark.parametrize( @@ -1188,9 +1250,13 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ): from litellm.utils import get_end_user_id_for_cost_tracking - litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only + litellm.enable_end_user_cost_tracking_prometheus_only = ( + enable_end_user_cost_tracking_prometheus_only + ) assert ( - get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus") + get_end_user_id_for_cost_tracking( + litellm_params=litellm_params, service_type="prometheus" + ) == expected_end_user_id ) @@ -1205,14 +1271,20 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ), # Test with only litellm_metadata field (new behavior) ( - {"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}}, + { + "litellm_metadata": { + "user_api_key_end_user_id": "user_from_litellm_metadata" + } + }, "user_from_litellm_metadata", ), # Test with both fields - metadata should take precedence for user_api_key fields ( { "metadata": {"user_api_key_end_user_id": "user_from_metadata"}, - "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, + "litellm_metadata": { + "user_api_key_end_user_id": "user_from_litellm_metadata" + }, }, "user_from_metadata", ), @@ -1228,7 +1300,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ( { "metadata": {}, - "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, + "litellm_metadata": { + "user_api_key_end_user_id": "user_from_litellm_metadata" + }, }, "user_from_litellm_metadata", ), @@ -1236,7 +1310,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ({}, None), ], ) -def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id): +def test_get_end_user_id_for_cost_tracking_metadata_handling( + litellm_params, expected_end_user_id +): """ Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata fields using the get_litellm_metadata_from_kwargs helper function. @@ -1383,7 +1459,9 @@ def test_get_valid_models_openai_proxy(monkeypatch): mock_response.status_code = 200 mock_response.json.return_value = mock_response_data - with patch.object(litellm.module_level_client, "get", return_value=mock_response) as mock_post: + with patch.object( + litellm.module_level_client, "get", return_value=mock_response + ) as mock_post: valid_models = get_valid_models(check_provider_endpoint=True) assert "litellm_proxy/gpt-5.5" in valid_models @@ -1460,11 +1538,16 @@ def test_get_valid_models_fireworks_ai(monkeypatch): mock_response.status_code = 200 mock_response.json.return_value = mock_response_data - with patch.object(litellm.module_level_client, "get", return_value=mock_response) as mock_post: + with patch.object( + litellm.module_level_client, "get", return_value=mock_response + ) as mock_post: valid_models = get_valid_models(check_provider_endpoint=True) print("valid_models", valid_models) mock_post.assert_called_once() - assert "fireworks_ai/accounts/fireworks/models/llama-3.1-8b-instruct" in valid_models + assert ( + "fireworks_ai/accounts/fireworks/models/llama-3.1-8b-instruct" + in valid_models + ) def test_get_valid_models_default(monkeypatch): @@ -1494,7 +1577,9 @@ def test_pick_cheapest_chat_model_from_llm_provider(): def test_get_num_retries(num_retries): from litellm.utils import _get_wrapper_num_retries - assert _get_wrapper_num_retries(kwargs={"num_retries": num_retries}, exception=Exception("test")) == ( + assert _get_wrapper_num_retries( + kwargs={"num_retries": num_retries}, exception=Exception("test") + ) == ( num_retries, { "num_retries": num_retries, @@ -1767,7 +1852,9 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch): assert len(litellm.success_callback) == curr_len_success_callback assert len(litellm.failure_callback) == curr_len_failure_callback - assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback) + assert any( + isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback + ) @pytest.mark.asyncio @@ -1794,13 +1881,20 @@ async def test_wrapper_kwargs_passthrough(): mock_original.assert_called_once() # get litellm logging object - litellm_logging_obj: LiteLLMLoggingObject = mock_original.call_args.kwargs.get("litellm_logging_obj") + litellm_logging_obj: LiteLLMLoggingObject = mock_original.call_args.kwargs.get( + "litellm_logging_obj" + ) assert litellm_logging_obj is not None - print(f"litellm_logging_obj.model_call_details: {litellm_logging_obj.model_call_details}") + print( + f"litellm_logging_obj.model_call_details: {litellm_logging_obj.model_call_details}" + ) # get base model - assert litellm_logging_obj.model_call_details["litellm_params"]["base_model"] == "gpt-5-mini" + assert ( + litellm_logging_obj.model_call_details["litellm_params"]["base_model"] + == "gpt-5-mini" + ) def test_dict_to_response_format_helper(): @@ -1854,7 +1948,7 @@ def test_validate_user_messages_invalid_content_type(): messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}] - with pytest.raises(Exception, match="Please ensure all messages are valid OpenAI chat completion") as e: + with pytest.raises(Exception, match='Please ensure all messages are valid OpenAI chat completion') as e: validate_chat_completion_user_messages(messages) assert "Invalid message" in str(e) @@ -1871,14 +1965,20 @@ from unittest.mock import Mock [ { "name": "default_on_guardrail", - "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)], + "callbacks": [ + CustomGuardrail(guardrail_name="test_guardrail", default_on=True) + ], "kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}}, "expected": ["test_guardrail"], }, { "name": "request_specific_guardrail", - "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], - "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}}, + "callbacks": [ + CustomGuardrail(guardrail_name="test_guardrail", default_on=False) + ], + "kwargs": { + "metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}} + }, "expected": ["test_guardrail"], }, { @@ -1887,12 +1987,18 @@ from unittest.mock import Mock CustomGuardrail(guardrail_name="default_guardrail", default_on=True), CustomGuardrail(guardrail_name="request_guardrail", default_on=False), ], - "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}}, + "kwargs": { + "metadata": { + "requester_metadata": {"guardrails": ["request_guardrail"]} + } + }, "expected": ["default_guardrail", "request_guardrail"], }, { "name": "empty_metadata", - "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], + "callbacks": [ + CustomGuardrail(guardrail_name="test_guardrail", default_on=False) + ], "kwargs": {}, "expected": [], }, @@ -1999,7 +2105,9 @@ def test_get_provider_audio_transcription_config(): from litellm.types.utils import LlmProviders for provider in LlmProviders: - config = ProviderConfigManager.get_provider_audio_transcription_config(model="whisper-1", provider=provider) + config = ProviderConfigManager.get_provider_audio_transcription_config( + model="whisper-1", provider=provider + ) @pytest.mark.parametrize( @@ -2042,7 +2150,9 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "123") - _model_cache.set_cached_model_info("openai", litellm_params=None, available_models=["gpt-5-mini"]) + _model_cache.set_cached_model_info( + "openai", litellm_params=None, available_models=["gpt-5-mini"] + ) monkeypatch.delenv("OPENAI_API_KEY") assert _model_cache.get_cached_model_info("openai") is None @@ -2131,8 +2241,12 @@ def test_delta_tool_calls_sequential_indices(): # Verify tool calls have sequential indices assert delta.tool_calls is not None, "Tool calls should not be None" assert len(delta.tool_calls) == 2 - assert delta.tool_calls[0].index == 0, f"First tool call should have index 0, got {delta.tool_calls[0].index}" - assert delta.tool_calls[1].index == 1, f"Second tool call should have index 1, got {delta.tool_calls[1].index}" + assert ( + delta.tool_calls[0].index == 0 + ), f"First tool call should have index 0, got {delta.tool_calls[0].index}" + assert ( + delta.tool_calls[1].index == 1 + ), f"Second tool call should have index 1, got {delta.tool_calls[1].index}" # Verify tool call details are preserved assert delta.tool_calls[0].function.name == "get_weather_for_dallas" @@ -2145,7 +2259,9 @@ def test_completion_with_no_model(): """ # test on empty with pytest.raises(TypeError): - response = litellm.completion(messages=[{"role": "user", "content": "Hello, how are you?"}]) + response = litellm.completion( + messages=[{"role": "user", "content": "Hello, how are you?"}] + ) def test_get_base_model_from_metadata(): @@ -2158,31 +2274,43 @@ def test_get_base_model_from_metadata(): from litellm.utils import _get_base_model_from_metadata # Test 1: base_model in metadata (Chat Completions API pattern) - model_call_details_with_metadata = {"litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-5.5"}}}} + model_call_details_with_metadata = { + "litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-5.5"}}} + } result = _get_base_model_from_metadata(model_call_details_with_metadata) assert result == "azure/gpt-5.5", f"Expected 'azure/gpt-5.5', got {result}" # Test 2: base_model in litellm_metadata (Responses API and generic API calls pattern) model_call_details_with_litellm_metadata = { - "litellm_params": {"litellm_metadata": {"model_info": {"base_model": "azure/gpt-5-mini"}}} + "litellm_params": { + "litellm_metadata": {"model_info": {"base_model": "azure/gpt-5-mini"}} + } } result = _get_base_model_from_metadata(model_call_details_with_litellm_metadata) assert result == "azure/gpt-5-mini", f"Expected 'azure/gpt-5-mini', got {result}" # Test 3: base_model in litellm_params (direct base_model) - model_call_details_with_direct_base_model = {"litellm_params": {"base_model": "azure/gpt-5-mini"}} + model_call_details_with_direct_base_model = { + "litellm_params": {"base_model": "azure/gpt-5-mini"} + } result = _get_base_model_from_metadata(model_call_details_with_direct_base_model) - assert result == "azure/gpt-5-mini", f"Expected 'azure/gpt-5-mini', got {result}" + assert ( + result == "azure/gpt-5-mini" + ), f"Expected 'azure/gpt-5-mini', got {result}" # Test 4: metadata takes precedence over litellm_metadata model_call_details_with_both = { "litellm_params": { "metadata": {"model_info": {"base_model": "azure/gpt-4-from-metadata"}}, - "litellm_metadata": {"model_info": {"base_model": "azure/gpt-4-from-litellm-metadata"}}, + "litellm_metadata": { + "model_info": {"base_model": "azure/gpt-4-from-litellm-metadata"} + }, } } result = _get_base_model_from_metadata(model_call_details_with_both) - assert result == "azure/gpt-4-from-metadata", f"Expected metadata to take precedence, got {result}" + assert ( + result == "azure/gpt-4-from-metadata" + ), f"Expected metadata to take precedence, got {result}" # Test 5: No base_model present model_call_details_without_base_model = {"litellm_params": {"metadata": {}}} diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index b8a53fefb5c..67b1a09c7ab 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -73,7 +73,9 @@ def test_azure_o3_streaming(): api_version="2024-02-15-preview", ) - with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_create: try: completion( model="azure/o3-mini", @@ -81,7 +83,9 @@ def test_azure_o3_streaming(): stream=True, client=client, ) - except Exception as e: # expect output translation error as mock response doesn't return a json + except ( + Exception + ) as e: # expect output translation error as mock response doesn't return a json print(e) assert mock_create.call_count == 1 assert "stream" in mock_create.call_args.kwargs @@ -100,7 +104,9 @@ def test_azure_o_series_routing(): api_version="2024-02-15-preview", ) - with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_create: try: completion( model="azure/o_series/my-random-deployment-name", @@ -108,7 +114,9 @@ def test_azure_o_series_routing(): stream=True, client=client, ) - except Exception as e: # expect output translation error as mock response doesn't return a json + except ( + Exception + ) as e: # expect output translation error as mock response doesn't return a json print(e) assert mock_create.call_count == 1 assert "stream" not in mock_create.call_args.kwargs @@ -175,7 +183,9 @@ async def test_azure_o1_series_response_format_extra_params(): ] response_format = {"type": "json_object"} tool_choice = "auto" - with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + with patch.object( + client.chat.completions.with_raw_response, "create" + ) as mock_client: try: await litellm.acompletion( client=client, diff --git a/tests/llm_translation/test_lambda_ai.py b/tests/llm_translation/test_lambda_ai.py index 78843fac052..e6f8b13d4ba 100644 --- a/tests/llm_translation/test_lambda_ai.py +++ b/tests/llm_translation/test_lambda_ai.py @@ -44,7 +44,9 @@ def test_lambda_ai_get_openai_compatible_provider_info(): os.environ, {"LAMBDA_API_KEY": "env-key", "LAMBDA_API_BASE": "https://env.lambda.ai/v1"}, ): - api_base, api_key = config._get_openai_compatible_provider_info("https://param.lambda.ai/v1", "param-key") + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.lambda.ai/v1", "param-key" + ) assert api_base == "https://param.lambda.ai/v1" assert api_key == "param-key" @@ -54,12 +56,16 @@ def test_get_llm_provider_lambda_ai(): from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider # Test with lambda_ai/model-name format - model, provider, api_key, api_base = get_llm_provider("lambda_ai/llama3.1-8b-instruct") + model, provider, api_key, api_base = get_llm_provider( + "lambda_ai/llama3.1-8b-instruct" + ) assert model == "llama3.1-8b-instruct" assert provider == "lambda_ai" # Test with api_base containing Lambda AI endpoint - model, provider, api_key, api_base = get_llm_provider("llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1") + model, provider, api_key, api_base = get_llm_provider( + "llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1" + ) assert model == "llama3.1-8b-instruct" assert provider == "lambda_ai" assert api_base == "https://api.lambda.ai/v1" @@ -94,3 +100,5 @@ async def test_lambda_ai_completion_call(): if "lambda_ai" not in str(e) and "provider" not in str(e).lower(): # Re-raise if it's not a provider-related error raise + + diff --git a/tests/llm_translation/test_perplexity_reasoning.py b/tests/llm_translation/test_perplexity_reasoning.py index 92d6a5d2ab3..0fdfdd79321 100644 --- a/tests/llm_translation/test_perplexity_reasoning.py +++ b/tests/llm_translation/test_perplexity_reasoning.py @@ -25,7 +25,9 @@ class TestPerplexityReasoning: ("perplexity/sonar-reasoning-pro", "high"), ], ) - def test_perplexity_reasoning_effort_parameter_mapping(self, model, reasoning_effort): + def test_perplexity_reasoning_effort_parameter_mapping( + self, model, reasoning_effort + ): """ Test that reasoning_effort parameter is correctly mapped for Perplexity Sonar reasoning models """ @@ -102,6 +104,7 @@ class TestPerplexityReasoning: "create", side_effect=_return_pydantic_obj, ) as mock_client: + response = completion( model=model, messages=[ @@ -127,7 +130,11 @@ class TestPerplexityReasoning: # Verify response structure assert response.choices[0].message.content is not None - assert response.choices[0].message.content == "This is a test response from the reasoning model." + assert ( + response.choices[0].message.content + == "This is a test response from the reasoning model." + ) + @pytest.mark.parametrize( "model,expected_api_base", @@ -136,14 +143,18 @@ class TestPerplexityReasoning: ("perplexity/sonar-reasoning-pro", "https://api.perplexity.ai"), ], ) - def test_perplexity_reasoning_api_base_configuration(self, model, expected_api_base): + def test_perplexity_reasoning_api_base_configuration( + self, model, expected_api_base + ): """ Test that Perplexity reasoning models use the correct API base """ from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig config = PerplexityChatConfig() - api_base, _ = config._get_openai_compatible_provider_info(api_base=None, api_key="test-key") + api_base, _ = config._get_openai_compatible_provider_info( + api_base=None, api_key="test-key" + ) assert api_base == expected_api_base @@ -154,6 +165,8 @@ class TestPerplexityReasoning: from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig config = PerplexityChatConfig() - supported_params = config.get_supported_openai_params(model="perplexity/sonar-reasoning") + supported_params = config.get_supported_openai_params( + model="perplexity/sonar-reasoning" + ) assert "reasoning_effort" in supported_params diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 0e04569bbdf..f40818b9bf1 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -149,6 +149,7 @@ def test_custom_pricing_as_completion_cost_param(): assert round(cost, 5) == round(expected_cost, 5) + # print(results) @@ -189,17 +190,23 @@ def test_cost_ft_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost(completion_response=resp, custom_llm_provider="openai") + cost = litellm.completion_cost( + completion_response=resp, custom_llm_provider="openai" + ) print("\n Calculated Cost for ft:gpt-3.5", cost) input_cost = model_cost["ft:gpt-3.5-turbo"]["input_cost_per_token"] output_cost = model_cost["ft:gpt-3.5-turbo"]["output_cost_per_token"] print(input_cost, output_cost) - expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) + expected_cost = (input_cost * resp.usage.prompt_tokens) + ( + output_cost * resp.usage.completion_tokens + ) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: print(f"Error: {e}") - pytest.fail(f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}") + pytest.fail( + f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}" + ) # test_cost_ft_gpt_35() @@ -228,11 +235,15 @@ def test_cost_azure_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost(completion_response=resp, model="azure/chatgpt-deployment-2") + cost = litellm.completion_cost( + completion_response=resp, model="azure/chatgpt-deployment-2" + ) print("\n Calculated Cost for azure/gpt-3.5-turbo", cost) input_cost = model_cost["azure/gpt-35-turbo"]["input_cost_per_token"] output_cost = model_cost["azure/gpt-35-turbo"]["output_cost_per_token"] - expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) + expected_cost = (input_cost * resp.usage.prompt_tokens) + ( + output_cost * resp.usage.completion_tokens + ) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: @@ -249,7 +260,9 @@ def test_cost_bedrock_pricing_actual_calls(): litellm.set_verbose = True model = "anthropic.claude-3-5-sonnet-20240620-v1:0" messages = [{"role": "user", "content": "Hey, how's it going?"}] - response = litellm.completion(model=model, messages=messages, mock_response="hello cool one") + response = litellm.completion( + model=model, messages=messages, mock_response="hello cool one" + ) print("response", response) cost = litellm.completion_cost( @@ -280,7 +293,8 @@ def test_whisper_openai(): print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] + * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -300,12 +314,15 @@ def test_whisper_azure(): _total_time_in_seconds = 3 setattr(transcription, "duration", _total_time_in_seconds) - cost = litellm.completion_cost(model="azure/azure-whisper", completion_response=transcription) + cost = litellm.completion_cost( + model="azure/azure-whisper", completion_response=transcription + ) print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] + * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -336,7 +353,9 @@ def test_dalle_3_azure_cost_tracking(): response.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} response._hidden_params = {"model": "dall-e-3", "model_id": None} print(f"response hidden params: {response._hidden_params}") - cost = litellm.completion_cost(completion_response=response, call_type="image_generation") + cost = litellm.completion_cost( + completion_response=response, call_type="image_generation" + ) assert cost > 0 @@ -368,7 +387,9 @@ def test_replicate_llama3_cost_tracking(): model="replicate/meta/meta-llama-3-8b-instruct", object="chat.completion", system_fingerprint=None, - usage=litellm.utils.Usage(prompt_tokens=48, completion_tokens=31, total_tokens=79), + usage=litellm.utils.Usage( + prompt_tokens=48, completion_tokens=31, total_tokens=79 + ), ) cost = litellm.completion_cost( completion_response=response, @@ -378,8 +399,14 @@ def test_replicate_llama3_cost_tracking(): print(f"cost: {cost}") cost = round(cost, 5) expected_cost = round( - litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["input_cost_per_token"] * 48 - + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["output_cost_per_token"] * 31, + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "input_cost_per_token" + ] + * 48 + + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ + "output_cost_per_token" + ] + * 31, 5, ) assert cost == expected_cost @@ -543,7 +570,9 @@ def test_vertex_ai_medlm_completion_cost(): model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider="vertex_ai") + predictive_cost = completion_cost( + model=model, messages=messages, custom_llm_provider="vertex_ai" + ) assert predictive_cost > 0 model = "vertex_ai/medlm-large" @@ -560,7 +589,9 @@ def test_vertex_ai_embedding_completion_cost(caplog): litellm.model_cost = litellm.get_model_cost_map(url="") text = "The quick brown fox jumps over the lazy dog." - input_tokens = litellm.token_counter(model="vertex_ai/text-embedding-004", text=text) + input_tokens = litellm.token_counter( + model="vertex_ai/text-embedding-004", text=text + ) model_info = litellm.get_model_info(model="vertex_ai/text-embedding-004") @@ -583,7 +614,10 @@ def test_vertex_ai_embedding_completion_cost(caplog): captured_logs = [rec.message for rec in caplog.records] for item in captured_logs: print("\nitem:{}\n".format(item)) - if "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " in item: + if ( + "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " + in item + ): raise Exception("Error log raised for calculating embedding cost") @@ -653,7 +687,9 @@ def test_vertex_ai_llama_predict_cost(): model = "meta/llama3-405b-instruct-maas" messages = [{"role": "user", "content": "Hey, hows it going???"}] custom_llm_provider = "vertex_ai" - predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider=custom_llm_provider) + predictive_cost = completion_cost( + model=model, messages=messages, custom_llm_provider=custom_llm_provider + ) assert predictive_cost == 0 @@ -667,7 +703,9 @@ def test_vertex_ai_mistral_predict_cost(usage): else: from openai.types.completion_usage import CompletionUsage - response_usage = CompletionUsage(prompt_tokens=32, completion_tokens=55, total_tokens=87) + response_usage = CompletionUsage( + prompt_tokens=32, completion_tokens=55, total_tokens=87 + ) response_object = ModelResponse( id="26c0ef045020429d9c5c9b078c01e564", choices=[ @@ -701,7 +739,9 @@ def test_vertex_ai_mistral_predict_cost(usage): assert predictive_cost > 0 -@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"]) +@pytest.mark.parametrize( + "model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"] +) def test_completion_cost_tts(model): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -801,7 +841,9 @@ def test_completion_cost_azure_common_deployment_name(): response._hidden_params["custom_llm_provider"] = "azure" print(response) - with patch.object(litellm.cost_calculator, "completion_cost", new=MagicMock()) as mock_client: + with patch.object( + litellm.cost_calculator, "completion_cost", new=MagicMock() + ) as mock_client: _ = litellm.response_cost_calculator( response_object=response, model="gpt-4-0314", @@ -861,7 +903,9 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): cost_1 = completion_cost(model=model, completion_response=response_1) - _model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + _model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) expected_cost = ( ( response_1.usage.prompt_tokens @@ -869,9 +913,12 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): - response_1.usage.prompt_tokens_details.cache_creation_tokens ) * _model_info["input_cost_per_token"] - + (response_1.usage.prompt_tokens_details.cached_tokens or 0) * _model_info["cache_read_input_token_cost"] - + (response_1.usage.cache_creation_input_tokens or 0) * _model_info["cache_creation_input_token_cost"] - + (response_1.usage.completion_tokens or 0) * _model_info["output_cost_per_token"] + + (response_1.usage.prompt_tokens_details.cached_tokens or 0) + * _model_info["cache_read_input_token_cost"] + + (response_1.usage.cache_creation_input_tokens or 0) + * _model_info["cache_creation_input_token_cost"] + + (response_1.usage.completion_tokens or 0) + * _model_info["output_cost_per_token"] ) # Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) assert round(expected_cost, 5) == round(cost_1, 5) @@ -987,7 +1034,9 @@ def test_completion_cost_databricks_embedding(model, monkeypatch): sync_handler = HTTPHandler() with patch.object(HTTPHandler, "post", return_value=mock_response): - resp = litellm.embedding(model=model, input=["hey, how's it going?"], client=sync_handler) + resp = litellm.embedding( + model=model, input=["hey, how's it going?"], client=sync_handler + ) print(resp) cost = completion_cost(completion_response=resp) @@ -1163,9 +1212,11 @@ def test_cost_openai_prompt_caching(): usage = response_2.usage _expected_cost2 = ( - (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) * model_info["input_cost_per_token"] + (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) + * model_info["input_cost_per_token"] + usage.completion_tokens * model_info["output_cost_per_token"] - + usage.prompt_tokens_details.cached_tokens * model_info["cache_read_input_token_cost"] + + usage.prompt_tokens_details.cached_tokens + * model_info["cache_read_input_token_cost"] ) print("_expected_cost2", _expected_cost2) @@ -1206,7 +1257,9 @@ def test_completion_cost_azure_ai_rerank(model): }, ) print("response", response) - cost = completion_cost(model=model, completion_response=response, call_type="arerank") + cost = completion_cost( + model=model, completion_response=response, call_type="arerank" + ) assert cost > 0 @@ -2158,7 +2211,9 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): completion_tokens=34, prompt_tokens=16, total_tokens=50, - completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=28, reasoning_tokens=0, text_tokens=6), + completion_tokens_details=CompletionTokensDetailsWrapper( + audio_tokens=28, reasoning_tokens=0, text_tokens=6 + ), prompt_tokens_details=PromptTokensDetailsWrapper( audio_tokens=0, cached_tokens=0, text_tokens=16, image_tokens=0 ), @@ -2197,15 +2252,27 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): print(f"model_info: {model_info}") ## input cost - input_audio_cost = model_info["input_cost_per_audio_token"] * usage_object.prompt_tokens_details.audio_tokens - input_text_cost = model_info["input_cost_per_token"] * usage_object.prompt_tokens_details.text_tokens + input_audio_cost = ( + model_info["input_cost_per_audio_token"] + * usage_object.prompt_tokens_details.audio_tokens + ) + input_text_cost = ( + model_info["input_cost_per_token"] + * usage_object.prompt_tokens_details.text_tokens + ) total_input_cost = input_audio_cost + input_text_cost ## output cost - output_audio_cost = model_info["output_cost_per_audio_token"] * usage_object.completion_tokens_details.audio_tokens - output_text_cost = model_info["output_cost_per_token"] * usage_object.completion_tokens_details.text_tokens + output_audio_cost = ( + model_info["output_cost_per_audio_token"] + * usage_object.completion_tokens_details.audio_tokens + ) + output_text_cost = ( + model_info["output_cost_per_token"] + * usage_object.completion_tokens_details.text_tokens + ) total_output_cost = output_audio_cost + output_text_cost @@ -2331,7 +2398,9 @@ def test_moderations(): litellm.add_known_models() assert "omni-moderation-latest" in litellm.model_cost - print(f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}") + print( + f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}" + ) assert "omni-moderation-latest" in litellm.open_ai_chat_completion_models response = moderation("I am a bad person", model="omni-moderation-latest") @@ -2368,7 +2437,9 @@ def test_cost_calculator_azure_embedding(): def test_add_known_models(): litellm.add_known_models() - assert "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models + assert ( + "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models + ) @pytest.mark.skip(reason="flaky test") @@ -2478,7 +2549,9 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg): } if base_model_arg == "litellm_param": - model_item["litellm_params"]["base_model"] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" + model_item["litellm_params"][ + "base_model" + ] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" elif base_model_arg == "model_info": model_item["model_info"] = { "base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 5c640aa22a6..37f4ece611d 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -114,13 +114,19 @@ def test_get_model_info_ft_model_with_provider_prefix(): assert info["key"] == "ft:gpt-3.5-turbo" -def _enforce_bedrock_converse_models(model_cost: List[Dict[str, Any]], whitelist_models: List[str]): +def _enforce_bedrock_converse_models( + model_cost: List[Dict[str, Any]], whitelist_models: List[str] +): """ Assert all new bedrock chat models are added as `bedrock_converse` unless explicitly whitelisted. """ # Check for unwhitelisted models for model, info in litellm.model_cost.items(): - if info["litellm_provider"] == "bedrock" and info["mode"] == "chat" and model not in whitelist_models: + if ( + info["litellm_provider"] == "bedrock" + and info["mode"] == "chat" + and model not in whitelist_models + ): raise AssertionError( f"New bedrock chat model detected: {model}. Please set `litellm_provider='bedrock_converse'` for this model." ) @@ -141,7 +147,9 @@ def test_model_info_bedrock_converse(monkeypatch): except FileNotFoundError: pytest.skip("whitelisted_bedrock_models.txt not found") - _enforce_bedrock_converse_models(model_cost=litellm.model_cost, whitelist_models=whitelist_models) + _enforce_bedrock_converse_models( + model_cost=litellm.model_cost, whitelist_models=whitelist_models + ) @pytest.mark.flaky(retries=6, delay=2) @@ -165,8 +173,10 @@ def test_model_info_bedrock_converse_enforcement(monkeypatch): # Check for unwhitelisted models with pytest.raises(AssertionError): - _enforce_bedrock_converse_models(model_cost=litellm.model_cost, whitelist_models=whitelist_models) - except FileNotFoundError: + _enforce_bedrock_converse_models( + model_cost=litellm.model_cost, whitelist_models=whitelist_models + ) + except FileNotFoundError as e: pytest.skip("whitelisted_bedrock_models.txt not found") @@ -203,7 +213,9 @@ def test_get_model_info_custom_provider(): # Get registered model info from litellm import get_model_info - get_model_info(model="my-custom-llm/my-fake-model") # 💥 "Exception: This model isn't mapped yet." in v1.56.10 + get_model_info( + model="my-custom-llm/my-fake-model" + ) # 💥 "Exception: This model isn't mapped yet." in v1.56.10 def test_get_model_info_custom_model_router(): @@ -255,7 +267,11 @@ def test_get_model_info_bedrock_models(): k = k.replace(f"{commitment}/", "") base_model = BedrockModelInfo.get_base_model(k) # get_base_model() returns model id without "bedrock/" prefix; cost map keys use "bedrock/" - base_model_key = base_model if base_model in litellm.model_cost else f"bedrock/{base_model}" + base_model_key = ( + base_model + if base_model in litellm.model_cost + else f"bedrock/{base_model}" + ) if base_model_key not in litellm.model_cost: continue base_model_info = litellm.model_cost[base_model_key] @@ -263,10 +279,12 @@ def test_get_model_info_bedrock_models(): if "invoke/" in k: continue if base_model_key.startswith("supports_"): - assert base_model_key in v, f"{base_model_key} is not in model cost map for {k}" - assert v[base_model_key] == base_model_value, ( - f"{base_model_key} is not equal to {base_model_value} for model {k}" - ) + assert ( + base_model_key in v + ), f"{base_model_key} is not in model cost map for {k}" + assert ( + v[base_model_key] == base_model_value + ), f"{base_model_key} is not equal to {base_model_value} for model {k}" def test_get_model_info_bedrock_cross_region_capability_parity(): @@ -294,7 +312,9 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): if not cap.startswith("supports_"): continue assert cap in v, f"{cap} is on {base_model_key} but missing from {k}" - assert v[cap] == base_value, f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}" + assert ( + v[cap] == base_value + ), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}" assert checked > 0, "no cross-region bedrock profiles found - the filter is inert" @@ -355,17 +375,23 @@ def test_get_model_info_case_insensitive_lookup(monkeypatch): ) # Test 1: Exact case should work - info = litellm.get_model_info(model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai") + info = litellm.get_model_info( + model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai" + ) assert info is not None assert info["supports_function_calling"] is True # Test 2: Lowercase should also work (case-insensitive lookup) - info_lower = litellm.get_model_info(model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai") + info_lower = litellm.get_model_info( + model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai" + ) assert info_lower is not None assert info_lower["supports_function_calling"] is True # Test 3: Mixed case should also work - info_mixed = litellm.get_model_info(model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai") + info_mixed = litellm.get_model_info( + model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai" + ) assert info_mixed is not None assert info_mixed["supports_function_calling"] is True @@ -393,7 +419,13 @@ def test_get_model_info_case_insensitive_supports_function_calling(monkeypatch): from litellm.utils import supports_function_calling # Exact case - assert supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") is True + assert ( + supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") + is True + ) # Lowercase (should now work with case-insensitive lookup) - assert supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") is True + assert ( + supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") + is True + ) diff --git a/tests/local_testing/test_register_model.py b/tests/local_testing/test_register_model.py index d78f2ac7811..5f334a27e35 100644 --- a/tests/local_testing/test_register_model.py +++ b/tests/local_testing/test_register_model.py @@ -41,7 +41,9 @@ def test_update_model_cost_via_completion(): input_cost_per_token=0.3, output_cost_per_token=0.4, ) - print(f"litellm.model_cost for gpt-3.5-turbo: {litellm.model_cost['gpt-3.5-turbo']}") + print( + f"litellm.model_cost for gpt-3.5-turbo: {litellm.model_cost['gpt-3.5-turbo']}" + ) assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.3 assert litellm.model_cost["gpt-3.5-turbo"]["output_cost_per_token"] == 0.4 except Exception as e: @@ -50,7 +52,11 @@ def test_update_model_cost_via_completion(): def test_no_test_invocation_at_module_scope(): tree = ast.parse(Path(__file__).read_text()) - defined = {node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + defined = { + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } invoked = [ node.value.func.id for node in tree.body diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 17b48063cce..7eecbb730dd 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1586,10 +1586,12 @@ class TestEnableAnthropicPromptCaching: points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") assert [p["index"] for p in points] == [None, -1] + def test_model_without_caching_support_not_injected(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert self._points(model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] + def test_stands_down_when_client_sent_cache_control(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) messages = [ @@ -1631,9 +1633,7 @@ class TestEnableAnthropicPromptCaching: """OpenAI-shaped tools nest cache_control under ``function``; the Anthropic chat transform honors that location, so the stand-down must see it too.""" monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - tools = [ - {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}} - ] + tools = [{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}] assert self._points(tools=tools) == [] def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): @@ -2218,7 +2218,9 @@ class TestAnthropicPromptCachingEnvVars: print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl])) """ ) - result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300 + ) assert result.returncode == 0, result.stderr enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1]) return enabled, ttl @@ -2429,9 +2431,7 @@ class TestOpenAIPromptCacheBreakpoint: assert kwargs == {} def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self): - messages = [ - {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]} - ] + messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} result, system = self._inject(messages, "sys", kwargs) assert result == messages @@ -2567,11 +2567,7 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: def test_tool_message_text_is_marked_on_chat_path(self): messages = [ {"role": "user", "content": "weather?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}], - }, + {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, ] out, params = self._chat(messages, [{"location": "message", "index": -1}]) @@ -2795,9 +2791,9 @@ class TestChatPathProviderStamp: class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): - messages = [ - {"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]} - ] + [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4)] + messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [ + {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4) + ] out, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( messages=messages, system="sys", @@ -2955,6 +2951,7 @@ class TestPromptCacheBreakpointCapability: yield litellm.utils._cached_get_model_info_helper.cache_clear() + def test_listed_model_uses_the_model_map_flag(self, monkeypatch): flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True} monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) @@ -2972,6 +2969,7 @@ class TestPromptCacheBreakpointCapability: ) assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False + def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch): unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index 0c2bb9ada71..aa2fc0b9a45 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -1,3 +1,4 @@ + import pytest import litellm diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 433117edb05..6b118c97082 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,3 +1,4 @@ + import pytest import litellm @@ -16,7 +17,9 @@ def test_web_search_cost_low(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"] + assert ( + cost == model_info["search_context_cost_per_query"]["search_context_size_low"] + ) def test_web_search_cost_medium(): @@ -27,7 +30,10 @@ def test_web_search_cost_medium(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"] + assert ( + cost + == model_info["search_context_cost_per_query"]["search_context_size_medium"] + ) def test_web_search_cost_high(): @@ -38,21 +44,33 @@ def test_web_search_cost_high(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"] + assert ( + cost == model_info["search_context_cost_per_query"]["search_context_size_high"] + ) # Test file search cost calculation def test_file_search_cost(): file_search = FileSearchTool(type="file_search") - cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search) + cost = StandardBuiltInToolCostTracking.get_cost_for_file_search( + file_search=file_search + ) assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call # Test edge cases def test_none_inputs(): # Test with None inputs - assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0 - assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0 + assert ( + StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=None, model_info=None + ) + == 0.0 + ) + assert ( + StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) + == 0.0 + ) # Test the main get_cost_for_built_in_tools method @@ -77,7 +95,9 @@ def test_get_cost_for_built_in_tools_file_search(): Test that the cost for a file search is 0.00 when no response object is provided """ model = "gpt-4" - standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search")) + standard_built_in_tools_params = StandardBuiltInToolsParams( + file_search=FileSearchTool(type="file_search") + ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, @@ -120,7 +140,9 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): usage = Usage(server_tool_use={"web_search_requests": 1}) assert isinstance(usage.server_tool_use, ServerToolUse) - assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( + response_object=None, usage=usage + ) def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): @@ -159,7 +181,9 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_serve standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] assert cost == per_query_cost * web_search_requests assert cost > 0.0 assert getattr(usage, "server_tool_use", None) is None @@ -197,7 +221,9 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] assert cost == per_query_cost * web_search_requests @@ -261,14 +287,18 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize("model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"]) +@pytest.mark.parametrize( + "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] +) def test_get_cost_for_gemini_web_search(model): """ Test that the cost for a web search is 0.00 when no response object is provided """ from litellm.types.utils import PromptTokensDetailsWrapper, Usage - usage = Usage(prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1)) + usage = Usage( + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) + ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, usage=usage, @@ -326,7 +356,9 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par ) assert web_search_cost > 0, "Web search cost should be non-zero" - assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})" + assert ( + cost >= web_search_cost + ), f"completion_cost ({cost}) should include web search cost ({web_search_cost})" def test_gemini_combined_search_and_maps_costs_are_additive(local_model_cost_map): @@ -362,6 +394,7 @@ def _openai_responses_with_web_search_calls(model, num_calls): ResponseFunctionWebSearch, ) + output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -392,7 +425,9 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) for num_calls in (1, 3): @@ -419,7 +454,9 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] response = ResponsesAPIResponse.model_validate( { @@ -428,7 +465,10 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): "model": model, "object": "response", "status": "completed", - "output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)], + "output": [ + {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} + for i in range(3) + ], } ) assert all(isinstance(item, dict) for item in response.output) @@ -441,7 +481,9 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): standard_built_in_tools_params=None, ) - assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" + assert cost == pytest.approx(3 * per_call), ( + f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" + ) # Note: File search integration test removed due to complex annotation detection logic @@ -519,3 +561,5 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( ) _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 + + diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py index 94e8b4bb7b0..83ee3437429 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py @@ -75,10 +75,12 @@ def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models( """Opus 4.7/4.8, Sonnet 4 and Sonnet 5 reject toolSpec.strict and additionalProperties.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) tool_spec = result[0]["toolSpec"] - assert "strict" not in tool_spec, f"strict leaked into toolSpec for {model_id}: {tool_spec}" - assert "additionalProperties" not in tool_spec["inputSchema"]["json"], ( - f"additionalProperties leaked into toolSpec for {model_id}: {tool_spec}" - ) + assert ( + "strict" not in tool_spec + ), f"strict leaked into toolSpec for {model_id}: {tool_spec}" + assert ( + "additionalProperties" not in tool_spec["inputSchema"]["json"] + ), f"additionalProperties leaked into toolSpec for {model_id}: {tool_spec}" @pytest.mark.parametrize( @@ -93,7 +95,9 @@ def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models( def test_bedrock_tools_pt_strict_kept_for_other_anthropic(model_id: str) -> None: """Sonnet 4.5/4.6 and Opus <=4.6 accept toolSpec.strict — keep forwarding it.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) - assert result[0]["toolSpec"]["strict"] is True, f"strict missing for {model_id}: {result[0]['toolSpec']}" + assert ( + result[0]["toolSpec"]["strict"] is True + ), f"strict missing for {model_id}: {result[0]['toolSpec']}" @pytest.mark.parametrize( @@ -113,7 +117,9 @@ def test_bedrock_tools_pt_falsy_strict_always_dropped(model_id: str) -> None: ones whose cost-map entry still allows ``strict: true`` through.""" result = _bedrock_tools_pt(_NON_STRICT_TOOL, model=model_id) tool_spec = result[0]["toolSpec"] - assert "strict" not in tool_spec, f"no-op strict: false leaked into toolSpec for {model_id}: {tool_spec}" + assert ( + "strict" not in tool_spec + ), f"no-op strict: false leaked into toolSpec for {model_id}: {tool_spec}" def test_responses_bridge_function_tool_does_not_reach_bedrock_with_strict() -> None: @@ -134,8 +140,10 @@ def test_responses_bridge_function_tool_does_not_reach_bedrock_with_strict() -> "required": ["city"], }, } - chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - [responses_tool] + chat_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + [responses_tool] + ) ) result = _bedrock_tools_pt(chat_tools, model="bedrock/us.anthropic.claude-sonnet-5") assert "strict" not in result[0]["toolSpec"] @@ -152,3 +160,5 @@ def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> Non """Non-Anthropic Bedrock families reject toolSpec.strict — must be dropped.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) assert "strict" not in result[0]["toolSpec"] + + diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 270df703dee..e963e40a51c 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -31,7 +31,9 @@ def _get_gemini_function_response_inline_data_parts(result): assert isinstance(result, list), "expected Gemini parts list" assert len(result) == 1, "multimodal function responses should stay in one part" function_response_part = result[0] - assert "inline_data" not in function_response_part, "inline_data should be nested under function_response.parts" + assert ( + "inline_data" not in function_response_part + ), "inline_data should be nested under function_response.parts" function_response = function_response_part["function_response"] nested_parts = function_response["parts"] return [part["inline_data"] for part in nested_parts if "inline_data" in part] @@ -47,9 +49,7 @@ def test_ollama_pt_simple_messages(): result = ollama_pt(model="llama2", messages=messages) - expected_prompt = ( - "### System:\nYou are a helpful assistant\n\n### Assistant:\nHow can I help you?\n\n### User:\nHello\n\n" - ) + expected_prompt = "### System:\nYou are a helpful assistant\n\n### Assistant:\nHow can I help you?\n\n### User:\nHello\n\n" assert isinstance(result, dict) assert result["prompt"] == expected_prompt assert result["images"] == [] @@ -104,7 +104,10 @@ async def test_anthropic_bedrock_thinking_blocks_with_none_content(): # verify the result assert len(result) == 2 - assert result[1]["content"][0]["reasoningContent"]["reasoningText"]["text"] == "This is a test thinking block" + assert ( + result[1]["content"][0]["reasoningContent"]["reasoningText"]["text"] + == "This is a test thinking block" + ) def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): @@ -172,7 +175,11 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): assert len(assistant_blocks) == 1 for block in assistant_blocks[0]["content"]: if "text" in block: - assert block["text"].strip(), f"Bedrock Converse rejects blank-text ContentBlocks; got {block!r}" + assert block[ + "text" + ].strip(), ( + f"Bedrock Converse rejects blank-text ContentBlocks; got {block!r}" + ) # toolUse blocks must still be present tool_use_blocks = [b for b in assistant_blocks[0]["content"] if "toolUse" in b] assert len(tool_use_blocks) == 2 @@ -213,16 +220,19 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): {"role": "user", "content": "Now what is 3+3?"}, ] - result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + result = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic" + ) assistant = next(m for m in result if m["role"] == "assistant") content = assistant["content"] - assert all(block.get("type") not in ("thinking", "redacted_thinking") for block in content), ( - f"unsignable thinking block must be dropped, got {content!r}" - ) - assert any(block.get("type") == "text" and block.get("text") == "2+2 equals 4." for block in content), ( - f"assistant answer text must be preserved, got {content!r}" - ) + assert all( + block.get("type") not in ("thinking", "redacted_thinking") for block in content + ), f"unsignable thinking block must be dropped, got {content!r}" + assert any( + block.get("type") == "text" and block.get("text") == "2+2 equals 4." + for block in content + ), f"assistant answer text must be preserved, got {content!r}" def test_anthropic_messages_pt_keeps_signed_thinking_block(): @@ -245,7 +255,9 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block(): {"role": "user", "content": "Now what is 3+3?"}, ] - result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") + result = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic" + ) assistant = next(m for m in result if m["role"] == "assistant") thinking_blocks = [b for b in assistant["content"] if b.get("type") == "thinking"] @@ -332,7 +344,9 @@ def test_bedrock_get_document_format_fallback_mimes(): """ # Test DOCX fallback - docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + docx_mime = ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ) supported_formats = ["pdf", "docx", "xlsx", "csv"] # Mock mimetypes.guess_all_extensions to return empty list (simulating Docker container scenario) @@ -356,11 +370,15 @@ def test_bedrock_get_document_format_mimetypes_success(): """ Test the _get_document_format method when mimetypes.guess_all_extensions works normally. """ - docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + docx_mime = ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ) supported_formats = ["pdf", "docx", "xlsx", "csv"] # Test normal mimetypes behavior (should not hit fallback) - result = BedrockImageProcessor._get_document_format(mime_type=docx_mime, supported_doc_formats=supported_formats) + result = BedrockImageProcessor._get_document_format( + mime_type=docx_mime, supported_doc_formats=supported_formats + ) assert result == "docx", f"Expected 'docx', got '{result}'" @@ -576,7 +594,9 @@ async def test_bedrock_process_image_async_factory(): image_url = "data:application/pdf; qs=0.001;base64,JVBERi0xLjQKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4" - content_block = await BedrockImageProcessor.process_image_async(image_url=image_url, format=None) + content_block = await BedrockImageProcessor.process_image_async( + image_url=image_url, format=None + ) print(f"content_block: {content_block}") @@ -619,7 +639,9 @@ def test_unpack_defs_resolves_nested_ref_inside_anyof_items(): items_schema = schema["properties"]["vatAmounts"]["anyOf"][0]["items"] # Assertions: items_schema should now be the resolved object, not an empty dict - assert isinstance(items_schema, dict), "Items schema should be a dict after unpacking" + assert isinstance( + items_schema, dict + ), "Items schema should be a dict after unpacking" assert items_schema.get("type") == "object" # Ensure essential properties are present assert set(items_schema.get("properties", {}).keys()) == {"vatRate", "vatAmount"} @@ -810,7 +832,9 @@ def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks(): last_message_with_tool_calls=last_message_with_tool_calls, ) inline_parts = _get_gemini_function_response_inline_data_parts(result) - assert len(inline_parts) == 2, f"expected 2 inline_data parts, got {len(inline_parts)}" + assert ( + len(inline_parts) == 2 + ), f"expected 2 inline_data parts, got {len(inline_parts)}" mime_types = {p["mime_type"] for p in inline_parts} assert mime_types == {"image/png", "image/jpeg"} @@ -846,7 +870,9 @@ def test_convert_gemini_tool_call_result_with_data_url_string(): last_message_with_tool_calls=last_message_with_tool_calls, ) inline_parts = _get_gemini_function_response_inline_data_parts(result) - assert len(inline_parts) == 1, "data-URL image string was not converted to inline_data" + assert ( + len(inline_parts) == 1 + ), "data-URL image string was not converted to inline_data" assert inline_parts[0]["mime_type"] == "image/png" assert inline_parts[0]["data"] == tiny_png_b64 @@ -882,9 +908,9 @@ def test_convert_gemini_tool_call_result_with_data_url_extra_params(): ) inline_parts = _get_gemini_function_response_inline_data_parts(result) assert len(inline_parts) == 1 - assert inline_parts[0]["mime_type"] == "image/png", ( - f"expected clean 'image/png', got '{inline_parts[0]['mime_type']}'" - ) + assert ( + inline_parts[0]["mime_type"] == "image/png" + ), f"expected clean 'image/png', got '{inline_parts[0]['mime_type']}'" def test_bedrock_tools_unpack_defs(): @@ -981,7 +1007,9 @@ def test_bedrock_tools_pt_strict_parameter(): }, } ] - result = _bedrock_tools_pt(tools_with_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0") + result = _bedrock_tools_pt( + tools_with_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0" + ) assert result[0]["toolSpec"]["strict"] is True assert result[0]["toolSpec"]["inputSchema"]["json"]["additionalProperties"] is False @@ -1003,7 +1031,9 @@ def test_bedrock_tools_pt_strict_parameter(): }, } ] - result = _bedrock_tools_pt(tools_without_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0") + result = _bedrock_tools_pt( + tools_without_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0" + ) assert "strict" not in result[0]["toolSpec"] assert "additionalProperties" not in result[0]["toolSpec"]["inputSchema"]["json"] @@ -1026,7 +1056,9 @@ def test_bedrock_image_processor_content_type_fallback_url_extension(): # Test with .png URL image_url = "https://example.com/test-image.png" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url + ) assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -1050,7 +1082,9 @@ def test_bedrock_image_processor_content_type_fallback_binary_detection(): # Test with URL without extension image_url = "https://example.com/test-image-without-extension" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url + ) assert content_type == "image/jpeg" assert base64_bytes == base64.b64encode(jpeg_content).decode("utf-8") @@ -1073,7 +1107,9 @@ def test_bedrock_image_processor_content_type_fallback_application_octet_stream( # Test with .gif URL image_url = "https://s3.amazonaws.com/bucket/image.gif" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url + ) assert content_type == "image/gif" assert base64_bytes == base64.b64encode(gif_content).decode("utf-8") @@ -1096,7 +1132,9 @@ def test_bedrock_image_processor_content_type_with_query_params(): # Test with URL containing query parameters (common in S3 signed URLs) image_url = "https://s3.amazonaws.com/bucket/image.webp?AWSAccessKeyId=123&Expires=456&Signature=789" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url + ) assert content_type == "image/webp" assert base64_bytes == base64.b64encode(webp_content).decode("utf-8") @@ -1118,7 +1156,9 @@ def test_bedrock_image_processor_content_type_normal_header(): mock_response.content = png_content image_url = "https://example.com/test-image.png" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url + ) assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -1138,7 +1178,7 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - with pytest.raises(ValueError, match="Unable to determine content type from URL: https") as excinfo: + with pytest.raises(ValueError, match='Unable to determine content type from URL: https') as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert "Unable to determine content type" in str(excinfo.value) @@ -1158,12 +1198,16 @@ def test_bedrock_image_processor_content_type_jpeg_variants(): # Test with .jpg extension image_url_jpg = "https://example.com/photo.jpg" - _, content_type_jpg = BedrockImageProcessor._post_call_image_processing(mock_response, image_url_jpg) + _, content_type_jpg = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url_jpg + ) assert content_type_jpg == "image/jpeg" # Test with .jpeg extension image_url_jpeg = "https://example.com/photo.jpeg" - _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing(mock_response, image_url_jpeg) + _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing( + mock_response, image_url_jpeg + ) assert content_type_jpeg == "image/jpeg" @@ -1185,7 +1229,9 @@ def test_bedrock_image_processor_content_type_pdf_document(): # Test with .pdf URL pdf_url = "https://s3.amazonaws.com/bucket/document.pdf" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, pdf_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, pdf_url + ) assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1218,8 +1264,12 @@ def test_bedrock_image_processor_content_type_document_formats(): ] for url, expected_mime in test_cases: - _, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, url) - assert content_type == expected_mime, f"Expected {expected_mime} for {url}, got {content_type}" + _, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, url + ) + assert ( + content_type == expected_mime + ), f"Expected {expected_mime} for {url}, got {content_type}" def test_bedrock_image_processor_content_type_s3_pdf_with_query(): @@ -1238,7 +1288,9 @@ def test_bedrock_image_processor_content_type_s3_pdf_with_query(): # S3 signed URL with query parameters s3_url = "https://my-bucket.s3.us-east-1.amazonaws.com/documents/report.pdf?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=abcdef123456" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, s3_url) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( + mock_response, s3_url + ) assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1347,8 +1399,12 @@ def test_bedrock_create_bedrock_block_normalized_base64(): base64_content = base64.b64encode(pdf_content).decode("utf-8") # Create versions with different whitespace - base64_with_newlines = "\n".join([base64_content[i : i + 64] for i in range(0, len(base64_content), 64)]) - base64_with_spaces = " ".join([base64_content[i : i + 32] for i in range(0, len(base64_content), 32)]) + base64_with_newlines = "\n".join( + [base64_content[i : i + 64] for i in range(0, len(base64_content), 64)] + ) + base64_with_spaces = " ".join( + [base64_content[i : i + 32] for i in range(0, len(base64_content), 32)] + ) # Create blocks block1 = BedrockImageProcessor._create_bedrock_block( @@ -1480,7 +1536,9 @@ def test_bedrock_create_bedrock_block_document_name_format(): # Check format: DocumentPDFmessages_{16_hex_chars}_{format} pattern = r"^DocumentPDFmessages_[0-9a-f]{16}_pdf$" - assert re.match(pattern, document_name), f"Document name format mismatch: {document_name}" + assert re.match( + pattern, document_name + ), f"Document name format mismatch: {document_name}" def test_bedrock_create_bedrock_block_different_document_formats(): @@ -1506,7 +1564,7 @@ def test_bedrock_create_bedrock_block_different_document_formats(): ) assert block.get("document") is not None - assert "DocumentPDFmessages_" in block["document"]["name"] + assert f"DocumentPDFmessages_" in block["document"]["name"] assert block["document"]["name"].endswith(f"_{format_type}") assert block["document"]["format"] == format_type @@ -1533,7 +1591,9 @@ def test_bedrock_nova_web_search_options_mapping(): assert system_tool["name"] == "nova_grounding" # Test with search_context_size (should be ignored for Nova) - result2 = config._map_web_search_options({"search_context_size": "high"}, "us.amazon.nova-premier-v1:0") + result2 = config._map_web_search_options( + {"search_context_size": "high"}, "us.amazon.nova-premier-v1:0" + ) assert result2 is not None system_tool2 = result2.get("systemTool") @@ -1599,7 +1659,9 @@ def test_bedrock_tools_pt_drops_unmappable_responses_builtin_tools(): {"type": "custom", "name": "free_form"}, ] - result = _bedrock_tools_pt(tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0") + result = _bedrock_tools_pt( + tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0" + ) names = [block["toolSpec"]["name"] for block in result if "toolSpec" in block] assert names == ["noop"] @@ -1629,7 +1691,9 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools(): }, ] - result = _bedrock_tools_pt(tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0") + result = _bedrock_tools_pt( + tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0" + ) names = [block["toolSpec"]["name"] for block in result if "toolSpec" in block] assert names == ["lookup"] @@ -1831,7 +1895,9 @@ def test_anthropic_messages_pt_server_tool_use_passthrough(): "tool_use_id": "srvtoolu_01ABC123", "content": { "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "get_time"}], + "tool_references": [ + {"type": "tool_reference", "tool_name": "get_time"} + ], }, }, {"type": "text", "text": "I found the time tool. How can I help you?"}, @@ -1859,14 +1925,20 @@ def test_anthropic_messages_pt_server_tool_use_passthrough(): # Verify server_tool_use block is preserved assert "server_tool_use" in content_types - server_tool_use_block = next(b for b in assistant_msg["content"] if b.get("type") == "server_tool_use") + server_tool_use_block = next( + b for b in assistant_msg["content"] if b.get("type") == "server_tool_use" + ) assert server_tool_use_block["id"] == "srvtoolu_01ABC123" assert server_tool_use_block["name"] == "tool_search_tool_regex" assert server_tool_use_block["input"] == {"query": ".*time.*"} # Verify tool_search_tool_result block is preserved assert "tool_search_tool_result" in content_types - tool_result_block = next(b for b in assistant_msg["content"] if b.get("type") == "tool_search_tool_result") + tool_result_block = next( + b + for b in assistant_msg["content"] + if b.get("type") == "tool_search_tool_result" + ) assert tool_result_block["tool_use_id"] == "srvtoolu_01ABC123" assert tool_result_block["content"]["type"] == "tool_search_tool_search_result" assert tool_result_block["content"]["tool_references"][0]["tool_name"] == "get_time" @@ -1918,7 +1990,9 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs(): "anyOf": [ {"$ref": "#/$defs/Literal"}, {"$ref": "#/$defs/FieldRef"}, - {"$ref": "#/$defs/Expression"}, # Circular: Operand -> Expression -> Operand + { + "$ref": "#/$defs/Expression" + }, # Circular: Operand -> Expression -> Operand ], }, "Literal": { @@ -2052,7 +2126,9 @@ def test_anthropic_messages_pt_file_block_cache_control_with_explicit_provider() file_block = content_blocks[0] assert file_block["type"] == "document" - assert "cache_control" in file_block, "cache_control should be preserved on file/document content blocks" + assert ( + "cache_control" in file_block + ), "cache_control should be preserved on file/document content blocks" assert file_block["cache_control"]["type"] == "ephemeral" text_block = content_blocks[1] @@ -2260,16 +2336,22 @@ def test_bedrock_tool_call_invoke_concatenated_json(): # First block keeps original tool id assert result[0]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN" assert result[0]["toolUse"]["name"] == "shell" - assert result[0]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009", "-m", "10"]} + assert result[0]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009", "-m", "10"] + } # Subsequent blocks get suffixed ids assert result[1]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_1" assert result[1]["toolUse"]["name"] == "shell" - assert result[1]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"]} + assert result[1]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"] + } assert result[2]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_2" assert result[2]["toolUse"]["name"] == "shell" - assert result[2]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"]} + assert result[2]["toolUse"]["input"] == { + "command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"] + } def test_bedrock_tool_call_invoke_concatenated_json_with_cache_control(): @@ -2424,7 +2506,9 @@ def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request def test_make_valid_bedrock_tool_name_preserves_hyphens(): assert make_valid_bedrock_tool_name("my-tool") == "my-tool" assert ( - make_valid_bedrock_tool_name("CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q") + make_valid_bedrock_tool_name( + "CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q" + ) == "CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q" ) @@ -2451,7 +2535,9 @@ def test_bedrock_tool_name_sanitized_consistently_in_tools_and_tool_use(): "function": {"name": raw_name, "arguments": "{}"}, } ] - tool_use_name = _convert_to_bedrock_tool_call_invoke(tool_calls)[0]["toolUse"]["name"] + tool_use_name = _convert_to_bedrock_tool_call_invoke(tool_calls)[0]["toolUse"][ + "name" + ] assert tool_spec_name == "foo_bar" assert tool_use_name == tool_spec_name @@ -2474,8 +2560,15 @@ def test_bedrock_converse_messages_pt_tool_use_matches_tool_spec_hyphen_name(): ], }, ] - translated = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") - tool_use_blocks = [block for msg in translated for block in msg.get("content", []) if "toolUse" in block] + translated = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + tool_use_blocks = [ + block + for msg in translated + for block in msg.get("content", []) + if "toolUse" in block + ] assert len(tool_use_blocks) == 1 assert tool_use_blocks[0]["toolUse"]["name"] == tool_name @@ -2572,7 +2665,11 @@ def test_sanitize_messages_deduplicates_tool_results(): result = sanitize_messages_for_tool_calling(messages) # Count tool messages with this ID — should be exactly 1 - tool_results = [m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123"] + tool_results = [ + m + for m in result + if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123" + ] assert len(tool_results) == 1 # Should keep the LAST occurrence (most complete) assert tool_results[0]["content"] == '{"temperature": 72, "condition": "sunny"}' @@ -2707,7 +2804,11 @@ def test_sanitize_messages_dedup_scoped_per_turn_preserves_cross_turn(): result = sanitize_messages_for_tool_calling(messages) # Both tool results must survive — one per turn - tool_results = [m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_X"] + tool_results = [ + m + for m in result + if m.get("role") == "tool" and m.get("tool_call_id") == "call_X" + ] assert len(tool_results) == 2, ( f"Expected 2 tool results (one per turn), got {len(tool_results)}. " "Dedup may be global instead of per-turn scoped." @@ -2761,26 +2862,32 @@ def test_sanitize_messages_combined_case_a_and_case_d(): tool_results = [m for m in result if m.get("role") in ("tool", "function")] # Case A: call_missing should have a dummy result injected - missing_results = [m for m in tool_results if m.get("tool_call_id") == "call_missing"] - assert len(missing_results) == 1, ( - f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" - ) + missing_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_missing" + ] + assert ( + len(missing_results) == 1 + ), f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" # Case D: call_duped should have exactly 1 result (the fresh one) - duped_results = [m for m in tool_results if m.get("tool_call_id") == "call_duped"] - assert len(duped_results) == 1, ( - f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" - ) - assert duped_results[0]["content"] == "fresh_result", ( - f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" - ) + duped_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_duped" + ] + assert ( + len(duped_results) == 1 + ), f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" + assert ( + duped_results[0]["content"] == "fresh_result" + ), f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" # Verify tool results immediately follow the assistant message asst_idx = next(i for i, m in enumerate(result) if m.get("role") == "assistant") - tool_msgs_after_asst = [m for m in result[asst_idx + 1 :] if m.get("role") in ("tool", "function")] - assert len(tool_msgs_after_asst) == 2, ( - f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" - ) + tool_msgs_after_asst = [ + m for m in result[asst_idx + 1 :] if m.get("role") in ("tool", "function") + ] + assert ( + len(tool_msgs_after_asst) == 2 + ), f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" # Both tool_call_ids should be present (order may vary) tool_ids = {m["tool_call_id"] for m in tool_msgs_after_asst} assert tool_ids == { @@ -2822,7 +2929,9 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): } ] - result = anthropic_messages_pt(messages, model="claude-sonnet-4-20250514", llm_provider="anthropic") + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-20250514", llm_provider="anthropic" + ) content_blocks = result[0]["content"] assert len(content_blocks) == 2 @@ -2830,7 +2939,9 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): # Document block (from file) should preserve cache_control doc_block = content_blocks[0] assert doc_block["type"] == "document" - assert "cache_control" in doc_block, "cache_control was dropped from file/document block" + assert ( + "cache_control" in doc_block + ), "cache_control was dropped from file/document block" assert doc_block["cache_control"]["type"] == "ephemeral" # Text block should also preserve cache_control @@ -2873,7 +2984,9 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): } # Claude 4.5 model: ttl should be preserved - result = add_cache_point_tool_block(tool_with_1h, model="jp.anthropic.claude-opus-4-7") + result = add_cache_point_tool_block( + tool_with_1h, model="jp.anthropic.claude-opus-4-7" + ) assert result is not None assert result["cachePoint"]["type"] == "default" assert result["cachePoint"]["ttl"] == "1h" @@ -2882,12 +2995,16 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): tool_with_5m = { "cache_control": {"type": "ephemeral", "ttl": "5m"}, } - result_5m = add_cache_point_tool_block(tool_with_5m, model="jp.anthropic.claude-opus-4-7") + result_5m = add_cache_point_tool_block( + tool_with_5m, model="jp.anthropic.claude-opus-4-7" + ) assert result_5m is not None assert result_5m["cachePoint"]["ttl"] == "5m" # Older model: ttl should be stripped - result_old = add_cache_point_tool_block(tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0") + result_old = add_cache_point_tool_block( + tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) assert result_old is not None assert result_old["cachePoint"]["type"] == "default" assert "ttl" not in result_old["cachePoint"] @@ -2906,7 +3023,9 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): # cache_control without ttl: returns default cachePoint (unchanged behavior) tool_no_ttl = {"cache_control": {"type": "ephemeral"}} - result_no_ttl = add_cache_point_tool_block(tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") + result_no_ttl = add_cache_point_tool_block( + tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" + ) assert result_no_ttl is not None assert result_no_ttl["cachePoint"]["type"] == "default" assert "ttl" not in result_no_ttl["cachePoint"] @@ -2957,7 +3076,9 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(monkeypatch): assert cache_blocks[0]["cachePoint"]["ttl"] == "1h" # Older model: cachePoint should not have ttl - result_old = _bedrock_tools_pt(tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0") + result_old = _bedrock_tools_pt( + tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) cache_blocks_old = [b for b in result_old if "cachePoint" in b] assert len(cache_blocks_old) == 1 assert "ttl" not in cache_blocks_old[0]["cachePoint"] @@ -3032,7 +3153,9 @@ def test_bedrock_converse_messages_pt_document_various_formats(): } ] - result = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + result = _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) doc_block = result[0]["content"][0] assert doc_block["document"]["format"] == expected_format, ( @@ -3059,8 +3182,12 @@ def test_bedrock_converse_messages_pt_document_deterministic_name(): } ] - result1 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") - result2 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + result1 = _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) + result2 = _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) name1 = result1[0]["content"][0]["document"]["name"] name2 = result2[0]["content"][0]["document"]["name"] @@ -3094,18 +3221,34 @@ def test_bedrock_converse_messages_pt_renames_duplicate_document_names(): }, ] - result1 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") - result2 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + result1 = _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) + result2 = _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) - names1 = [block["document"]["name"] for message in result1 for block in message["content"] if "document" in block] - names2 = [block["document"]["name"] for message in result2 for block in message["content"] if "document" in block] + names1 = [ + block["document"]["name"] + for message in result1 + for block in message["content"] + if "document" in block + ] + names2 = [ + block["document"]["name"] + for message in result2 + for block in message["content"] + if "document" in block + ] assert len(names1) == 2 assert len(set(names1)) == 2 assert names1[1] == f"{names1[0]}_2" assert names1 == names2 - single_turn = _bedrock_converse_messages_pt([messages[0]], "anthropic.claude-sonnet-4-6", "bedrock") + single_turn = _bedrock_converse_messages_pt( + [messages[0]], "anthropic.claude-sonnet-4-6", "bedrock" + ) assert names1[0] == single_turn[0]["content"][0]["document"]["name"] @@ -3127,10 +3270,14 @@ def test_rename_duplicate_bedrock_document_names_skips_organic_suffixes(): def _names(contents): return [block["document"]["name"] for block in contents[0]["content"]] - organic_first = _rename_duplicate_bedrock_document_names(_contents(["report", "report_2", "report"])) + organic_first = _rename_duplicate_bedrock_document_names( + _contents(["report", "report_2", "report"]) + ) assert _names(organic_first) == ["report", "report_2", "report_3"] - organic_last = _rename_duplicate_bedrock_document_names(_contents(["report", "report", "report_2"])) + organic_last = _rename_duplicate_bedrock_document_names( + _contents(["report", "report", "report_2"]) + ) assert _names(organic_last) == ["report", "report_3", "report_2"] @@ -3152,11 +3299,18 @@ def test_bedrock_converse_messages_pt_document_rejects_url_source(): ] with pytest.raises(ValueError, match="only supports base64-encoded"): - _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + _bedrock_converse_messages_pt( + messages, "anthropic.claude-sonnet-4-6", "bedrock" + ) def _collect_cache_points(blocks): - return [block["cachePoint"] for message in blocks for block in message["content"] if "cachePoint" in block] + return [ + block["cachePoint"] + for message in blocks + for block in message["content"] + if "cachePoint" in block + ] @pytest.mark.parametrize( @@ -3420,7 +3574,9 @@ def test_bedrock_converse_pdf_only_user_message_gets_text_block(): } ] - result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") + result = _bedrock_converse_messages_pt( + messages, "anthropic.claude-haiku-4-5", "bedrock" + ) assert len(result) == 1 assert any("document" in block for block in result[0]["content"]) @@ -3438,7 +3594,9 @@ def test_bedrock_converse_document_with_text_gets_no_extra_text_block(): } ] - result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") + result = _bedrock_converse_messages_pt( + messages, "anthropic.claude-haiku-4-5", "bedrock" + ) assert _text_blocks(result[0]) == ["summarize this"] @@ -3451,7 +3609,9 @@ def test_bedrock_converse_image_only_user_message_gets_no_text_block(): } ] - result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") + result = _bedrock_converse_messages_pt( + messages, "anthropic.claude-haiku-4-5", "bedrock" + ) assert any("image" in block for block in result[0]["content"]) assert _text_blocks(result[0]) == [] @@ -3494,7 +3654,9 @@ def test_bedrock_converse_tool_round_trip_document_injects_text_before_cache_poi }, ] - result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") + result = _bedrock_converse_messages_pt( + messages, "anthropic.claude-haiku-4-5", "bedrock" + ) assert _text_blocks(result[0]) == ["read the pdf"] document_message = result[-1] diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index e17216b7b34..21cba74fba5 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -922,3 +922,5 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr assert model not in litellm.model_cost info = litellm.get_model_info(model, custom_llm_provider=provider) assert info.get("supports_tool_search") is tool_search, model + + diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 28ba46a7e75..63d4571fe8d 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -395,6 +395,7 @@ class TestGetRouterDeploymentModelInfo: logging_obj.litellm_params = {"api_base": ""} assert logging_obj.get_router_deployment_model_info() is None + def test_merging_does_not_mutate_the_cached_model_info(self) -> None: """The published-rate merge must not write into get_model_info's lru-cached dict. @@ -3887,7 +3888,9 @@ def test_get_standard_logging_object_payload_carries_matched_access_groups(loggi "model": "gpt-4o", "messages": [], "litellm_params": { - "metadata": {"user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"]}, + "metadata": { + "user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"] + }, "proxy_server_request": {"body": {}}, }, }, @@ -3971,7 +3974,9 @@ def _model_router_response(selected_model: str, stamp: bool): from litellm.types.utils import ModelResponse response = ModelResponse(model=selected_model) - response._hidden_params = {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {} + response._hidden_params = ( + {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {} + ) return response @@ -3995,7 +4000,9 @@ def test_standard_logging_payload_uses_stamped_model_router_model(logging_obj): "messages": [], "litellm_params": {"metadata": {}}, }, - init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=True), + init_response_obj=_model_router_response( + "azure_ai/grok-4-1-fast-reasoning", stamp=True + ), start_time=now, end_time=now, logging_obj=logging_obj, @@ -4027,7 +4034,9 @@ def test_standard_logging_payload_keeps_requested_model_without_router_stamp( "messages": [], "litellm_params": {"metadata": {}}, }, - init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=False), + init_response_obj=_model_router_response( + "azure_ai/grok-4-1-fast-reasoning", stamp=False + ), start_time=now, end_time=now, logging_obj=logging_obj, @@ -5515,7 +5524,9 @@ class TestNonInferenceCallTypesAreNotBilled: init_response_obj=self._retrieved_response(), start_time=now, end_time=now, - logging_obj=self._logging_obj("aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA), + logging_obj=self._logging_obj( + "aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA + ), status="success", ) @@ -5761,7 +5772,9 @@ async def test_streaming_success_callbacks_survive_cost_calculation_failure(): releasing.async_log_success_event = AsyncMock() patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing]) - with patcher, patch.object(logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block")): + with patcher, patch.object( + logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block") + ): await logging_obj.async_success_handler(result=_assembled_stream_result()) assert logging_obj.model_call_details["response_cost"] is None @@ -5774,9 +5787,8 @@ async def test_streaming_success_callbacks_survive_standard_logging_payload_fail releasing.async_log_success_event = AsyncMock() patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing]) - with ( - patcher, - patch.object(logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream")), + with patcher, patch.object( + logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream") ): await logging_obj.async_success_handler(result=_assembled_stream_result()) @@ -6124,8 +6136,6 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa ) for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) - - def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch): """With LITELLM_OTEL_V2 on and operator credentials present, the "newrelic" callback builds the OTel v2 logger (per-team credential routing); with the @@ -6281,9 +6291,7 @@ def test_get_error_information_skips_traceback_for_budget_rejection_with_provide from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup assert litellm.log_client_error_tracebacks is False - over_budget = _raise_and_catch( - litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic") - ) + over_budget = _raise_and_catch(litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic")) result = StandardLoggingPayloadSetup.get_error_information(over_budget) assert result["error_code"] == "429" assert result["llm_provider"] == "anthropic" @@ -6856,7 +6864,9 @@ def test_passthrough_embeddings_result_swapped_for_callbacks(): ], "model": "EmbeddingsGigaR", }, - request=httpx.Request("POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"), + request=httpx.Request( + "POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings" + ), ) _, _, swapped_result = logging_obj._success_handler_helper_fn( @@ -6875,14 +6885,12 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene request-level guardrail_status but never mask an intervention.""" flagged = {"guardrail_status": "guardrail_flagged"} - assert ( - _get_status_fields("success", [{"guardrail_status": "success"}, flagged], None)["guardrail_status"] - == "guardrail_flagged" - ) - assert ( - _get_status_fields("success", [flagged, {"guardrail_status": "guardrail_intervened"}], None)["guardrail_status"] - == "guardrail_intervened" - ) + assert _get_status_fields( + "success", [{"guardrail_status": "success"}, flagged], None + )["guardrail_status"] == "guardrail_flagged" + assert _get_status_fields( + "success", [flagged, {"guardrail_status": "guardrail_intervened"}], None + )["guardrail_status"] == "guardrail_intervened" def test_get_error_information_redacts_provider_key_from_upstream_url(): @@ -6935,41 +6943,22 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non return httpx.Response(200, json=mock_responses_api_response(content).model_dump()) if provider == "anthropic": - return httpx.Response( - 200, - json={ - "id": "msg-audit", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5", - "content": [{"type": "text", "text": content}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 10, "output_tokens": 5}, - }, - ) + return httpx.Response(200, json={ + "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) if provider == "bedrock": - return httpx.Response( - 200, - json={ - "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, - "stopReason": "end_turn", - "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, - "metrics": {"latencyMs": 1}, - }, - ) - return httpx.Response( - 200, - json={ - "id": "chatcmpl-audit", - "object": "chat.completion", - "created": 0, - "model": "gpt-5.6", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - }, - ) + return httpx.Response(200, json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }) + return httpx.Response(200, json={ + "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }) async def capture(kwargs, response_obj, start_time, end_time): logs.put_nowait(kwargs["standard_logging_object"]) @@ -6980,15 +6969,11 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler.client = http_client client: Final = ( AsyncAzureOpenAI( - api_key="transport-only", - azure_endpoint="https://azure.invalid", - api_version="2025-04-01-preview", - http_client=http_client, + api_key="transport-only", azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", http_client=http_client, ) - if provider == "azure" - else AsyncOpenAI(api_key="transport-only", http_client=http_client) - if provider == "openai" - else handler + if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" else handler ) model: Final = { "openai": "openai/gpt-5.6", @@ -7001,44 +6986,23 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non async def run(marker: str) -> None: if provider == "responses": await litellm.aresponses( - model=model, - api_key="transport-only", - client=client, - max_output_tokens=128, - instructions="classifier-rubric", - input=marker, + model=model, api_key="transport-only", client=client, max_output_tokens=128, + instructions="classifier-rubric", input=marker, metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], - num_retries=0, + success_callback=[capture], num_retries=0, ) return await litellm.acompletion( - model=model, - api_key="transport-only", - client=client, - max_tokens=128, - aws_access_key_id="transport-only", - aws_secret_access_key="transport-only", - aws_region_name="us-east-1", + model=model, api_key="transport-only", client=client, max_tokens=128, + aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], - num_retries=0, - **( - {"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} - if provider == "azure" - else {} - ), - **( - { - "extra_body": {"audit_context": "provider-extra"}, - "extra_headers": {"X-Audit": "header-only-secret"}, - } - if provider in ("openai", "azure") - else {} - ), + success_callback=[capture], num_retries=0, + **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), + **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} + if provider in ("openai", "azure") else {}), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -7062,17 +7026,14 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non @pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("call_type", ["completion", "acompletion", "responses", "aresponses"]) -def test_classifier_audit_obeys_message_logging_before_payload_emission( - logging_obj, monkeypatch, redaction, status, call_type -): +def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status, call_type): from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") params: Final = { - "metadata": { - "internal_call_origin": "autorouter_classifier", - **({"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}), - }, + "metadata": {"internal_call_origin": "autorouter_classifier", **( + {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} + )}, "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, } logging_obj.call_type = call_type @@ -7085,12 +7046,8 @@ def test_classifier_audit_obeys_message_logging_before_payload_emission( ) now: Final = datetime.datetime.now() payload: Final = get_standard_logging_object_payload( - kwargs={**logging_obj.model_call_details, "call_type": call_type}, - init_response_obj={}, - start_time=now, - end_time=now, - logging_obj=logging_obj, - status=status, + kwargs={**logging_obj.model_call_details, "call_type": call_type}, init_response_obj={}, + start_time=now, end_time=now, logging_obj=logging_obj, status=status, ) assert payload is not None if redaction == "none": diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 7a70a146667..c2f0cfcc32e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -187,7 +187,11 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block1"}]), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block1"} + ] + ), make_chunk( thinking_blocks=[ { @@ -205,10 +209,16 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block2"}]), + make_chunk( + thinking_blocks=[ + {"type": "thinking", "thinking": None, "signature": "sig_block2"} + ] + ), ] - thinking_chunks = [chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")] + thinking_chunks = [ + chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks") + ] processor = ChunkProcessor(chunks=chunks) result = processor.get_combined_thinking_content(thinking_chunks) @@ -253,7 +263,9 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=11779, total_tokens=11784, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=11775), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=11775 + ), cache_creation_input_tokens=4, cache_read_input_tokens=11775, ), @@ -287,7 +299,9 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=0, total_tokens=214, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=0 + ), cache_creation_input_tokens=0, cache_read_input_tokens=0, ), @@ -347,7 +361,10 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown(): ) # Sanity: the delta event genuinely lacks the breakdown - this is the input # condition that used to defeat cost calc. - assert getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) is None + assert ( + getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) def _usage_chunk(usage, finish_reason): return ModelResponseStream( @@ -466,7 +483,9 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): prompt_tokens=1234, total_tokens=1239, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=543).model_dump(), + prompt_tokens_details=PromptTokensDetails( + audio_tokens=None, cached_tokens=543 + ).model_dump(), ), index=2, ) @@ -483,7 +502,6 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): assert usage.prompt_tokens_details.cached_tokens == 543 - def test_stream_chunk_builder_litellm_usage_chunks(): """ Validate ChunkProcessor.calculate_usage uses provided usage fields from streaming chunks @@ -557,7 +575,9 @@ def test_stream_chunk_builder_litellm_usage_chunks(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage(chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="") + usage = processor.calculate_usage( + chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="" + ) assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -601,11 +621,15 @@ def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): provider_specific_fields=None, stream_options={"include_usage": True}, ) - usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) + usage_chunk.usage = CompletionUsage( + prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 + ) assert type(usage_chunk.usage) is CompletionUsage chunks = [content_chunk, usage_chunk] - usage = ChunkProcessor(chunks=chunks).calculate_usage(chunks=chunks, model="mantle-claude", completion_output="") + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="mantle-claude", completion_output="" + ) assert usage.prompt_tokens == 20 assert usage.completion_tokens == 60 @@ -628,7 +652,9 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4.1-nano-2025-04-14", "id": "chatcmpl-123", "choices": []}, ] - result = ChunkProcessor._get_model_from_chunks(chunks=chunks, first_chunk_model="azure-model-router") + result = ChunkProcessor._get_model_from_chunks( + chunks=chunks, first_chunk_model="azure-model-router" + ) # Should return the actual model, not the request model assert result == "gpt-4.1-nano-2025-04-14" @@ -639,7 +665,9 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4", "id": "chatcmpl-456", "choices": []}, ] - result_same = ChunkProcessor._get_model_from_chunks(chunks=chunks_same_model, first_chunk_model="gpt-4") + result_same = ChunkProcessor._get_model_from_chunks( + chunks=chunks_same_model, first_chunk_model="gpt-4" + ) # Should return the first chunk's model when all are the same assert result_same == "gpt-4" @@ -715,7 +743,9 @@ def test_stream_chunk_builder_anthropic_web_search(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage(chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="") + usage = processor.calculate_usage( + chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" + ) assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -867,11 +897,15 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields(): ], ) chunk_dict = chunk.model_dump() - chunk_dict["_hidden_params"] = {"provider_specific_fields": {"traffic_type": "default"}} + chunk_dict["_hidden_params"] = { + "provider_specific_fields": {"traffic_type": "default"} + } response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" + assert ( + response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" + ) def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): @@ -916,7 +950,10 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): assert getattr(response, "vertex_ai_grounding_metadata") == grounding_metadata assert getattr(response, "vertex_ai_url_context_metadata") == url_context_metadata assert response._hidden_params["vertex_ai_grounding_metadata"] == grounding_metadata - assert response._hidden_params["vertex_ai_url_context_metadata"] == url_context_metadata + assert ( + response._hidden_params["vertex_ai_url_context_metadata"] + == url_context_metadata + ) dumped = response.model_dump() assert dumped["vertex_ai_grounding_metadata"] == grounding_metadata @@ -963,7 +1000,9 @@ def test_stream_chunk_builder_uses_assembled_model_for_provider_metadata(): def test_stream_chunk_builder_propagates_vertex_ai_safety_results(): """Assembled response must expose safety data under the non-streaming field name.""" - safety_ratings = [[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]] + safety_ratings = [ + [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}] + ] chunk = ModelResponseStream( id="chatcmpl-vertex-safety", @@ -1005,12 +1044,18 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks(): ) ], ).model_dump() - chunk_dict["_hidden_params"] = {"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]} + chunk_dict["_hidden_params"] = { + "vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}] + } response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert getattr(response, "vertex_ai_grounding_metadata") == [{"webSearchQueries": ["test query"]}] - assert response.model_dump()["vertex_ai_grounding_metadata"] == [{"webSearchQueries": ["test query"]}] + assert getattr(response, "vertex_ai_grounding_metadata") == [ + {"webSearchQueries": ["test query"]} + ] + assert response.model_dump()["vertex_ai_grounding_metadata"] == [ + {"webSearchQueries": ["test query"]} + ] def test_cost_field_in_usage_chunks(): @@ -1019,21 +1064,29 @@ def test_cost_field_in_usage_chunks(): id="chatcmpl-1", created=1745513206, model="openrouter/claude", - choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], usage=chunk1_usage, ) - chunk2_usage = Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025) + chunk2_usage = Usage( + completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 + ) chunk2 = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openrouter/claude", - choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], usage=chunk2_usage, ) processor = ChunkProcessor(chunks=[chunk1, chunk2]) - usage = processor.calculate_usage(chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi") + usage = processor.calculate_usage( + chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi" + ) assert hasattr(usage, "cost") assert usage.cost == 0.00025 @@ -1077,19 +1130,25 @@ def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): id="chatcmpl-1", created=1745513206, model="openai/gpt-5.6-sol", - choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) + ], usage=Usage( prompt_tokens=6017, completion_tokens=4, total_tokens=6021, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=6004, cache_write_tokens=10), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=6004, cache_write_tokens=10 + ), ), ) chunk_without_details = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openai/gpt-5.6-sol", - choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), ) @@ -1372,7 +1431,9 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens_details.text_tokens == expected_text_tokens -def _openai_chunk(choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None) -> dict[str, object]: +def _openai_chunk( + choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None +) -> dict[str, object]: base: Final = { "id": "chatcmpl-lit6552", "object": "chat.completion.chunk", diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a4b05da7023..89cc1a3fb76 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,3 +1,4 @@ + import pytest from unittest.mock import MagicMock, patch @@ -32,9 +33,13 @@ def test_response_format_transformation_unit_test(): "additionalProperties": False, } - result = config._create_json_tool_call_for_response_format(json_schema=response_format_json_schema) + result = config._create_json_tool_call_for_response_format( + json_schema=response_format_json_schema + ) - assert result["input_schema"]["properties"] == {"agent_doing": {"title": "Agent Doing", "type": "string"}} + assert result["input_schema"]["properties"] == { + "agent_doing": {"title": "Agent Doing", "type": "string"} + } print(result) @@ -545,7 +550,9 @@ def test_extract_response_content_with_citations(): }, } - _, citations, _, _, _, _, _, _ = config.extract_response_content(completion_response) + _, citations, _, _, _, _, _, _ = config.extract_response_content( + completion_response + ) assert citations == [ [ { @@ -618,8 +625,12 @@ def test_web_search_tool_transformation(): assert anthropic_web_search_tool["user_location"]["city"] == "San Francisco" -@pytest.mark.parametrize("search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)]) -def test_web_search_tool_transformation_with_search_context_size(search_context_size, expected_max_uses): +@pytest.mark.parametrize( + "search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)] +) +def test_web_search_tool_transformation_with_search_context_size( + search_context_size, expected_max_uses +): from litellm.types.llms.openai import OpenAIWebSearchOptions config = AnthropicConfig() @@ -794,7 +805,10 @@ def test_web_search_tool_result_in_provider_specific_fields(): assert "web_search_results" in provider_fields assert len(provider_fields["web_search_results"]) == 1 assert provider_fields["web_search_results"][0]["type"] == "web_search_tool_result" - assert provider_fields["web_search_results"][0]["tool_use_id"] == "srvtoolu_provider_test" + assert ( + provider_fields["web_search_results"][0]["tool_use_id"] + == "srvtoolu_provider_test" + ) def test_multiple_web_search_tool_results(): @@ -1018,7 +1032,10 @@ def test_transform_response_with_prefix_prompt(): ) assert result is not None - assert result.choices[0].message.content == "You are a helpful assistant. The grass is green." + assert ( + result.choices[0].message.content + == "You are a helpful assistant. The grass is green." + ) def test_get_supported_params_thinking(): @@ -1133,12 +1150,18 @@ def test_anthropic_beta_header_merging_with_output_format(): } } - result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params) + result_headers = config.update_headers_with_optional_anthropic_beta( + headers, optional_params + ) # Both beta headers should be present beta_value = result_headers["anthropic-beta"] - assert "context-1m-2025-08-07" in beta_value, f"User's context-1m beta header missing from: {beta_value}" - assert "structured-outputs-2025-11-13" in beta_value, f"Structured output beta header missing from: {beta_value}" + assert ( + "context-1m-2025-08-07" in beta_value + ), f"User's context-1m beta header missing from: {beta_value}" + assert ( + "structured-outputs-2025-11-13" in beta_value + ), f"Structured output beta header missing from: {beta_value}" def test_anthropic_beta_header_merging_with_multiple_features(): @@ -1160,7 +1183,9 @@ def test_anthropic_beta_header_merging_with_multiple_features(): "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}], } - result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params) + result_headers = config.update_headers_with_optional_anthropic_beta( + headers, optional_params + ) beta_value = result_headers["anthropic-beta"] @@ -1203,7 +1228,9 @@ def test_anthropic_structured_output_beta_header(): "strict": True, "schema": { "description": 'Progress report for the thinking process\n\nThis model represents a snapshot of the agent\'s current progress during\nthe thinking process, providing a brief description of the current activity.\n\nAttributes:\n agent_doing: Brief description of what the agent is currently doing.\n Should be kept under 10 words. Example: "Learning about home automation"', - "properties": {"agent_doing": {"title": "Agent Doing", "type": "string"}}, + "properties": { + "agent_doing": {"title": "Agent Doing", "type": "string"} + }, "required": ["agent_doing"], "title": "ThinkingStep", "type": "object", @@ -1217,7 +1244,10 @@ def test_anthropic_structured_output_beta_header(): assert response is not None print(f"response: {response}") print(f"raw_request_headers: {response['raw_request_headers']}") - assert "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] + assert ( + "structured-outputs-2025-11-13" + in response["raw_request_headers"]["anthropic-beta"] + ) @pytest.mark.parametrize( @@ -1353,7 +1383,9 @@ def test_tool_search_regex_detection(): config = AnthropicModelInfo() # Test with tool search regex tool - tools = [{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}] + tools = [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} + ] assert config.is_tool_search_used(tools) is True # Test without tool search @@ -1368,7 +1400,9 @@ def test_tool_search_bm25_detection(): config = AnthropicModelInfo() # Test with tool search BM25 tool - tools = [{"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}] + tools = [ + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} + ] assert config.is_tool_search_used(tools) is True @@ -1560,7 +1594,9 @@ def test_tool_search_complete_response_parsing(): "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "content": { "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], + "tool_references": [ + {"type": "tool_reference", "tool_name": "get_weather"} + ], }, }, {"type": "text", "text": "Great! I found a weather tool."}, @@ -1611,7 +1647,9 @@ def test_tool_search_complete_response_parsing(): assert usage.server_tool_use is not None assert usage.server_tool_use.web_search_requests == 0 - assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + assert ( + usage.server_tool_use.tool_search_requests == 1 + ) # Counted from server_tool_use blocks def test_allowed_callers_field_preservation(): @@ -1663,7 +1701,9 @@ def test_programmatic_tool_calling_beta_header(): assert is_programmatic is True # Test header generation - headers = model_info.get_anthropic_headers(api_key="test-key", programmatic_tool_calling_used=True) + headers = model_info.get_anthropic_headers( + api_key="test-key", programmatic_tool_calling_used=True + ) assert "anthropic-beta" in headers assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] @@ -1807,7 +1847,9 @@ def test_input_examples_beta_header(): assert is_examples_used is True # Test header generation - headers = model_info.get_anthropic_headers(api_key="test-key", input_examples_used=True) + headers = model_info.get_anthropic_headers( + api_key="test-key", input_examples_used=True + ) assert "anthropic-beta" in headers assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] @@ -1893,7 +1935,10 @@ def test_input_examples_empty_list_not_added(): transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None # Empty list should not be added - assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + assert ( + "input_examples" not in transformed_tool + or len(transformed_tool.get("input_examples", [])) == 0 + ) # ============ Effort Parameter Tests ============ @@ -1953,7 +1998,9 @@ def test_effort_beta_header_injection(): effort_used = model_info.is_effort_used(optional_params=optional_params, custom_llm_provider="anthropic") assert effort_used is True - headers = model_info.get_anthropic_headers(api_key="test-key", effort_used=effort_used) + headers = model_info.get_anthropic_headers( + api_key="test-key", effort_used=effort_used + ) assert "anthropic-beta" in headers assert "effort-2025-11-24" in headers["anthropic-beta"] @@ -1979,7 +2026,9 @@ def test_effort_validation(): optional_params = {"output_config": {"effort": "invalid"}} - with pytest.raises(litellm.exceptions.BadRequestError, match="Invalid effort value"): + with pytest.raises( + litellm.exceptions.BadRequestError, match="Invalid effort value" + ): config.transform_request( model="claude-opus-4-5-20251101", messages=messages, @@ -2215,8 +2264,16 @@ def test_anthropic_model_supports_speed_param_rejects_non_anthropic_providers( ): """Fast mode is direct-Anthropic-only. Vertex/Azure/Bedrock strip their prefix before the shared transform runs, so the bare Opus id must still be rejected.""" - assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", custom_llm_provider) is False - assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") is True + assert ( + AnthropicConfig._model_supports_speed_param( + "claude-opus-4-8", custom_llm_provider + ) + is False + ) + assert ( + AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") + is True + ) def test_vertex_anthropic_drops_speed_for_opus_with_drop_params(monkeypatch): @@ -2464,7 +2521,9 @@ def test_supports_effort_level_handles_provider_prefixes(model, level, expected) ("claude-opus-4-5-20251101", None, False), ], ) -def test_validate_effort_for_model_centralises_per_model_gating(model, effort, expect_error): +def test_validate_effort_for_model_centralises_per_model_gating( + model, effort, expect_error +): err = AnthropicConfig._validate_effort_for_model(model, effort, "anthropic") if expect_error: assert err is not None @@ -2513,7 +2572,11 @@ def test_transform_request_injects_dummy_tool_without_tools_param(): litellm.modify_params = prev_modify_params assert "tools" in result - names = [t.get("name") for t in result["tools"] if isinstance(t, dict) and t.get("name") is not None] + names = [ + t.get("name") + for t in result["tools"] + if isinstance(t, dict) and t.get("name") is not None + ] assert "dummy_tool" in names @@ -2579,9 +2642,13 @@ def test_calculate_usage_completion_tokens_details_with_reasoning(): "output_tokens": 500, } # Simulating reasoning content that would count as ~50 tokens - reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens + reasoning_content = ( + "Let me think about this step by step. " * 10 + ) # Roughly 50 tokens - usage = config.calculate_usage(usage_object=usage_object, reasoning_content=reasoning_content) + usage = config.calculate_usage( + usage_object=usage_object, reasoning_content=reasoning_content + ) # completion_tokens_details should be populated with both reasoning and text tokens assert usage.completion_tokens_details is not None @@ -2632,7 +2699,9 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): # reasoning_effort should not be in the result (it's transformed to thinking) assert "reasoning_effort" not in result # Should set output_config with the mapped effort value - assert "output_config" in result, f"output_config missing for {model} with effort={effort}" + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort_map[effort] @@ -2733,7 +2802,9 @@ def test_raw_adaptive_thinking_untouched_for_46_plus_model(): ("gpt-4o", False), ], ) -def test_is_adaptive_thinking_model_is_sourced_from_cost_map(local_model_cost_map, model, expected): +def test_is_adaptive_thinking_model_is_sourced_from_cost_map( + local_model_cost_map, model, expected +): """Adaptive thinking resolves from the cost map first (an explicit supports_adaptive_thinking entry, or the anthropic-claude fallback rule for unmapped future Claudes), then from a date-safe opus/sonnet/haiku >= 4.6 name version as a @@ -2849,7 +2920,9 @@ def test_reasoning_effort_sets_output_config_for_46_models(): drop_params=False, ) - assert "output_config" in result, f"output_config missing for {model} with effort={effort}" + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort @@ -2888,7 +2961,9 @@ def test_reasoning_effort_does_not_set_output_config_for_older_models(): drop_params=False, ) - assert "output_config" not in result, f"output_config should not be set for {model}" + assert ( + "output_config" not in result + ), f"output_config should not be set for {model}" @pytest.mark.parametrize( @@ -2928,10 +3003,14 @@ def test_reasoning_effort_accepts_dict_shape_for_adaptive_model(reasoning_effort ) # thinking must be set (adaptive for 4.6+) - assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}" + assert ( + "thinking" in result + ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "adaptive" # output_config must carry the mapped effort - assert "output_config" in result, f"output_config missing for reasoning_effort={reasoning_effort_value!r}" + assert ( + "output_config" in result + ), f"output_config missing for reasoning_effort={reasoning_effort_value!r}" assert result["output_config"]["effort"] == "low" @@ -2960,13 +3039,16 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( drop_params=False, ) - assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}" + assert ( + "thinking" in result + ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "enabled" assert "budget_tokens" in result["thinking"] assert result["thinking"]["budget_tokens"] > 0 # Older models must not get adaptive-thinking output_config assert "output_config" not in result, ( - f"output_config should not be set for non-adaptive model (reasoning_effort={reasoning_effort_value!r})" + f"output_config should not be set for non-adaptive model " + f"(reasoning_effort={reasoning_effort_value!r})" ) @@ -3017,8 +3099,12 @@ def test_reasoning_effort_unparseable_dict_is_dropped(bad_value): model="claude-sonnet-4-6-20260219", drop_params=False, ) - assert "thinking" not in result, f"thinking should not be set for bad value {bad_value!r}" - assert "output_config" not in result, f"output_config should not be set for bad value {bad_value!r}" + assert ( + "thinking" not in result + ), f"thinking should not be set for bad value {bad_value!r}" + assert ( + "output_config" not in result + ), f"output_config should not be set for bad value {bad_value!r}" @pytest.mark.parametrize( @@ -3128,7 +3214,9 @@ def test_reasoning_effort_garbage_raises_bad_request(effort): ("max", DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET), ], ) -def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model(effort, expected_budget): +def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model( + effort, expected_budget +): """``xhigh`` / ``max`` extend the budget_tokens progression on budget-mode models.""" config = AnthropicConfig() @@ -3258,11 +3346,17 @@ def test_code_execution_tool_results_extraction(): # Verify first tool call assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[0].function.name + == "bash_code_execution" + ) # Verify second tool call assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[1].function.name + == "text_editor_code_execution" + ) # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields @@ -3285,7 +3379,10 @@ def test_code_execution_tool_results_extraction(): assert editor_result["content"]["is_file_update"] is False # Verify text content is properly concatenated - assert "I'll calculate that for you." in transformed_response.choices[0].message.content + assert ( + "I'll calculate that for you." + in transformed_response.choices[0].message.content + ) assert "Done!" in transformed_response.choices[0].message.content @@ -3353,7 +3450,10 @@ def test_code_execution_tool_results_in_hidden_params(): assert "provider_specific_fields" in hidden assert "tool_results" in hidden["provider_specific_fields"] assert len(hidden["provider_specific_fields"]["tool_results"]) == 1 - assert hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] == "hello\n" + assert ( + hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] + == "hello\n" + ) def test_tool_search_tool_result_not_in_tool_results(): @@ -3549,7 +3649,10 @@ def test_compaction_block_in_provider_specific_fields(): assert "compaction_blocks" in provider_fields assert len(provider_fields["compaction_blocks"]) == 1 assert provider_fields["compaction_blocks"][0]["type"] == "compaction" - assert "Summary of the conversation" in provider_fields["compaction_blocks"][0]["content"] + assert ( + "Summary of the conversation" + in provider_fields["compaction_blocks"][0]["content"] + ) def test_multiple_compaction_blocks(): @@ -3597,7 +3700,9 @@ def test_compaction_block_request_transformation(): {"role": "user", "content": "What is the weather in San Francisco?"}, { "role": "assistant", - "content": [{"type": "text", "text": "I don't have access to real-time data."}], + "content": [ + {"type": "text", "text": "I don't have access to real-time data."} + ], "provider_specific_fields": { "compaction_blocks": [ { @@ -3610,7 +3715,9 @@ def test_compaction_block_request_transformation(): {"role": "user", "content": "What about New York?"}, ] - result = anthropic_messages_pt(messages=messages, model="claude-opus-4-6", llm_provider="anthropic") + result = anthropic_messages_pt( + messages=messages, model="claude-opus-4-6", llm_provider="anthropic" + ) # Find the assistant message assistant_message = None @@ -3724,7 +3831,9 @@ def test_map_openai_context_management_to_anthropic(): "instructions": "Focus on preserving code snippets", } ] - result = config.map_openai_context_management_to_anthropic(openai_format_with_instructions) + result = config.map_openai_context_management_to_anthropic( + openai_format_with_instructions + ) assert result is not None assert result["edits"][0]["trigger"]["value"] == 150000 @@ -3751,7 +3860,9 @@ def test_map_openai_params_with_context_management(): config = AnthropicConfig() # Test with OpenAI list format - non_default_params = {"context_management": [{"type": "compaction", "compact_threshold": 200000}]} + non_default_params = { + "context_management": [{"type": "compaction", "compact_threshold": 200000}] + } optional_params = {} result = config.map_openai_params( @@ -3788,7 +3899,10 @@ def test_map_openai_params_with_context_management(): ) assert "context_management" in result - assert result["context_management"] == non_default_params_anthropic["context_management"] + assert ( + result["context_management"] + == non_default_params_anthropic["context_management"] + ) def test_cache_control_in_supported_params(): @@ -3899,7 +4013,10 @@ def test_compaction_block_empty_list_not_added(): # Verify compaction_blocks is not in provider_specific_fields when there are none provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: - assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None + assert ( + "compaction_blocks" not in provider_fields + or provider_fields.get("compaction_blocks") is None + ) def test_fast_mode_beta_header(): @@ -3948,7 +4065,9 @@ def test_fast_mode_usage_calculation(): "output_tokens": 500, } - usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None, speed="fast") + usage = config.calculate_usage( + usage_object=usage_object, reasoning_content=None, speed="fast" + ) assert usage.prompt_tokens == 1000 assert usage.completion_tokens == 500 @@ -3969,7 +4088,9 @@ def test_fast_mode_with_inference_geo(): base_completion = 0.025 with ( - patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, + patch( + "litellm.llms.anthropic.cost_calculation.generic_cost_per_token" + ) as mock_cost, patch("litellm.get_model_info") as mock_info, ): mock_cost.return_value = (base_prompt, base_completion) @@ -4160,7 +4281,9 @@ def test_map_tool_helper_enforces_object_type_when_missing(): "name": "search_code", "description": "Search for code patterns", "parameters": { - "properties": {"query": {"type": "string", "description": "Search query"}}, + "properties": { + "query": {"type": "string", "description": "Search query"} + }, "required": ["query"], }, }, @@ -4173,9 +4296,9 @@ def test_map_tool_helper_enforces_object_type_when_missing(): assert "properties" in result["input_schema"] assert "query" in result["input_schema"]["properties"] # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_enforces_object_type_when_wrong_type(): @@ -4201,13 +4324,13 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type(): result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" - assert result["input_schema"].get("properties") == {}, ( - "properties should be injected as {} when schema has non-object type and no properties key" - ) + assert ( + result["input_schema"].get("properties") == {} + ), "properties should be injected as {} when schema has non-object type and no properties key" # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_preserves_valid_object_schema(): @@ -4274,8 +4397,12 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "Hello"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_null) - assert thinking_blocks is not None, "thinking blocks should not be None when thinking=null" + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_null + ) + assert ( + thinking_blocks is not None + ), "thinking blocks should not be None when thinking=null" assert len(thinking_blocks) == 1 assert "Hello" in text @@ -4286,8 +4413,12 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "World"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_missing) - assert thinking_blocks is not None, "thinking blocks should not be None when thinking key is absent" + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_missing + ) + assert ( + thinking_blocks is not None + ), "thinking blocks should not be None when thinking key is absent" assert len(thinking_blocks) == 1 assert "World" in text @@ -4298,7 +4429,9 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "Done"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_text) + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_text + ) assert thinking_blocks is not None assert len(thinking_blocks) == 1 assert thinking_blocks[0]["thinking"] == "Let me think..." @@ -4357,8 +4490,12 @@ def test_advisor_beta_header_injected(): } ] } - result = config.update_headers_with_optional_anthropic_beta(headers, optional_params) - assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get("anthropic-beta", "") + result = config.update_headers_with_optional_anthropic_beta( + headers, optional_params + ) + assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get( + "anthropic-beta", "" + ) def test_advisor_beta_header_not_injected_without_tool(): @@ -4366,7 +4503,9 @@ def test_advisor_beta_header_not_injected_without_tool(): config = AnthropicConfig() headers: dict = {} optional_params: dict = {"tools": []} - result = config.update_headers_with_optional_anthropic_beta(headers, optional_params) + result = config.update_headers_with_optional_anthropic_beta( + headers, optional_params + ) assert "advisor-tool-2026-03-01" not in result.get("anthropic-beta", "") @@ -4393,7 +4532,9 @@ def test_advisor_tool_result_preserved_in_response(): {"type": "text", "text": "Here is the implementation."}, ] } - text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content(completion_response) + text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content( + completion_response + ) assert "Consulting advisor." in text assert "Here is the implementation." in text # server_tool_use (advisor) should be a tool_call @@ -4508,7 +4649,9 @@ def test_basic_sanitize_anthropic_tool_name_replaces_invalid_chars(): ) assert ( - _basic_sanitize_anthropic_tool_name("github_openapi_mcp-actions/download-job-logs-for-workflow-run") + _basic_sanitize_anthropic_tool_name( + "github_openapi_mcp-actions/download-job-logs-for-workflow-run" + ) == "github_openapi_mcp-actions_download-job-logs-for-workflow-run" ) # other punctuation @@ -4537,7 +4680,9 @@ def test_build_anthropic_tool_name_maps_no_collisions(): ] ) assert forward == { - "actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run"), + "actions/download-job-logs-for-workflow-run": ( + "actions_download-job-logs-for-workflow-run" + ), "pulls/list-files": "pulls_list-files", } assert reverse == {v: k for k, v in forward.items()} @@ -4588,7 +4733,9 @@ def test_build_anthropic_tool_name_maps_three_way_collision(): _build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar", "foo.bar"]) + forward, reverse = _build_anthropic_tool_name_maps( + ["foo_bar", "foo/bar", "foo.bar"] + ) assert "foo_bar" not in forward # untouched assert forward["foo/bar"] == "foo_bar_2" assert forward["foo.bar"] == "foo_bar_3" @@ -4661,13 +4808,16 @@ def test_map_openai_params_does_not_pollute_optional_params_with_internal_keys() ) # No internal keys may appear in optional_params for ANY input. for key in optional_params: - assert not key.startswith("_anthropic_tool_name"), ( - f"optional_params leaked internal key {key!r}: {optional_params}" - ) + assert not key.startswith( + "_anthropic_tool_name" + ), f"optional_params leaked internal key {key!r}: {optional_params}" # And no key starting with `_` either; optional_params should only # contain documented Anthropic Messages API parameters. for key in optional_params: - assert not key.startswith("_"), f"optional_params leaked underscore-prefixed key {key!r}: {optional_params}" + assert not key.startswith("_"), ( + f"optional_params leaked underscore-prefixed key {key!r}: " + f"{optional_params}" + ) def test_map_openai_params_no_maps_when_all_names_already_valid(): @@ -4696,7 +4846,11 @@ def test_map_openai_params_no_maps_when_all_names_already_valid(): def test_rewrite_tool_names_in_messages_uses_forward_map(): config = AnthropicConfig() - forward_map = {"actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run")} + forward_map = { + "actions/download-job-logs-for-workflow-run": ( + "actions_download-job-logs-for-workflow-run" + ) + } messages = [ {"role": "user", "content": "go"}, { @@ -4719,9 +4873,15 @@ def test_rewrite_tool_names_in_messages_uses_forward_map(): out = config._rewrite_tool_names_in_messages(messages, forward_map) # input list must not be mutated - assert messages[1]["tool_calls"][0]["function"]["name"] == "actions/download-job-logs-for-workflow-run" + assert ( + messages[1]["tool_calls"][0]["function"]["name"] + == "actions/download-job-logs-for-workflow-run" + ) # output rewritten according to forward map - assert out[1]["tool_calls"][0]["function"]["name"] == "actions_download-job-logs-for-workflow-run" + assert ( + out[1]["tool_calls"][0]["function"]["name"] + == "actions_download-job-logs-for-workflow-run" + ) # non-tool-call messages pass through unchanged (same object) assert out[0] is messages[0] assert out[2] is messages[2] @@ -4797,7 +4957,9 @@ def test_sanitize_tool_names_in_request_does_not_mutate_caller_tool_dicts(): caller_tools = [caller_tool] optional_params: dict = {"tools": caller_tools} - forward, reverse = config._sanitize_tool_names_in_request(optional_params=optional_params) + forward, reverse = config._sanitize_tool_names_in_request( + optional_params=optional_params + ) assert forward.get(original_name) sanitized = forward[original_name] @@ -4946,7 +5108,10 @@ def test_streaming_iterator_reverse_maps_tool_use_name(): parsed = iterator.chunk_parser(chunk=chunk) tool_calls = parsed.choices[0].delta.tool_calls assert tool_calls is not None and len(tool_calls) == 1 - assert tool_calls[0]["function"]["name"] == "actions/download-job-logs-for-workflow-run" + assert ( + tool_calls[0]["function"]["name"] + == "actions/download-job-logs-for-workflow-run" + ) def test_streaming_iterator_passthrough_when_name_not_in_map(): @@ -5042,9 +5207,9 @@ def test_transform_request_does_not_leak_internal_keys_into_body(): for tool in data.get("tools", []): name = tool.get("name") assert isinstance(name, str) - assert _re.fullmatch(r"[a-zA-Z0-9_-]{1,128}", name), ( - f"sanitized tool name {name!r} still violates Anthropic regex" - ) + assert _re.fullmatch( + r"[a-zA-Z0-9_-]{1,128}", name + ), f"sanitized tool name {name!r} still violates Anthropic regex" # Sent name for the bad tool is the disambiguated form, valid name passes through. sent_names = {t["name"] for t in data["tools"]} @@ -5180,7 +5345,9 @@ def test_transform_request_rewrites_tool_names_in_history(): for block in content: if isinstance(block, dict) and block.get("type") == "tool_use": tool_use_names.append(block.get("name")) - assert tool_use_names, "expected at least one tool_use block in transformed messages" + assert ( + tool_use_names + ), "expected at least one tool_use block in transformed messages" for name in tool_use_names: assert name == "actions_download-job-logs-for-workflow-run", ( f"history tool_use.name {name!r} not rewritten -- Anthropic will " @@ -5204,12 +5371,19 @@ def test_sanitize_tool_names_in_request_skips_hosted_tools(): } forward, reverse = AnthropicConfig._sanitize_tool_names_in_request(optional_params) # Only the custom tool was rewritten. - assert forward == {"actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run"} - assert reverse == {"actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run"} + assert forward == { + "actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run" + } + assert reverse == { + "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run" + } # Hosted tool's name unchanged. assert optional_params["tools"][0]["name"] == "web_search" # Custom tool's name updated in place. - assert optional_params["tools"][1]["name"] == "actions_download-job-logs-for-workflow-run" + assert ( + optional_params["tools"][1]["name"] + == "actions_download-job-logs-for-workflow-run" + ) def test_sanitize_tool_names_in_request_no_tools_is_noop(): @@ -5443,7 +5617,9 @@ def test_translate_system_message_keeps_billing_header_for_first_party_anthropic assert config.should_strip_billing_metadata() is False result = config.translate_system_message( - messages=_system_with_billing_header("You are Claude Code, Anthropic's official CLI for Claude.") + messages=_system_with_billing_header( + "You are Claude Code, Anthropic's official CLI for Claude." + ) ) texts = [block["text"] for block in result] @@ -5459,7 +5635,9 @@ def test_translate_system_message_strips_billing_header_for_bedrock(): config = BedrockClaudePlatformConfig() assert config.should_strip_billing_metadata() is True - result = config.translate_system_message(messages=_system_with_billing_header("real system prompt")) + result = config.translate_system_message( + messages=_system_with_billing_header("real system prompt") + ) texts = [block["text"] for block in result] assert all(not t.startswith("x-anthropic-billing-header:") for t in texts) @@ -5525,7 +5703,9 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): config = AmazonAnthropicClaudeConfig() assert config.should_strip_billing_metadata() is True - result = config.translate_system_message(messages=_system_with_billing_header("real system prompt")) + result = config.translate_system_message( + messages=_system_with_billing_header("real system prompt") + ) texts = [block["text"] for block in result] assert all(not t.startswith("x-anthropic-billing-header:") for t in texts) @@ -5579,7 +5759,9 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): ), ], ) -def test_should_strip_billing_metadata_by_provider(module_path, class_name, expected_strip): +def test_should_strip_billing_metadata_by_provider( + module_path, class_name, expected_strip +): import importlib config_cls = getattr(importlib.import_module(module_path), class_name) @@ -5847,7 +6029,9 @@ def test_is_anthropic_usage_object_rejects_responses_api_usage(): ("claude-sonnet-4-5-20250929", False), ], ) -def test_disabled_thinking_omitted_only_for_always_on_models(local_model_cost_map, model, expected_dropped): +def test_disabled_thinking_omitted_only_for_always_on_models( + local_model_cost_map, model, expected_dropped +): """``thinking={"type": "disabled"}`` is omitted for always-on-thinking models (Fable/Mythos, which 400 on it: the API remedy is to omit the param) and is forwarded verbatim for every model that accepts it.""" @@ -5893,7 +6077,9 @@ def test_forced_tool_choice_raises_clean_error_on_fable_5_1_without_drop_params( "tool_choice", ["required", {"type": "required"}, {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params(local_model_cost_map, tool_choice): +def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params( + local_model_cost_map, tool_choice +): config = AnthropicConfig() result = config.map_openai_params( @@ -5920,7 +6106,9 @@ def test_forced_tool_choice_downgrade_keeps_parallel_tool_calls_flag(local_model @pytest.mark.parametrize("tool_choice, expected_type", [("auto", "auto"), ("none", "none")]) -def test_unforced_tool_choice_forwarded_on_fable_5_1(local_model_cost_map, tool_choice, expected_type, monkeypatch): +def test_unforced_tool_choice_forwarded_on_fable_5_1( + local_model_cost_map, tool_choice, expected_type, monkeypatch +): monkeypatch.setattr(litellm, "drop_params", False) config = AnthropicConfig() @@ -5935,7 +6123,9 @@ def test_unforced_tool_choice_forwarded_on_fable_5_1(local_model_cost_map, tool_ @pytest.mark.parametrize("model", ["claude-fable-5", "claude-opus-5", "claude-sonnet-5"]) -def test_forced_tool_choice_forwarded_on_models_that_support_it(local_model_cost_map, model, monkeypatch): +def test_forced_tool_choice_forwarded_on_models_that_support_it( + local_model_cost_map, model, monkeypatch +): monkeypatch.setattr(litellm, "drop_params", False) config = AnthropicConfig() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py index 03cbc98dcb9..1c05f0adcf7 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py @@ -161,7 +161,9 @@ class TestAdapterAdaptiveThinking: ) adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_anthropic_thinking_to_reasoning_effort({"type": "adaptive"}) + result = adapter.translate_anthropic_thinking_to_reasoning_effort( + {"type": "adaptive"} + ) assert result == "medium" def test_messages_adapter_adaptive_overridden_by_output_config(self): diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 3c4bf91fc97..133d6e502f4 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1974,6 +1974,7 @@ class TestClaudeOpus48AdaptiveThinking: assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True + @pytest.mark.parametrize( "model", [ diff --git a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py index f929c97ba39..b447645bae8 100644 --- a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py +++ b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py @@ -66,7 +66,11 @@ def test_azure_speech_audio_transcription_uses_dedicated_api_base_env(monkeypatc monkeypatch.setattr( "litellm.llms.azure.audio_transcription.transformation.get_secret_str", - lambda key: "https://centralus.api.cognitive.microsoft.com" if key == "AZURE_SPEECH_API_BASE" else None, + lambda key: ( + "https://centralus.api.cognitive.microsoft.com" + if key == "AZURE_SPEECH_API_BASE" + else None + ), ) url = config.get_complete_url( @@ -220,3 +224,5 @@ def test_azure_speech_transcription_routes_through_provider_config(monkeypatch): AzureSpeechAudioTranscriptionConfig, ) assert audio_handler.call_args.kwargs["custom_llm_provider"] == "azure" + + diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index f128954a338..8a832e176a6 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -252,7 +252,8 @@ def test_azure_model_router_response_shows_actual_model(): # Verify that the response contains the actual model used, not the router model assert result.model == "azure_ai/gpt-5-nano-2025-08-07", ( - f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), but got '{result.model}'" + f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), " + f"but got '{result.model}'" ) @@ -310,11 +311,19 @@ def test_azure_model_router_stamps_selected_model_on_hidden_params(): ) assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == result.model - assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == "azure_ai/grok-4-1-fast-reasoning" - assert AzureFoundryModelInfo.get_model_router_selected_model(result._hidden_params) == ( - "azure_ai/grok-4-1-fast-reasoning" + assert ( + result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] + == "azure_ai/grok-4-1-fast-reasoning" + ) + assert AzureFoundryModelInfo.get_model_router_selected_model( + result._hidden_params + ) == ("azure_ai/grok-4-1-fast-reasoning") + assert ( + AzureFoundryModelInfo.is_model_router_call( + model="smart-pick", hidden_params=result._hidden_params + ) + is True ) - assert AzureFoundryModelInfo.is_model_router_call(model="smart-pick", hidden_params=result._hidden_params) is True def test_azure_model_router_stamp_does_not_leak_across_responses(): @@ -352,10 +361,14 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): mock_response.text = error_text mock_response.json.return_value = json.loads(error_text) mock_response.status_code = 400 - e = httpx.HTTPStatusError(message="400", request=MagicMock(), response=mock_response) + e = httpx.HTTPStatusError( + message="400", request=MagicMock(), response=mock_response + ) assert config._error_has_tool_level_extra_fields(error_text) is True - assert config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True + assert ( + config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True + ) request_data = { "model": "FW-Kimi-K2.6", @@ -478,7 +491,9 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): { "role": "assistant", "content": "I can help.", - "thinking_blocks": [{"type": "thinking", "thinking": "Reading the file.", "signature": "sig"}], + "thinking_blocks": [ + {"type": "thinking", "thinking": "Reading the file.", "signature": "sig"} + ], "provider_specific_fields": {"thought_signature": "sig-top"}, "tool_calls": [ { diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 87b9fb8b307..9753605888e 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -3,7 +3,9 @@ import json import os import sys -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) from unittest.mock import patch @@ -37,7 +39,9 @@ class TestAzureAnthropicMessagesConfig: litellm_params = {"api_key": "test-api-key"} api_key = "test-api-key" - with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -68,7 +72,9 @@ class TestAzureAnthropicMessagesConfig: optional_params = {} litellm_params = {"api_key": "test-api-key"} - with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -92,7 +98,9 @@ class TestAzureAnthropicMessagesConfig: optional_params = {} litellm_params = {"api_key": "test-api-key"} - with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -165,6 +173,7 @@ class TestAzureAnthropicMessagesConfig: assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + def test_get_complete_url_with_base_url_without_anthropic(self): """Test get_complete_url with base URL without /anthropic""" config = AzureAnthropicMessagesConfig() @@ -258,7 +267,9 @@ class TestAzureAnthropicMessagesConfig: assert "scope" not in result["system"][0]["cache_control"] assert result["system"][0]["cache_control"]["type"] == "ephemeral" assert "scope" not in result["messages"][0]["content"][0]["cache_control"] - assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" + assert ( + result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" + ) class TestProviderConfigManagerAzureAnthropicMessages: @@ -365,7 +376,9 @@ class TestAzureAnthropicMidConversationSystem: {"role": "assistant", "content": "reading"}, {"role": "user", "content": "continue"}, ] - result = _azure_transform("claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}]) + result = _azure_transform( + "claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}] + ) assert result["messages"] == [ {"role": "user", "content": "read the file"}, { @@ -396,7 +409,9 @@ def test_azure_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_fl import litellm - cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + cost_map_path = os.path.join( + os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" + ) with open(cost_map_path) as f: cost_map = json.load(f) rules = cost_map["fallback_generalizations"]["rules"] diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index d8c3a458082..70cb8bd1e66 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -28,11 +28,16 @@ def test_transform_usage(): openai_usage = config.transform_usage(usage) assert ( openai_usage.prompt_tokens - == usage["inputTokens"] + usage["cacheReadInputTokens"] + usage["cacheWriteInputTokens"] + == usage["inputTokens"] + + usage["cacheReadInputTokens"] + + usage["cacheWriteInputTokens"] ) assert openai_usage.completion_tokens == usage["outputTokens"] assert openai_usage.total_tokens == usage["totalTokens"] - assert openai_usage.prompt_tokens_details.cached_tokens == usage["cacheReadInputTokens"] + assert ( + openai_usage.prompt_tokens_details.cached_tokens + == usage["cacheReadInputTokens"] + ) assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] # completion_tokens_details should always be populated @@ -80,7 +85,10 @@ def test_transform_usage_with_mismatched_cache_details_falls_back(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None + assert ( + getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) def test_transform_usage_without_cache_details_stays_none(): @@ -96,7 +104,10 @@ def test_transform_usage_without_cache_details_stays_none(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None + assert ( + getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) + is None + ) def test_bedrock_converse_1h_cache_write_billed_at_1h_rate(monkeypatch): @@ -328,10 +339,14 @@ def test_apply_tool_call_transformation_if_needed(): role="user", content=json.dumps(tool_response), ) - transformed_message, _ = config.apply_tool_call_transformation_if_needed(message, tool_calls) + transformed_message, _ = config.apply_tool_call_transformation_if_needed( + message, tool_calls + ) assert len(transformed_message.tool_calls) == 1 assert transformed_message.tool_calls[0].function.name == "test_function" - assert transformed_message.tool_calls[0].function.arguments == json.dumps(tool_response["parameters"]) + assert transformed_message.tool_calls[0].function.arguments == json.dumps( + tool_response["parameters"] + ) def test_transform_tool_call_with_cache_control(): @@ -380,7 +395,12 @@ def test_transform_tool_call_with_cache_control(): print(function_out_msg) assert function_out_msg["toolSpec"]["name"] == "get_location" assert function_out_msg["toolSpec"]["description"] == "Get the user's location" - assert function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"]["type"] == "string" + assert ( + function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"][ + "type" + ] + == "string" + ) transformed_cache_msg = result["toolConfig"]["tools"][1] assert "cachePoint" in transformed_cache_msg @@ -515,7 +535,9 @@ def test_reasoning_effort_none_omits_thinking_for_anthropic_converse(model): ("bedrock/converse/us.anthropic.claude-sonnet-4-6", "minimal", "low"), ], ) -def test_reasoning_effort_sets_output_config_for_adaptive_models_converse(model, effort, expected_effort): +def test_reasoning_effort_sets_output_config_for_adaptive_models_converse( + model, effort, expected_effort +): """Adaptive Claude 4.6 / 4.7 on Bedrock Converse routes the tier via ``output_config.effort``.""" config = AmazonConverseConfig() @@ -743,7 +765,9 @@ def test_output_config_format_translated_to_native_output_config_converse(): assert additional.get("output_config") == {"effort": "xhigh"} assert "format" not in additional["output_config"] assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - parsed_schema = json.loads(result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) + parsed_schema = json.loads( + result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] + ) assert parsed_schema == {**schema, "additionalProperties": False} @@ -779,7 +803,10 @@ def test_output_config_format_dropped_on_unsupported_converse_model_warns(caplog ) assert "outputConfig" not in result - assert any("dropping `output_config.format`" in record.getMessage() for record in caplog.records) + assert any( + "dropping `output_config.format`" in record.getMessage() + for record in caplog.records + ) def test_output_config_normalized_marker_does_not_leak_into_optional_params(): @@ -815,7 +842,9 @@ def test_output_config_normalized_marker_does_not_leak_into_optional_params(): ("bedrock/converse/us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_output_config_effort_normalized_for_bedrock_converse_opus(model, expected_effort): +def test_output_config_effort_normalized_for_bedrock_converse_opus( + model, expected_effort +): """Bedrock Converse accepts ``xhigh`` and forwards the provider-safe effort.""" config = AmazonConverseConfig() @@ -1088,13 +1117,17 @@ def test_get_supported_openai_params_bedrock_converse(): for model in litellm.BEDROCK_CONVERSE_MODELS: print(f"Testing model: {model}") config = AmazonConverseConfig() - supported_params_without_prefix = config.get_supported_openai_params(model=model) - - supported_params_with_prefix = config.get_supported_openai_params(model=f"bedrock/converse/{model}") - - assert set(supported_params_without_prefix) == set(supported_params_with_prefix), ( - f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" + supported_params_without_prefix = config.get_supported_openai_params( + model=model ) + + supported_params_with_prefix = config.get_supported_openai_params( + model=f"bedrock/converse/{model}" + ) + + assert set(supported_params_without_prefix) == set( + supported_params_with_prefix + ), f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" print(f"✅ Passed for model: {model}") @@ -1586,7 +1619,9 @@ def test_transform_response_with_structured_response_calling_tool(): "output": { "message": { "content": [ - {"text": "I'll check the current weather in San Francisco for you."}, + { + "text": "I'll check the current weather in San Francisco for you." + }, { "toolUse": { "input": { @@ -2096,7 +2131,9 @@ def test_transform_request_with_function_tool(): } ] - messages = [{"role": "user", "content": "What's the weather like in San Francisco?"}] + messages = [ + {"role": "user", "content": "What's the weather like in San Francisco?"} + ] # Transform request request_data = config.transform_request( @@ -2204,18 +2241,22 @@ async def test_assistant_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2261,10 +2302,12 @@ async def test_assistant_message_list_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2317,10 +2360,12 @@ async def test_tool_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2334,7 +2379,10 @@ async def test_tool_message_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather data: sunny, 25°C" + assert ( + tool_message_content[0]["toolResult"]["content"][0]["text"] + == "Weather data: sunny, 25°C" + ) # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -2376,10 +2424,12 @@ async def test_tool_message_string_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2390,7 +2440,10 @@ async def test_tool_message_string_content_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather: sunny, 25°C" + assert ( + tool_message_content[0]["toolResult"]["content"][0]["text"] + == "Weather: sunny, 25°C" + ) # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -2430,7 +2483,9 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() "source": "Great Source of Information About Apptio", "title": "12adbd74-46bd-4a88-88b2-0048755f6eb5", "content": [ - {"text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM"} + { + "text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM" + } ], "citations": {"enabled": True}, } @@ -2443,10 +2498,12 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2455,7 +2512,10 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() assert tool_result["status"] == "success" assert len(tool_result["content"]) == 1 assert "searchResult" in tool_result["content"][0] - assert tool_result["content"][0]["searchResult"]["title"] == "12adbd74-46bd-4a88-88b2-0048755f6eb5" + assert ( + tool_result["content"][0]["searchResult"]["title"] + == "12adbd74-46bd-4a88-88b2-0048755f6eb5" + ) @pytest.mark.asyncio @@ -2492,10 +2552,12 @@ async def test_tool_message_empty_search_results_falls_back_to_content(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2657,10 +2719,12 @@ async def test_assistant_tool_calls_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2715,10 +2779,12 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2764,10 +2830,12 @@ async def test_no_cache_control_no_cache_point(): llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -2937,7 +3005,10 @@ def test_guarded_text_with_mixed_content_types(): # Third should be guardContent assert "guardContent" in content[2] - assert content[2]["guardContent"]["text"]["text"] == "This sensitive content should be guarded" + assert ( + content[2]["guardContent"]["text"]["text"] + == "This sensitive content should be guarded" + ) @pytest.mark.asyncio @@ -3032,7 +3103,10 @@ def test_guarded_text_with_tool_calls(): # Second should be guardContent assert "guardContent" in content[1] - assert content[1]["guardContent"]["text"]["text"] == "Please be careful with sensitive information" + assert ( + content[1]["guardContent"]["text"]["text"] + == "Please be careful with sensitive information" + ) # Other messages should not have guardContent for i in range(1, 3): @@ -3093,36 +3167,52 @@ def test_auto_convert_last_user_message_to_guarded_text(): } ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" + assert ( + converted_messages[0]["content"][0]["text"] + == "What is the main topic of this legal document?" + ) def test_auto_convert_last_user_message_string_content(): """Test that last user message with string content is automatically converted to guarded_text when guardrailConfig is present.""" config = AmazonConverseConfig() - messages = [{"role": "user", "content": "What is the main topic of this legal document?"}] + messages = [ + {"role": "user", "content": "What is the main topic of this legal document?"} + ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" + assert ( + converted_messages[0]["content"][0]["text"] + == "What is the main topic of this legal document?" + ) def test_no_conversion_when_no_guardrail_config(): @@ -3144,7 +3234,9 @@ def test_no_conversion_when_no_guardrail_config(): optional_params = {} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify no conversion happened assert converted_messages == messages @@ -3161,10 +3253,14 @@ def test_no_conversion_when_guarded_text_already_present(): } ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify no conversion happened assert converted_messages == messages @@ -3190,10 +3286,14 @@ def test_auto_convert_with_mixed_content(): } ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion assert len(converted_messages) == 1 @@ -3202,11 +3302,17 @@ def test_auto_convert_with_mixed_content(): # First element should be converted to guarded_text assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" + assert ( + converted_messages[0]["content"][0]["text"] + == "What is the main topic of this legal document?" + ) # Second element should remain unchanged assert converted_messages[0]["content"][1]["type"] == "image_url" - assert converted_messages[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg" + assert ( + converted_messages[0]["content"][1]["image_url"]["url"] + == "https://example.com/image.jpg" + ) def test_auto_convert_in_full_transformation(): @@ -3225,7 +3331,9 @@ def test_auto_convert_in_full_transformation(): } ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the full transformation result = config._transform_request( @@ -3245,7 +3353,10 @@ def test_auto_convert_in_full_transformation(): assert "content" in message assert len(message["content"]) == 1 assert "guardContent" in message["content"][0] - assert message["content"][0]["guardContent"]["text"]["text"] == "What is the main topic of this legal document?" + assert ( + message["content"][0]["guardContent"]["text"]["text"] + == "What is the main topic of this legal document?" + ) def test_convert_consecutive_user_messages_to_guarded_text(): @@ -3259,10 +3370,14 @@ def test_convert_consecutive_user_messages_to_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion - only the last two user messages should be converted assert len(converted_messages) == 4 @@ -3297,10 +3412,14 @@ def test_convert_all_user_messages_when_all_consecutive(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify all three user messages are converted assert len(converted_messages) == 3 @@ -3324,10 +3443,14 @@ def test_convert_consecutive_user_messages_with_string_content(): {"role": "user", "content": "Second user message"}, ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion assert len(converted_messages) == 3 @@ -3360,10 +3483,14 @@ def test_skip_consecutive_user_messages_with_existing_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Should be converted"}]}, ] - optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} + optional_params = { + "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} + } # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) # Verify the conversion assert len(converted_messages) == 2 @@ -3922,22 +4049,24 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): optional_params = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Verify the condition is detected - assert last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks), ( - "Should detect missing thinking_blocks" - ) + assert last_assistant_with_tool_calls_has_no_thinking_blocks( + messages_without_thinking_blocks + ), "Should detect missing thinking_blocks" # Simulate what _transform_request_helper does if ( optional_params.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) + and last_assistant_with_tool_calls_has_no_thinking_blocks( + messages_without_thinking_blocks + ) ): if litellm.modify_params: optional_params.pop("thinking", None) - assert "thinking" not in optional_params, ( - "thinking param should be dropped when modify_params=True and thinking_blocks are missing" - ) + assert ( + "thinking" not in optional_params + ), "thinking param should be dropped when modify_params=True and thinking_blocks are missing" # Test case 2: thinking should NOT be dropped when thinking_blocks are present messages_with_thinking_blocks = [ @@ -3952,46 +4081,58 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): "function": {"name": "search", "arguments": "{}"}, } ], - "thinking_blocks": [{"type": "thinking", "thinking": "Let me search for weather..."}], + "thinking_blocks": [ + {"type": "thinking", "thinking": "Let me search for weather..."} + ], }, {"role": "tool", "content": "Weather is sunny", "tool_call_id": "call_123"}, ] - optional_params_with_thinking = {"thinking": {"type": "enabled", "budget_tokens": 1000}} + optional_params_with_thinking = { + "thinking": {"type": "enabled", "budget_tokens": 1000} + } # Verify the condition is NOT detected when thinking_blocks are present - assert not last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks), ( - "Should NOT detect missing thinking_blocks when they are present" - ) + assert not last_assistant_with_tool_calls_has_no_thinking_blocks( + messages_with_thinking_blocks + ), "Should NOT detect missing thinking_blocks when they are present" # Simulate what _transform_request_helper does if ( optional_params_with_thinking.get("thinking") is not None and messages_with_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks) + and last_assistant_with_tool_calls_has_no_thinking_blocks( + messages_with_thinking_blocks + ) ): if litellm.modify_params: optional_params_with_thinking.pop("thinking", None) - assert "thinking" in optional_params_with_thinking, ( - "thinking param should NOT be dropped when thinking_blocks are present" - ) + assert ( + "thinking" in optional_params_with_thinking + ), "thinking param should NOT be dropped when thinking_blocks are present" # Test case 3: thinking should NOT be dropped when modify_params=False litellm.modify_params = False - optional_params_no_modify = {"thinking": {"type": "enabled", "budget_tokens": 1000}} + optional_params_no_modify = { + "thinking": {"type": "enabled", "budget_tokens": 1000} + } # Simulate what _transform_request_helper does if ( optional_params_no_modify.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) + and last_assistant_with_tool_calls_has_no_thinking_blocks( + messages_without_thinking_blocks + ) ): if litellm.modify_params: optional_params_no_modify.pop("thinking", None) - assert "thinking" in optional_params_no_modify, "thinking param should NOT be dropped when modify_params=False" + assert ( + "thinking" in optional_params_no_modify + ), "thinking param should NOT be dropped when modify_params=False" finally: # Restore original modify_params setting @@ -4075,14 +4216,19 @@ def test_translate_response_format_native_output_config(monkeypatch): assert "fake_stream" not in result # Verify the schema content (additionalProperties: false is added by normalization) - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ + "schema" + ] parsed_schema = json.loads(schema_str) expected_schema = { **response_format["json_schema"]["schema"], "additionalProperties": False, } assert parsed_schema == expected_schema - assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "WeatherResult" + assert ( + result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] + == "WeatherResult" + ) finally: litellm.model_cost = old_cost if old_env is None: @@ -4160,7 +4306,9 @@ def test_native_structured_output_no_fake_stream(monkeypatch): assert "fake_stream" not in result # Verify the schema content - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ + "schema" + ] assert json.loads(schema_str) == { "type": "object", "properties": {"answer": {"type": "string"}}, @@ -4213,7 +4361,10 @@ def test_transform_request_with_output_config(): assert "outputConfig" in result assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" + assert ( + result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] + == "TestSchema" + ) def test_transform_request_strips_anthropic_output_config(): @@ -4334,7 +4485,10 @@ def test_transform_response_native_structured_output(): ) # Content should be the JSON text directly - assert result.choices[0].message.content == '{"temp": 62, "description": "Mild and foggy"}' + assert ( + result.choices[0].message.content + == '{"temp": 62, "description": "Mild and foggy"}' + ) # Should NOT have tool_calls assert result.choices[0].message.tool_calls is None assert result.choices[0].finish_reason == "stop" @@ -4447,7 +4601,10 @@ def test_add_additional_properties_definitions(): # definitions object assert result["definitions"]["Item"]["additionalProperties"] is False # Nested object inside definitions - assert result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] is False + assert ( + result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] + is False + ) def test_json_object_no_schema_skips_tool_injection(monkeypatch): @@ -4504,7 +4661,9 @@ def test_output_config_applies_additional_properties(): output_config = AmazonConverseConfig._create_output_config_for_response_format( json_schema=schema, name="test_schema" ) - parsed = json.loads(output_config["textFormat"]["structure"]["jsonSchema"]["schema"]) + parsed = json.loads( + output_config["textFormat"]["structure"]["jsonSchema"]["schema"] + ) assert parsed["additionalProperties"] is False assert parsed["properties"]["nested"]["additionalProperties"] is False @@ -4553,7 +4712,12 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True + assert ( + request_data["additionalModelRequestFields"]["tool_choice"][ + "disable_parallel_tool_use" + ] + is True + ) assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] @@ -4585,7 +4749,12 @@ def test_parallel_tool_calls_flag_decoupled_from_ttl_pricing(monkeypatch): headers={}, ) - assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True + assert ( + request_data["additionalModelRequestFields"]["tool_choice"][ + "disable_parallel_tool_use" + ] + is True + ) def test_parallel_tool_calls_older_model_drops_disable_flag(): @@ -4732,7 +4901,9 @@ def test_parallel_tool_use_merge_preserves_user_tool_choice_type(): class TestBedrockMinThinkingBudgetTokens: """Test that thinking.budget_tokens is clamped to the Bedrock minimum (1024).""" - def _map_params(self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0"): + def _map_params( + self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0" + ): """Helper to call map_openai_params with the given thinking value.""" config = AmazonConverseConfig() non_default_params = {"thinking": thinking_value} @@ -4959,7 +5130,9 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 2: json_tool_call delta — should become text, not tool_use json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"temp": 62}'}) - text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) + text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) assert text_2 == '{"temp": 62}' assert tool_use_2 is None @@ -4983,7 +5156,9 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 5: real tool delta real_delta = ContentBlockDeltaEvent(toolUse={"input": '{"location": "SF"}'}) - text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event(real_delta, index=1) + text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event( + real_delta, index=1 + ) assert text_5 == "" assert tool_use_5 is not None assert tool_use_5["function"]["arguments"] == '{"location": "SF"}' @@ -5016,7 +5191,9 @@ def test_streaming_without_json_mode_passes_all_tools(): # json_tool_call delta — should be a tool_use, not text json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"data": 1}'}) - text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) + text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event( + json_delta, index=0 + ) assert text == "" assert tool_use_delta is not None assert tool_use_delta["function"]["arguments"] == '{"data": 1}' @@ -5500,7 +5677,11 @@ def test_transform_response_citation_null_source_title_become_empty_strings(): "content": [ { "citationsContent": { - "content": [{"text": "Apptio is a company that makes calls to Bedrock"}], + "content": [ + { + "text": "Apptio is a company that makes calls to Bedrock" + } + ], "citations": [ { "location": { @@ -5635,11 +5816,15 @@ def test_transform_response_citations_offset_tracks_text_only_blocks(): message = result.choices[0].message expected_start = len(leading_text) assert message.content == leading_text + cited_text - assert message.content[expected_start : expected_start + len(cited_text)] == cited_text + assert ( + message.content[expected_start : expected_start + len(cited_text)] == cited_text + ) assert message.annotations is not None assert len(message.annotations) == 1 assert message.annotations[0]["url_citation"]["start_index"] == expected_start - assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len(cited_text) + assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len( + cited_text + ) def test_transform_response_stitches_citations_for_whitespace_punctuation_text(): @@ -5749,7 +5934,9 @@ def test_bedrock_tool_message_openai_file_pdf_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_1" @@ -5791,7 +5978,9 @@ def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_img_1" @@ -5848,7 +6037,9 @@ def test_bedrock_tool_message_file_id_http_url_becomes_document(): "process_image_sync", return_value=fake_document_block, ) as mock_proc: - translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) mock_proc.assert_called_once() assert mock_proc.call_args.kwargs["image_url"] == pdf_url @@ -5919,7 +6110,9 @@ def test_bedrock_tool_message_image_url_png_still_becomes_image(): }, ] - translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert len(tool_result["content"]) == 1 @@ -6114,10 +6307,12 @@ async def test_grounding_source_and_query_rendered_as_text(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -6161,7 +6356,9 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_no_tools(): (#24158, #27138).""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={} + ) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -6182,7 +6379,9 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_tools_empty(tools_value): structured tool blocks with no toolConfig.""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={"tools": tools_value}) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={"tools": tools_value} + ) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -6198,7 +6397,9 @@ def test_neutralize_orphaned_tool_blocks_rewrites_tool_result_only_history(): {"role": "tool", "tool_call_id": "call_xyz", "content": "lookup result"}, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={} + ) assert not any(m.get("role") in ("tool", "function") for m in result) serialized = json.dumps(result) @@ -6235,9 +6436,13 @@ def test_neutralize_orphaned_tool_blocks_non_text_result_marked_not_empty(): }, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={} + ) - rewritten = next(m for m in result if m.get("role") == "user" and m is not messages[0]) + rewritten = next( + m for m in result if m.get("role") == "user" and m is not messages[0] + ) text = rewritten["content"] assert text.strip() # never empty assert "non-text tool result omitted" in text @@ -6260,7 +6465,9 @@ def test_neutralize_orphaned_tool_blocks_noop_when_no_tool_history(): """Plain conversation with no tool blocks is returned unchanged.""" messages = [{"role": "user", "content": "hi"}] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={} + ) assert result is messages @@ -6271,9 +6478,14 @@ def test_neutralize_orphaned_tool_blocks_logs_warning(caplog): messages = _orphaned_tool_history_messages() with caplog.at_level("WARNING"): - AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) + AmazonConverseConfig._neutralize_orphaned_tool_blocks( + messages, optional_params={} + ) - assert any("neutralizing orphaned tool blocks" in record.getMessage() for record in caplog.records) + assert any( + "neutralizing orphaned tool blocks" in record.getMessage() + for record in caplog.records + ) def _assert_no_structured_tool_blocks(result): @@ -6391,7 +6603,9 @@ def test_transform_request_neutralized_tool_output_is_guarded(monkeypatch): }, {"role": "tool", "tool_call_id": "c1", "content": "secret tool output"}, ], - optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, + optional_params={ + "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} + }, litellm_params={}, headers={}, ) @@ -6431,19 +6645,23 @@ def test_transform_request_neutralized_tool_output_guarded_mid_history(monkeypat {"role": "assistant", "content": "Here is the summary."}, {"role": "user", "content": "thanks"}, ], - optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, + optional_params={ + "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} + }, litellm_params={}, headers={}, ) _assert_no_structured_tool_blocks(result) blocks = [block for message in result["messages"] for block in message["content"]] - guarded_texts = [block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block] + guarded_texts = [ + block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block + ] plain_texts = [block["text"] for block in blocks if "text" in block and "guardContent" not in block] assert any("malware" in text for text in guarded_texts), "mid-history tool output must be guarded" - assert not any("malware" in text for text in plain_texts), ( - "mid-history tool output must not reach the model as unguarded text" - ) + assert not any( + "malware" in text for text in plain_texts + ), "mid-history tool output must not reach the model as unguarded text" @pytest.mark.asyncio @@ -6579,7 +6797,10 @@ def _agentic_messages_with_ttl(ttl_target: str): def _collect_cache_points(result): return [ - block["cachePoint"] for message in result for block in message.get("content") or [] if "cachePoint" in block + block["cachePoint"] + for message in result + for block in message.get("content") or [] + if "cachePoint" in block ] @@ -6605,10 +6826,12 @@ async def test_message_level_cache_control_honors_ttl_for_supported_model( model="global.anthropic.claude-opus-4-7", llm_provider="bedrock_converse", ) - async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="global.anthropic.claude-opus-4-7", - llm_provider="bedrock_converse", + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="global.anthropic.claude-opus-4-7", + llm_provider="bedrock_converse", + ) ) assert result == async_result @@ -6920,7 +7143,9 @@ def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_cras ("us.anthropic.claude-opus-4-8", False), ], ) -def test_disabled_thinking_omitted_for_always_on_models_converse(local_model_cost_map, model, expected_dropped): +def test_disabled_thinking_omitted_for_always_on_models_converse( + local_model_cost_map, model, expected_dropped +): """Bedrock Converse: ``thinking={"type": "disabled"}`` is omitted for always-on-thinking models and forwarded verbatim for models that accept it.""" config = AmazonConverseConfig() @@ -6939,7 +7164,6 @@ def test_disabled_thinking_omitted_for_always_on_models_converse(local_model_cos else: assert additional.get("thinking") == {"type": "disabled"} - @pytest.mark.parametrize( "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], @@ -6948,10 +7172,14 @@ def test_disabled_thinking_omitted_for_always_on_models_converse(local_model_cos "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse(local_model_cost_map, model, tool_choice): +def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse( + local_model_cost_map, model, tool_choice +): config = AmazonConverseConfig() - result = config.map_tool_choice_values(model=model, tool_choice=tool_choice, drop_params=True) + result = config.map_tool_choice_values( + model=model, tool_choice=tool_choice, drop_params=True + ) assert result == {"auto": {}} @@ -6960,12 +7188,16 @@ def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse(local_model "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse(local_model_cost_map, tool_choice, monkeypatch): +def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse( + local_model_cost_map, tool_choice, monkeypatch +): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() with pytest.raises(litellm.utils.UnsupportedParamsError, match="forced tool use"): - config.map_tool_choice_values(model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False) + config.map_tool_choice_values( + model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False + ) @pytest.mark.parametrize("tool_choice", ["auto", "none"]) @@ -6983,7 +7215,9 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], ) -def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(local_model_cost_map, model): +def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( + local_model_cost_map, model +): """Regression: Bedrock rejects both ``outputConfig`` structured output and forced tool_choice for Fable 5.1, so response_format must map to a tool without a forced tool_choice.""" @@ -7010,11 +7244,15 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve assert result.get("json_mode") is True -def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(local_model_cost_map, monkeypatch): +def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( + local_model_cost_map, monkeypatch +): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() - result = config.map_tool_choice_values(model="anthropic.claude-fable-5", tool_choice="required", drop_params=False) + result = config.map_tool_choice_values( + model="anthropic.claude-fable-5", tool_choice="required", drop_params=False + ) assert result == {"any": {}} diff --git a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py index ebecd615605..122dd5b555a 100644 --- a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py +++ b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py @@ -203,7 +203,9 @@ def test_transform_request_image_pathlike_input(tmp_path): ) assert body["taskType"] == "IMAGE_VARIATION" - assert body["imageVariationParams"]["images"][0] == base64.b64encode(image_bytes).decode("utf-8") + assert body["imageVariationParams"]["images"][0] == base64.b64encode( + image_bytes + ).decode("utf-8") def test_transform_request_inpainting_with_mask(): @@ -364,7 +366,9 @@ def test_transform_request_inpainting_explicit_task_without_mask_raises(): """INPAINTING taskType without mask or maskPrompt must fail fast.""" config = BedrockAmazonNovaCanvasImageEditConfig() img = io.BytesIO(b"img") - with pytest.raises(ValueError, match="INPAINTING requires either maskPrompt or maskImage"): + with pytest.raises( + ValueError, match="INPAINTING requires either maskPrompt or maskImage" + ): config.transform_image_edit_request( model="amazon.nova-canvas-v1:0", prompt="fix it", diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 7d243594cb3..575d0b881c3 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -48,7 +48,9 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): _dummy_stream(), litellm_logging_obj=LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", - messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], + messages=[ + {"role": "user", "content": "Hello, can you tell me a short joke?"} + ], stream=True, call_type="chat", start_time=datetime.now(), @@ -225,7 +227,9 @@ async def test_bedrock_sse_wrapper_keeps_usage_in_message_start_and_message_delt def test_chunk_parser_usage_transformation(): """Ensure Bedrock invocation metrics are transformed to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0") + decoder = AmazonAnthropicClaudeMessagesStreamDecoder( + model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0" + ) chunk = { "type": "message_delta", @@ -254,7 +258,9 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): fields and cache tokens end up billed at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") + decoder = AmazonAnthropicClaudeMessagesStreamDecoder( + model="bedrock/invoke/anthropic.claude-sonnet-4-6" + ) chunk = { "type": "message_stop", @@ -280,7 +286,9 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): """Cache itemization inside invocationMetrics maps to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") + decoder = AmazonAnthropicClaudeMessagesStreamDecoder( + model="bedrock/invoke/anthropic.claude-sonnet-4-6" + ) chunk = { "type": "message_stop", @@ -303,7 +311,9 @@ def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): def test_chunk_parser_keeps_existing_token_counts_over_invocation_metrics(): """Token counts reported in the chunk's own usage block win over invocationMetrics.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") + decoder = AmazonAnthropicClaudeMessagesStreamDecoder( + model="bedrock/invoke/anthropic.claude-sonnet-4-6" + ) chunk = { "type": "message_stop", @@ -338,7 +348,9 @@ async def test_bedrock_sse_wrapper_preserves_cache_usage_with_invocation_metrics final usage billed cache reads and writes at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") + decoder = AmazonAnthropicClaudeMessagesStreamDecoder( + model="bedrock/invoke/anthropic.claude-sonnet-4-6" + ) cfg = AmazonAnthropicClaudeMessagesConfig() raw_chunks = [ @@ -548,7 +560,11 @@ def test_normalize_custom_field_on_tools(): assert request4["tools"] is None # Case 5: an explicit top-level flag wins over a conflicting wrapped one - request5 = {"tools": [{"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}}]} + request5 = { + "tools": [ + {"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}} + ] + } normalize_custom_field_on_tools(request5) assert request5["tools"][0] == {"name": "Read", "defer_loading": False} @@ -569,7 +585,9 @@ def test_normalize_custom_field_on_tools(): assert request7["tools"] == [{"name": "Read"}, {"name": "Write"}] -@pytest.mark.parametrize("deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}]) +@pytest.mark.parametrize( + "deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}] +) def test_bedrock_invoke_messages_transform_emits_top_level_defer_loading( deferred_marker, ): @@ -702,7 +720,9 @@ def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled( "max_tokens": 32000, "stream": False, "thinking": {"type": "enabled", "budget_tokens": 2048}, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } result = cfg.transform_anthropic_messages_request( model="global.anthropic.claude-sonnet-4-6-v1:0", @@ -804,7 +824,9 @@ def test_remove_ttl_from_cache_control_processes_tools(local_model_cost_map): "messages": [], } - cfg._remove_ttl_from_cache_control(request, model="anthropic.claude-3-5-sonnet-20241022-v2:0") + cfg._remove_ttl_from_cache_control( + request, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) # Tool ttl should be stripped assert "ttl" not in request["tools"][0]["cache_control"] @@ -840,7 +862,9 @@ def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(local_ ], } - cfg._remove_ttl_from_cache_control(request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") + cfg._remove_ttl_from_cache_control( + request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" + ) # Both tools and system should preserve ttl for Claude 4.5 assert request["tools"][0]["cache_control"]["ttl"] == "1h" @@ -924,7 +948,9 @@ def test_bedrock_messages_strips_output_config(): headers={}, ) - assert "output_config" not in result, "output_config should be stripped for models that don't support it" + assert "output_config" not in result, ( + "output_config should be stripped for models that don't support it" + ) assert result.get("max_tokens") == 4096 @@ -957,7 +983,9 @@ def test_bedrock_messages_preserves_output_config_for_claude_4_6(): headers={}, ) - assert "output_config" in result, "output_config should be preserved for supported models" + assert "output_config" in result, ( + "output_config should be preserved for supported models" + ) assert result["output_config"] == {"effort": "high"} assert result.get("max_tokens") == 4096 @@ -1109,7 +1137,9 @@ def test_bedrock_messages_converts_output_config_format_to_inline_schema(): ("anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bedrock_messages_normalizes_output_config_effort_for_opus(model, expected_effort): +def test_bedrock_messages_normalizes_output_config_effort_for_opus( + model, expected_effort +): """Bedrock /v1/messages accepts ``xhigh`` and forwards the provider-safe effort.""" from unittest.mock import patch @@ -1167,7 +1197,9 @@ def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema headers={}, ) - assert caller_messages == [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] + assert caller_messages == [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + ] assert caller_message == { "role": "user", "content": [{"type": "text", "text": "Hello"}], @@ -1483,7 +1515,9 @@ def test_bedrock_messages_strips_context_management(): messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] optional_params = { "max_tokens": 4096, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } result = cfg.transform_anthropic_messages_request( @@ -1494,7 +1528,9 @@ def test_bedrock_messages_strips_context_management(): headers={}, ) - assert "context_management" not in result, "context_management should be stripped — Bedrock Invoke rejects it" + assert "context_management" not in result, ( + "context_management should be stripped — Bedrock Invoke rejects it" + ) assert result.get("max_tokens") == 4096 @@ -1641,8 +1677,12 @@ def test_bedrock_messages_filters_user_provided_unsupported_beta_header(): ) betas = result.get("anthropic_beta") or [] - assert "advisor-tool-2026-03-01" not in betas, "user-provided beta not in the Bedrock mapping must be dropped" - assert "context-1m-2025-08-07" in betas, "user-provided beta that IS in the Bedrock mapping should survive" + assert "advisor-tool-2026-03-01" not in betas, ( + "user-provided beta not in the Bedrock mapping must be dropped" + ) + assert "context-1m-2025-08-07" in betas, ( + "user-provided beta that IS in the Bedrock mapping should survive" + ) def test_bedrock_messages_renames_user_provided_aliased_beta_header(): @@ -1670,7 +1710,9 @@ def test_bedrock_messages_renames_user_provided_aliased_beta_header(): assert "advanced-tool-use-2025-11-20" not in betas, ( "Anthropic-direct spelling should be rewritten, not forwarded verbatim" ) - assert "tool-search-tool-2025-10-19" in betas, "user-provided beta should be renamed to the Bedrock-side spelling" + assert "tool-search-tool-2025-10-19" in betas, ( + "user-provided beta should be renamed to the Bedrock-side spelling" + ) @pytest.mark.asyncio @@ -1932,7 +1974,9 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46(): "global.anthropic.claude-fable-5", ], ) -def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models(local_model_cost_map, model): +def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models( + local_model_cost_map, model +): """clear_thinking_20251015 without a top-level ``thinking`` field must inject ``thinking.type=adaptive`` plus ``output_config.effort`` on adaptive-thinking models (Opus 4.7/4.8, Fable 5). The legacy ``thinking.type=enabled`` shape is @@ -1942,7 +1986,9 @@ def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -1965,7 +2011,9 @@ def test_bedrock_clear_thinking_converts_legacy_enabled_budget_to_effort(): "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, }, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -1983,7 +2031,10 @@ def test_resolve_clear_thinking_budget_tokens_honors_explicit_zero(): and only fall back to the minimum when the caller omits the budget.""" cfg = AmazonAnthropicClaudeMessagesConfig() assert cfg._resolve_clear_thinking_budget_tokens(0) == 0 - assert cfg._resolve_clear_thinking_budget_tokens(None) == BEDROCK_MIN_THINKING_BUDGET_TOKENS + assert ( + cfg._resolve_clear_thinking_budget_tokens(None) + == BEDROCK_MIN_THINKING_BUDGET_TOKENS + ) assert cfg._resolve_clear_thinking_budget_tokens(12000) == 12000 @@ -1993,7 +2044,9 @@ def test_bedrock_clear_thinking_keeps_enabled_for_non_adaptive_models(): cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2018,7 +2071,9 @@ def test_bedrock_invoke_transform_emits_adaptive_thinking_for_opus_4_8(): optional_params = { "max_tokens": 32000, "stream": False, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } result = cfg.transform_anthropic_messages_request( @@ -2055,7 +2110,9 @@ def test_bedrock_invoke_transform_normalizes_system_role_message_into_system(): assert all(m.get("role") != "system" for m in result["messages"]) assert result["messages"] == [{"role": "user", "content": "hi"}] - assert result["system"] == [{"type": "text", "text": "You are a careful assistant."}] + assert result["system"] == [ + {"type": "text", "text": "You are a careful assistant."} + ] def test_bedrock_invoke_transform_merges_system_role_into_existing_system(): @@ -2170,7 +2227,9 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(lo ) assert result["messages"] == messages - assert result["system"] == [{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}] + assert result["system"] == [ + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} + ] def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): @@ -2353,13 +2412,13 @@ def test_bedrock_invoke_transform_converted_system_carries_only_its_content(loca assert result["messages"][2] == { "role": "user", "content": [ - { - "type": "text", - "text": ( - "Operator note (not from the user): the following was " - "originally a mid-conversation system-role reminder." - ), - }, + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, ], } @@ -2495,7 +2554,10 @@ def test_as_system_content_blocks_handles_each_shape(): def test_effort_from_thinking_budget_tiers(budget_tokens, expected_effort): """The budget -> effort mapping pins each tier boundary so a shifted threshold is caught.""" - assert AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) == expected_effort + assert ( + AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) + == expected_effort + ) def test_inject_adaptive_thinking_preserves_existing_effort(): @@ -2504,7 +2566,9 @@ def test_inject_adaptive_thinking_preserves_existing_effort(): cfg = AmazonAnthropicClaudeMessagesConfig() request = {"output_config": {"effort": "max", "other": "keep"}} - cfg._inject_adaptive_thinking_for_clear_thinking(request, budget_tokens=24000, model="us.anthropic.claude-fable-5") + cfg._inject_adaptive_thinking_for_clear_thinking( + request, budget_tokens=24000, model="us.anthropic.claude-fable-5" + ) assert request["thinking"] == {"type": "adaptive"} assert request["output_config"] == {"effort": "max", "other": "keep"} @@ -2517,7 +2581,9 @@ def test_bedrock_clear_thinking_noops_when_thinking_already_adaptive(): request = { "max_tokens": 32000, "thinking": {"type": "adaptive"}, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2537,7 +2603,9 @@ def test_bedrock_clear_thinking_replaces_disabled_thinking_on_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "disabled"}, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2557,7 +2625,9 @@ def test_bedrock_clear_thinking_leaves_enabled_thinking_on_non_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 8000}, - "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, + "context_management": { + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + }, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2592,7 +2662,9 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] optional_params = { "max_tokens": 4096, - "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}] + }, } result = cfg.transform_anthropic_messages_request( @@ -2603,11 +2675,12 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ headers={}, ) - assert result.get("context_management") == {"edits": [{"type": "clear_tool_uses_20250919"}]}, ( - "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" - ) + assert result.get("context_management") == { + "edits": [{"type": "clear_tool_uses_20250919"}] + }, "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" assert "context-management-2025-06-27" in result.get("anthropic_beta", []), ( - "context-management-2025-06-27 beta must reach the InvokeModel body so the tool-call-clearing edit is accepted" + "context-management-2025-06-27 beta must reach the InvokeModel body so " + "the tool-call-clearing edit is accepted" ) @@ -2684,9 +2757,9 @@ def test_bedrock_messages_filters_clear_thinking_keeps_clear_tool_uses( cm = result.get("context_management") assert cm is not None - assert [e.get("type") for e in cm["edits"]] == ["clear_tool_uses_20250919"], ( - "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" - ) + assert [e.get("type") for e in cm["edits"]] == [ + "clear_tool_uses_20250919" + ], "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" betas = result.get("anthropic_beta", []) assert "context-management-2025-06-27" in betas diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 8deb16bceb2..4cdca97bbff 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -1,3 +1,4 @@ + import pytest @@ -29,7 +30,9 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: + with patch.object( + mod, "_load_bedrock_response_stream_shape", return_value=sentinel + ) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -76,7 +79,9 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert ( + loaded_shape is not None + ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -141,7 +146,9 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") + bedrock_route = bedrock_model_info.get_bedrock_route( + model="bedrock/us.deepseek.r1-v1:0" + ) assert bedrock_route == "converse" @@ -214,19 +221,27 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") + base_model = bedrock_model_info.get_base_model( + model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" + ) assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") + base_model = bedrock_model_info.get_base_model( + model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" + ) assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") + base_model = bedrock_model_info.get_base_model( + model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" + ) assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") + base_model = bedrock_model_info.get_base_model( + model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" + ) assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -240,14 +255,23 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" - assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" + assert ( + get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") + == "anthropic.claude-opus-4-6-v1" + ) + assert ( + get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") + == "anthropic.claude-sonnet-4-6" + ) assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" + assert ( + get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") + == "anthropic.claude-opus-4-6-v1" + ) # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -287,7 +311,9 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( + model, expected_ceiling +): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -306,24 +332,54 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "invoke" - assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False + assert ( + BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + ) + assert ( + BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "invoke" + ) + assert ( + BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") + is False + ) # A genuine mantle route still resolves, via the startswith branch... - assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" + assert ( + BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") + == "mantle" + ) # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" + assert ( + BedrockModelInfo.get_bedrock_route( + "bedrock/mantle/anthropic.claude-mythos-preview" + ) + == "mantle" + ) def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True + assert ( + BedrockModelInfo._model_has_route_prefix( + "mantle/anthropic.claude-mythos-preview", "mantle/" + ) + is True + ) # f"/{prefix}" boundary branch - assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True + assert ( + BedrockModelInfo._model_has_route_prefix( + "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" + ) + is True + ) # neither branch: the token only appears glued to another segment - assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False + assert ( + BedrockModelInfo._model_has_route_prefix( + "bedrock_mantle/openai.gpt-5.5", "mantle/" + ) + is False + ) @pytest.mark.parametrize( @@ -373,10 +429,16 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False + assert ( + BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") + is False + ) # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True + assert ( + BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") + is True + ) def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index df67ee7d5ae..3ad0d7308f7 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -52,7 +52,10 @@ class TestBedrockMantleResponsesURL: api_base="https://bedrock-mantle.us-east-2.api.aws/v1/", litellm_params={}, ) - assert url_trailing == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + assert ( + url_trailing + == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" + ) def test_url_does_not_double_openai_v1(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -112,7 +115,9 @@ class TestBedrockMantleResponsesURL: with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, - litellm_params={"aws_region_name": "us-east-1.api.aws.attacker.example/"}, + litellm_params={ + "aws_region_name": "us-east-1.api.aws.attacker.example/" + }, ) def test_url_region_default_us_east_1(self, monkeypatch): @@ -165,7 +170,9 @@ class TestBedrockMantleResponsesURL: class TestBedrockMantleGetLlmProviderRegion: - def test_get_llm_provider_uses_supplemental_litellm_params(self, monkeypatch, local_cost_map): + def test_get_llm_provider_uses_supplemental_litellm_params( + self, monkeypatch, local_cost_map + ): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -182,7 +189,9 @@ class TestBedrockMantleGetLlmProviderRegion: # the resolved chat base) is on the /openai/v1 base per the AWS card. assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_get_llm_provider_uses_aws_region_from_litellm_params(self, monkeypatch, local_cost_map): + def test_get_llm_provider_uses_aws_region_from_litellm_params( + self, monkeypatch, local_cost_map + ): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -216,14 +225,18 @@ class TestBedrockMantleResponsesAuth: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert headers["Authorization"] == "Bearer env-key" def test_bedrock_bearer_token_fallback(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "bearer-key") cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert headers["Authorization"] == "Bearer bearer-key" def test_missing_bearer_does_not_raise_in_validate_environment(self, monkeypatch): @@ -231,7 +244,9 @@ class TestBedrockMantleResponsesAuth: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) + headers = cfg.validate_environment( + headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() + ) assert "Authorization" not in headers def test_project_id_sets_openai_project_header(self): @@ -239,7 +254,9 @@ class TestBedrockMantleResponsesAuth: headers = cfg.validate_environment( headers={}, model="openai.gpt-5.5", - litellm_params=GenericLiteLLMParams(api_key="fake-key", aws_bedrock_project_id="proj_abc123def456"), + litellm_params=GenericLiteLLMParams( + api_key="fake-key", aws_bedrock_project_id="proj_abc123def456" + ), ) assert headers["OpenAI-Project"] == "proj_abc123def456" @@ -340,7 +357,9 @@ class TestBedrockMantleResponsesTools: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" + ) as mock_warning: cfg.map_openai_params( response_api_optional_params={"tools": [{"type": "file_search", "vector_store_ids": ["vs_123"]}]}, model="openai.gpt-5.5", @@ -541,7 +560,9 @@ class TestBedrockMantleServiceTier: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" + ) as mock_warning: cfg.map_openai_params( response_api_optional_params={"service_tier": "priority"}, model="openai.gpt-5.5", @@ -630,9 +651,7 @@ class TestBedrockMantleReasoningSummary: model="openai.gpt-5.6-sol", drop_params=True, ) - warnings = [ - record for record in caplog.records if "dropping unsupported reasoning.summary" in record.getMessage() - ] + warnings = [record for record in caplog.records if "dropping unsupported reasoning.summary" in record.getMessage()] assert len(warnings) == 1 assert "detailed" in warnings[0].getMessage() @@ -806,7 +825,9 @@ class TestBedrockMantleCodexAdditionalTools: def test_hoist_is_logged_at_debug_level(self): from unittest.mock import patch - with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug") as mock_debug: + with patch( + "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug" + ) as mock_debug: self._transform( input=[ {"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS}, @@ -963,13 +984,7 @@ class TestBedrockMantleCodexInputItemNormalization: {"type": "function_call", "name": "shell", "arguments": "{}", "call_id": "call_2"}, {"type": "function_call_output", "call_id": "call_2", "output": "ok"}, {"type": "tool_search_call", "call_id": "call_3", "execution": "server", "arguments": {"query": "x"}}, - { - "type": "tool_search_output", - "call_id": "call_3", - "status": "completed", - "execution": "server", - "tools": [], - }, + {"type": "tool_search_output", "call_id": "call_3", "status": "completed", "execution": "server", "tools": []}, {"type": "compaction_trigger"}, ] body = self._transform(input=copy.deepcopy(supported_items)) @@ -983,12 +998,7 @@ class TestBedrockMantleCodexInputItemNormalization: with caplog.at_level(logging.WARNING, logger="LiteLLM"): body = self._transform( input=[ - { - "type": "agent_message", - "author": "a", - "recipient": "b", - "content": [{"type": "input_text", "text": "hi"}], - }, + {"type": "agent_message", "author": "a", "recipient": "b", "content": [{"type": "input_text", "text": "hi"}]}, self._USER_MESSAGE, ] ) @@ -1127,7 +1137,9 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_price_map_flag_routes_non_gpt_name_to_openai_path(self, restore_model_cost): + def test_price_map_flag_routes_non_gpt_name_to_openai_path( + self, restore_model_cost + ): # Data-driven onboarding: a frontier model whose name does NOT match the # openai.gpt- convention can still be routed to /openai/v1/responses by # declaring use_openai_responses_path in its price-map entry, with no code @@ -1150,6 +1162,7 @@ class TestBedrockMantleResponsesRegistry: assert isinstance(cfg, BedrockMantleResponsesAPIConfig) assert cfg.use_openai_path is True + @pytest.mark.parametrize( "model", [ @@ -1183,7 +1196,9 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_declared_responses_non_openai_routes_to_standard_path(self, restore_model_cost): + def test_declared_responses_non_openai_routes_to_standard_path( + self, restore_model_cost + ): # New feature: a non-OpenAI model declared mode=responses (e.g. via a # user's proxy model_info block) must route to the STANDARD /v1/responses # path, not the frontier /openai/v1/responses path. Fails before the @@ -1301,7 +1316,9 @@ class TestBedrockMantlePerModelResponsesURL: model=model, ) assert isinstance(cfg, BedrockMantleResponsesAPIConfig) - return cfg.get_complete_url(api_base=None, litellm_params={"aws_region_name": region}) + return cfg.get_complete_url( + api_base=None, litellm_params={"aws_region_name": region} + ) def test_gpt_oss_uses_standard_responses_path(self, local_cost_map): url = self._url_for("openai.gpt-oss-120b") @@ -1400,7 +1417,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, signed_body = cfg.sign_request( @@ -1422,7 +1441,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1444,7 +1465,9 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("get_credentials must not run for bearer auth") + ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1590,7 +1613,9 @@ class TestBedrockMantleResponsesSigV4: } cfg = BedrockMantleResponsesAPIConfig(aws_signer=BaseAWSLLM()) url = cfg.get_complete_url(api_base=None, litellm_params=params) - assert url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" + assert ( + url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" + ) headers, _ = cfg.sign_request( headers={}, @@ -1601,7 +1626,9 @@ class TestBedrockMantleResponsesSigV4: ) assert "/ap-southeast-2/bedrock/aws4_request" in headers["Authorization"] - def test_injected_default_region_base_does_not_override_aws_region_name(self, monkeypatch): + def test_injected_default_region_base_does_not_override_aws_region_name( + self, monkeypatch + ): """2nd-round adversarial regression: responses/main.py auto-injects litellm_params.api_base = https://bedrock-mantle..api.aws/v1 (default region, ignoring aws_region_name). The config must still pin BOTH the URL host @@ -1704,7 +1731,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1734,7 +1761,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1759,7 +1786,9 @@ class TestBedrockMantleResponsesSigV4: signer = BaseAWSLLM() signer.get_credentials = MagicMock( - side_effect=ConnectTimeoutError(endpoint_url="https://sts.us-east-2.amazonaws.com") + side_effect=ConnectTimeoutError( + endpoint_url="https://sts.us-east-2.amazonaws.com" + ) ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) @@ -1777,6 +1806,8 @@ class TestBedrockMantleResponsesSigV4: class TestBedrockMantleResponsesPricing: + + def test_models_registered(self, local_cost_map): assert "bedrock_mantle/openai.gpt-5.5" in litellm.bedrock_mantle_models assert "bedrock_mantle/openai.gpt-5.4" in litellm.bedrock_mantle_models diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 97465d8c49e..0cc3963358f 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -98,7 +98,9 @@ class TestBedrockMantleConfig: cfg._get_openai_compatible_provider_info( None, None, - litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), + litellm_params=GenericLiteLLMParams( + aws_region_name="us-east-1.api.aws.attacker.example/" + ), ) def test_get_llm_provider_rejects_malicious_aws_region_name(self, monkeypatch): @@ -111,10 +113,14 @@ class TestBedrockMantleConfig: litellm.get_llm_provider( model="openai.gpt-5.5", custom_llm_provider="bedrock_mantle", - litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), + litellm_params=GenericLiteLLMParams( + aws_region_name="us-east-1.api.aws.attacker.example/" + ), ) - def test_get_llm_provider_uses_aws_region_name_for_responses(self, monkeypatch, local_cost_map): + def test_get_llm_provider_uses_aws_region_name_for_responses( + self, monkeypatch, local_cost_map + ): from litellm.types.router import GenericLiteLLMParams monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) @@ -172,14 +178,18 @@ class TestBedrockMantleConfig: monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model="openai.gpt-oss-120b") + api_base, _ = cfg._get_openai_compatible_provider_info( + None, None, model="openai.gpt-oss-120b" + ) assert api_base == "https://bedrock-mantle.us-east-2.api.aws/v1" @pytest.mark.parametrize( "model_id", ["google.gemma-4-31b", "google.gemma-4-26b-a4b", "google.gemma-4-e2b"], ) - def test_chat_base_for_gemma_4_uses_openai_v1(self, monkeypatch, local_cost_map, model_id): + def test_chat_base_for_gemma_4_uses_openai_v1( + self, monkeypatch, local_cost_map, model_id + ): # The chat-config bug the Gemma 4 cards exposed: gemma-4-* is served on the # /openai/v1 base, not the hardcoded /v1. Driven by the price-map # use_openai_responses_path flag (loaded by local_cost_map). Fails before @@ -187,16 +197,22 @@ class TestBedrockMantleConfig: monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model=model_id) + api_base, _ = cfg._get_openai_compatible_provider_info( + None, None, model=model_id + ) assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_chat_base_explicit_api_base_wins_over_derived(self, monkeypatch, local_cost_map): + def test_chat_base_explicit_api_base_wins_over_derived( + self, monkeypatch, local_cost_map + ): # An explicit api_base must not be overridden by the data-driven default, # even for a model whose default differs (gemma-4 -> openai/v1). monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) custom_base = "https://bedrock-mantle.us-west-2.api.aws/v1" cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info(custom_base, None, model="google.gemma-4-31b") + api_base, _ = cfg._get_openai_compatible_provider_info( + custom_base, None, model="google.gemma-4-31b" + ) assert api_base == custom_base def test_api_key_from_env(self, monkeypatch): @@ -251,7 +267,9 @@ class TestBedrockMantleChatAuth: from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM signer = BaseAWSLLM() - signer.get_credentials = MagicMock(side_effect=AssertionError("SigV4 must not run when a Bearer token exists")) + signer.get_credentials = MagicMock( + side_effect=AssertionError("SigV4 must not run when a Bearer token exists") + ) return signer def test_bearer_token_skips_sigv4(self, monkeypatch): @@ -368,7 +386,9 @@ class TestBedrockMantleChatAuth: assert "/eu-west-1/bedrock/aws4_request" in headers["Authorization"] - def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees(self, monkeypatch): + def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees( + self, monkeypatch + ): # If a caller (e.g. proxy) passes a stale api_base in one region and an # aws_region_name in a different region, the SigV4 credential scope must # match the URL host or Bedrock rejects the request with 401. Without the @@ -456,7 +476,7 @@ class TestBedrockMantleChatAuth: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleChatConfig(aws_signer=signer) - with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: + with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -482,7 +502,9 @@ class TestBedrockMantleChatAuth: ): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0") + monkeypatch.setenv( + "AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0" + ) monkeypatch.setenv("AWS_REGION", "us-east-2") requests = [] @@ -512,7 +534,9 @@ class TestBedrockMantleChatAuth: request=httpx.Request("POST", url), ) - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post + ): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -556,9 +580,7 @@ class TestBedrockMantleChatAuth: "object": "chat.completion", "created": 1733529600, "model": "google.gemma-4-31b", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, }, request=httpx.Request("POST", url), @@ -624,7 +646,9 @@ class TestBedrockMantleProjectHeader: def mock_post(self, url, data=None, headers=None, **kwargs): raw_body = data.decode("utf-8") if isinstance(data, bytes) else data - requests.append({"headers": headers or {}, "body": json.loads(raw_body or "{}")}) + requests.append( + {"headers": headers or {}, "body": json.loads(raw_body or "{}")} + ) return httpx.Response( status_code=200, json={ @@ -648,7 +672,9 @@ class TestBedrockMantleProjectHeader: request=httpx.Request("POST", url), ) - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post + ): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -664,15 +690,20 @@ class TestBedrockMantleProjectHeader: class TestBedrockMantleProviderResolution: def test_get_llm_provider_resolves_correctly(self): - model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-120b") + model, provider, _, _ = litellm.get_llm_provider( + "bedrock_mantle/openai.gpt-oss-120b" + ) assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-120b" def test_get_llm_provider_20b(self): - model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-20b") + model, provider, _, _ = litellm.get_llm_provider( + "bedrock_mantle/openai.gpt-oss-20b" + ) assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-20b" + def test_get_llm_provider_strips_region_prefix(self, monkeypatch, local_cost_map): for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"): monkeypatch.delenv(var, raising=False) @@ -705,9 +736,7 @@ class TestBedrockMantleProviderResolution: "object": "chat.completion", "created": 1733529600, "model": "xai.grok-4.3", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58}, }, request=request, diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 80372418026..718d00222aa 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -103,3 +103,5 @@ def test_crusoe_provider_detection_by_prefix(): model, provider, _, _ = get_llm_provider("crusoe/meta-llama/Llama-3.3-70B-Instruct") assert provider == "crusoe" assert model == "meta-llama/Llama-3.3-70B-Instruct" + + diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index 17bbf9852e7..344b0cf127a 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -42,7 +42,9 @@ class TestDashscopeCostCalculator: """ usage = Usage(prompt_tokens=1000, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-max", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-max", usage=usage + ) model_info = litellm.get_model_info("dashscope/qwen-max") expected_prompt_cost = 1000 * model_info["input_cost_per_token"] @@ -58,7 +60,9 @@ class TestDashscopeCostCalculator: """ # Tier 1 for qwen-flash is [0, 256,000] tokens usage = Usage(prompt_tokens=100000, completion_tokens=50000) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-flash", usage=usage + ) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1_pricing = model_info["tiered_pricing"][0] @@ -76,7 +80,9 @@ class TestDashscopeCostCalculator: """ # Tiering for qwen-flash: Tier 1: [0, 256k], Tier 2: [256k, 1M] usage = Usage(prompt_tokens=300000, completion_tokens=300000) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-flash", usage=usage + ) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1 = model_info["tiered_pricing"][0] @@ -88,7 +94,9 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) - graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (44000 * tier_2["input_cost_per_token"]) + graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + ( + 44000 * tier_2["input_cost_per_token"] + ) assert prompt_cost > graduated_prompt_cost def test_dashscope_tiered_pricing_boundary_stays_in_lower_tier(self): @@ -97,12 +105,18 @@ class TestDashscopeCostCalculator: official `0 < Token <= 256K` phrasing. """ usage = Usage(prompt_tokens=256000, completion_tokens=1000) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-flash", usage=usage + ) tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose(prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10) - assert math.isclose(completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10) + assert math.isclose( + prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10 + ) + assert math.isclose( + completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10 + ) def test_dashscope_tiered_pricing_output_uses_input_selected_tier(self): """ @@ -114,7 +128,9 @@ class TestDashscopeCostCalculator: tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose(completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10) + assert math.isclose( + completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10 + ) def test_dashscope_tiered_pricing_with_caching(self): """ @@ -143,13 +159,17 @@ class TestDashscopeCostCalculator: """ Requests above the highest declared range bill entirely at the last tier's rate. """ - usage = Usage(prompt_tokens=1200000, completion_tokens=1000) # Max defined range for qwen-flash is 1M + usage = Usage( + prompt_tokens=1200000, completion_tokens=1000 + ) # Max defined range for qwen-flash is 1M prompt_cost, _ = dashscope_cost_per_token(model="qwen-flash", usage=usage) tier_2 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][1] - assert math.isclose(prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10) + assert math.isclose( + prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10 + ) def _register_tiered_model(self, model_key: str, tiered_pricing: list[dict]) -> None: litellm.model_cost[model_key] = { @@ -184,7 +204,9 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-str-tier-test", usage=usage + ) assert math.isclose(prompt_cost, 500 * float("4e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 200 * float("1.6e-06"), rel_tol=1e-10) @@ -197,7 +219,9 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=2500, completion_tokens=3000) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-str-tier-test", usage=usage + ) assert math.isclose(prompt_cost, 2500 * float("8e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 3000 * float("3.2e-06"), rel_tol=1e-10) @@ -230,12 +254,18 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=300000, # 200k new + 60k cache creation + 40k cache read completion_tokens=1000, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=40000, cache_creation_tokens=60000 + ), ) - prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-write-test", usage=usage) + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-cache-write-test", usage=usage + ) - expected_prompt_cost = (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) + expected_prompt_cost = ( + (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) + ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -272,9 +302,13 @@ class TestDashscopeCostCalculator: completion_tokens_details={"reasoning_tokens": 170}, ) - prompt_cost, _ = dashscope_cost_per_token(model="qwen-nested-cache-write-test", usage=usage) + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-nested-cache-write-test", usage=usage + ) - assert math.isclose(prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10) + assert math.isclose( + prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10 + ) def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self): """ @@ -298,7 +332,9 @@ class TestDashscopeCostCalculator: prompt_tokens_details=PromptTokensDetailsWrapper(cache_creation_tokens=4000), ) - prompt_cost, _ = dashscope_cost_per_token(model="qwen-no-cache-write-test", usage=usage) + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-no-cache-write-test", usage=usage + ) assert math.isclose(prompt_cost, 10000 * 3.25e-07, rel_tol=1e-10) @@ -316,12 +352,18 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=10000, completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=2000, cache_creation_tokens=3000), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=2000, cache_creation_tokens=3000 + ), ) - prompt_cost, _ = dashscope_cost_per_token(model="qwen-flat-cache-write-test", usage=usage) + prompt_cost, _ = dashscope_cost_per_token( + model="qwen-flat-cache-write-test", usage=usage + ) - expected_prompt_cost = (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) + expected_prompt_cost = ( + (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) + ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -338,7 +380,9 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-input-only-tier-test", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-input-only-tier-test", usage=usage + ) assert math.isclose(prompt_cost, 500 * 4e-07, rel_tol=1e-10) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) @@ -361,9 +405,13 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token(model="qwen-input-only-reasoning-test", usage=usage) + _, completion_cost = dashscope_cost_per_token( + model="qwen-input-only-reasoning-test", usage=usage + ) - assert math.isclose(completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10) + assert math.isclose( + completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10 + ) def test_dashscope_tier_output_rate_wins_over_the_model_reasoning_rate(self): """ @@ -388,10 +436,13 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token(model="qwen-tier-output-reasoning-test", usage=usage) + _, completion_cost = dashscope_cost_per_token( + model="qwen-tier-output-reasoning-test", usage=usage + ) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) + def test_dashscope_tier_zero_reasoning_rate_bills_reasoning_free(self): """ Regression: a tier declaring an explicit zero reasoning rate had it treated as @@ -415,7 +466,9 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token(model="qwen-tier-zero-reasoning-test", usage=usage) + _, completion_cost = dashscope_cost_per_token( + model="qwen-tier-zero-reasoning-test", usage=usage + ) assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10) @@ -444,7 +497,9 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=0, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-zero-input-test", usage=usage) + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-zero-input-test", usage=usage + ) assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index ea55980a558..68c52f9be72 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -216,7 +216,9 @@ def test_validate_environment_raises_without_api_key(monkeypatch): def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): assert ( - get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}) + get_fireworks_session_id( + {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"} + ) == "session-123" ) @@ -268,18 +270,25 @@ def test_handle_message_content_with_tool_calls(): }, } ] - updated_message = config._handle_message_content_with_tool_calls(message, tool_calls) + updated_message = config._handle_message_content_with_tool_calls( + message, tool_calls + ) assert updated_message.tool_calls is not None assert len(updated_message.tool_calls) == 1 assert updated_message.tool_calls[0].function.name == "get_current_weather" - assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments + assert ( + updated_message.tool_calls[0].function.arguments + == expected_tool_call.function.arguments + ) def test_get_supported_openai_params_reasoning_effort(): """Test that reasoning_effort is only included in supported params for models that support it.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") + supported_params = config.get_supported_openai_params( + "fireworks_ai/accounts/fireworks/models/glm-5p1" + ) assert "reasoning_effort" in supported_params assert "thinking" in supported_params @@ -294,7 +303,9 @@ def test_get_supported_openai_params_parallel_tool_calls(): """Test that parallel_tool_calls is included for models that support function calling.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") + supported_params = config.get_supported_openai_params( + "fireworks_ai/accounts/fireworks/models/glm-5p1" + ) assert "parallel_tool_calls" in supported_params assert "tools" in supported_params assert "tool_choice" in supported_params @@ -308,7 +319,9 @@ def test_get_supported_openai_params_parallel_tool_calls(): def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813") + supported_params = config.get_supported_openai_params( + "fireworks_ai/deepseek-v4-pro-0813" + ) assert "tool_choice" in supported_params assert "reasoning_effort" in supported_params @@ -317,7 +330,9 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_ def test_get_supported_openai_params_preserves_generic_reasoning_fallback(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash") + supported_params = config.get_supported_openai_params( + "fireworks_ai/accounts/fireworks/models/glm-5p3-flash" + ) assert "reasoning_effort" in supported_params @@ -351,10 +366,14 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): mock_response = MagicMock() mock_response.status_code = 200 - mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]} + mock_response.json.return_value = { + "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] + } with ( - patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, + patch( + "litellm.module_level_client.get", return_value=mock_response + ) as mock_get, patch( "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", side_effect=lambda key: { @@ -366,9 +385,13 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): ): result = config.get_models(api_key="test-key", api_base=api_base) - called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get( + "url", "" + ) assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}" + assert called_url.startswith( + expected_url_prefix + ), f"URL {called_url} does not start with {expected_url_prefix}" assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] @@ -396,7 +419,9 @@ def test_transform_messages_helper_removes_provider_specific_fields(): }, ] # Call helper - out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={}) + out = config._transform_messages_helper( + messages, model="fireworks/test", litellm_params={} + ) for msg in out: assert "provider_specific_fields" not in msg @@ -409,11 +434,15 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co { "role": "assistant", "content": "I can help.", - "thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}], + "thinking_blocks": [ + {"type": "thinking", "thinking": "internal", "signature": ""} + ], "reasoning_content": "internal", }, ] - out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}) + out = config._transform_messages_helper( + messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} + ) assert "thinking_blocks" not in out[1] assert out[1]["reasoning_content"] == "internal" assert out[1]["content"] == "I can help." @@ -903,7 +932,9 @@ def test_transform_messages_helper_rejects_file_blocks(): litellm.BadRequestError, match="Fireworks AI chat completions does not support file content blocks", ): - config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}) + config._transform_messages_helper( + messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={} + ) def test_transform_messages_helper_rejects_non_vision_image_inputs(): @@ -915,14 +946,18 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" + }, }, ], } ] with pytest.raises(litellm.BadRequestError, match="does not support image inputs"): - config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) + config._transform_messages_helper( + messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} + ) def test_transform_messages_helper_allows_vision_image_inputs(): @@ -934,7 +969,9 @@ def test_transform_messages_helper_allows_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" + }, }, ], } @@ -958,7 +995,9 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): custom_model = "accounts/myorg/models/custom-glm-5p2" assert config._get_model_cost_capability(custom_model, "supports_vision") is False - assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None + assert ( + config._get_model_cost_capability_exact(custom_model, "supports_vision") is None + ) messages = [ { @@ -966,12 +1005,16 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): "content": [ { "type": "image_url", - "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" + }, }, ], } ] - out = config._transform_messages_helper(messages, model=custom_model, litellm_params={}) + out = config._transform_messages_helper( + messages, model=custom_model, litellm_params={} + ) assert out == messages @@ -984,7 +1027,9 @@ def test_transform_messages_helper_skips_non_dict_content(): } ] - out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) + out = config._transform_messages_helper( + messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} + ) assert out == messages @@ -1204,7 +1249,9 @@ def test_streaming_surfaces_fireworks_response_fields(): surfaced: dict = {} for chunk in stream: fields = getattr(chunk, "provider_specific_fields", None) or {} - surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")}) + surfaced.update( + {k: v for k, v in fields.items() if k.startswith("fireworks_")} + ) assert surfaced["fireworks_token_ids"] == [[123]] assert surfaced["fireworks_raw_outputs"] == [raw_output] @@ -1257,7 +1304,9 @@ def test_transform_request_direct_route_passthrough(): def test_map_extra_body_params_translates_truncate_prompt_tokens(): config = FireworksAIConfig() - result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL) + result = config.map_extra_body_params( + {"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL + ) assert result == {"prompt_truncate_len": 4096} @@ -1416,7 +1465,9 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped(): def test_map_extra_body_params_guided_json(): config = FireworksAIConfig() schema = {"type": "object", "properties": {"x": {"type": "string"}}} - result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL) + result = config.map_extra_body_params( + {"extra_body": {"guided_json": schema}}, _REASONING_MODEL + ) assert result == { "response_format": { "type": "json_schema", @@ -1427,10 +1478,16 @@ def test_map_extra_body_params_guided_json(): def test_map_extra_body_params_guided_grammar_and_choice(): config = FireworksAIConfig() - grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL) - assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}} + grammar = config.map_extra_body_params( + {"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL + ) + assert grammar == { + "response_format": {"type": "grammar", "grammar": "root ::= 'hello'"} + } - choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL) + choice = config.map_extra_body_params( + {"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL + ) assert choice == { "response_format": { "type": "json_schema", @@ -1516,7 +1573,9 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, config = FireworksAIConfig() with caplog.at_level(logging.DEBUG): - result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL) + result = config.map_extra_body_params( + {"extra_body": {param: value}}, _REASONING_MODEL + ) assert result == {} assert param in caplog.text @@ -1608,7 +1667,10 @@ def test_in_schema_unsupported_params_still_raise(): def test_streaming_preserves_selected_model_for_private_accounting(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" + requested_route = ( + "accounts/fireworks/routers/firerouter/" + "kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" + ) selected_model = "deepseek-v4-flash-0731" sse_lines = [ "data: " @@ -1662,14 +1724,19 @@ def test_streaming_preserves_selected_model_for_private_accounting(): assert chunks assert {chunk.model for chunk in chunks} == {requested_route} - assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model} + assert { + chunk._hidden_params.get("provider_response_model") for chunk in chunks + } == {selected_model} assembled = litellm.stream_chunk_builder(chunks=chunks) assert assembled is not None assert assembled.model == requested_route assert assembled._hidden_params["provider_response_model"] == selected_model selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"] - expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"] + expected_cost = ( + 5 * selected_model_info["input_cost_per_token"] + + selected_model_info["output_cost_per_token"] + ) assert litellm.completion_cost( completion_response=assembled, custom_llm_provider="fireworks_ai", diff --git a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py index c4c023077fc..1d12be2adee 100644 --- a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py +++ b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py @@ -188,15 +188,21 @@ def test_inception_does_not_leak_key_to_caller_api_base(): caller also supplies their own key. """ config = InceptionChatConfig() - with mock.patch.dict(os.environ, {"INCEPTION_API_KEY": "server-secret"}, clear=True): + with mock.patch.dict( + os.environ, {"INCEPTION_API_KEY": "server-secret"}, clear=True + ): with mock.patch.object(litellm, "inception_key", "module-secret"): # caller overrides api_base without a key -> server key withheld - api_base, api_key = config._get_openai_compatible_provider_info("https://attacker.example/v1", None) + api_base, api_key = config._get_openai_compatible_provider_info( + "https://attacker.example/v1", None + ) assert api_base == "https://attacker.example/v1" assert api_key is None # caller overrides api_base AND supplies their own key -> used as-is - _, api_key = config._get_openai_compatible_provider_info("https://attacker.example/v1", "caller-key") + _, api_key = config._get_openai_compatible_provider_info( + "https://attacker.example/v1", "caller-key" + ) assert api_key == "caller-key" # default/server base -> server-managed key resolved @@ -211,7 +217,9 @@ def test_get_llm_provider_inception(): assert model == "mercury-2" assert provider == "inception" - model, provider, _, api_base = get_llm_provider("mercury-2", api_base="https://api.inceptionlabs.ai/v1") + model, provider, _, api_base = get_llm_provider( + "mercury-2", api_base="https://api.inceptionlabs.ai/v1" + ) assert model == "mercury-2" assert provider == "inception" assert api_base == "https://api.inceptionlabs.ai/v1" @@ -285,3 +293,5 @@ def test_inception_completion_targets_inception_endpoint(): assert captured["body"]["model"] == "mercury-2" assert captured["body"]["tool_choice"] == "auto" assert response.choices[0].message.content == "hi" + + diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py index a75883d7846..f8242aa3d2b 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -305,3 +305,5 @@ class TestOCIEmbeddingConfig: optional_params={}, litellm_params={}, ) + + diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 1df47223f06..ca737c0bb80 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -109,7 +109,9 @@ class TestOpenAIResponsesAPIConfig: # Check expected fields have correct values for field, value in expected_fields.items(): assert field in params, f"Missing expected field: {field}" - assert params[field] == value, f"Field {field} has value {params[field]}, expected {value}" + assert ( + params[field] == value + ), f"Field {field} has value {params[field]}, expected {value}" def test_transform_responses_api_request(self): """Test request transformation""" @@ -457,7 +459,9 @@ class TestOpenAIResponsesAPIConfig: } # Mock the get_event_model_class to avoid validation issues in tests - with patch.object(OpenAIResponsesAPIConfig, "get_event_model_class") as mock_get_class: + with patch.object( + OpenAIResponsesAPIConfig, "get_event_model_class" + ) as mock_get_class: mock_get_class.return_value = ResponseCompletedEvent result = self.config.transform_streaming_response( @@ -476,7 +480,9 @@ class TestOpenAIResponsesAPIConfig: headers = {} api_key = "test_api_key" litellm_params = GenericLiteLLMParams(api_key=api_key) - result = self.config.validate_environment(headers=headers, model=self.model, litellm_params=litellm_params) + result = self.config.validate_environment( + headers=headers, model=self.model, litellm_params=litellm_params + ) assert "Authorization" in result assert result["Authorization"] == f"Bearer {api_key}" @@ -487,7 +493,9 @@ class TestOpenAIResponsesAPIConfig: with patch("litellm.api_key", "litellm_api_key"): litellm_params = GenericLiteLLMParams() - result = self.config.validate_environment(headers=headers, model=self.model, litellm_params=litellm_params) + result = self.config.validate_environment( + headers=headers, model=self.model, litellm_params=litellm_params + ) assert "Authorization" in result assert result["Authorization"] == "Bearer litellm_api_key" @@ -593,7 +601,10 @@ class TestOpenAIResponsesAPIConfig: headers={}, ) - assert url == "https://custom-openai.example.com/v1/responses/..%2F..%2Ffiles%3Fx%3D1%23frag/input_items" + assert ( + url + == "https://custom-openai.example.com/v1/responses/..%2F..%2Ffiles%3Fx%3D1%23frag/input_items" + ) assert data["limit"] == 20 def test_get_event_model_class_generic_event(self): @@ -668,7 +679,9 @@ class TestOpenAIResponsesAPIConfig: ) assert isinstance(result, ImageGenerationPartialImageEvent) - assert result.type == ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE + assert ( + result.type == ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE + ) assert result.partial_image_index == idx assert result.b64_json == chunk["b64_json"] @@ -883,7 +896,9 @@ class TestOpenAIResponsesAPIConfig: "namespace": "drop", }, ] - out = BaseResponsesAPIConfig.strip_custom_tool_call_namespace_from_responses_input(inp) + out = BaseResponsesAPIConfig.strip_custom_tool_call_namespace_from_responses_input( + inp + ) assert out[0]["namespace"] == "keep" assert "namespace" not in out[1] @@ -956,21 +971,30 @@ class TestAzureResponsesAPIConfig: api_base=base_url, litellm_params={"api_version": "preview"}, ) - assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" + assert ( + result_preview + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" + ) # Test with latest version - should use openai/v1/responses result_latest = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "latest"}, ) - assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" + assert ( + result_latest + == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" + ) # Test with date-based version - should use openai/responses result_date = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "2025-01-01"}, ) - assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + assert ( + result_date + == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" + ) def test_azure_transform_then_normalize_strips_custom_tool_call_namespace(self): """Same as OpenAI path: ``normalize_responses_api_request_dict`` strips custom_tool_call only.""" @@ -1137,7 +1161,10 @@ class TestTransformListInputItemsRequest: ) # Assert - assert url == "https://test.openai.azure.com/openai/responses/compact?api-version=2024-05-01-preview" + assert ( + url + == "https://test.openai.azure.com/openai/responses/compact?api-version=2024-05-01-preview" + ) assert data["model"] == "gpt-5.2-codex" assert data["input"] == "hello" @@ -1224,7 +1251,9 @@ class TestTransformListInputItemsRequest: assert params == expected_params @patch("litellm.router.Router") - def test_mock_litellm_router_with_transform_list_input_items_request(self, mock_router): + def test_mock_litellm_router_with_transform_list_input_items_request( + self, mock_router + ): """Mock test using litellm.router for transform_list_input_items_request""" # Setup mock router mock_router_instance = Mock() @@ -1238,7 +1267,9 @@ class TestTransformListInputItemsRequest: ) # Setup router mock - mock_router_instance.get_provider_responses_api_config.return_value = mock_provider_config + mock_router_instance.get_provider_responses_api_config.return_value = ( + mock_provider_config + ) # Test parameters response_id = "resp_test123" @@ -1554,7 +1585,9 @@ class TestPhaseParameter: phase = getattr(output_item, "phase", None) expected = "commentary" if idx == 0 else "final_answer" - assert phase == expected, f"output[{idx}] phase={phase!r}, expected {expected!r}" + assert ( + phase == expected + ), f"output[{idx}] phase={phase!r}, expected {expected!r}" def test_streaming_output_item_done_preserves_phase(self): """OutputItemDoneEvent must preserve phase on its item.""" @@ -1688,7 +1721,9 @@ class TestPhaseParameter: if isinstance(item, dict): input_items.append(item) else: - input_items.append(item.model_dump() if hasattr(item, "model_dump") else dict(item)) + input_items.append( + item.model_dump() if hasattr(item, "model_dump") else dict(item) + ) input_items.append( { @@ -1785,7 +1820,9 @@ class TestResponsesSurfaceSharesTheEffortRule: ("gpt-6-astra", "low", False), ], ) - def test_temperature_follows_the_resolved_effort(self, local_model_cost_map, model, effort, temperature_survives): + def test_temperature_follows_the_resolved_effort( + self, local_model_cost_map, model, effort, temperature_survives + ): params = {"temperature": 0} if effort is not None: params["reasoning"] = {"effort": effort} @@ -2189,6 +2226,7 @@ class TestReasoningFollowsModelSupport: ) assert mapped["reasoning"] == reasoning + def test_azure_deployments_keep_reasoning_even_on_a_non_reasoning_model_name(self, local_model_cost_map): mapped = AzureOpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params={"reasoning": {"effort": "medium"}}, diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 63bd5f6e1ed..a82d07fa6be 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -27,7 +27,9 @@ def gpt5_config() -> OpenAIGPT5Config: @pytest.fixture(autouse=True) def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) + monkeypatch.setattr( + litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url) + ) litellm.add_known_models(model_cost_map=litellm.model_cost) @@ -37,7 +39,9 @@ def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): def test_gpt5_chat_does_not_support_reasoning_effort(config: OpenAIConfig): - assert "reasoning_effort" not in config.get_supported_openai_params(model="gpt-5-chat-latest") + assert "reasoning_effort" not in config.get_supported_openai_params( + model="gpt-5-chat-latest" + ) def test_gpt5_chat_supports_temperature(config: OpenAIConfig): @@ -447,7 +451,9 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): """Dict with effort='minimal' triggers minimal model-support validation.""" with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "minimal", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4-mini", drop_params=False, @@ -457,7 +463,9 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='minimal' passes through for gpt-5.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "minimal", "summary": "detailed"} + }, optional_params={}, model="gpt-5", drop_params=False, @@ -472,11 +480,21 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): Models with supports_minimal_reasoning_effort=true (or missing) → not disabled. Provider-prefixed models (openai/gpt-5.4-mini) are normalized before lookup. """ - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-mini", "minimal") - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-nano", "minimal") - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("openai/gpt-5.4-mini", "minimal") - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4", "minimal") - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-pro", "minimal") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-mini", "minimal" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-nano", "minimal" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "openai/gpt-5.4-mini", "minimal" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4", "minimal" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4-pro", "minimal" + ) def test_is_explicitly_disabled_factory_minimal(): @@ -571,16 +589,26 @@ def test_gpt5_unknown_model_passes_through_low(config: OpenAIConfig): def test_gpt5_low_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): """supports_low_reasoning_effort=false → disabled; missing/true → not disabled.""" - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5-pro", "low") - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5-pro-2026-04-23", "low") - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5", "low") - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4", "low") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.5-pro", "low" + ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.5-pro-2026-04-23", "low" + ) + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.5", "low" + ) + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( + "gpt-5.4", "low" + ) def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig): """Dict with summary/generate_summary is normalized for chat completions.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "high", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -596,7 +624,9 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): """ with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} + }, optional_params={}, model="gpt-5.1", drop_params=False, @@ -606,7 +636,9 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='xhigh' passes through for gpt-5.4+.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + non_default_params={ + "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} + }, optional_params={}, model="gpt-5.4", drop_params=False, @@ -661,7 +693,9 @@ def test_gpt5_normalizes_reasoning_effort_dict_with_summary_from_optional_params """reasoning_effort dict with summary in optional_params is normalized.""" params = config.map_openai_params( non_default_params={}, - optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, + optional_params={ + "reasoning_effort": {"effort": "medium", "summary": "detailed"} + }, model="gpt-5.4", drop_params=False, ) @@ -911,7 +945,9 @@ def test_gpt5_search_supported_params(gpt5_config: OpenAIGPT5Config): "reasoning_effort", ] for param in rejected: - assert param not in supported, f"{param} should not be supported for search models" + assert ( + param not in supported + ), f"{param} should not be supported for search models" def test_gpt5_search_has_expected_params(gpt5_config: OpenAIGPT5Config): @@ -997,15 +1033,21 @@ def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): optional_params = {"reasoningSummary": False, "reasoning_summary": "ignored"} assert peek_reasoning_summary_aliases(optional_params) is False - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) assert rs_val is False assert stripped == {} - optional_params = {"extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"}} + optional_params = { + "extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"} + } assert peek_reasoning_summary_aliases(optional_params) is False - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) assert rs_val is False assert stripped == {} @@ -1019,7 +1061,9 @@ def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): } assert peek_reasoning_summary_aliases(optional_params) == "auto" - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( + optional_params + ) assert rs_val == "auto" assert stripped == {"extra_body": {"metadata": "ok"}} @@ -1038,7 +1082,9 @@ def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex", "gpt-5.1", "gpt-5.2"]: supported = config.get_supported_openai_params(model=model) for param in rejected_params: - assert param not in supported, f"{param} should not be supported for {model}" + assert ( + param not in supported + ), f"{param} should not be supported for {model}" def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): @@ -1047,16 +1093,22 @@ def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): supported = config.get_supported_openai_params(model=model) assert "logprobs" in supported, f"logprobs should be supported for {model}" assert "top_p" in supported, f"top_p should be supported for {model}" - assert "top_logprobs" in supported, f"top_logprobs should be supported for {model}" + assert ( + "top_logprobs" in supported + ), f"top_logprobs should be supported for {model}" def test_gpt5_base_does_not_support_logprobs_top_p(config: OpenAIConfig): """Base gpt-5/gpt-5-mini do NOT support logprobs, top_p, top_logprobs.""" for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex"]: supported = config.get_supported_openai_params(model=model) - assert "logprobs" not in supported, f"logprobs should not be supported for {model}" + assert ( + "logprobs" not in supported + ), f"logprobs should not be supported for {model}" assert "top_p" not in supported, f"top_p should not be supported for {model}" - assert "top_logprobs" not in supported, f"top_logprobs should not be supported for {model}" + assert ( + "top_logprobs" not in supported + ), f"top_logprobs should not be supported for {model}" def test_gpt5_1_logprobs_passthrough(config: OpenAIConfig): diff --git a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py index 68cd33bf745..6cc5ffa2dae 100644 --- a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py +++ b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py @@ -7,7 +7,9 @@ import sys from unittest.mock import patch -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) class TestSimpleProviderConfigSupportedEndpoints: @@ -17,7 +19,9 @@ class TestSimpleProviderConfigSupportedEndpoints: """supported_endpoints defaults to [] (chat always enabled, nothing else)""" from litellm.llms.openai_like.json_loader import SimpleProviderConfig - config = SimpleProviderConfig("test", {"base_url": "https://example.com", "api_key_env": "TEST_KEY"}) + config = SimpleProviderConfig( + "test", {"base_url": "https://example.com", "api_key_env": "TEST_KEY"} + ) assert config.supported_endpoints == [] def test_custom_supported_endpoints(self): @@ -53,11 +57,15 @@ class TestSimpleProviderConfigSupportedEndpoints: class TestJSONProviderRegistryResponsesAPI: """Test supports_responses_api on JSONProviderRegistry.""" + def test_nonexistent_provider(self): """Non-existent provider returns False""" from litellm.llms.openai_like.json_loader import JSONProviderRegistry - assert JSONProviderRegistry.supports_responses_api("nonexistent_provider_xyz") is False + assert ( + JSONProviderRegistry.supports_responses_api("nonexistent_provider_xyz") + is False + ) class TestCreateResponsesConfigClass: @@ -110,7 +118,9 @@ class TestCreateResponsesConfigClass: config_cls = create_responses_config_class(provider) config = config_cls() - url = config.get_complete_url(api_base="https://custom.api.com/v1", litellm_params={}) + url = config.get_complete_url( + api_base="https://custom.api.com/v1", litellm_params={} + ) assert url == "https://custom.api.com/v1/responses" def test_generated_class_get_complete_url_strips_trailing_slash(self): @@ -123,7 +133,9 @@ class TestCreateResponsesConfigClass: config_cls = create_responses_config_class(provider) config = config_cls() - url = config.get_complete_url(api_base="https://custom.api.com/v1/", litellm_params={}) + url = config.get_complete_url( + api_base="https://custom.api.com/v1/", litellm_params={} + ) assert url == "https://custom.api.com/v1/responses" def test_generated_class_validate_environment(self): @@ -140,7 +152,9 @@ class TestCreateResponsesConfigClass: "litellm.llms.openai_like.dynamic_config.get_secret_str", return_value="sk-test-key-123", ): - headers = config.validate_environment(headers={}, model="test-model", litellm_params=None) + headers = config.validate_environment( + headers={}, model="test-model", litellm_params=None + ) assert headers["Authorization"] == "Bearer sk-test-key-123" def test_generated_class_validate_environment_litellm_params_override(self): @@ -155,7 +169,9 @@ class TestCreateResponsesConfigClass: config = config_cls() litellm_params = GenericLiteLLMParams(api_key="sk-override-key") - headers = config.validate_environment(headers={}, model="test-model", litellm_params=litellm_params) + headers = config.validate_environment( + headers={}, model="test-model", litellm_params=litellm_params + ) assert headers["Authorization"] == "Bearer sk-override-key" def test_generated_class_inherits_openai_responses_methods(self): diff --git a/tests/test_litellm/llms/openai_like/test_cognition_provider.py b/tests/test_litellm/llms/openai_like/test_cognition_provider.py index 9a38456da16..81895d7dc42 100644 --- a/tests/test_litellm/llms/openai_like/test_cognition_provider.py +++ b/tests/test_litellm/llms/openai_like/test_cognition_provider.py @@ -110,6 +110,8 @@ class TestCognitionProviderIdentity: class TestCognitionCostTracking: + + def test_supported_endpoints_matrix(self): matrix = json.loads((Path(litellm.__file__).parent / "provider_endpoints_support_backup.json").read_text()) @@ -118,3 +120,5 @@ class TestCognitionCostTracking: assert endpoints["messages"] is True assert endpoints["responses"] is True assert endpoints["embeddings"] is False + + diff --git a/tests/test_litellm/llms/openai_like/test_meta_provider.py b/tests/test_litellm/llms/openai_like/test_meta_provider.py index 359416b581c..20f5af2567c 100644 --- a/tests/test_litellm/llms/openai_like/test_meta_provider.py +++ b/tests/test_litellm/llms/openai_like/test_meta_provider.py @@ -24,6 +24,7 @@ class TestMetaProviderConfig: assert meta.api_key_env == "META_API_KEY" assert meta.api_base_env == "META_API_BASE" + def test_meta_in_openai_compatible_providers(self): from litellm.constants import openai_compatible_providers @@ -90,7 +91,9 @@ class TestMetaProviderConfig: class TestMetaReasoningParams: def test_muse_spark_supports_reasoning_effort(self): - params = litellm.get_supported_openai_params(model="muse-spark-1.1", custom_llm_provider="meta") + params = litellm.get_supported_openai_params( + model="muse-spark-1.1", custom_llm_provider="meta" + ) assert params is not None assert "reasoning_effort" in params @@ -109,7 +112,9 @@ class TestMetaReasoningParams: def test_reasoning_effort_gated_on_capability(self): """A meta model without reasoning metadata must not advertise reasoning_effort.""" - params = litellm.get_supported_openai_params(model="some-non-reasoning-model", custom_llm_provider="meta") + params = litellm.get_supported_openai_params( + model="some-non-reasoning-model", custom_llm_provider="meta" + ) assert params is not None assert "reasoning_effort" not in params @@ -181,3 +186,5 @@ class TestMetaAnthropicMessages: ) assert headers["authorization"] == "Bearer sk-env-key" assert headers["anthropic-version"] == "2023-06-01" + + diff --git a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py index 76e818bfc49..15cc6a34de9 100644 --- a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py +++ b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py @@ -154,6 +154,7 @@ class TestSCXAIModelMetadata: with open(json_path) as f: return json.load(f) + def test_scx_ai_models_synced_to_backup(self): model_cost = self._load(("model_prices_and_context_window.json",)) backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) diff --git a/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py b/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py index 1ff70142719..620e6e1a836 100644 --- a/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py +++ b/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py @@ -79,6 +79,7 @@ class TestTensormeshProviderConfig: matching the text_completion flag in provider_endpoints_support.json.""" assert "tensormesh" in litellm.openai_text_completion_compatible_providers + def test_tensormesh_router_config(self): """Test that tensormesh can be used in Router configuration""" from litellm import Router @@ -115,6 +116,7 @@ class TestTensormeshCostMap: litellm.model_cost = original_model_cost litellm.get_model_info.cache_clear() + def test_reasoning_flag_matches_expected_set(self): reasoning_models = { "tensormesh/deepseek-ai/DeepSeek-V4-Flash", @@ -129,3 +131,4 @@ class TestTensormeshCostMap: } for model in TENSORMESH_MODELS: assert litellm.supports_reasoning(model) is (model in reasoning_models), model + diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index f4828a19fc1..4069a32793f 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -204,6 +204,7 @@ class TestPerplexityCostCalculator: assert math.isclose(prompt_cost, (1000 * 1e-07) + (100 * 2e-06), rel_tol=1e-10) assert math.isclose(completion_cost, (150 * 2e-07) + (50 * 3e-06) + 0.005, rel_tol=1e-10) + def test_provider_stated_cost_still_wins_inside_an_off_peak_window(self): """A response that carries Perplexity's own metered cost bills that cost whatever the window says; the caller strips it when the deployment carries custom pricing.""" diff --git a/tests/test_litellm/llms/reducto/test_model_info.py b/tests/test_litellm/llms/reducto/test_model_info.py index 548e5a308d4..499adf0d179 100644 --- a/tests/test_litellm/llms/reducto/test_model_info.py +++ b/tests/test_litellm/llms/reducto/test_model_info.py @@ -1,8 +1,13 @@ + import litellm def test_reducto_provider_registration(): - model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="reducto/parse-v3") + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model="reducto/parse-v3" + ) assert model == "parse-v3" assert custom_llm_provider == "reducto" + + diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 11b08081568..60514e19c33 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -212,9 +212,13 @@ def test_build_vertex_schema(): "properties": { "tags": {"items": {"type": "string"}, "type": "array"}, "metadata": {"type": "object"}, - "callbacks": {"anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}]}, + "callbacks": { + "anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}] + }, "run_name": {"type": "string"}, - "max_concurrency": {"anyOf": [{"type": "integer"}, {"type": "null"}]}, + "max_concurrency": { + "anyOf": [{"type": "integer"}, {"type": "null"}] + }, "recursion_limit": {"type": "integer"}, "configurable": {"type": "object"}, "run_id": { @@ -258,7 +262,9 @@ def test_build_vertex_schema(): ] }, "run_name": {"type": "string"}, - "max_concurrency": {"anyOf": [{"type": "integer", "nullable": True}]}, + "max_concurrency": { + "anyOf": [{"type": "integer", "nullable": True}] + }, "recursion_limit": {"type": "integer"}, "configurable": {"type": "object"}, "run_id": {"anyOf": [{"type": "string", "nullable": True}]}, @@ -359,7 +365,9 @@ def test_build_vertex_schema_array_branch_missing_items_in_anyof(): array_branches = [b for b in callbacks_anyof if b.get("type") == "array"] assert array_branches, "expected an array branch to remain after transform" for branch in array_branches: - assert branch.get("items") == {"type": "object"}, f"array branch must have items synthesized; got {branch}" + assert branch.get("items") == { + "type": "object" + }, f"array branch must have items synthesized; got {branch}" def test_vertex_ai_complex_response_schema(): @@ -745,7 +753,9 @@ def test_convert_schema_types_type_array_conversion(): assert anyof_types[1]["type"] == "number" # 4. Other properties preserved - assert input_schema["properties"]["studio"]["description"] == "The studio ID or name" + assert ( + input_schema["properties"]["studio"]["description"] == "The studio ID or name" + ) assert input_schema["required"] == ["studio"] @@ -912,9 +922,7 @@ def test_construct_target_url_with_version_prefix(): ), ], ) -def test_construct_target_url_versionless_project_route_gets_api_version( - requested_route: str, expected_url: str -) -> None: +def test_construct_target_url_versionless_project_route_gets_api_version(requested_route: str, expected_url: str) -> None: from litellm.llms.vertex_ai.common_utils import construct_target_url target_url = construct_target_url( @@ -1047,7 +1055,10 @@ def test_fix_enum_types(): # 2. Non-string enums are removed assert "enum" not in input_schema["properties"]["maxLength"] assert "enum" not in input_schema["properties"]["enabled"] - assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] + assert ( + "enum" + not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] + ) # 3. anyOf with string type keeps enum, non-string removes it assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0] @@ -1251,7 +1262,9 @@ async def test_vertex_ai_token_counter_converts_messages_to_contents_for_gemini( token_counter = VertexAITokenCounter() - with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens: + with patch( + "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens" + ) as mock_acount_tokens: mock_acount_tokens.return_value = { "totalTokens": 42, "tokenizer_used": "gemini", @@ -1293,7 +1306,9 @@ async def test_vertex_ai_token_counter_returns_none_when_api_omits_total_tokens( token_counter = VertexAITokenCounter() - with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens: + with patch( + "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens" + ) as mock_acount_tokens: mock_acount_tokens.return_value = {"tokenizer_used": "gemini"} result = await token_counter.count_tokens( @@ -1336,7 +1351,9 @@ async def test_vertex_ai_partner_model_detection(): # Test Minimax models assert VertexAIPartnerModels.is_vertex_partner_model("minimaxai/minimax-m2-maas") # Test Moonshot models - assert VertexAIPartnerModels.is_vertex_partner_model("moonshotai/kimi-k2-thinking-maas") + assert VertexAIPartnerModels.is_vertex_partner_model( + "moonshotai/kimi-k2-thinking-maas" + ) # Test Gemini models (should NOT be detected as partner model) assert not VertexAIPartnerModels.is_vertex_partner_model("gemini-1.5-pro") @@ -1367,7 +1384,9 @@ def test_vertex_ai_moonshot_uses_openai_handler(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.should_use_openai_handler("moonshotai/kimi-k2-thinking-maas") + assert VertexAIPartnerModels.should_use_openai_handler( + "moonshotai/kimi-k2-thinking-maas" + ) def test_vertex_ai_zai_uses_openai_handler(): @@ -1402,7 +1421,9 @@ def test_vertex_ai_gemma_maas_is_partner_model(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.is_vertex_partner_model("google/gemma-4-26b-a4b-it-maas") + assert VertexAIPartnerModels.is_vertex_partner_model( + "google/gemma-4-26b-a4b-it-maas" + ) def test_vertex_ai_gemma_maas_uses_openai_handler(): @@ -1413,7 +1434,9 @@ def test_vertex_ai_gemma_maas_uses_openai_handler(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.should_use_openai_handler("google/gemma-4-26b-a4b-it-maas") + assert VertexAIPartnerModels.should_use_openai_handler( + "google/gemma-4-26b-a4b-it-maas" + ) def test_vertex_ai_gemma_maas_routes_to_partner_models(): @@ -1495,24 +1518,36 @@ def test_build_vertex_schema_empty_properties(): # Verify the transformation removed empty properties # Navigate to the go_back schema - go_back_schema = result["properties"]["action"]["items"]["anyOf"][0]["properties"]["go_back"] + go_back_schema = result["properties"]["action"]["items"]["anyOf"][0]["properties"][ + "go_back" + ] # Verify empty properties was removed assert "properties" not in go_back_schema, "Empty properties should be removed" # Verify type is kept as object (Gemini requires type: object even without properties) - assert go_back_schema.get("type") == "object", "Type should be kept as object when properties is empty" + assert ( + go_back_schema.get("type") == "object" + ), "Type should be kept as object when properties is empty" # Verify required was also removed - assert "required" not in go_back_schema, "Required should be removed when properties is empty" + assert ( + "required" not in go_back_schema + ), "Required should be removed when properties is empty" # Verify description is preserved - assert go_back_schema.get("description") == "Go back", "Description should be preserved" + assert ( + go_back_schema.get("description") == "Go back" + ), "Description should be preserved" # Verify parent schema still has proper structure parent_schema = result["properties"]["action"]["items"]["anyOf"][0] - assert parent_schema["type"] == "object", "Parent schema should still have object type" - assert "go_back" in parent_schema["properties"], "go_back should still be in parent properties" + assert ( + parent_schema["type"] == "object" + ), "Parent schema should still have object type" + assert ( + "go_back" in parent_schema["properties"] + ), "go_back should still be in parent properties" def test_add_object_type_schema_with_no_properties_and_no_type(): @@ -1603,8 +1638,12 @@ def test_pop_vertex_request_labels_prefers_explicit_labels_then_metadata(): def test_pop_vertex_request_labels_uses_litellm_metadata_when_metadata_absent(): optional: dict = {} - litellm_params = {"litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}}} - assert pop_vertex_request_labels(optional, litellm_params) == {"team": "from_litellm_meta"} + litellm_params = { + "litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}} + } + assert pop_vertex_request_labels(optional, litellm_params) == { + "team": "from_litellm_meta" + } def test_vertex_text_embedding_request_includes_labels_from_metadata(): @@ -1614,7 +1653,9 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata(): input="hi", optional_params={}, model="text-embedding-004", - litellm_params={"metadata": {"requester_metadata": {"project_id": "cost-center-1"}}}, + litellm_params={ + "metadata": {"requester_metadata": {"project_id": "cost-center-1"}} + }, ) assert req.get("labels") == {"project_id": "cost-center-1"} @@ -1642,3 +1683,5 @@ def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(mode from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info assert get_vertex_ai_lyria_model_info(model=model) is None + + diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 387cc405f02..ee80aed6f47 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -181,6 +181,7 @@ class TestVertexAILyriaTextToSpeechConfig: assert isinstance(config, VertexAILyriaTextToSpeechConfig) + def test_vertex_chirp_does_not_select_lyria_config(self): config = ProviderConfigManager.get_provider_text_to_speech_config( model="chirp", @@ -208,7 +209,9 @@ class TestVertexAILyriaTextToSpeechConfig: ) def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None: - injected: Final = "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" + injected: Final = ( + "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" + ) encoded: Final = ( "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle" "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D" diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 6b50dadbb38..028a7cc4b05 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1,3 +1,4 @@ + import pytest from litellm.anthropic_beta_headers_manager import ( @@ -15,7 +16,9 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im ], ) def test_vertex_ai_anthropic_thinking_param(model, expected_thinking): - supported_openai_params = VertexAIAnthropicConfig().get_supported_openai_params(model=model) + supported_openai_params = VertexAIAnthropicConfig().get_supported_openai_params( + model=model + ) if expected_thinking: assert "thinking" in supported_openai_params @@ -116,12 +119,14 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): }, "is_vertex_request": True, } - result_vertex = config.update_headers_with_optional_anthropic_beta(headers_vertex, optional_params_vertex) - - assert "anthropic-beta" not in result_vertex, ( - f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + result_vertex = config.update_headers_with_optional_anthropic_beta( + headers_vertex, optional_params_vertex ) + assert ( + "anthropic-beta" not in result_vertex + ), f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + # Test case 2: Non-Vertex request with output_format SHOULD add beta header headers_non_vertex = {} optional_params_non_vertex = { @@ -138,12 +143,12 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): headers_non_vertex, optional_params_non_vertex ) - assert "anthropic-beta" in result_non_vertex, ( - "Non-Vertex request SHOULD have anthropic-beta header for structured output" - ) - assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", ( - f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" - ) + assert ( + "anthropic-beta" in result_non_vertex + ), "Non-Vertex request SHOULD have anthropic-beta header for structured output" + assert ( + result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13" + ), f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): @@ -198,7 +203,9 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # Should have tools and tool_choice (tool-based approach) assert "tools" in result_params, "Tools should be present for structured output" - assert "tool_choice" in result_params, "Tool choice should be present for structured output" + assert ( + "tool_choice" in result_params + ), "Tool choice should be present for structured output" assert "json_mode" in result_params, "JSON mode should be enabled" # Verify the tool is the response format tool @@ -223,7 +230,9 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # Mock the parent transform_request to return data with output_format original_transform = config.__class__.__bases__[0].transform_request - def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): + def mock_transform_request( + self, model, messages, optional_params, litellm_params, headers + ): # Return test data that includes output_format return test_data.copy() @@ -245,7 +254,9 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # callers who explicitly requested them. assert "output_format" in final_data assert final_data["output_format"]["type"] == "json_schema" - assert "model" not in final_data, "model is still stripped (Vertex routes by URL)" + assert ( + "model" not in final_data + ), "model is still stripped (Vertex routes by URL)" assert "tools" in final_data, "tools should still be present" assert "tool_choice" in final_data, "tool_choice should still be present" @@ -281,7 +292,9 @@ def test_vertex_ai_anthropic_other_models_still_use_tools(): ) # Should still use tool-based approach - assert "tools" in result_params, "Claude 3 Sonnet should also use tool-based structured output" + assert ( + "tools" in result_params + ), "Claude 3 Sonnet should also use tool-based structured output" assert "tool_choice" in result_params, "Tool choice should be present" assert "json_mode" in result_params, "JSON mode should be enabled" @@ -409,18 +422,28 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea # This beta header should be removed PROMPT_CACHING_BETA_HEADER = "prompt-caching-scope-2026-01-05" - headers = {"anthropic-beta": f"other-feature,{PROMPT_CACHING_BETA_HEADER},web-search-2025-03-05"} + headers = { + "anthropic-beta": f"other-feature,{PROMPT_CACHING_BETA_HEADER},web-search-2025-03-05" + } headers = update_headers_with_filtered_beta(headers, "vertex_ai") beta_header = headers.get("anthropic-beta") - assert PROMPT_CACHING_BETA_HEADER not in (beta_header or ""), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" - assert "other-feature" not in (beta_header or ""), "Other non-excluded beta headers should remain" - assert "web-search-2025-03-05" in (beta_header or ""), "Other non-excluded beta headers should remain" + assert PROMPT_CACHING_BETA_HEADER not in ( + beta_header or "" + ), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" + assert "other-feature" not in ( + beta_header or "" + ), "Other non-excluded beta headers should remain" + assert "web-search-2025-03-05" in ( + beta_header or "" + ), "Other non-excluded beta headers should remain" # If prompt-caching was the only value, header should be removed completely headers2 = {"anthropic-beta": PROMPT_CACHING_BETA_HEADER} headers2 = update_headers_with_filtered_beta(headers2, "vertex_ai") - assert "anthropic-beta" not in headers2, "Header should be removed if no supported values remain" + assert ( + "anthropic-beta" not in headers2 + ), "Header should be removed if no supported values remain" def test_vertex_ai_anthropic_output_config_effort_only_forwarded(): @@ -566,7 +589,9 @@ def test_vertex_ai_anthropic_output_format_and_output_config_effort_preserved(): original_transform = config.__class__.__bases__[0].transform_request - def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): + def mock_transform_request( + self, model, messages, optional_params, litellm_params, headers + ): return test_data.copy() config.__class__.__bases__[0].transform_request = mock_transform_request diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py index a8da13e2f36..e9b58622a4b 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py @@ -48,6 +48,37 @@ _GEMMA_MODEL_COST_ENTRY = { # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _reset_litellm_http_client_cache(): + """Ensure each test gets a fresh async HTTP client mock.""" + from litellm import in_memory_llm_clients_cache + + in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture(autouse=True) +def clean_vertex_env(): + """Clear Google/Vertex AI environment variables before each test to prevent test isolation issues.""" + saved_env = {} + env_vars_to_clear = [ + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "VERTEXAI_PROJECT", + "VERTEX_PROJECT", + "VERTEX_LOCATION", + "VERTEX_AI_PROJECT", + ] + for var in env_vars_to_clear: + if var in os.environ: + saved_env[var] = os.environ[var] + del os.environ[var] + + yield + + for var, value in saved_env.items(): + os.environ[var] = value + + # --------------------------------------------------------------------------- # Unit tests: region and URL construction # --------------------------------------------------------------------------- @@ -61,7 +92,11 @@ class TestVertexBaseGetVertexRegionGemma: with patch.dict( litellm.model_cost, - {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, + { + "vertex_ai/google/gemma-4-26b-a4b-it-maas": { + "supported_regions": ["global"] + } + }, clear=False, ): result = vertex_base.get_vertex_region( @@ -75,7 +110,11 @@ class TestVertexBaseGetVertexRegionGemma: with patch.dict( litellm.model_cost, - {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, + { + "vertex_ai/google/gemma-4-26b-a4b-it-maas": { + "supported_regions": ["global"] + } + }, clear=False, ): result = vertex_base.get_vertex_region( @@ -101,9 +140,9 @@ class TestCreateVertexURLGemma: which in turn generates the /endpoints/openapi URL shape. If this mapping ever changes, the URL-shape tests below become misleading. """ - assert VertexAIPartnerModels.should_use_openai_handler("google/gemma-4-26b-a4b-it-maas"), ( - "Gemma MaaS must use the OpenAI-compatible handler (VertexPartnerProvider.llama path)" - ) + assert VertexAIPartnerModels.should_use_openai_handler( + "google/gemma-4-26b-a4b-it-maas" + ), "Gemma MaaS must use the OpenAI-compatible handler (VertexPartnerProvider.llama path)" def test_global_location_url_format(self): # VertexPartnerProvider.llama is correct: Gemma MaaS reaches create_vertex_url @@ -174,37 +213,6 @@ _MOCK_RESPONSE_JSON = { } -@pytest.fixture(autouse=True) -def _reset_litellm_http_client_cache(): - """Ensure each test gets a fresh async HTTP client mock.""" - from litellm import in_memory_llm_clients_cache - - in_memory_llm_clients_cache.flush_cache() - - -@pytest.fixture(autouse=True) -def clean_vertex_env(): - """Clear Google/Vertex AI environment variables before each test to prevent test isolation issues.""" - saved_env = {} - env_vars_to_clear = [ - "GOOGLE_APPLICATION_CREDENTIALS", - "GOOGLE_CLOUD_PROJECT", - "VERTEXAI_PROJECT", - "VERTEX_PROJECT", - "VERTEX_LOCATION", - "VERTEX_AI_PROJECT", - ] - for var in env_vars_to_clear: - if var in os.environ: - saved_env[var] = os.environ[var] - del os.environ[var] - - yield - - for var, value in saved_env.items(): - os.environ[var] = value - - @pytest.mark.asyncio async def test_vertex_ai_gemma_global_endpoint_url(): """ @@ -220,7 +228,9 @@ async def test_vertex_ai_gemma_global_endpoint_url(): mock_vertexai.preview = MagicMock() with ( - patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" + ) as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), @@ -231,7 +241,11 @@ async def test_vertex_ai_gemma_global_endpoint_url(): ), patch.dict( litellm.model_cost, - {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, + { + "vertex_ai/google/gemma-4-26b-a4b-it-maas": { + "supported_regions": ["global"] + } + }, clear=False, ), ): @@ -290,7 +304,9 @@ async def test_vertex_ai_gemma_function_calling_passthrough(): mock_vertexai.preview = MagicMock() with ( - patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" + ) as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), @@ -361,7 +377,9 @@ async def test_vertex_ai_gemma_vision_passthrough(): mock_vertexai.preview = MagicMock() with ( - patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, + patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" + ) as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py index ae2c60c1781..763103ea1f0 100644 --- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py @@ -21,8 +21,14 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject VEO_31_LITE_VERTEX_MODEL = "vertex_ai/veo-3.1-lite-generate-001" -ROOT_MODEL_COST_PATH = Path(__file__).parents[5] / "model_prices_and_context_window.json" -BACKUP_MODEL_COST_PATH = Path(__file__).parents[5] / "litellm" / "model_prices_and_context_window_backup.json" +ROOT_MODEL_COST_PATH = ( + Path(__file__).parents[5] / "model_prices_and_context_window.json" +) +BACKUP_MODEL_COST_PATH = ( + Path(__file__).parents[5] + / "litellm" + / "model_prices_and_context_window_backup.json" +) ModelCostMap = Mapping[str, Mapping[str, object]] @@ -76,7 +82,9 @@ class TestVertexAIVideoConfig: "vertex_location": "us-central1", } - url = self.config.get_complete_url(model="vertex_ai/veo-002", api_base=None, litellm_params=litellm_params) + url = self.config.get_complete_url( + model="vertex_ai/veo-002", api_base=None, litellm_params=litellm_params + ) expected = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/veo-002" assert url == expected @@ -109,7 +117,10 @@ class TestVertexAIVideoConfig: monkeypatch.setattr(litellm, "vertex_project", None) with pytest.raises(ValueError, match="vertex_project is required"): - self.config.get_complete_url(model="veo-002", api_base=None, litellm_params={}) + self.config.get_complete_url( + model="veo-002", api_base=None, litellm_params={} + ) + def test_transform_video_create_request(self): """Test transformation of video creation request.""" @@ -250,7 +261,9 @@ class TestVertexAIVideoConfig: assert mapped["aspectRatio"] == "16:9" assert "resolution" not in mapped - def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3(self, monkeypatch: pytest.MonkeyPatch): + def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3( + self, monkeypatch: pytest.MonkeyPatch + ): model = "veo-3.1-generate-001" model_key = f"vertex_ai/{model}" model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) @@ -423,7 +436,9 @@ class TestVertexAIVideoConfig: "raiMediaFilteredCount": 0, "videos": [ { - "bytesBase64Encoded": base64.b64encode(b"fake_video_data").decode(), + "bytesBase64Encoded": base64.b64encode( + b"fake_video_data" + ).decode(), "mimeType": "video/mp4", } ], @@ -489,7 +504,9 @@ class TestVertexAIVideoConfig: "done": True, "response": { "@type": "type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse", - "videos": [{"bytesBase64Encoded": encoded_video, "mimeType": "video/mp4"}], + "videos": [ + {"bytesBase64Encoded": encoded_video, "mimeType": "video/mp4"} + ], }, } @@ -509,7 +526,9 @@ class TestVertexAIVideoConfig: } with pytest.raises(ValueError, match="Video generation is not complete yet"): - self.config.transform_video_content_response(raw_response=mock_response, logging_obj=self.mock_logging_obj) + self.config.transform_video_content_response( + raw_response=mock_response, logging_obj=self.mock_logging_obj + ) def test_transform_video_content_response_missing_video_data(self): """Test that missing video data raises error.""" @@ -521,7 +540,9 @@ class TestVertexAIVideoConfig: } with pytest.raises(ValueError, match="No video data found"): - self.config.transform_video_content_response(raw_response=mock_response, logging_obj=self.mock_logging_obj) + self.config.transform_video_content_response( + raw_response=mock_response, logging_obj=self.mock_logging_obj + ) def test_get_video_edit_prefetch_params(self): """Test that prefetch params returns the fetchPredictOperation URL and body.""" @@ -547,7 +568,9 @@ class TestVertexAIVideoConfig: prefetched = { "done": True, - "response": {"videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}]}, + "response": { + "videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}] + }, } url, data, files = self.config.transform_video_edit_request( @@ -574,7 +597,9 @@ class TestVertexAIVideoConfig: prefetched = { "done": True, - "response": {"videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}]}, + "response": { + "videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}] + }, } _, data, _ = self.config.transform_video_edit_request( @@ -700,7 +725,9 @@ class TestVertexAIVideoConfig: def test_get_error_class(self): """Test error class generation.""" - error = self.config.get_error_class(error_message="Test error", status_code=500, headers={}) + error = self.config.get_error_class( + error_message="Test error", status_code=500, headers={} + ) # Should return VertexAIError from litellm.llms.vertex_ai.common_utils import VertexAIError @@ -912,7 +939,10 @@ class TestImageAndParametersPassthrough: # instances contains prompt + image assert len(data["instances"]) == 1 instance = data["instances"][0] - assert instance["prompt"] == "Cinematic drone shot moving forward along the beach boardwalk" + assert ( + instance["prompt"] + == "Cinematic drone shot moving forward along the beach boardwalk" + ) assert instance["image"] == image # parameters block is correct and not double-nested diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index 02f22a4135d..ef669db5864 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -75,6 +75,7 @@ def wandb_request_mock(respx_mock: respx.MockRouter) -> respx.Route: class TestWandbConfig: """Test class for WandB Inference functionality""" + def test_default_api_base(self): """Test that default API base is used when none is provided""" config = WandbConfig() @@ -107,7 +108,9 @@ class TestWandbConfig: This test mocks the actual HTTP request to test the integration properly. """ - litellm.disable_aiohttp_transport = True # since this uses respx, we need to set use_aiohttp_transport to False + litellm.disable_aiohttp_transport = ( + True # since this uses respx, we need to set use_aiohttp_transport to False + ) # Set up environment variables for the test api_key = "fake-wandb-key" @@ -144,7 +147,9 @@ class TestWandbConfig: # Make the actual API call through LiteLLM response = completion( model=model, - messages=[{"role": "user", "content": "write code for saying hey from LiteLLM"}], + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ], api_key=api_key, api_base=api_base, ) @@ -223,6 +228,7 @@ class TestWandbConfig: assert request_body["max_tokens"] == 64 assert "max_completion_tokens" not in request_body + @pytest.mark.respx() def test_wandb_completion_keeps_reasoning_effort_for_an_unregistered_model( self, wandb_test_config, wandb_request_mock: respx.Route diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/test_litellm/llms/xai/test_xai_model_registry.py index 47c91e24f14..969f1e56770 100644 --- a/tests/test_litellm/llms/xai/test_xai_model_registry.py +++ b/tests/test_litellm/llms/xai/test_xai_model_registry.py @@ -7,7 +7,6 @@ from __future__ import annotations import json from pathlib import Path - REPO_ROOT = Path(__file__).parents[4] PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 10873c4772a..3c6733cb86d 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -11,7 +11,9 @@ def test_get_team_models_for_all_models_and_team_only_models(): model_access_groups = {} include_model_access_groups = False - result = get_team_models(team_models, proxy_model_list, model_access_groups, include_model_access_groups) + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups + ) combined_models = team_models + proxy_model_list assert set(result) == set(combined_models) @@ -244,7 +246,9 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): +def test_get_complete_model_list_order( + key_models, team_models, proxy_model_list, model_list, expected +): """ Test that get_complete_model_list preserves order """ @@ -397,7 +401,9 @@ def test_wildcard_credential_hydration_preserves_deployment_params( captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") + captured_params["has_unexpected_field"] = hasattr( + litellm_params, "unexpected_field" + ) return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -442,7 +448,9 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), + litellm_params=LiteLLM_Params( + model="ollama_chat/*", custom_llm_provider="ollama_chat" + ), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -469,7 +477,9 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), + litellm_params=LiteLLM_Params( + model="huggingface/*", custom_llm_provider="huggingface" + ), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] @@ -831,7 +841,9 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} + model_cost_map={ + fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} + } ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 94ce8019b1b..2e4c0853c07 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -28,11 +28,7 @@ def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], c assert usage.prompt_tokens_details.cached_tokens == 0 selected_cost: Final = 0.013 assert compute_autorouter_savings( - "claude-opus-5", - "claude-sonnet-5", - "anthropic", - usage, - conversation_continuing=continuing, + "claude-opus-5", "claude-sonnet-5", "anthropic", usage, conversation_continuing=continuing, cost_breakdown={"input_cost": 0.01, "output_cost": 0.003}, ) == pytest.approx(sum(anthropic_cost_per_token("claude-opus-5", expected)) - selected_cost) @@ -40,17 +36,11 @@ def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], c def test_anthropic_baseline_keeps_negotiated_prices_with_provider_multiplier() -> None: info: Final = { **litellm.get_model_info("claude-opus-5", "anthropic"), - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "cache_read_input_token_cost": 3e-7, + "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "cache_read_input_token_cost": 3e-7, } usage: Final = _usage(1000, 1000, 0, 100).model_copy(update={"speed": "fast"}) assert compute_autorouter_savings( - "claude-opus-5", - "claude-sonnet-5", - "anthropic", - usage, - baseline_info=info, + "claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info, cost_breakdown={"input_cost": 0.01, "output_cost": 0.003}, ) == pytest.approx(0.0015 * 2 - 0.013) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 9b8c55d51bb..ea3feae00ec 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -2226,9 +2226,7 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data={"metadata": {}}, - original_exception=HTTPException( - status_code=400, detail="Upstream passthrough request failed with status 400" - ), + original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"), user_api_key_dict=UserAPIKeyAuth(), traceback_str=upstream_traceback, ) @@ -2292,13 +2290,9 @@ def test_mcp_conversion_preserves_request_policy_and_isolates_guardrail_data(buc parent = { "model": "parent-model", bucket: { - "guardrails": ["policy-rule"], - "guardrail_config": {"language": "en"}, - "applied_policies": ["parent-policy"], - "policy_sources": {"parent-policy": "model"}, - "_guardrail_pipelines": [], - "_pipeline_managed_guardrails": ["pipeline-rule"], - "tags": ["review"], + "guardrails": ["policy-rule"], "guardrail_config": {"language": "en"}, + "applied_policies": ["parent-policy"], "policy_sources": {"parent-policy": "model"}, + "_guardrail_pipelines": [], "_pipeline_managed_guardrails": ["pipeline-rule"], "tags": ["review"], }, "guardrails": [{"request-rule": {"extra_body": {"threshold": 0.9}}}], "guardrail_config": {"entities": ["EMAIL_ADDRESS"]}, @@ -2328,26 +2322,13 @@ def test_mcp_conversion_honors_only_authenticated_global_guardrail_opt_outs(opt_ from litellm.responses.mcp.request_context import MCPRequestContext auth = UserAPIKeyAuth(metadata={"opted_out_global_guardrails": ["global-rule"] if opt_out else []}) - context = MCPRequestContext.resolve( - kwargs={ - "metadata": { - "user_api_key_auth": auth, - "disable_global_guardrails": True, - "user_api_key_metadata": {"disable_global_guardrails": True}, - } - }, - tools=None, - ) + context = MCPRequestContext.resolve(kwargs={"metadata": { + "user_api_key_auth": auth, "disable_global_guardrails": True, + "user_api_key_metadata": {"disable_global_guardrails": True}, + }}, tools=None) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) - kwargs = { - "name": "execute", - "arguments": {}, - "user_api_key_auth": auth, - "guardrail_context": context.guardrail_context, - } - synthetic = proxy_logging._convert_mcp_to_llm_format( - proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs - ) + kwargs = {"name": "execute", "arguments": {}, "user_api_key_auth": auth, "guardrail_context": context.guardrail_context} + synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) guardrail = CustomGuardrail(guardrail_name="global-rule", event_hook="pre_mcp_call", default_on=True) assert guardrail.should_run_guardrail(synthetic, GuardrailEventHooks.pre_mcp_call) is (not opt_out) synthetic["metadata"]["user_api_key_metadata"]["opted_out_global_guardrails"].append("unrelated") @@ -2361,25 +2342,18 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyGuardrails registry = policy_registry.PolicyRegistry() - registry._policies = { - "model-policy": Policy( - condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) - ) - } + registry._policies = {"model-policy": Policy( + condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) + )} registry._initialized = True monkeypatch.setattr(policy_registry, "_policy_registry", registry) monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) kwargs = { - "name": "execute", - "arguments": {}, + "name": "execute", "arguments": {}, "user_api_key_auth": UserAPIKeyAuth(metadata={"policies": ["model-policy"]}), - "guardrail_context": MCPRequestContext.resolve_guardrail_context( - {"model": model, "guardrails": ["request-rule"]} - ), + "guardrail_context": MCPRequestContext.resolve_guardrail_context({"model": model, "guardrails": ["request-rule"]}), } - synthetic = proxy_logging._convert_mcp_to_llm_format( - proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs - ) + synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"] diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index 9a1cbe73ae8..9b811e6f1ce 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -325,6 +325,8 @@ KIMI_K3_PERPLEXITY_KEY = "perplexity/perplexity/kimi-k3" class TestKimiK3AdvertisesItsDocumentedLevels: + + @pytest.mark.parametrize("model, provider", [("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai")]) def test_the_declaration_survives_model_info_hydration(self, local_model_cost_map, model, provider): """The hydration line is the load-bearing seam: without it the key the map carries never diff --git a/tests/test_litellm/test_claude_fable_5_config.py b/tests/test_litellm/test_claude_fable_5_config.py index a0ed8d856dc..dfbda795c7a 100644 --- a/tests/test_litellm/test_claude_fable_5_config.py +++ b/tests/test_litellm/test_claude_fable_5_config.py @@ -92,3 +92,5 @@ def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model): from litellm.llms.anthropic.common_utils import AnthropicModelInfo assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True + + diff --git a/tests/test_litellm/test_claude_opus_4_6_config.py b/tests/test_litellm/test_claude_opus_4_6_config.py index f7d264ec5ae..7bded3b6ed3 100644 --- a/tests/test_litellm/test_claude_opus_4_6_config.py +++ b/tests/test_litellm/test_claude_opus_4_6_config.py @@ -2,6 +2,7 @@ Validate Claude Opus 4.6 model configuration entries. """ + import litellm diff --git a/tests/test_litellm/test_claude_opus_4_8_config.py b/tests/test_litellm/test_claude_opus_4_8_config.py index f41e6616c83..9471ef4ef4f 100644 --- a/tests/test_litellm/test_claude_opus_4_8_config.py +++ b/tests/test_litellm/test_claude_opus_4_8_config.py @@ -21,3 +21,5 @@ REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") def test_opus_4_8_registered_for_bedrock_converse(): assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS + + diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index 3327b2795ce..aaf179e0216 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -56,3 +56,5 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map): def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS + + diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 702da61a438..5e7d5797a62 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -33,3 +33,5 @@ ALL_SONNET_5_VARIANTS = ( def test_sonnet_5_registered_for_bedrock_converse(): assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS + + diff --git a/tests/test_litellm/test_dashscope_image_generation.py b/tests/test_litellm/test_dashscope_image_generation.py index 397cc9b313a..1dd0b322623 100644 --- a/tests/test_litellm/test_dashscope_image_generation.py +++ b/tests/test_litellm/test_dashscope_image_generation.py @@ -70,7 +70,9 @@ class TestDashScopeImageGenerationConfig: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", ], ) - def test_get_complete_url_ignores_chat_compatible_mode_base(self, chat_api_base: str): + def test_get_complete_url_ignores_chat_compatible_mode_base( + self, chat_api_base: str + ): url = self.cfg.get_complete_url(chat_api_base, None, "qwen-image-3.0", {}, {}) assert url == DEFAULT_API_BASE @@ -131,7 +133,9 @@ class TestDashScopeImageGenerationConfig: headers={}, ) assert req["model"] == model - assert req["input"]["messages"][0]["content"][0]["text"] == ("a poster with small multilingual text") + assert req["input"]["messages"][0]["content"][0]["text"] == ( + "a poster with small multilingual text" + ) assert req["parameters"]["size"] == "2048*2048" assert req["parameters"]["n"] == 6 @@ -396,7 +400,11 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): "finish_reason": "stop", "message": { "role": "assistant", - "content": [{"image": "https://dashscope-result.oss.aliyuncs.com/test.png"}], + "content": [ + { + "image": "https://dashscope-result.oss.aliyuncs.com/test.png" + } + ], }, } ] @@ -410,7 +418,9 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): }, } - with patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") as mock_post: + with patch( + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" + ) as mock_post: mock_http_response = MagicMock() mock_http_response.json.return_value = mock_response_body mock_http_response.status_code = 200 @@ -427,11 +437,15 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): assert response is not None assert response.data is not None assert len(response.data) == 1 - assert response.data[0].url == "https://dashscope-result.oss.aliyuncs.com/test.png" + assert ( + response.data[0].url == "https://dashscope-result.oss.aliyuncs.com/test.png" + ) # Verify the HTTP call was made to the DashScope endpoint call_args = mock_post.call_args - called_url = call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") + called_url = ( + call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") + ) assert called_url == DEFAULT_API_BASE # Verify request body contains DashScope format diff --git a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py index bc400bfa362..0632441e1b5 100644 --- a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py +++ b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py @@ -3,7 +3,6 @@ from pathlib import Path import pytest - REPO_ROOT = Path(__file__).parents[2] MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 3d9534628cb..4f2daa56b81 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1525,6 +1525,7 @@ class TestProxyFunctionCalling: # For now, we expect False (current behavior), but document the limitation assert proxy_result is False, f"Current limitation: {proxy_model_with_hints} returns False without inference" + def test_litellm_utils_supports_function_calling_import(self): """Test that supports_function_calling can be imported from litellm.utils.""" try: @@ -1544,6 +1545,7 @@ class TestProxyFunctionCalling: except Exception as e: pytest.fail(f"Failed to access litellm.supports_function_calling: {e}") + def test_edge_cases_and_malformed_proxy_models(self): """Test edge cases and malformed proxy model names.""" test_cases = [ @@ -5656,3 +5658,5 @@ def test_get_model_info_gemini(monkeypatch): ): assert info.get("tpm") is not None, f"{model} does not have tpm" assert info.get("rpm") is not None, f"{model} does not have rpm" + + diff --git a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py index 4a9a429801c..bb1843c5d05 100644 --- a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py +++ b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py @@ -22,3 +22,5 @@ def test_grok_models_with_cache_read_price_advertise_prompt_caching() -> None: assert missing_flag == (), ( f"grok models with cache_read_input_token_cost fail supports_prompt_caching: {missing_flag}" ) + +