diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index b68c2cb3d65..72713a36831 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -22,11 +22,7 @@ from litellm.litellm_core_utils.duration_parser import ( ) from litellm.utils import ( check_valid_key, - create_pretrained_tokenizer, - create_tokenizer, - function_to_dict, get_llm_provider, - get_max_tokens, get_supported_openai_params, get_token_count, get_valid_models, @@ -38,6 +34,9 @@ from unittest.mock import AsyncMock, MagicMock, patch # Assuming your trim_messages, shorten_message_to_fit_limit, and get_token_count functions are all in a module named 'message_utils' + + +# Test 1: Check trimming of normal message @pytest.fixture(autouse=True) def reset_mock_cache(): from litellm.utils import _model_cache @@ -45,7 +44,6 @@ def reset_mock_cache(): _model_cache.flush_cache() -# Test 1: Check trimming of normal message def test_basic_trimming(): litellm._turn_on_debug() messages = [ @@ -75,9 +73,7 @@ def test_basic_trimming_no_max_tokens_specified(): print("trimmed messages for gpt-4") print(trimmed_messages) # print(get_token_count(messages=trimmed_messages, model="claude-2")) - assert ( - get_token_count(messages=trimmed_messages, model="gpt-4") - ) <= litellm.model_cost["gpt-4"]["max_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_tokens"] # test_basic_trimming_no_max_tokens_specified() @@ -94,9 +90,7 @@ def test_multiple_messages_trimming(): "content": "This is another long message that will also exceed the limit.", }, ] - trimmed_messages = trim_messages( - messages=messages, model="gpt-3.5-turbo", max_tokens=20 - ) + trimmed_messages = trim_messages(messages=messages, model="gpt-3.5-turbo", max_tokens=20) # print(get_token_count(messages=trimmed_messages, model="gpt-3.5-turbo")) assert (get_token_count(messages=trimmed_messages, model="gpt-3.5-turbo")) <= 20 @@ -115,9 +109,7 @@ def test_multiple_messages_no_trimming(): "content": "This is another long message that will also exceed the limit.", }, ] - trimmed_messages = trim_messages( - messages=messages, model="gpt-3.5-turbo", max_tokens=100 - ) + trimmed_messages = trim_messages(messages=messages, model="gpt-3.5-turbo", max_tokens=100) print("Trimmed messages") print(trimmed_messages) assert messages == trimmed_messages @@ -144,9 +136,7 @@ def test_large_trimming_multiple_messages(): def test_large_trimming_single_message(): - messages = [ - {"role": "user", "content": "This is a singlelongwordthatexceedsthelimit."} - ] + messages = [{"role": "user", "content": "This is a singlelongwordthatexceedsthelimit."}] trimmed_messages = trim_messages(messages, max_tokens=5, model="gpt-4-0613") assert (get_token_count(messages=trimmed_messages, model="gpt-4-0613")) <= 5 assert (get_token_count(messages=trimmed_messages, model="gpt-4-0613")) > 0 @@ -277,10 +267,7 @@ def test_trimming_with_model_cost_max_input_tokens(model): }, ] trimmed_messages = trim_messages(messages, model=model) - assert ( - get_token_count(trimmed_messages, model=model) - < litellm.model_cost[model]["max_input_tokens"] - ) + assert get_token_count(trimmed_messages, model=model) < litellm.model_cost[model]["max_input_tokens"] def test_trimming_with_untokenizable_field(caplog: pytest.LogCaptureFixture) -> None: @@ -333,9 +320,7 @@ def test_aget_valid_models(): print(valid_models) # list of openai supported llms on litellm - expected_models = ( - litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models - ) + expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models assert set(valid_models) == set(expected_models) @@ -357,9 +342,7 @@ def test_get_valid_models_with_custom_llm_provider(custom_llm_provider): provider=LlmProviders(custom_llm_provider), ) assert provider_config is not None - valid_models = get_valid_models( - check_provider_endpoint=True, custom_llm_provider=custom_llm_provider - ) + valid_models = get_valid_models(check_provider_endpoint=True, custom_llm_provider=custom_llm_provider) print(valid_models) assert len(valid_models) > 0 assert set(provider_config.get_models()) == set(valid_models) @@ -392,9 +375,7 @@ def test_validate_environment_empty_model(): def test_validate_environment_api_key(): response_obj = validate_environment(model="gpt-5-mini", api_key="sk-my-test-key") - assert ( - response_obj["keys_in_environment"] is True - ), f"Missing keys={response_obj['missing_keys']}" + assert response_obj["keys_in_environment"] is True, f"Missing keys={response_obj['missing_keys']}" def test_validate_environment_api_version(): @@ -404,9 +385,7 @@ def test_validate_environment_api_version(): api_base="https://fake.openai.azure.com/", api_version="2024-02-15", ) - assert ( - response_obj["keys_in_environment"] is True - ), f"Missing keys={response_obj['missing_keys']}" + assert response_obj["keys_in_environment"] is True, f"Missing keys={response_obj['missing_keys']}" def test_validate_environment_api_base_dynamic(): @@ -481,18 +460,14 @@ def test_function_to_dict(): assert function_json["description"] == expected_output["description"] assert function_json["parameters"]["type"] == expected_output["parameters"]["type"] assert ( - function_json["parameters"]["properties"]["location"] - == expected_output["parameters"]["properties"]["location"] + function_json["parameters"]["properties"]["location"] == expected_output["parameters"]["properties"]["location"] ) # the enum can change it can be - which is why we don't assert on unit # {'type': 'string', 'description': 'Temperature unit', 'enum': "['fahrenheit', 'celsius']"} # {'type': 'string', 'description': 'Temperature unit', 'enum': "['celsius', 'fahrenheit']"} - assert ( - function_json["parameters"]["required"] - == expected_output["parameters"]["required"] - ) + assert function_json["parameters"]["required"] == expected_output["parameters"]["required"] print("passed") @@ -500,74 +475,6 @@ def test_function_to_dict(): # test_function_to_dict() -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("gpt-3.5-turbo", True), - ("azure/gpt-4-1106-preview", True), - ("groq/gemma-7b-it", True), - ("gemini/gemini-2.5-flash", True), - ], -) -def test_supports_function_calling(model, expected_bool): - try: - assert litellm.supports_function_calling(model=model) == expected_bool - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("gpt-4o-mini-search-preview", True), - ("openai/gpt-4o-mini-search-preview", True), - ("gpt-4o-search-preview", True), - ("openai/gpt-4o-search-preview", True), - ("groq/deepseek-r1-distill-llama-70b", False), - ("groq/llama-3.3-70b-versatile", False), - ("codestral/codestral-latest", False), - ], -) -def test_supports_web_search(model, expected_bool): - try: - assert litellm.supports_web_search(model=model) == expected_bool - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("openai/o3-mini", True), - ("o3-mini", True), - ("xai/grok-3-mini-beta", True), - ("xai/grok-3-mini-fast-beta", True), - ("xai/grok-2", False), - ("gpt-3.5-turbo", False), - ], -) -def test_supports_reasoning(model, expected_bool): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - try: - assert litellm.supports_reasoning(model=model) == expected_bool - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_get_max_token_unit_test(): - """ - More complete testing in `test_completion_cost.py` - """ - model = "bedrock/anthropic.claude-3-haiku-20240307-v1:0" - - max_tokens = get_max_tokens( - model - ) # Returns a number instead of throwing an Exception - - assert isinstance(max_tokens, int) - - def test_get_supported_openai_params() -> None: # Mapped provider assert isinstance(get_supported_openai_params("gpt-4"), list) @@ -602,9 +509,7 @@ def test_get_chat_completion_prompt(): prompt_variables=None, ) - assert litellm_logging_obj.messages == [ - {"role": "user", "content": updated_message} - ] + assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}] def test_redact_msgs_from_logs(): @@ -676,9 +581,7 @@ def test_redact_embedding_response(): litellm.turn_off_message_logging = True # Create a test EmbeddingResponse with usage data - original_usage = litellm.Usage( - prompt_tokens=10, completion_tokens=0, total_tokens=10 - ) + original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10) original_data = [ {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]}, {"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]}, @@ -714,9 +617,7 @@ def test_redact_embedding_response(): # Assert the redacted response preserves critical metadata assert _redacted_response_obj.usage == original_usage # usage should be preserved - assert ( - _redacted_response_obj.model == "text-embedding-3-small" - ) # model should be preserved + assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved assert _redacted_response_obj.object == "list" # object should be preserved # Assert sensitive data is cleared @@ -770,12 +671,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): ) # Test Case 1: standard_callback_dynamic_params = False (or not set) - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=False - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -784,12 +681,8 @@ def test_redact_msgs_from_logs_with_dynamic_params(): assert _redacted_response_obj.choices[0].message.content == test_content # Test Case 2: standard_callback_dynamic_params = True - standard_callback_dynamic_params = StandardCallbackDynamicParams( - turn_off_message_logging=True - ) - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -800,9 +693,7 @@ def test_redact_msgs_from_logs_with_dynamic_params(): # Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging # since litellm.turn_off_message_logging is True redaction should occur standard_callback_dynamic_params = StandardCallbackDynamicParams() - litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = ( - standard_callback_dynamic_params - ) + litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params _redacted_response_obj = redact_message_input_output_from_logging( result=response_obj, model_call_details=litellm_logging_obj.model_call_details, @@ -907,9 +798,7 @@ def test_get_llm_provider_ft_models(): @pytest.mark.parametrize("langfuse_trace_id", [None, "my-unique-trace-id"]) -@pytest.mark.parametrize( - "langfuse_existing_trace_id", [None, "my-unique-existing-trace-id"] -) +@pytest.mark.parametrize("langfuse_existing_trace_id", [None, "my-unique-existing-trace-id"]) def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): """ - Unit test for `_get_trace_id` function in Logging obj @@ -948,22 +837,13 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): ## if existing_trace_id exists if langfuse_existing_trace_id is not None: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == langfuse_existing_trace_id - ) + assert litellm_logging_obj._get_trace_id(service_name="langfuse") == langfuse_existing_trace_id ## if trace_id exists elif langfuse_trace_id is not None: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == langfuse_trace_id - ) + assert litellm_logging_obj._get_trace_id(service_name="langfuse") == langfuse_trace_id ## if no trace_id or existing_trace_id is provided, use litellm_trace_id else: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == litellm_logging_obj.litellm_trace_id - ) + assert litellm_logging_obj._get_trace_id(service_name="langfuse") == litellm_logging_obj.litellm_trace_id def test_convert_model_response_object(): @@ -1041,73 +921,6 @@ def test_parse_content_for_reasoning(content, expected_reasoning, expected_conte ) -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("vertex_ai/gemini-2.5-pro", True), - ("gemini/gemini-2.5-pro", True), - ("predibase/llama3-8b-instruct", True), - ("databricks/databricks-meta-llama-3-1-70b-instruct", True), - ("gpt-3.5-turbo", False), - ("groq/llama-3.3-70b-versatile", False), - ], -) -def test_supports_response_schema(model, expected_bool): - """ - Unit tests for 'supports_response_schema' helper function. - - Should be true for gemini-2.5-pro on google ai studio / vertex ai AND predibase models - Should be false otherwise - """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - from litellm.utils import supports_response_schema - - response = supports_response_schema(model=model, custom_llm_provider=None) - - assert expected_bool == response - - -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("gpt-3.5-turbo", True), - ("gpt-4", True), - ("command-nightly", False), - ("gemini-2.5-pro", True), - ], -) -def test_supports_function_calling_v2(model, expected_bool): - """ - Unit test for 'supports_function_calling' helper function. - """ - from litellm.utils import supports_function_calling - - response = supports_function_calling(model=model, custom_llm_provider=None) - assert expected_bool == response - - -@pytest.mark.parametrize( - "model, expected_bool", - [ - ("gpt-4o", True), - ("gpt-3.5-turbo", False), - ("claude-sonnet-4-6", True), - ("gemini-2.5-flash", True), - ("command-nightly", False), - ], -) -def test_supports_vision(model, expected_bool): - """ - Unit test for 'supports_vision' helper function. - """ - from litellm.utils import supports_vision - - response = supports_vision(model=model, custom_llm_provider=None) - assert expected_bool == response - - def test_usage_object_null_tokens(): """ Unit test. @@ -1146,7 +959,6 @@ def test_is_base64_encoded(): clear=True, ) def test_async_http_handler(mock_async_client): - import httpx import ssl timeout = 120 @@ -1154,9 +966,7 @@ def test_async_http_handler(mock_async_client): concurrent_limit = 2 # Mock the transport creation to return a specific transport - with mock.patch.object( - AsyncHTTPHandler, "_create_async_transport" - ) as mock_create_transport: + with mock.patch.object(AsyncHTTPHandler, "_create_async_transport") as mock_create_transport: mock_transport = mock.MagicMock() mock_create_transport.return_value = mock_transport @@ -1221,20 +1031,6 @@ def test_async_http_handler_force_ipv4(mock_async_client): litellm.force_ipv4 = False -@pytest.mark.parametrize( - "model, expected_bool", [("gpt-3.5-turbo", False), ("gpt-4o-audio-preview", True)] -) -def test_supports_audio_input(model, expected_bool): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - from litellm.utils import supports_audio_input, supports_audio_output - - supports_pc = supports_audio_input(model=model) - - assert supports_pc == expected_bool - - def test_is_base64_encoded_2(): from litellm.utils import is_base64_encoded @@ -1277,9 +1073,7 @@ def test_is_base64_encoded_2(): [ { "role": "user", - "content": [ - {"type": "image_url", "url": "https://example.com/image.png"} - ], + "content": [{"type": "image_url", "url": "https://example.com/image.png"}], } ], True, @@ -1355,10 +1149,7 @@ def test_models_by_provider(): continue elif k == "sample_spec": continue - elif ( - v["litellm_provider"] == "sagemaker" - or v["litellm_provider"] == "bedrock_converse" - ): + elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse": continue elif v.get("mode") in ("search", "evaluation"): continue @@ -1366,9 +1157,7 @@ def test_models_by_provider(): providers.add(v["litellm_provider"]) for provider in providers: - assert provider in models_by_provider.keys() or JSONProviderRegistry.exists( - provider - ) + assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider) @pytest.mark.parametrize( @@ -1379,16 +1168,11 @@ def test_models_by_provider(): ({"user_api_key_end_user_id": "123"}, True, None), ], ) -def test_get_end_user_id_for_cost_tracking( - litellm_params, disable_end_user_cost_tracking, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id): from litellm.utils import get_end_user_id_for_cost_tracking litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking - assert ( - get_end_user_id_for_cost_tracking(litellm_params=litellm_params) - == expected_end_user_id - ) + assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id @pytest.mark.parametrize( @@ -1404,13 +1188,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ): from litellm.utils import get_end_user_id_for_cost_tracking - litellm.enable_end_user_cost_tracking_prometheus_only = ( - enable_end_user_cost_tracking_prometheus_only - ) + litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only assert ( - get_end_user_id_for_cost_tracking( - litellm_params=litellm_params, service_type="prometheus" - ) + get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus") == expected_end_user_id ) @@ -1425,20 +1205,14 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ), # Test with only litellm_metadata field (new behavior) ( - { - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - } - }, + {"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}}, "user_from_litellm_metadata", ), # Test with both fields - metadata should take precedence for user_api_key fields ( { "metadata": {"user_api_key_end_user_id": "user_from_metadata"}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_metadata", ), @@ -1454,9 +1228,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ( { "metadata": {}, - "litellm_metadata": { - "user_api_key_end_user_id": "user_from_litellm_metadata" - }, + "litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}, }, "user_from_litellm_metadata", ), @@ -1464,9 +1236,7 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only( ({}, None), ], ) -def test_get_end_user_id_for_cost_tracking_metadata_handling( - litellm_params, expected_end_user_id -): +def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id): """ Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata fields using the get_litellm_metadata_from_kwargs helper function. @@ -1569,23 +1339,6 @@ def test_token_counter_with_image_url_with_detail_high(): assert _tokens == DEFAULT_IMAGE_TOKEN_COUNT + 7 -def test_fireworks_ai_vision_capability_from_cost_map(monkeypatch): - """ - Fireworks deprecated document inlining on 2025-06-30, so vision/PDF support is - no longer hardcoded to True for every Fireworks model. Capabilities are read - from the model cost map: unmapped models no longer advertise vision or PDF - support, while mapped VLMs still do. - """ - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - from litellm.utils import supports_pdf_input, supports_vision - - assert supports_vision("fireworks_ai/llama-3.1-8b-instruct") is False - assert supports_pdf_input("fireworks_ai/llama-3.1-8b-instruct") is False - - assert supports_vision("fireworks_ai/minimax-m3") is True - - def test_logprobs_type(): from litellm.types.utils import Logprobs @@ -1630,9 +1383,7 @@ def test_get_valid_models_openai_proxy(monkeypatch): mock_response.status_code = 200 mock_response.json.return_value = mock_response_data - with patch.object( - litellm.module_level_client, "get", return_value=mock_response - ) as mock_post: + with patch.object(litellm.module_level_client, "get", return_value=mock_response) as mock_post: valid_models = get_valid_models(check_provider_endpoint=True) assert "litellm_proxy/gpt-5.5" in valid_models @@ -1709,16 +1460,11 @@ def test_get_valid_models_fireworks_ai(monkeypatch): mock_response.status_code = 200 mock_response.json.return_value = mock_response_data - with patch.object( - litellm.module_level_client, "get", return_value=mock_response - ) as mock_post: + with patch.object(litellm.module_level_client, "get", return_value=mock_response) as mock_post: valid_models = get_valid_models(check_provider_endpoint=True) print("valid_models", valid_models) mock_post.assert_called_once() - assert ( - "fireworks_ai/accounts/fireworks/models/llama-3.1-8b-instruct" - in valid_models - ) + assert "fireworks_ai/accounts/fireworks/models/llama-3.1-8b-instruct" in valid_models def test_get_valid_models_default(monkeypatch): @@ -1728,21 +1474,12 @@ def test_get_valid_models_default(monkeypatch): Prevent regression for existing usage. """ from litellm.utils import get_valid_models - import litellm monkeypatch.setenv("FIREWORKS_API_KEY", "sk-1234") valid_models = get_valid_models() assert len(valid_models) > 0 -def test_supports_vision_gemini(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - from litellm.utils import supports_vision - - assert supports_vision("gemini-2.5-pro") is True - - def test_pick_cheapest_chat_model_from_llm_provider(): from litellm.litellm_core_utils.llm_request_utils import ( pick_cheapest_chat_models_from_llm_provider, @@ -1757,9 +1494,7 @@ def test_pick_cheapest_chat_model_from_llm_provider(): def test_get_num_retries(num_retries): from litellm.utils import _get_wrapper_num_retries - assert _get_wrapper_num_retries( - kwargs={"num_retries": num_retries}, exception=Exception("test") - ) == ( + assert _get_wrapper_num_retries(kwargs={"num_retries": num_retries}, exception=Exception("test")) == ( num_retries, { "num_retries": num_retries, @@ -2032,9 +1767,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch): assert len(litellm.success_callback) == curr_len_success_callback assert len(litellm.failure_callback) == curr_len_failure_callback - assert any( - isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback - ) + assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback) @pytest.mark.asyncio @@ -2061,20 +1794,13 @@ async def test_wrapper_kwargs_passthrough(): mock_original.assert_called_once() # get litellm logging object - litellm_logging_obj: LiteLLMLoggingObject = mock_original.call_args.kwargs.get( - "litellm_logging_obj" - ) + litellm_logging_obj: LiteLLMLoggingObject = mock_original.call_args.kwargs.get("litellm_logging_obj") assert litellm_logging_obj is not None - print( - f"litellm_logging_obj.model_call_details: {litellm_logging_obj.model_call_details}" - ) + print(f"litellm_logging_obj.model_call_details: {litellm_logging_obj.model_call_details}") # get base model - assert ( - litellm_logging_obj.model_call_details["litellm_params"]["base_model"] - == "gpt-5-mini" - ) + assert litellm_logging_obj.model_call_details["litellm_params"]["base_model"] == "gpt-5-mini" def test_dict_to_response_format_helper(): @@ -2128,7 +1854,7 @@ def test_validate_user_messages_invalid_content_type(): messages = [{"content": [{"type": "invalid_type", "text": "Hello"}]}] - with pytest.raises(Exception, match='Please ensure all messages are valid OpenAI chat completion') as e: + with pytest.raises(Exception, match="Please ensure all messages are valid OpenAI chat completion") as e: validate_chat_completion_user_messages(messages) assert "Invalid message" in str(e) @@ -2145,20 +1871,14 @@ from unittest.mock import Mock [ { "name": "default_on_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=True) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)], "kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}}, "expected": ["test_guardrail"], }, { "name": "request_specific_guardrail", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], - "kwargs": { - "metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}} - }, + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}}, "expected": ["test_guardrail"], }, { @@ -2167,18 +1887,12 @@ from unittest.mock import Mock CustomGuardrail(guardrail_name="default_guardrail", default_on=True), CustomGuardrail(guardrail_name="request_guardrail", default_on=False), ], - "kwargs": { - "metadata": { - "requester_metadata": {"guardrails": ["request_guardrail"]} - } - }, + "kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}}, "expected": ["default_guardrail", "request_guardrail"], }, { "name": "empty_metadata", - "callbacks": [ - CustomGuardrail(guardrail_name="test_guardrail", default_on=False) - ], + "callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)], "kwargs": {}, "expected": [], }, @@ -2285,9 +1999,7 @@ def test_get_provider_audio_transcription_config(): from litellm.types.utils import LlmProviders for provider in LlmProviders: - config = ProviderConfigManager.get_provider_audio_transcription_config( - model="whisper-1", provider=provider - ) + config = ProviderConfigManager.get_provider_audio_transcription_config(model="whisper-1", provider=provider) @pytest.mark.parametrize( @@ -2330,9 +2042,7 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "123") - _model_cache.set_cached_model_info( - "openai", litellm_params=None, available_models=["gpt-5-mini"] - ) + _model_cache.set_cached_model_info("openai", litellm_params=None, available_models=["gpt-5-mini"]) monkeypatch.delenv("OPENAI_API_KEY") assert _model_cache.get_cached_model_info("openai") is None @@ -2421,12 +2131,8 @@ def test_delta_tool_calls_sequential_indices(): # Verify tool calls have sequential indices assert delta.tool_calls is not None, "Tool calls should not be None" assert len(delta.tool_calls) == 2 - assert ( - delta.tool_calls[0].index == 0 - ), f"First tool call should have index 0, got {delta.tool_calls[0].index}" - assert ( - delta.tool_calls[1].index == 1 - ), f"Second tool call should have index 1, got {delta.tool_calls[1].index}" + assert delta.tool_calls[0].index == 0, f"First tool call should have index 0, got {delta.tool_calls[0].index}" + assert delta.tool_calls[1].index == 1, f"Second tool call should have index 1, got {delta.tool_calls[1].index}" # Verify tool call details are preserved assert delta.tool_calls[0].function.name == "get_weather_for_dallas" @@ -2439,9 +2145,7 @@ def test_completion_with_no_model(): """ # test on empty with pytest.raises(TypeError): - response = litellm.completion( - messages=[{"role": "user", "content": "Hello, how are you?"}] - ) + response = litellm.completion(messages=[{"role": "user", "content": "Hello, how are you?"}]) def test_get_base_model_from_metadata(): @@ -2454,43 +2158,31 @@ def test_get_base_model_from_metadata(): from litellm.utils import _get_base_model_from_metadata # Test 1: base_model in metadata (Chat Completions API pattern) - model_call_details_with_metadata = { - "litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-5.5"}}} - } + model_call_details_with_metadata = {"litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-5.5"}}}} result = _get_base_model_from_metadata(model_call_details_with_metadata) assert result == "azure/gpt-5.5", f"Expected 'azure/gpt-5.5', got {result}" # Test 2: base_model in litellm_metadata (Responses API and generic API calls pattern) model_call_details_with_litellm_metadata = { - "litellm_params": { - "litellm_metadata": {"model_info": {"base_model": "azure/gpt-5-mini"}} - } + "litellm_params": {"litellm_metadata": {"model_info": {"base_model": "azure/gpt-5-mini"}}} } result = _get_base_model_from_metadata(model_call_details_with_litellm_metadata) assert result == "azure/gpt-5-mini", f"Expected 'azure/gpt-5-mini', got {result}" # Test 3: base_model in litellm_params (direct base_model) - model_call_details_with_direct_base_model = { - "litellm_params": {"base_model": "azure/gpt-5-mini"} - } + model_call_details_with_direct_base_model = {"litellm_params": {"base_model": "azure/gpt-5-mini"}} result = _get_base_model_from_metadata(model_call_details_with_direct_base_model) - assert ( - result == "azure/gpt-5-mini" - ), f"Expected 'azure/gpt-5-mini', got {result}" + assert result == "azure/gpt-5-mini", f"Expected 'azure/gpt-5-mini', got {result}" # Test 4: metadata takes precedence over litellm_metadata model_call_details_with_both = { "litellm_params": { "metadata": {"model_info": {"base_model": "azure/gpt-4-from-metadata"}}, - "litellm_metadata": { - "model_info": {"base_model": "azure/gpt-4-from-litellm-metadata"} - }, + "litellm_metadata": {"model_info": {"base_model": "azure/gpt-4-from-litellm-metadata"}}, } } result = _get_base_model_from_metadata(model_call_details_with_both) - assert ( - result == "azure/gpt-4-from-metadata" - ), f"Expected metadata to take precedence, got {result}" + assert result == "azure/gpt-4-from-metadata", f"Expected metadata to take precedence, got {result}" # Test 5: No base_model present model_call_details_without_base_model = {"litellm_params": {"metadata": {}}} diff --git a/tests/llm_translation/test_azure_o_series.py b/tests/llm_translation/test_azure_o_series.py index ce7e614cbe2..b8a53fefb5c 100644 --- a/tests/llm_translation/test_azure_o_series.py +++ b/tests/llm_translation/test_azure_o_series.py @@ -1,15 +1,12 @@ import json import os -from datetime import datetime -from unittest.mock import AsyncMock, patch, MagicMock +from unittest.mock import patch - -import httpx import pytest import litellm -from litellm import Choices, Message, ModelResponse +from litellm import ModelResponse from base_llm_unit_tests import BaseLLMChatTest, BaseOSeriesModelsTest @@ -44,36 +41,6 @@ class TestAzureOpenAIO3Mini(BaseOSeriesModelsTest, BaseLLMChatTest): """Temporary override. o1 prompt caching is not working.""" pass - def test_override_fake_stream(self): - """Test that native streaming is not supported for o1.""" - router = litellm.Router( - model_list=[ - { - "model_name": "azure/o1-preview", - "litellm_params": { - "model": "azure/o1-preview", - "api_key": "my-fake-o1-key", - "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com", - }, - "model_info": { - "supports_native_streaming": True, - }, - } - ] - ) - - ## check model info - - model_info = litellm.get_model_info( - model="azure/o1-preview", custom_llm_provider="azure" - ) - assert model_info["supports_native_streaming"] is True - - fake_stream = litellm.AzureOpenAIO1Config().should_fake_stream( - model="azure/o1-preview", stream=True - ) - assert fake_stream is False - class TestAzureOpenAIO3(BaseOSeriesModelsTest): def get_base_completion_call_args(self): @@ -106,9 +73,7 @@ def test_azure_o3_streaming(): api_version="2024-02-15-preview", ) - with patch.object( - client.chat.completions.with_raw_response, "create" - ) as mock_create: + with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: try: completion( model="azure/o3-mini", @@ -116,9 +81,7 @@ def test_azure_o3_streaming(): stream=True, client=client, ) - except ( - Exception - ) as e: # expect output translation error as mock response doesn't return a json + except Exception as e: # expect output translation error as mock response doesn't return a json print(e) assert mock_create.call_count == 1 assert "stream" in mock_create.call_args.kwargs @@ -137,9 +100,7 @@ def test_azure_o_series_routing(): api_version="2024-02-15-preview", ) - with patch.object( - client.chat.completions.with_raw_response, "create" - ) as mock_create: + with patch.object(client.chat.completions.with_raw_response, "create") as mock_create: try: completion( model="azure/o_series/my-random-deployment-name", @@ -147,9 +108,7 @@ def test_azure_o_series_routing(): stream=True, client=client, ) - except ( - Exception - ) as e: # expect output translation error as mock response doesn't return a json + except Exception as e: # expect output translation error as mock response doesn't return a json print(e) assert mock_create.call_count == 1 assert "stream" not in mock_create.call_args.kwargs @@ -216,9 +175,7 @@ async def test_azure_o1_series_response_format_extra_params(): ] response_format = {"type": "json_object"} tool_choice = "auto" - with patch.object( - client.chat.completions.with_raw_response, "create" - ) as mock_client: + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: try: await litellm.acompletion( client=client, diff --git a/tests/llm_translation/test_lambda_ai.py b/tests/llm_translation/test_lambda_ai.py index edba459b352..78843fac052 100644 --- a/tests/llm_translation/test_lambda_ai.py +++ b/tests/llm_translation/test_lambda_ai.py @@ -44,9 +44,7 @@ def test_lambda_ai_get_openai_compatible_provider_info(): os.environ, {"LAMBDA_API_KEY": "env-key", "LAMBDA_API_BASE": "https://env.lambda.ai/v1"}, ): - api_base, api_key = config._get_openai_compatible_provider_info( - "https://param.lambda.ai/v1", "param-key" - ) + api_base, api_key = config._get_openai_compatible_provider_info("https://param.lambda.ai/v1", "param-key") assert api_base == "https://param.lambda.ai/v1" assert api_key == "param-key" @@ -56,16 +54,12 @@ def test_get_llm_provider_lambda_ai(): from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider # Test with lambda_ai/model-name format - model, provider, api_key, api_base = get_llm_provider( - "lambda_ai/llama3.1-8b-instruct" - ) + model, provider, api_key, api_base = get_llm_provider("lambda_ai/llama3.1-8b-instruct") assert model == "llama3.1-8b-instruct" assert provider == "lambda_ai" # Test with api_base containing Lambda AI endpoint - model, provider, api_key, api_base = get_llm_provider( - "llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1" - ) + model, provider, api_key, api_base = get_llm_provider("llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1") assert model == "llama3.1-8b-instruct" assert provider == "lambda_ai" assert api_base == "https://api.lambda.ai/v1" @@ -100,37 +94,3 @@ async def test_lambda_ai_completion_call(): if "lambda_ai" not in str(e) and "provider" not in str(e).lower(): # Re-raise if it's not a provider-related error raise - - -def test_lambda_ai_model_list_populated(): - """Test that lambda_ai_models list is populated correctly""" - # Ensure we're using local model cost map and repopulate models - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - # Clear and repopulate all model lists after reloading model_cost - litellm.lambda_ai_models = set() - litellm.add_known_models() - - # This should be populated by the add_known_models function - assert ( - len(litellm.lambda_ai_models) > 0 - ), "lambda_ai_models list should not be empty" - - # Check that all models in the list are Lambda AI models - for model in litellm.lambda_ai_models: - assert model.startswith( - "lambda_ai/" - ), f"Model {model} should start with 'lambda_ai/'" - - # Check some expected models are in the list - expected_models = [ - "lambda_ai/llama3.1-8b-instruct", - "lambda_ai/hermes3-405b", - "lambda_ai/deepseek-v3-0324", - ] - - for model in expected_models: - assert ( - model in litellm.lambda_ai_models - ), f"{model} should be in lambda_ai_models list" diff --git a/tests/llm_translation/test_perplexity_reasoning.py b/tests/llm_translation/test_perplexity_reasoning.py index 61fbc9d7824..92d6a5d2ab3 100644 --- a/tests/llm_translation/test_perplexity_reasoning.py +++ b/tests/llm_translation/test_perplexity_reasoning.py @@ -1,4 +1,3 @@ -import json import os from unittest.mock import patch, MagicMock @@ -26,9 +25,7 @@ class TestPerplexityReasoning: ("perplexity/sonar-reasoning-pro", "high"), ], ) - def test_perplexity_reasoning_effort_parameter_mapping( - self, model, reasoning_effort - ): + def test_perplexity_reasoning_effort_parameter_mapping(self, model, reasoning_effort): """ Test that reasoning_effort parameter is correctly mapped for Perplexity Sonar reasoning models """ @@ -105,7 +102,6 @@ class TestPerplexityReasoning: "create", side_effect=_return_pydantic_obj, ) as mock_client: - response = completion( model=model, messages=[ @@ -131,55 +127,7 @@ class TestPerplexityReasoning: # Verify response structure assert response.choices[0].message.content is not None - assert ( - response.choices[0].message.content - == "This is a test response from the reasoning model." - ) - - def test_perplexity_reasoning_models_support_reasoning(self): - """ - Test that Perplexity Sonar reasoning models are correctly identified as supporting reasoning - """ - from litellm.utils import supports_reasoning - - # Set up local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - reasoning_models = [ - "perplexity/sonar-reasoning", - "perplexity/sonar-reasoning-pro", - ] - - for model in reasoning_models: - assert supports_reasoning(model, None), f"{model} should support reasoning" - - def test_perplexity_non_reasoning_models_dont_support_reasoning(self): - """ - Test that non-reasoning Perplexity models don't support reasoning - """ - from litellm.utils import supports_reasoning - - # Set up local model cost map - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - non_reasoning_models = [ - "perplexity/sonar", - "perplexity/sonar-pro", - "perplexity/llama-3.1-sonar-large-128k-chat", - "perplexity/mistral-7b-instruct", - ] - - for model in non_reasoning_models: - # These models should not support reasoning (should return False or raise exception) - try: - result = supports_reasoning(model, None) - # If it doesn't raise an exception, it should return False - assert result is False, f"{model} should not support reasoning" - except Exception: - # If it raises an exception, that's also acceptable behavior - pass + assert response.choices[0].message.content == "This is a test response from the reasoning model." @pytest.mark.parametrize( "model,expected_api_base", @@ -188,18 +136,14 @@ class TestPerplexityReasoning: ("perplexity/sonar-reasoning-pro", "https://api.perplexity.ai"), ], ) - def test_perplexity_reasoning_api_base_configuration( - self, model, expected_api_base - ): + def test_perplexity_reasoning_api_base_configuration(self, model, expected_api_base): """ Test that Perplexity reasoning models use the correct API base """ from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig config = PerplexityChatConfig() - api_base, _ = config._get_openai_compatible_provider_info( - api_base=None, api_key="test-key" - ) + api_base, _ = config._get_openai_compatible_provider_info(api_base=None, api_key="test-key") assert api_base == expected_api_base @@ -210,8 +154,6 @@ class TestPerplexityReasoning: from litellm.llms.perplexity.chat.transformation import PerplexityChatConfig config = PerplexityChatConfig() - supported_params = config.get_supported_openai_params( - model="perplexity/sonar-reasoning" - ) + supported_params = config.get_supported_openai_params(model="perplexity/sonar-reasoning") assert "reasoning_effort" in supported_params diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index d900dcb6f27..0e04569bbdf 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -6,8 +6,7 @@ import litellm.cost_calculator import asyncio import time from typing import Optional -from unittest.mock import AsyncMock, MagicMock, patch -import base64 +from unittest.mock import MagicMock, patch import pytest import litellm @@ -15,9 +14,7 @@ from litellm import ( TranscriptionResponse, completion_cost, cost_per_token, - get_max_tokens, model_cost, - open_ai_chat_completion_models, ) from litellm.llms.custom_httpx.http_handler import HTTPHandler import json @@ -152,7 +149,6 @@ def test_custom_pricing_as_completion_cost_param(): assert round(cost, 5) == round(expected_cost, 5) - # print(results) @@ -162,12 +158,6 @@ def test_custom_pricing_as_completion_cost_param(): # test_get_palm_tokens() -def test_zephyr_hf_tokens(): - max_tokens = get_max_tokens("huggingface/HuggingFaceH4/zephyr-7b-beta") - print(max_tokens) - assert max_tokens == 32768 - - # test_zephyr_hf_tokens() @@ -199,23 +189,17 @@ def test_cost_ft_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost( - completion_response=resp, custom_llm_provider="openai" - ) + cost = litellm.completion_cost(completion_response=resp, custom_llm_provider="openai") print("\n Calculated Cost for ft:gpt-3.5", cost) input_cost = model_cost["ft:gpt-3.5-turbo"]["input_cost_per_token"] output_cost = model_cost["ft:gpt-3.5-turbo"]["output_cost_per_token"] print(input_cost, output_cost) - expected_cost = (input_cost * resp.usage.prompt_tokens) + ( - output_cost * resp.usage.completion_tokens - ) + expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: print(f"Error: {e}") - pytest.fail( - f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}" - ) + pytest.fail(f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}") # test_cost_ft_gpt_35() @@ -244,15 +228,11 @@ def test_cost_azure_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost( - completion_response=resp, model="azure/chatgpt-deployment-2" - ) + cost = litellm.completion_cost(completion_response=resp, model="azure/chatgpt-deployment-2") print("\n Calculated Cost for azure/gpt-3.5-turbo", cost) input_cost = model_cost["azure/gpt-35-turbo"]["input_cost_per_token"] output_cost = model_cost["azure/gpt-35-turbo"]["output_cost_per_token"] - expected_cost = (input_cost * resp.usage.prompt_tokens) + ( - output_cost * resp.usage.completion_tokens - ) + expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: @@ -269,9 +249,7 @@ def test_cost_bedrock_pricing_actual_calls(): litellm.set_verbose = True model = "anthropic.claude-3-5-sonnet-20240620-v1:0" messages = [{"role": "user", "content": "Hey, how's it going?"}] - response = litellm.completion( - model=model, messages=messages, mock_response="hello cool one" - ) + response = litellm.completion(model=model, messages=messages, mock_response="hello cool one") print("response", response) cost = litellm.completion_cost( @@ -302,8 +280,7 @@ def test_whisper_openai(): print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] - * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -323,15 +300,12 @@ def test_whisper_azure(): _total_time_in_seconds = 3 setattr(transcription, "duration", _total_time_in_seconds) - cost = litellm.completion_cost( - model="azure/azure-whisper", completion_response=transcription - ) + cost = litellm.completion_cost(model="azure/azure-whisper", completion_response=transcription) print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] - * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -362,9 +336,7 @@ def test_dalle_3_azure_cost_tracking(): response.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} response._hidden_params = {"model": "dall-e-3", "model_id": None} print(f"response hidden params: {response._hidden_params}") - cost = litellm.completion_cost( - completion_response=response, call_type="image_generation" - ) + cost = litellm.completion_cost(completion_response=response, call_type="image_generation") assert cost > 0 @@ -396,9 +368,7 @@ def test_replicate_llama3_cost_tracking(): model="replicate/meta/meta-llama-3-8b-instruct", object="chat.completion", system_fingerprint=None, - usage=litellm.utils.Usage( - prompt_tokens=48, completion_tokens=31, total_tokens=79 - ), + usage=litellm.utils.Usage(prompt_tokens=48, completion_tokens=31, total_tokens=79), ) cost = litellm.completion_cost( completion_response=response, @@ -408,14 +378,8 @@ def test_replicate_llama3_cost_tracking(): print(f"cost: {cost}") cost = round(cost, 5) expected_cost = round( - litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ - "input_cost_per_token" - ] - * 48 - + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ - "output_cost_per_token" - ] - * 31, + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["input_cost_per_token"] * 48 + + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["output_cost_per_token"] * 31, 5, ) assert cost == expected_cost @@ -426,10 +390,8 @@ def test_groq_response_cost_tracking(is_streaming): from litellm.utils import ( CallTypes, Choices, - Delta, Message, ModelResponse, - StreamingChoices, Usage, ) @@ -548,12 +510,6 @@ def test_gemini_completion_cost(provider): assert calculated_output_cost == output_cost -def _count_characters(text): - # Remove white spaces and count characters - filtered_text = "".join(char for char in text if not char.isspace()) - return len(filtered_text) - - def test_vertex_ai_completion_cost(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -587,9 +543,7 @@ def test_vertex_ai_medlm_completion_cost(): model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider="vertex_ai" - ) + predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider="vertex_ai") assert predictive_cost > 0 model = "vertex_ai/medlm-large" @@ -606,9 +560,7 @@ def test_vertex_ai_embedding_completion_cost(caplog): litellm.model_cost = litellm.get_model_cost_map(url="") text = "The quick brown fox jumps over the lazy dog." - input_tokens = litellm.token_counter( - model="vertex_ai/text-embedding-004", text=text - ) + input_tokens = litellm.token_counter(model="vertex_ai/text-embedding-004", text=text) model_info = litellm.get_model_info(model="vertex_ai/text-embedding-004") @@ -631,10 +583,7 @@ def test_vertex_ai_embedding_completion_cost(caplog): captured_logs = [rec.message for rec in caplog.records] for item in captured_logs: print("\nitem:{}\n".format(item)) - if ( - "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " - in item - ): + if "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " in item: raise Exception("Error log raised for calculating embedding cost") @@ -704,9 +653,7 @@ def test_vertex_ai_llama_predict_cost(): model = "meta/llama3-405b-instruct-maas" messages = [{"role": "user", "content": "Hey, hows it going???"}] custom_llm_provider = "vertex_ai" - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider=custom_llm_provider - ) + predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider=custom_llm_provider) assert predictive_cost == 0 @@ -720,9 +667,7 @@ def test_vertex_ai_mistral_predict_cost(usage): else: from openai.types.completion_usage import CompletionUsage - response_usage = CompletionUsage( - prompt_tokens=32, completion_tokens=55, total_tokens=87 - ) + response_usage = CompletionUsage(prompt_tokens=32, completion_tokens=55, total_tokens=87) response_object = ModelResponse( id="26c0ef045020429d9c5c9b078c01e564", choices=[ @@ -756,9 +701,7 @@ def test_vertex_ai_mistral_predict_cost(usage): assert predictive_cost > 0 -@pytest.mark.parametrize( - "model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"] -) +@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"]) def test_completion_cost_tts(model): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -817,10 +760,8 @@ def test_completion_cost_azure_common_deployment_name(): from litellm.utils import ( CallTypes, Choices, - Delta, Message, ModelResponse, - StreamingChoices, Usage, ) @@ -860,9 +801,7 @@ def test_completion_cost_azure_common_deployment_name(): response._hidden_params["custom_llm_provider"] = "azure" print(response) - with patch.object( - litellm.cost_calculator, "completion_cost", new=MagicMock() - ) as mock_client: + with patch.object(litellm.cost_calculator, "completion_cost", new=MagicMock()) as mock_client: _ = litellm.response_cost_calculator( response_object=response, model="gpt-4-0314", @@ -922,9 +861,7 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): cost_1 = completion_cost(model=model, completion_response=response_1) - _model_info = litellm.get_model_info( - model=model, custom_llm_provider=custom_llm_provider - ) + _model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) expected_cost = ( ( response_1.usage.prompt_tokens @@ -932,12 +869,9 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): - response_1.usage.prompt_tokens_details.cache_creation_tokens ) * _model_info["input_cost_per_token"] - + (response_1.usage.prompt_tokens_details.cached_tokens or 0) - * _model_info["cache_read_input_token_cost"] - + (response_1.usage.cache_creation_input_tokens or 0) - * _model_info["cache_creation_input_token_cost"] - + (response_1.usage.completion_tokens or 0) - * _model_info["output_cost_per_token"] + + (response_1.usage.prompt_tokens_details.cached_tokens or 0) * _model_info["cache_read_input_token_cost"] + + (response_1.usage.cache_creation_input_tokens or 0) * _model_info["cache_creation_input_token_cost"] + + (response_1.usage.completion_tokens or 0) * _model_info["output_cost_per_token"] ) # Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) assert round(expected_cost, 5) == round(cost_1, 5) @@ -1053,9 +987,7 @@ def test_completion_cost_databricks_embedding(model, monkeypatch): sync_handler = HTTPHandler() with patch.object(HTTPHandler, "post", return_value=mock_response): - resp = litellm.embedding( - model=model, input=["hey, how's it going?"], client=sync_handler - ) + resp = litellm.embedding(model=model, input=["hey, how's it going?"], client=sync_handler) print(resp) cost = completion_cost(completion_response=resp) @@ -1231,11 +1163,9 @@ def test_cost_openai_prompt_caching(): usage = response_2.usage _expected_cost2 = ( - (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) - * model_info["input_cost_per_token"] + (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) * model_info["input_cost_per_token"] + usage.completion_tokens * model_info["output_cost_per_token"] - + usage.prompt_tokens_details.cached_tokens - * model_info["cache_read_input_token_cost"] + + usage.prompt_tokens_details.cached_tokens * model_info["cache_read_input_token_cost"] ) print("_expected_cost2", _expected_cost2) @@ -1252,7 +1182,7 @@ def test_cost_openai_prompt_caching(): ], ) def test_completion_cost_azure_ai_rerank(model): - from litellm import RerankResponse, rerank + from litellm import RerankResponse os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -1276,14 +1206,12 @@ def test_completion_cost_azure_ai_rerank(model): }, ) print("response", response) - cost = completion_cost( - model=model, completion_response=response, call_type="arerank" - ) + cost = completion_cost(model=model, completion_response=response, call_type="arerank") assert cost > 0 def test_together_ai_embedding_completion_cost(): - from litellm.utils import Choices, EmbeddingResponse, Message, ModelResponse, Usage + from litellm.utils import EmbeddingResponse, Usage os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -2222,7 +2150,6 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): ModelResponse, Usage, ChatCompletionAudioResponse, - PromptTokensDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, ) @@ -2231,9 +2158,7 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): completion_tokens=34, prompt_tokens=16, total_tokens=50, - completion_tokens_details=CompletionTokensDetailsWrapper( - audio_tokens=28, reasoning_tokens=0, text_tokens=6 - ), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=28, reasoning_tokens=0, text_tokens=6), prompt_tokens_details=PromptTokensDetailsWrapper( audio_tokens=0, cached_tokens=0, text_tokens=16, image_tokens=0 ), @@ -2272,27 +2197,15 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): print(f"model_info: {model_info}") ## input cost - input_audio_cost = ( - model_info["input_cost_per_audio_token"] - * usage_object.prompt_tokens_details.audio_tokens - ) - input_text_cost = ( - model_info["input_cost_per_token"] - * usage_object.prompt_tokens_details.text_tokens - ) + input_audio_cost = model_info["input_cost_per_audio_token"] * usage_object.prompt_tokens_details.audio_tokens + input_text_cost = model_info["input_cost_per_token"] * usage_object.prompt_tokens_details.text_tokens total_input_cost = input_audio_cost + input_text_cost ## output cost - output_audio_cost = ( - model_info["output_cost_per_audio_token"] - * usage_object.completion_tokens_details.audio_tokens - ) - output_text_cost = ( - model_info["output_cost_per_token"] - * usage_object.completion_tokens_details.text_tokens - ) + output_audio_cost = model_info["output_cost_per_audio_token"] * usage_object.completion_tokens_details.audio_tokens + output_text_cost = model_info["output_cost_per_token"] * usage_object.completion_tokens_details.text_tokens total_output_cost = output_audio_cost + output_text_cost @@ -2418,9 +2331,7 @@ def test_moderations(): litellm.add_known_models() assert "omni-moderation-latest" in litellm.model_cost - print( - f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}" - ) + print(f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}") assert "omni-moderation-latest" in litellm.open_ai_chat_completion_models response = moderation("I am a bad person", model="omni-moderation-latest") @@ -2457,14 +2368,11 @@ def test_cost_calculator_azure_embedding(): def test_add_known_models(): litellm.add_known_models() - assert ( - "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models - ) + assert "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models @pytest.mark.skip(reason="flaky test") def test_bedrock_cost_calc_with_region(): - from litellm import completion from litellm import ModelResponse @@ -2570,9 +2478,7 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg): } if base_model_arg == "litellm_param": - model_item["litellm_params"][ - "base_model" - ] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" + model_item["litellm_params"]["base_model"] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" elif base_model_arg == "model_info": model_item["model_info"] = { "base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 38ccfd91f95..5c640aa22a6 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -47,12 +47,6 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch): assert model_info["input_cost_per_token"] == 0.0 -def test_get_model_info_gemini_pro(): - info = litellm.get_model_info("gemini-2.0-flash") - print("info", info) - assert info["key"] == "gemini-2.0-flash" - - def test_get_model_info_ollama_chat(): from litellm.llms.ollama.completion.transformation import OllamaConfig @@ -120,19 +114,13 @@ def test_get_model_info_ft_model_with_provider_prefix(): assert info["key"] == "ft:gpt-3.5-turbo" -def _enforce_bedrock_converse_models( - model_cost: List[Dict[str, Any]], whitelist_models: List[str] -): +def _enforce_bedrock_converse_models(model_cost: List[Dict[str, Any]], whitelist_models: List[str]): """ Assert all new bedrock chat models are added as `bedrock_converse` unless explicitly whitelisted. """ # Check for unwhitelisted models for model, info in litellm.model_cost.items(): - if ( - info["litellm_provider"] == "bedrock" - and info["mode"] == "chat" - and model not in whitelist_models - ): + if info["litellm_provider"] == "bedrock" and info["mode"] == "chat" and model not in whitelist_models: raise AssertionError( f"New bedrock chat model detected: {model}. Please set `litellm_provider='bedrock_converse'` for this model." ) @@ -153,9 +141,7 @@ def test_model_info_bedrock_converse(monkeypatch): except FileNotFoundError: pytest.skip("whitelisted_bedrock_models.txt not found") - _enforce_bedrock_converse_models( - model_cost=litellm.model_cost, whitelist_models=whitelist_models - ) + _enforce_bedrock_converse_models(model_cost=litellm.model_cost, whitelist_models=whitelist_models) @pytest.mark.flaky(retries=6, delay=2) @@ -179,10 +165,8 @@ def test_model_info_bedrock_converse_enforcement(monkeypatch): # Check for unwhitelisted models with pytest.raises(AssertionError): - _enforce_bedrock_converse_models( - model_cost=litellm.model_cost, whitelist_models=whitelist_models - ) - except FileNotFoundError as e: + _enforce_bedrock_converse_models(model_cost=litellm.model_cost, whitelist_models=whitelist_models) + except FileNotFoundError: pytest.skip("whitelisted_bedrock_models.txt not found") @@ -219,9 +203,7 @@ def test_get_model_info_custom_provider(): # Get registered model info from litellm import get_model_info - get_model_info( - model="my-custom-llm/my-fake-model" - ) # 💥 "Exception: This model isn't mapped yet." in v1.56.10 + get_model_info(model="my-custom-llm/my-fake-model") # 💥 "Exception: This model isn't mapped yet." in v1.56.10 def test_get_model_info_custom_model_router(): @@ -273,11 +255,7 @@ def test_get_model_info_bedrock_models(): k = k.replace(f"{commitment}/", "") base_model = BedrockModelInfo.get_base_model(k) # get_base_model() returns model id without "bedrock/" prefix; cost map keys use "bedrock/" - base_model_key = ( - base_model - if base_model in litellm.model_cost - else f"bedrock/{base_model}" - ) + base_model_key = base_model if base_model in litellm.model_cost else f"bedrock/{base_model}" if base_model_key not in litellm.model_cost: continue base_model_info = litellm.model_cost[base_model_key] @@ -285,12 +263,10 @@ def test_get_model_info_bedrock_models(): if "invoke/" in k: continue if base_model_key.startswith("supports_"): - assert ( - base_model_key in v - ), f"{base_model_key} is not in model cost map for {k}" - assert ( - v[base_model_key] == base_model_value - ), f"{base_model_key} is not equal to {base_model_value} for model {k}" + assert base_model_key in v, f"{base_model_key} is not in model cost map for {k}" + assert v[base_model_key] == base_model_value, ( + f"{base_model_key} is not equal to {base_model_value} for model {k}" + ) def test_get_model_info_bedrock_cross_region_capability_parity(): @@ -318,9 +294,7 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): if not cap.startswith("supports_"): continue assert cap in v, f"{cap} is on {base_model_key} but missing from {k}" - assert ( - v[cap] == base_value - ), f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}" + assert v[cap] == base_value, f"{cap} is {v[cap]} on {k} but {base_value} on {base_model_key}" assert checked > 0, "no cross-region bedrock profiles found - the filter is inert" @@ -354,27 +328,6 @@ def test_get_model_info_huggingface_models(monkeypatch): ) -@pytest.mark.parametrize( - "model, provider", - [ - ("bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0", None), - ( - "bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0", - "bedrock", - ), - ], -) -def test_get_model_info_cost_calculator_bedrock_region_cris_stripped(model, provider): - """ - ensure cross region inferencing model is used correctly - Relevant Issue: https://github.com/BerriAI/litellm/issues/8115 - """ - info = get_model_info(model=model, custom_llm_provider=provider) - print("info", info) - assert info["key"] == "us.anthropic.claude-3-haiku-20240307-v1:0" - assert info["litellm_provider"] == "bedrock" - - def test_get_model_info_case_insensitive_lookup(monkeypatch): """ Test that model info lookup is case-insensitive. @@ -402,23 +355,17 @@ def test_get_model_info_case_insensitive_lookup(monkeypatch): ) # Test 1: Exact case should work - info = litellm.get_model_info( - model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai" - ) + info = litellm.get_model_info(model="Qwen/Qwen3-Next-80B-A3B-Thinking", custom_llm_provider="together_ai") assert info is not None assert info["supports_function_calling"] is True # Test 2: Lowercase should also work (case-insensitive lookup) - info_lower = litellm.get_model_info( - model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai" - ) + info_lower = litellm.get_model_info(model="qwen/qwen3-next-80b-a3b-thinking", custom_llm_provider="together_ai") assert info_lower is not None assert info_lower["supports_function_calling"] is True # Test 3: Mixed case should also work - info_mixed = litellm.get_model_info( - model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai" - ) + info_mixed = litellm.get_model_info(model="QWEN/qwen3-NEXT-80b-a3b-thinking", custom_llm_provider="together_ai") assert info_mixed is not None assert info_mixed["supports_function_calling"] is True @@ -446,13 +393,7 @@ def test_get_model_info_case_insensitive_supports_function_calling(monkeypatch): from litellm.utils import supports_function_calling # Exact case - assert ( - supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") - is True - ) + assert supports_function_calling("TestModel-ABC", custom_llm_provider="test_provider") is True # Lowercase (should now work with case-insensitive lookup) - assert ( - supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") - is True - ) + assert supports_function_calling("testmodel-abc", custom_llm_provider="test_provider") is True diff --git a/tests/local_testing/test_prompt_caching.py b/tests/local_testing/test_prompt_caching.py deleted file mode 100644 index f6b3fb89e9e..00000000000 --- a/tests/local_testing/test_prompt_caching.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Asserts that prompt caching information is correctly returned for Anthropic, OpenAI, and Deepseek""" - -import io - - -import litellm -import pytest - - -def _usage_format_tests(usage: litellm.Usage): - """ - OpenAI prompt caching - - prompt_tokens = sum of non-cache hit tokens + cache-hit tokens - - total_tokens = prompt_tokens + completion_tokens - - Example - ``` - "usage": { - "prompt_tokens": 2006, - "completion_tokens": 300, - "total_tokens": 2306, - "prompt_tokens_details": { - "cached_tokens": 1920 - }, - "completion_tokens_details": { - "reasoning_tokens": 0 - } - # ANTHROPIC_ONLY # - "cache_creation_input_tokens": 0 - } - ``` - """ - assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - - assert usage.prompt_tokens > usage.prompt_tokens_details.cached_tokens - - -def test_supports_prompt_caching(): - from litellm.utils import supports_prompt_caching - - supports_pc = supports_prompt_caching(model="anthropic/claude-sonnet-4-5-20250929") - - assert supports_pc diff --git a/tests/local_testing/test_register_model.py b/tests/local_testing/test_register_model.py index eddd697974c..d78f2ac7811 100644 --- a/tests/local_testing/test_register_model.py +++ b/tests/local_testing/test_register_model.py @@ -2,8 +2,6 @@ # This tests calling batch_completions by running 100 messages together import ast -import sys, os -import traceback from pathlib import Path import pytest @@ -32,16 +30,6 @@ def test_update_model_cost(): # test_update_model_cost() -def test_update_model_cost_map_url(): - try: - litellm.register_model( - model_cost="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" - ) - assert litellm.model_cost["gpt-4"]["input_cost_per_token"] == 0.00003 - except Exception as e: - pytest.fail(f"An error occurred: {e}") - - # test_update_model_cost_map_url() @@ -53,9 +41,7 @@ def test_update_model_cost_via_completion(): input_cost_per_token=0.3, output_cost_per_token=0.4, ) - print( - f"litellm.model_cost for gpt-3.5-turbo: {litellm.model_cost['gpt-3.5-turbo']}" - ) + print(f"litellm.model_cost for gpt-3.5-turbo: {litellm.model_cost['gpt-3.5-turbo']}") assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.3 assert litellm.model_cost["gpt-3.5-turbo"]["output_cost_per_token"] == 0.4 except Exception as e: @@ -64,11 +50,7 @@ def test_update_model_cost_via_completion(): def test_no_test_invocation_at_module_scope(): tree = ast.parse(Path(__file__).read_text()) - defined = { - node.name - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } + defined = {node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} invoked = [ node.value.func.id for node in tree.body diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py index 80b830369e6..96751cebe01 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py @@ -3,7 +3,6 @@ from __future__ import annotations import base64 from collections.abc import Callable from datetime import date -from pathlib import Path from typing import Final, cast from unittest.mock import patch from urllib.parse import parse_qs, urlparse @@ -262,20 +261,6 @@ def _reducto_document() -> ReductoDocumentUrlDocument: ) -def test_fixture_catalogs_match_active_registered_ocr_models() -> None: - registry_path: Final = Path(__file__).resolve().parents[6] / "model_prices_and_context_window.json" - registry: Final = MODEL_REGISTRY.validate_json(registry_path.read_text(encoding="utf-8")) - active_registered: Final = frozenset( - model - for model, raw_metadata in registry.items() - if raw_metadata.get("mode") == "ocr" and raw_metadata.get("litellm_provider") in SUPPORTED_OCR_PROVIDERS - for metadata in (_ModelRegistryEntry.model_validate(raw_metadata),) - if metadata.deprecation_date is None or metadata.deprecation_date > date.today() - ) - - assert ACTIVE_OCR_MODELS == active_registered - - @pytest.mark.parametrize( ("fixture_model", "provider_config", "model"), ( diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 6b7780acd20..17b48063cce 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1,15 +1,12 @@ import copy -import datetime import json import os import subprocess import sys import textwrap -import unittest from typing import List, Optional, Tuple -from unittest.mock import ANY, MagicMock, Mock, patch +from unittest.mock import MagicMock, patch -import httpx import pytest import litellm @@ -19,7 +16,6 @@ from litellm.integrations.anthropic_cache_control_hook import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import StandardCallbackDynamicParams @pytest.fixture(autouse=True) @@ -1590,41 +1586,10 @@ class TestEnableAnthropicPromptCaching: points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") assert [p["index"] for p in points] == [None, -1] - @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai"), ("gemini-2.0-flash", "gemini")]) - def test_non_anthropic_providers_never_injected(self, monkeypatch, model, provider): - """These report supports_prompt_caching=True but never consume cache_control markers.""" - from litellm.utils import supports_prompt_caching - - monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - assert supports_prompt_caching(model=model, custom_llm_provider=provider) is True - assert self._points(model=model, provider=provider) == [] - - def test_databricks_claude_not_injected_despite_caching_support(self, monkeypatch, local_model_cost_map): - from litellm.utils import supports_prompt_caching - - monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - model = "databricks/databricks-claude-sonnet-4-5" - assert supports_prompt_caching(model=model, custom_llm_provider="databricks") is True - assert self._points(model=model, provider="databricks") == [] - def test_model_without_caching_support_not_injected(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) assert self._points(model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] - @pytest.mark.parametrize("model", ["us.xai.grok-4.6", "global.xai.grok-4.6"]) - def test_bedrock_grok_not_injected(self, monkeypatch, local_model_cost_map, model): - """Bedrock supports only implicit prompt caching for Grok: explicit cachePoint - breakpoints make it reject the whole request ("You invoked an unsupported model - or your request did not allow prompt caching"), so supports_prompt_caching stays - false, while implicit cache hits still bill at the cache-read rate.""" - from litellm.utils import supports_prompt_caching - - monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - assert supports_prompt_caching(model=model, custom_llm_provider="bedrock") is False - assert self._points(model=model, provider="bedrock") == [] - entry = litellm.model_cost[model] - assert 0 < entry["cache_read_input_token_cost"] < entry["input_cost_per_token"] - def test_stands_down_when_client_sent_cache_control(self, monkeypatch): monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) messages = [ @@ -1666,7 +1631,9 @@ class TestEnableAnthropicPromptCaching: """OpenAI-shaped tools nest cache_control under ``function``; the Anthropic chat transform honors that location, so the stand-down must see it too.""" monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) - tools = [{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}] + tools = [ + {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}} + ] assert self._points(tools=tools) == [] def test_seed_stands_down_when_only_tools_carry_cache_control(self, monkeypatch): @@ -2251,9 +2218,7 @@ class TestAnthropicPromptCachingEnvVars: print(json.dumps([litellm.enable_anthropic_prompt_caching, litellm.anthropic_prompt_caching_ttl])) """ ) - result = subprocess.run( - [sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300 - ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, env=env, timeout=300) assert result.returncode == 0, result.stderr enabled, ttl = json.loads(result.stdout.strip().splitlines()[-1]) return enabled, ttl @@ -2464,7 +2429,9 @@ class TestOpenAIPromptCacheBreakpoint: assert kwargs == {} def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self): - messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}] + messages = [ + {"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]} + ] kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} result, system = self._inject(messages, "sys", kwargs) assert result == messages @@ -2600,7 +2567,11 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: def test_tool_message_text_is_marked_on_chat_path(self): messages = [ {"role": "user", "content": "weather?"}, - {"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}], + }, {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, ] out, params = self._chat(messages, [{"location": "message", "index": -1}]) @@ -2824,9 +2795,9 @@ class TestChatPathProviderStamp: class TestClientBreakpointsCountedOnce: def test_client_message_breakpoints_are_not_double_counted(self): - messages = [{"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]}] + [ - {"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4) - ] + messages = [ + {"role": "user", "content": [{"type": "text", "text": "m0", "cache_control": {"type": "ephemeral"}}]} + ] + [{"role": "user", "content": [{"type": "text", "text": f"m{i}"}]} for i in range(1, 4)] out, system, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( messages=messages, system="sys", @@ -2984,19 +2955,6 @@ class TestPromptCacheBreakpointCapability: yield litellm.utils._cached_get_model_info_helper.cache_clear() - def test_public_helper_reads_the_model_map(self): - from litellm.utils import supports_prompt_cache_breakpoint - - assert supports_prompt_cache_breakpoint("gpt-5.6") is True - assert supports_prompt_cache_breakpoint("openai/gpt-5.6-sol") is True - assert supports_prompt_cache_breakpoint("gpt-5.6", custom_llm_provider="openai") is True - assert supports_prompt_cache_breakpoint("gpt-4.1") is False - - @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) - def test_model_map_flags_every_openai_gpt_5_6_entry(self, model): - assert litellm.model_cost[model]["litellm_provider"] == "openai" - assert litellm.model_cost[model]["supports_prompt_cache_breakpoint"] is True - def test_listed_model_uses_the_model_map_flag(self, monkeypatch): flagged = {**litellm.model_cost["gpt-4.1"], "supports_prompt_cache_breakpoint": True} monkeypatch.setitem(litellm.model_cost, "gpt-4.1", flagged) @@ -3014,10 +2972,6 @@ class TestPromptCacheBreakpointCapability: ) assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False - def test_listed_gpt_model_without_the_flag_follows_the_version_rule(self): - assert "supports_prompt_cache_breakpoint" not in litellm.model_cost["gpt-4.1"] - assert supports_openai_prompt_cache_breakpoint("gpt-4.1") is False - def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch): unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} monkeypatch.setitem(litellm.model_cost, "gpt-5.6", unflagged) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py index af2f169157e..0c2bb9ada71 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_guardrail_cost.py @@ -1,5 +1,3 @@ -import os - import pytest import litellm @@ -121,22 +119,6 @@ def test_billed_guardrail_cost_by_unit_treats_none_in_spend_as_billed(): assert billed_guardrail_cost_by_unit(entry) == {"contentPolicyUnits": 0.15} -def test_shipped_bedrock_guardrail_prices_match_aws_pricing_page(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - assert litellm.model_cost["bedrock/guardrails"]["guardrail_cost_per_unit"] == { - "automatedReasoningPolicyUnits": 0.00017, - "contentPolicyImageUnits": 0.00075, - "contentPolicyUnits": 0.00015, - "contextualGroundingPolicyUnits": 0.0001, - "sensitiveInformationPolicyFreeUnits": 0.0, - "sensitiveInformationPolicyUnits": 0.0001, - "topicPolicyUnits": 0.00015, - "wordPolicyUnits": 0.0, - } - assert "bedrock/guardrails" not in litellm.bedrock_models - - def test_guardrail_information_cost_sums_entries(): entries = [ {"guardrail_name": "a", "guardrail_cost": 0.0003}, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 5775656301d..776d78a04e0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1575,59 +1575,6 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): litellm.model_cost.pop(model, None) -def test_gpt_5_6_alias_prices_match_sol(local_model_cost_map): - """Regression: the bare gpt-5.6 alias routes to GPT-5.6 Sol, so every cost field on - the two entries has to hold the same value. They drifted once before, when Sol took - its promotional cut and gpt-5.6 was left on the pre-cut rates, overbilling callers - who used the alias.""" - alias = litellm.model_cost["gpt-5.6"] - sol = litellm.model_cost["gpt-5.6-sol"] - - cost_fields = sorted(field for field in sol if "cost" in field) - assert len(cost_fields) == 27 - - for field in cost_fields: - assert alias.get(field) == sol.get(field), field - - -@pytest.mark.parametrize( - "model,expected_none,expected_xhigh,expected_minimal", - [ - # Verified against OpenAI's live API on 2026-04-24: - # gpt-5.5 -> supports: none, low, medium, high, xhigh - # gpt-5.5-pro -> supports: medium, high, xhigh - # Neither supports "minimal"; gpt-5.5-pro additionally does not support "none". - # The JSON must reflect this so LiteLLM rejects unsupported values locally - # (or drops them with drop_params=True) instead of round-tripping to OpenAI - # for a 400. - ("gpt-5.5", True, True, False), - ("gpt-5.5-2026-04-23", True, True, False), - ("gpt-5.5-pro", False, True, False), - ("gpt-5.5-pro-2026-04-23", False, True, False), - ], -) -def test_gpt55_reasoning_effort_flags_match_live_openai_api( - _local_model_cost_map, model, expected_none, expected_xhigh, expected_minimal -): - """Pin reasoning_effort capability flags to OpenAI's actual API contract. - - Observed via `POST /v1/chat/completions` with reasoning_effort=minimal: - ``Unsupported value: 'reasoning_effort' does not support 'minimal' with - this model``. gpt-5.5-pro additionally rejects 'none' and 'low'. - """ - - m = litellm.model_cost[model] - assert m.get("supports_none_reasoning_effort") is expected_none, ( - f"{model}: supports_none_reasoning_effort expected {expected_none}" - ) - assert m.get("supports_xhigh_reasoning_effort") is expected_xhigh, ( - f"{model}: supports_xhigh_reasoning_effort expected {expected_xhigh}" - ) - assert m.get("supports_minimal_reasoning_effort") is expected_minimal, ( - f"{model}: supports_minimal_reasoning_effort expected {expected_minimal}" - ) - - @pytest.mark.parametrize( "base_model,dated_model", [ @@ -1662,29 +1609,6 @@ def test_gpt55_dated_variants_match_base_reasoning_effort_capabilities(_local_mo ) -@pytest.mark.parametrize( - "model,expected_none,expected_minimal,expected_xhigh", - [ - # Mirror live OpenAI API contract (verified via openai/gpt-5.5* on - # 2026-04-24): chat accepts {none, low, medium, high, xhigh} but NOT - # minimal; pro accepts {medium, high, xhigh} only. - # NOTE: openai/gpt-5.5* entries currently set supports_minimal=true on - # main (pre #26456). Once that PR lands, OpenAI + Azure flags align. - ("azure/gpt-5.5", True, False, True), - ("azure/gpt-5.5-pro", False, False, True), - ], -) -def test_azure_gpt55_reasoning_effort_flags_match_live_openai_api( - _local_model_cost_map, model, expected_none, expected_minimal, expected_xhigh -): - """Azure entries pin reasoning_effort flags to OpenAI's actual API contract.""" - - m = litellm.model_cost[model] - assert m.get("supports_none_reasoning_effort") is expected_none - assert m.get("supports_minimal_reasoning_effort") is expected_minimal - assert m.get("supports_xhigh_reasoning_effort") is expected_xhigh - - def test_string_cost_values(): """Test that cost values defined as strings are properly converted to floats.""" from unittest.mock import patch @@ -3413,14 +3337,6 @@ GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH = ( ) -@pytest.mark.parametrize("prefix", ["", "gemini/", "vertex_ai/"]) -def test_gemini_38_flash_matches_37_flash_promotional_pricing(prefix, _local_model_cost_map): - new_model = litellm.model_cost[f"{prefix}gemini-3.8-flash"] - old_model = litellm.model_cost[f"{prefix}gemini-3.7-flash"] - for field in GEMINI_38_FLASH_FIELDS_SHARED_WITH_37_FLASH: - assert new_model[field] == old_model[field], field - - @pytest.mark.parametrize( ("response_quality", "requested_quality", "expected_cost"), [ diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 761eed868b5..433117edb05 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,5 +1,3 @@ -from collections.abc import Mapping, Sequence - import pytest import litellm @@ -18,9 +16,7 @@ def test_web_search_cost_low(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost == model_info["search_context_cost_per_query"]["search_context_size_low"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"] def test_web_search_cost_medium(): @@ -31,10 +27,7 @@ def test_web_search_cost_medium(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost - == model_info["search_context_cost_per_query"]["search_context_size_medium"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"] def test_web_search_cost_high(): @@ -45,33 +38,21 @@ def test_web_search_cost_high(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost == model_info["search_context_cost_per_query"]["search_context_size_high"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"] # Test file search cost calculation def test_file_search_cost(): file_search = FileSearchTool(type="file_search") - cost = StandardBuiltInToolCostTracking.get_cost_for_file_search( - file_search=file_search - ) + cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search) assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call # Test edge cases def test_none_inputs(): # Test with None inputs - assert ( - StandardBuiltInToolCostTracking.get_cost_for_web_search( - web_search_options=None, model_info=None - ) - == 0.0 - ) - assert ( - StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) - == 0.0 - ) + assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0 + assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0 # Test the main get_cost_for_built_in_tools method @@ -96,9 +77,7 @@ def test_get_cost_for_built_in_tools_file_search(): Test that the cost for a file search is 0.00 when no response object is provided """ model = "gpt-4" - standard_built_in_tools_params = StandardBuiltInToolsParams( - file_search=FileSearchTool(type="file_search") - ) + standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search")) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, @@ -141,9 +120,7 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): usage = Usage(server_tool_use={"web_search_requests": 1}) assert isinstance(usage.server_tool_use, ServerToolUse) - assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( - response_object=None, usage=usage - ) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage) def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): @@ -182,9 +159,7 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_serve standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] assert cost == per_query_cost * web_search_requests assert cost > 0.0 assert getattr(usage, "server_tool_use", None) is None @@ -222,9 +197,7 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] assert cost == per_query_cost * web_search_requests @@ -288,18 +261,14 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize( - "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] -) +@pytest.mark.parametrize("model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"]) def test_get_cost_for_gemini_web_search(model): """ Test that the cost for a web search is 0.00 when no response object is provided """ from litellm.types.utils import PromptTokensDetailsWrapper, Usage - usage = Usage( - prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) - ) + usage = Usage(prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1)) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, usage=usage, @@ -357,61 +326,7 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par ) assert web_search_cost > 0, "Web search cost should be non-zero" - assert ( - cost >= web_search_cost - ), f"completion_cost ({cost}) should include web search cost ({web_search_cost})" - - -@pytest.mark.parametrize( - "model", - [ - "vertex_ai/gemini-3.1-flash-lite", # resolves directly via get_model_info - "gemini/gemini-3.1-flash-lite", # provider-prefixed, resolves via model_cost fallback - ], -) -def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map): - """ - Gemini 3.x bills web search per individual query (web_search_billing_unit == "per_query"), - so N searches cost N * $0.014. - - Regression for the bug where the billing unit was dropped between the pricing JSON and the - cost calculator: the field was missing from the ModelInfoBase TypedDict and from the - ModelInfoBase(...) constructor in _get_model_info_helper, so get_model_info returned it as - None and cost_per_web_search_request fell back to the per_prompt clamp, collapsing N queries - to a single charge. The "gemini/..." case additionally covers response_cost_calculator - resolving a provider-prefixed model name that get_model_info cannot map under vertex_ai. - """ - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - web_search_requests = 2 - model_info = litellm.get_model_info(model) - assert model_info["web_search_billing_unit"] == "per_query" - per_query_cost = model_info["search_context_cost_per_query"][ - "search_context_size_medium" - ] - expected_cost = per_query_cost * web_search_requests - - usage = Usage( - prompt_tokens=11, - completion_tokens=100, - total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=web_search_requests - ), - ) - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - custom_llm_provider="vertex_ai", - standard_built_in_tools_params=None, - ) - - assert cost == pytest.approx(expected_cost), ( - f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} " - f"per_query search fee, got ${cost}" - ) + assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})" def test_gemini_combined_search_and_maps_costs_are_additive(local_model_cost_map): @@ -441,94 +356,12 @@ def test_gemini_combined_search_and_maps_costs_are_additive(local_model_cost_map assert cost == pytest.approx(search_rate * 2 + maps_rate) -def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map): - """ - Gemini 2.x bills web search per grounded prompt: multiple internal queries are one flat - $0.035 fee. Guards the per_prompt clamp against the per_query plumbing, which makes - web_search_billing_unit always present on the resolved ModelInfo (None for 2.x), so the - clamp must treat a None billing unit as per_prompt rather than skipping the clamp. - """ - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - model = "vertex_ai/gemini-2.5-flash" - model_info = litellm.get_model_info(model) - assert not model_info.get("web_search_billing_unit") - expected_cost = model_info["search_context_cost_per_query"][ - "search_context_size_medium" - ] - - usage = Usage( - prompt_tokens=11, - completion_tokens=100, - total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=2 - ), - ) - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - custom_llm_provider="vertex_ai", - standard_built_in_tools_params=None, - ) - - assert cost == pytest.approx(expected_cost), ( - f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), " - f"got ${cost}" - ) - - -def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( - local_model_cost_map, -): - """ - Regression for the provider-prefix fallback in _handle_web_search_cost. When the initial - get_model_info lookup fails for a "/"-containing model, the retry re-resolves model_info from - the prefix and must adopt that prefix's provider for routing. Otherwise an unrelated model - (here OpenRouter, which carries no web search pricing) is re-resolved but still routed through - the request's vertex_ai Gemini calculator, which charges its $0.035 per_prompt default for a - model that should cost nothing for web search. - """ - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - model = "openrouter/google/gemini-3.1-flash-lite" - model_info = litellm.get_model_info(model) - assert model_info["litellm_provider"] == "openrouter" - assert not model_info.get("search_context_cost_per_query") - - usage = Usage( - prompt_tokens=11, - completion_tokens=100, - total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=2 - ), - ) - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - custom_llm_provider="vertex_ai", - standard_built_in_tools_params=None, - ) - - assert cost == 0.0, ( - "A non-Gemini provider-prefixed model with no web search pricing must not be charged " - f"the vertex_ai per_prompt default via the prefix fallback, got ${cost}" - ) - - def _openai_responses_with_web_search_calls(model, num_calls): from openai.types.responses.response_function_web_search import ( ActionSearch, ResponseFunctionWebSearch, ) - from litellm.types.llms.openai import ResponsesAPIResponse - output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -559,9 +392,7 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) for num_calls in (1, 3): @@ -585,13 +416,10 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): counter must read their "type" key like the detection gate does, instead of flooring a multi-search response to a single billable search. """ - from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] response = ResponsesAPIResponse.model_validate( { @@ -600,10 +428,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): "model": model, "object": "response", "status": "completed", - "output": [ - {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} - for i in range(3) - ], + "output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)], } ) assert all(isinstance(item, dict) for item in response.output) @@ -616,9 +441,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): standard_built_in_tools_params=None, ) - assert cost == pytest.approx(3 * per_call), ( - f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" - ) + assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" # Note: File search integration test removed due to complex annotation detection logic @@ -631,7 +454,6 @@ def test_response_includes_output_type_reads_dict_output_items(): items without an "action" field) stay plain dicts in the output union. The gate must read their "type" key instead of returning False and skipping the web search fee. """ - from litellm.types.llms.openai import ResponsesAPIResponse response = ResponsesAPIResponse.model_validate( { @@ -697,36 +519,3 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( ) _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 - - -def _responses_with_web_search( - model: str, actions: Sequence[Mapping[str, str]], tool_usage: Mapping[str, object] | None = None -) -> ResponsesAPIResponse: - payload = { - "id": "resp_1", - "created_at": 1756900000, - "model": model.split("/", 1)[-1], - "object": "response", - "status": "completed", - "output": [ - {"type": "web_search_call", "id": f"ws_{i}", "status": "completed", "action": action} - for i, action in enumerate(actions) - ], - } - return ResponsesAPIResponse.model_validate( - payload if tool_usage is None else {**payload, "tool_usage": tool_usage} - ) - - -def _web_search_cost(model: str, response: ResponsesAPIResponse, custom_llm_provider: str) -> float: - from litellm.types.utils import Usage - - return StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - response_object=response, - usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - custom_llm_provider=custom_llm_provider, - standard_built_in_tools_params=None, - ) - - diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py index 370ec4b6f60..94e8b4bb7b0 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py @@ -14,7 +14,6 @@ rather than forwarded as a no-op the provider can reject. See BerriAI/litellm#33 import pytest from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt -from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools _STRICT_TOOL = [ { @@ -76,12 +75,10 @@ def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models( """Opus 4.7/4.8, Sonnet 4 and Sonnet 5 reject toolSpec.strict and additionalProperties.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) tool_spec = result[0]["toolSpec"] - assert ( - "strict" not in tool_spec - ), f"strict leaked into toolSpec for {model_id}: {tool_spec}" - assert ( - "additionalProperties" not in tool_spec["inputSchema"]["json"] - ), f"additionalProperties leaked into toolSpec for {model_id}: {tool_spec}" + assert "strict" not in tool_spec, f"strict leaked into toolSpec for {model_id}: {tool_spec}" + assert "additionalProperties" not in tool_spec["inputSchema"]["json"], ( + f"additionalProperties leaked into toolSpec for {model_id}: {tool_spec}" + ) @pytest.mark.parametrize( @@ -96,9 +93,7 @@ def test_bedrock_tools_pt_strict_dropped_for_strict_unsupported_models( def test_bedrock_tools_pt_strict_kept_for_other_anthropic(model_id: str) -> None: """Sonnet 4.5/4.6 and Opus <=4.6 accept toolSpec.strict — keep forwarding it.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) - assert ( - result[0]["toolSpec"]["strict"] is True - ), f"strict missing for {model_id}: {result[0]['toolSpec']}" + assert result[0]["toolSpec"]["strict"] is True, f"strict missing for {model_id}: {result[0]['toolSpec']}" @pytest.mark.parametrize( @@ -118,9 +113,7 @@ def test_bedrock_tools_pt_falsy_strict_always_dropped(model_id: str) -> None: ones whose cost-map entry still allows ``strict: true`` through.""" result = _bedrock_tools_pt(_NON_STRICT_TOOL, model=model_id) tool_spec = result[0]["toolSpec"] - assert ( - "strict" not in tool_spec - ), f"no-op strict: false leaked into toolSpec for {model_id}: {tool_spec}" + assert "strict" not in tool_spec, f"no-op strict: false leaked into toolSpec for {model_id}: {tool_spec}" def test_responses_bridge_function_tool_does_not_reach_bedrock_with_strict() -> None: @@ -141,10 +134,8 @@ def test_responses_bridge_function_tool_does_not_reach_bedrock_with_strict() -> "required": ["city"], }, } - chat_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - [responses_tool] - ) + chat_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + [responses_tool] ) result = _bedrock_tools_pt(chat_tools, model="bedrock/us.anthropic.claude-sonnet-5") assert "strict" not in result[0]["toolSpec"] @@ -161,78 +152,3 @@ def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> Non """Non-Anthropic Bedrock families reject toolSpec.strict — must be dropped.""" result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) assert "strict" not in result[0]["toolSpec"] - - -def test_bedrock_converse_supports_strict_tools_helper() -> None: - """Direct check for the gate helper used by factory.py.""" - assert ( - bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-7") - is False - ) - assert ( - bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-8") - is False - ) - assert ( - bedrock_converse_supports_strict_tools( - "anthropic.claude-sonnet-4-5-20250929-v1:0" - ) - is True - ) - assert ( - bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-6") - is True - ) - assert bedrock_converse_supports_strict_tools("us.amazon.nova-micro-v1:0") is False - assert bedrock_converse_supports_strict_tools("") is False - # Sonnet 4 also rejects strict on Bedrock Converse - assert ( - bedrock_converse_supports_strict_tools( - "anthropic.claude-sonnet-4-20250514-v1:0" - ) - is False - ) - assert ( - bedrock_converse_supports_strict_tools( - "bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0" - ) - is False - ) - assert bedrock_converse_supports_strict_tools("anthropic.claude-sonnet-5") is False - assert ( - bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-sonnet-5") - is False - ) - assert ( - bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") - is True - ) - - -@pytest.mark.parametrize( - "cost_map_key", - [ - "anthropic.claude-opus-4-7", - "us.anthropic.claude-opus-4-7", - "anthropic.claude-opus-4-8", - "us.anthropic.claude-opus-4-8", - "anthropic.claude-sonnet-4-20250514-v1:0", - "global.anthropic.claude-sonnet-4-20250514-v1:0", - "us.anthropic.claude-sonnet-4-20250514-v1:0", - "eu.anthropic.claude-sonnet-4-20250514-v1:0", - "apac.anthropic.claude-sonnet-4-20250514-v1:0", - "anthropic.claude-sonnet-5", - "global.anthropic.claude-sonnet-5", - "us.anthropic.claude-sonnet-5", - "eu.anthropic.claude-sonnet-5", - "au.anthropic.claude-sonnet-5", - "jp.anthropic.claude-sonnet-5", - ], -) -def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None: - """The gate is driven by ``bedrock_converse_supports_strict_tools: false`` in - ``model_prices_and_context_window.json``, not hardcoded model patterns.""" - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - cost_map = GetModelCostMap.load_local_model_cost_map() - assert cost_map[cost_map_key]["bedrock_converse_supports_strict_tools"] is False diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 034062826f6..270df703dee 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1,5 +1,4 @@ import base64 -import json import logging import os import re @@ -10,7 +9,6 @@ import pytest import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( - BAD_MESSAGE_ERROR_STR, BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, BedrockConverseMessagesProcessor, BedrockImageProcessor, @@ -33,9 +31,7 @@ def _get_gemini_function_response_inline_data_parts(result): assert isinstance(result, list), "expected Gemini parts list" assert len(result) == 1, "multimodal function responses should stay in one part" function_response_part = result[0] - assert ( - "inline_data" not in function_response_part - ), "inline_data should be nested under function_response.parts" + assert "inline_data" not in function_response_part, "inline_data should be nested under function_response.parts" function_response = function_response_part["function_response"] nested_parts = function_response["parts"] return [part["inline_data"] for part in nested_parts if "inline_data" in part] @@ -51,7 +47,9 @@ def test_ollama_pt_simple_messages(): result = ollama_pt(model="llama2", messages=messages) - expected_prompt = "### System:\nYou are a helpful assistant\n\n### Assistant:\nHow can I help you?\n\n### User:\nHello\n\n" + expected_prompt = ( + "### System:\nYou are a helpful assistant\n\n### Assistant:\nHow can I help you?\n\n### User:\nHello\n\n" + ) assert isinstance(result, dict) assert result["prompt"] == expected_prompt assert result["images"] == [] @@ -106,10 +104,7 @@ async def test_anthropic_bedrock_thinking_blocks_with_none_content(): # verify the result assert len(result) == 2 - assert ( - result[1]["content"][0]["reasoningContent"]["reasoningText"]["text"] - == "This is a test thinking block" - ) + assert result[1]["content"][0]["reasoningContent"]["reasoningText"]["text"] == "This is a test thinking block" def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): @@ -177,11 +172,7 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): assert len(assistant_blocks) == 1 for block in assistant_blocks[0]["content"]: if "text" in block: - assert block[ - "text" - ].strip(), ( - f"Bedrock Converse rejects blank-text ContentBlocks; got {block!r}" - ) + assert block["text"].strip(), f"Bedrock Converse rejects blank-text ContentBlocks; got {block!r}" # toolUse blocks must still be present tool_use_blocks = [b for b in assistant_blocks[0]["content"] if "toolUse" in b] assert len(tool_use_blocks) == 2 @@ -222,19 +213,16 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): {"role": "user", "content": "Now what is 3+3?"}, ] - result = anthropic_messages_pt( - messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic" - ) + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") assistant = next(m for m in result if m["role"] == "assistant") content = assistant["content"] - assert all( - block.get("type") not in ("thinking", "redacted_thinking") for block in content - ), f"unsignable thinking block must be dropped, got {content!r}" - assert any( - block.get("type") == "text" and block.get("text") == "2+2 equals 4." - for block in content - ), f"assistant answer text must be preserved, got {content!r}" + assert all(block.get("type") not in ("thinking", "redacted_thinking") for block in content), ( + f"unsignable thinking block must be dropped, got {content!r}" + ) + assert any(block.get("type") == "text" and block.get("text") == "2+2 equals 4." for block in content), ( + f"assistant answer text must be preserved, got {content!r}" + ) def test_anthropic_messages_pt_keeps_signed_thinking_block(): @@ -257,9 +245,7 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block(): {"role": "user", "content": "Now what is 3+3?"}, ] - result = anthropic_messages_pt( - messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic" - ) + result = anthropic_messages_pt(messages=messages, model="claude-sonnet-4-6", llm_provider="anthropic") assistant = next(m for m in result if m["role"] == "assistant") thinking_blocks = [b for b in assistant["content"] if b.get("type") == "thinking"] @@ -346,9 +332,7 @@ def test_bedrock_get_document_format_fallback_mimes(): """ # Test DOCX fallback - docx_mime = ( - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ) + docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" supported_formats = ["pdf", "docx", "xlsx", "csv"] # Mock mimetypes.guess_all_extensions to return empty list (simulating Docker container scenario) @@ -372,15 +356,11 @@ def test_bedrock_get_document_format_mimetypes_success(): """ Test the _get_document_format method when mimetypes.guess_all_extensions works normally. """ - docx_mime = ( - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - ) + docx_mime = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" supported_formats = ["pdf", "docx", "xlsx", "csv"] # Test normal mimetypes behavior (should not hit fallback) - result = BedrockImageProcessor._get_document_format( - mime_type=docx_mime, supported_doc_formats=supported_formats - ) + result = BedrockImageProcessor._get_document_format(mime_type=docx_mime, supported_doc_formats=supported_formats) assert result == "docx", f"Expected 'docx', got '{result}'" @@ -596,9 +576,7 @@ async def test_bedrock_process_image_async_factory(): image_url = "data:application/pdf; qs=0.001;base64,JVBERi0xLjQKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURlY29kZT4" - content_block = await BedrockImageProcessor.process_image_async( - image_url=image_url, format=None - ) + content_block = await BedrockImageProcessor.process_image_async(image_url=image_url, format=None) print(f"content_block: {content_block}") @@ -641,9 +619,7 @@ def test_unpack_defs_resolves_nested_ref_inside_anyof_items(): items_schema = schema["properties"]["vatAmounts"]["anyOf"][0]["items"] # Assertions: items_schema should now be the resolved object, not an empty dict - assert isinstance( - items_schema, dict - ), "Items schema should be a dict after unpacking" + assert isinstance(items_schema, dict), "Items schema should be a dict after unpacking" assert items_schema.get("type") == "object" # Ensure essential properties are present assert set(items_schema.get("properties", {}).keys()) == {"vatRate", "vatAmount"} @@ -834,9 +810,7 @@ def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks(): last_message_with_tool_calls=last_message_with_tool_calls, ) inline_parts = _get_gemini_function_response_inline_data_parts(result) - assert ( - len(inline_parts) == 2 - ), f"expected 2 inline_data parts, got {len(inline_parts)}" + assert len(inline_parts) == 2, f"expected 2 inline_data parts, got {len(inline_parts)}" mime_types = {p["mime_type"] for p in inline_parts} assert mime_types == {"image/png", "image/jpeg"} @@ -872,9 +846,7 @@ def test_convert_gemini_tool_call_result_with_data_url_string(): last_message_with_tool_calls=last_message_with_tool_calls, ) inline_parts = _get_gemini_function_response_inline_data_parts(result) - assert ( - len(inline_parts) == 1 - ), "data-URL image string was not converted to inline_data" + assert len(inline_parts) == 1, "data-URL image string was not converted to inline_data" assert inline_parts[0]["mime_type"] == "image/png" assert inline_parts[0]["data"] == tiny_png_b64 @@ -910,9 +882,9 @@ def test_convert_gemini_tool_call_result_with_data_url_extra_params(): ) inline_parts = _get_gemini_function_response_inline_data_parts(result) assert len(inline_parts) == 1 - assert ( - inline_parts[0]["mime_type"] == "image/png" - ), f"expected clean 'image/png', got '{inline_parts[0]['mime_type']}'" + assert inline_parts[0]["mime_type"] == "image/png", ( + f"expected clean 'image/png', got '{inline_parts[0]['mime_type']}'" + ) def test_bedrock_tools_unpack_defs(): @@ -1009,9 +981,7 @@ def test_bedrock_tools_pt_strict_parameter(): }, } ] - result = _bedrock_tools_pt( - tools_with_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + result = _bedrock_tools_pt(tools_with_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0") assert result[0]["toolSpec"]["strict"] is True assert result[0]["toolSpec"]["inputSchema"]["json"]["additionalProperties"] is False @@ -1033,9 +1003,7 @@ def test_bedrock_tools_pt_strict_parameter(): }, } ] - result = _bedrock_tools_pt( - tools_without_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + result = _bedrock_tools_pt(tools_without_strict, model="anthropic.claude-sonnet-4-5-20250929-v1:0") assert "strict" not in result[0]["toolSpec"] assert "additionalProperties" not in result[0]["toolSpec"]["inputSchema"]["json"] @@ -1058,9 +1026,7 @@ def test_bedrock_image_processor_content_type_fallback_url_extension(): # Test with .png URL image_url = "https://example.com/test-image.png" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -1084,9 +1050,7 @@ def test_bedrock_image_processor_content_type_fallback_binary_detection(): # Test with URL without extension image_url = "https://example.com/test-image-without-extension" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert content_type == "image/jpeg" assert base64_bytes == base64.b64encode(jpeg_content).decode("utf-8") @@ -1109,9 +1073,7 @@ def test_bedrock_image_processor_content_type_fallback_application_octet_stream( # Test with .gif URL image_url = "https://s3.amazonaws.com/bucket/image.gif" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert content_type == "image/gif" assert base64_bytes == base64.b64encode(gif_content).decode("utf-8") @@ -1134,9 +1096,7 @@ def test_bedrock_image_processor_content_type_with_query_params(): # Test with URL containing query parameters (common in S3 signed URLs) image_url = "https://s3.amazonaws.com/bucket/image.webp?AWSAccessKeyId=123&Expires=456&Signature=789" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert content_type == "image/webp" assert base64_bytes == base64.b64encode(webp_content).decode("utf-8") @@ -1158,9 +1118,7 @@ def test_bedrock_image_processor_content_type_normal_header(): mock_response.content = png_content image_url = "https://example.com/test-image.png" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert content_type == "image/png" assert base64_bytes == base64.b64encode(png_content).decode("utf-8") @@ -1180,7 +1138,7 @@ def test_bedrock_image_processor_content_type_fallback_failure(): # Test with URL without recognizable extension image_url = "https://example.com/unknown-file" - with pytest.raises(ValueError, match='Unable to determine content type from URL: https') as excinfo: + with pytest.raises(ValueError, match="Unable to determine content type from URL: https") as excinfo: BedrockImageProcessor._post_call_image_processing(mock_response, image_url) assert "Unable to determine content type" in str(excinfo.value) @@ -1200,16 +1158,12 @@ def test_bedrock_image_processor_content_type_jpeg_variants(): # Test with .jpg extension image_url_jpg = "https://example.com/photo.jpg" - _, content_type_jpg = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url_jpg - ) + _, content_type_jpg = BedrockImageProcessor._post_call_image_processing(mock_response, image_url_jpg) assert content_type_jpg == "image/jpeg" # Test with .jpeg extension image_url_jpeg = "https://example.com/photo.jpeg" - _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing( - mock_response, image_url_jpeg - ) + _, content_type_jpeg = BedrockImageProcessor._post_call_image_processing(mock_response, image_url_jpeg) assert content_type_jpeg == "image/jpeg" @@ -1231,9 +1185,7 @@ def test_bedrock_image_processor_content_type_pdf_document(): # Test with .pdf URL pdf_url = "https://s3.amazonaws.com/bucket/document.pdf" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, pdf_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, pdf_url) assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1243,7 +1195,6 @@ def test_bedrock_image_processor_content_type_document_formats(): """ Test that _post_call_image_processing handles various document formats """ - import base64 # Create mock response mock_response = MagicMock() @@ -1267,12 +1218,8 @@ def test_bedrock_image_processor_content_type_document_formats(): ] for url, expected_mime in test_cases: - _, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, url - ) - assert ( - content_type == expected_mime - ), f"Expected {expected_mime} for {url}, got {content_type}" + _, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, url) + assert content_type == expected_mime, f"Expected {expected_mime} for {url}, got {content_type}" def test_bedrock_image_processor_content_type_s3_pdf_with_query(): @@ -1291,9 +1238,7 @@ def test_bedrock_image_processor_content_type_s3_pdf_with_query(): # S3 signed URL with query parameters s3_url = "https://my-bucket.s3.us-east-1.amazonaws.com/documents/report.pdf?AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Expires=1234567890&Signature=abcdef123456" - base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing( - mock_response, s3_url - ) + base64_bytes, content_type = BedrockImageProcessor._post_call_image_processing(mock_response, s3_url) assert content_type == "application/pdf" assert base64_bytes == base64.b64encode(pdf_content).decode("utf-8") @@ -1402,12 +1347,8 @@ def test_bedrock_create_bedrock_block_normalized_base64(): base64_content = base64.b64encode(pdf_content).decode("utf-8") # Create versions with different whitespace - base64_with_newlines = "\n".join( - [base64_content[i : i + 64] for i in range(0, len(base64_content), 64)] - ) - base64_with_spaces = " ".join( - [base64_content[i : i + 32] for i in range(0, len(base64_content), 32)] - ) + base64_with_newlines = "\n".join([base64_content[i : i + 64] for i in range(0, len(base64_content), 64)]) + base64_with_spaces = " ".join([base64_content[i : i + 32] for i in range(0, len(base64_content), 32)]) # Create blocks block1 = BedrockImageProcessor._create_bedrock_block( @@ -1539,9 +1480,7 @@ def test_bedrock_create_bedrock_block_document_name_format(): # Check format: DocumentPDFmessages_{16_hex_chars}_{format} pattern = r"^DocumentPDFmessages_[0-9a-f]{16}_pdf$" - assert re.match( - pattern, document_name - ), f"Document name format mismatch: {document_name}" + assert re.match(pattern, document_name), f"Document name format mismatch: {document_name}" def test_bedrock_create_bedrock_block_different_document_formats(): @@ -1567,7 +1506,7 @@ def test_bedrock_create_bedrock_block_different_document_formats(): ) assert block.get("document") is not None - assert f"DocumentPDFmessages_" in block["document"]["name"] + assert "DocumentPDFmessages_" in block["document"]["name"] assert block["document"]["name"].endswith(f"_{format_type}") assert block["document"]["format"] == format_type @@ -1594,9 +1533,7 @@ def test_bedrock_nova_web_search_options_mapping(): assert system_tool["name"] == "nova_grounding" # Test with search_context_size (should be ignored for Nova) - result2 = config._map_web_search_options( - {"search_context_size": "high"}, "us.amazon.nova-premier-v1:0" - ) + result2 = config._map_web_search_options({"search_context_size": "high"}, "us.amazon.nova-premier-v1:0") assert result2 is not None system_tool2 = result2.get("systemTool") @@ -1662,9 +1599,7 @@ def test_bedrock_tools_pt_drops_unmappable_responses_builtin_tools(): {"type": "custom", "name": "free_form"}, ] - result = _bedrock_tools_pt( - tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + result = _bedrock_tools_pt(tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0") names = [block["toolSpec"]["name"] for block in result if "toolSpec" in block] assert names == ["noop"] @@ -1694,9 +1629,7 @@ def test_bedrock_tools_pt_keeps_anthropic_input_schema_tools(): }, ] - result = _bedrock_tools_pt( - tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + result = _bedrock_tools_pt(tools=tools, model="anthropic.claude-sonnet-4-5-20250929-v1:0") names = [block["toolSpec"]["name"] for block in result if "toolSpec" in block] assert names == ["lookup"] @@ -1898,9 +1831,7 @@ def test_anthropic_messages_pt_server_tool_use_passthrough(): "tool_use_id": "srvtoolu_01ABC123", "content": { "type": "tool_search_tool_search_result", - "tool_references": [ - {"type": "tool_reference", "tool_name": "get_time"} - ], + "tool_references": [{"type": "tool_reference", "tool_name": "get_time"}], }, }, {"type": "text", "text": "I found the time tool. How can I help you?"}, @@ -1928,20 +1859,14 @@ def test_anthropic_messages_pt_server_tool_use_passthrough(): # Verify server_tool_use block is preserved assert "server_tool_use" in content_types - server_tool_use_block = next( - b for b in assistant_msg["content"] if b.get("type") == "server_tool_use" - ) + server_tool_use_block = next(b for b in assistant_msg["content"] if b.get("type") == "server_tool_use") assert server_tool_use_block["id"] == "srvtoolu_01ABC123" assert server_tool_use_block["name"] == "tool_search_tool_regex" assert server_tool_use_block["input"] == {"query": ".*time.*"} # Verify tool_search_tool_result block is preserved assert "tool_search_tool_result" in content_types - tool_result_block = next( - b - for b in assistant_msg["content"] - if b.get("type") == "tool_search_tool_result" - ) + tool_result_block = next(b for b in assistant_msg["content"] if b.get("type") == "tool_search_tool_result") assert tool_result_block["tool_use_id"] == "srvtoolu_01ABC123" assert tool_result_block["content"]["type"] == "tool_search_tool_search_result" assert tool_result_block["content"]["tool_references"][0]["tool_name"] == "get_time" @@ -1993,9 +1918,7 @@ def test_bedrock_tools_unpack_defs_no_oom_with_nested_refs(): "anyOf": [ {"$ref": "#/$defs/Literal"}, {"$ref": "#/$defs/FieldRef"}, - { - "$ref": "#/$defs/Expression" - }, # Circular: Operand -> Expression -> Operand + {"$ref": "#/$defs/Expression"}, # Circular: Operand -> Expression -> Operand ], }, "Literal": { @@ -2129,9 +2052,7 @@ def test_anthropic_messages_pt_file_block_cache_control_with_explicit_provider() file_block = content_blocks[0] assert file_block["type"] == "document" - assert ( - "cache_control" in file_block - ), "cache_control should be preserved on file/document content blocks" + assert "cache_control" in file_block, "cache_control should be preserved on file/document content blocks" assert file_block["cache_control"]["type"] == "ephemeral" text_block = content_blocks[1] @@ -2339,22 +2260,16 @@ def test_bedrock_tool_call_invoke_concatenated_json(): # First block keeps original tool id assert result[0]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN" assert result[0]["toolUse"]["name"] == "shell" - assert result[0]["toolUse"]["input"] == { - "command": ["curl", "-i", "http://localhost:9009", "-m", "10"] - } + assert result[0]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009", "-m", "10"]} # Subsequent blocks get suffixed ids assert result[1]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_1" assert result[1]["toolUse"]["name"] == "shell" - assert result[1]["toolUse"]["input"] == { - "command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"] - } + assert result[1]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009/robots.txt", "-m", "5"]} assert result[2]["toolUse"]["toolUseId"] == "tooluse_L7I3TewYAUhoheJZQEuwVN_2" assert result[2]["toolUse"]["name"] == "shell" - assert result[2]["toolUse"]["input"] == { - "command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"] - } + assert result[2]["toolUse"]["input"] == {"command": ["curl", "-i", "http://localhost:9009/sitemap.xml", "-m", "5"]} def test_bedrock_tool_call_invoke_concatenated_json_with_cache_control(): @@ -2509,9 +2424,7 @@ def test_bedrock_tool_call_invoke_unconvertible_raises_non_retryable_bad_request def test_make_valid_bedrock_tool_name_preserves_hyphens(): assert make_valid_bedrock_tool_name("my-tool") == "my-tool" assert ( - make_valid_bedrock_tool_name( - "CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q" - ) + make_valid_bedrock_tool_name("CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q") == "CreateCaseKnowledgeArticle_foTWsqR6yDt-OnSsvR5e6Q" ) @@ -2538,9 +2451,7 @@ def test_bedrock_tool_name_sanitized_consistently_in_tools_and_tool_use(): "function": {"name": raw_name, "arguments": "{}"}, } ] - tool_use_name = _convert_to_bedrock_tool_call_invoke(tool_calls)[0]["toolUse"][ - "name" - ] + tool_use_name = _convert_to_bedrock_tool_call_invoke(tool_calls)[0]["toolUse"]["name"] assert tool_spec_name == "foo_bar" assert tool_use_name == tool_spec_name @@ -2563,15 +2474,8 @@ def test_bedrock_converse_messages_pt_tool_use_matches_tool_spec_hyphen_name(): ], }, ] - translated = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) - tool_use_blocks = [ - block - for msg in translated - for block in msg.get("content", []) - if "toolUse" in block - ] + translated = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + tool_use_blocks = [block for msg in translated for block in msg.get("content", []) if "toolUse" in block] assert len(tool_use_blocks) == 1 assert tool_use_blocks[0]["toolUse"]["name"] == tool_name @@ -2668,11 +2572,7 @@ def test_sanitize_messages_deduplicates_tool_results(): result = sanitize_messages_for_tool_calling(messages) # Count tool messages with this ID — should be exactly 1 - tool_results = [ - m - for m in result - if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123" - ] + tool_results = [m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123"] assert len(tool_results) == 1 # Should keep the LAST occurrence (most complete) assert tool_results[0]["content"] == '{"temperature": 72, "condition": "sunny"}' @@ -2807,11 +2707,7 @@ def test_sanitize_messages_dedup_scoped_per_turn_preserves_cross_turn(): result = sanitize_messages_for_tool_calling(messages) # Both tool results must survive — one per turn - tool_results = [ - m - for m in result - if m.get("role") == "tool" and m.get("tool_call_id") == "call_X" - ] + tool_results = [m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_X"] assert len(tool_results) == 2, ( f"Expected 2 tool results (one per turn), got {len(tool_results)}. " "Dedup may be global instead of per-turn scoped." @@ -2865,32 +2761,26 @@ def test_sanitize_messages_combined_case_a_and_case_d(): tool_results = [m for m in result if m.get("role") in ("tool", "function")] # Case A: call_missing should have a dummy result injected - missing_results = [ - m for m in tool_results if m.get("tool_call_id") == "call_missing" - ] - assert ( - len(missing_results) == 1 - ), f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" + missing_results = [m for m in tool_results if m.get("tool_call_id") == "call_missing"] + assert len(missing_results) == 1, ( + f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" + ) # Case D: call_duped should have exactly 1 result (the fresh one) - duped_results = [ - m for m in tool_results if m.get("tool_call_id") == "call_duped" - ] - assert ( - len(duped_results) == 1 - ), f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" - assert ( - duped_results[0]["content"] == "fresh_result" - ), f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" + duped_results = [m for m in tool_results if m.get("tool_call_id") == "call_duped"] + assert len(duped_results) == 1, ( + f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" + ) + assert duped_results[0]["content"] == "fresh_result", ( + f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" + ) # Verify tool results immediately follow the assistant message asst_idx = next(i for i, m in enumerate(result) if m.get("role") == "assistant") - tool_msgs_after_asst = [ - m for m in result[asst_idx + 1 :] if m.get("role") in ("tool", "function") - ] - assert ( - len(tool_msgs_after_asst) == 2 - ), f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" + tool_msgs_after_asst = [m for m in result[asst_idx + 1 :] if m.get("role") in ("tool", "function")] + assert len(tool_msgs_after_asst) == 2, ( + f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" + ) # Both tool_call_ids should be present (order may vary) tool_ids = {m["tool_call_id"] for m in tool_msgs_after_asst} assert tool_ids == { @@ -2932,9 +2822,7 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): } ] - result = anthropic_messages_pt( - messages, model="claude-sonnet-4-20250514", llm_provider="anthropic" - ) + result = anthropic_messages_pt(messages, model="claude-sonnet-4-20250514", llm_provider="anthropic") content_blocks = result[0]["content"] assert len(content_blocks) == 2 @@ -2942,9 +2830,7 @@ def test_anthropic_messages_pt_file_block_preserves_cache_control(): # Document block (from file) should preserve cache_control doc_block = content_blocks[0] assert doc_block["type"] == "document" - assert ( - "cache_control" in doc_block - ), "cache_control was dropped from file/document block" + assert "cache_control" in doc_block, "cache_control was dropped from file/document block" assert doc_block["cache_control"]["type"] == "ephemeral" # Text block should also preserve cache_control @@ -2987,9 +2873,7 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): } # Claude 4.5 model: ttl should be preserved - result = add_cache_point_tool_block( - tool_with_1h, model="jp.anthropic.claude-opus-4-7" - ) + result = add_cache_point_tool_block(tool_with_1h, model="jp.anthropic.claude-opus-4-7") assert result is not None assert result["cachePoint"]["type"] == "default" assert result["cachePoint"]["ttl"] == "1h" @@ -2998,16 +2882,12 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): tool_with_5m = { "cache_control": {"type": "ephemeral", "ttl": "5m"}, } - result_5m = add_cache_point_tool_block( - tool_with_5m, model="jp.anthropic.claude-opus-4-7" - ) + result_5m = add_cache_point_tool_block(tool_with_5m, model="jp.anthropic.claude-opus-4-7") assert result_5m is not None assert result_5m["cachePoint"]["ttl"] == "5m" # Older model: ttl should be stripped - result_old = add_cache_point_tool_block( - tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) + result_old = add_cache_point_tool_block(tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0") assert result_old is not None assert result_old["cachePoint"]["type"] == "default" assert "ttl" not in result_old["cachePoint"] @@ -3026,9 +2906,7 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): # cache_control without ttl: returns default cachePoint (unchanged behavior) tool_no_ttl = {"cache_control": {"type": "ephemeral"}} - result_no_ttl = add_cache_point_tool_block( - tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + result_no_ttl = add_cache_point_tool_block(tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") assert result_no_ttl is not None assert result_no_ttl["cachePoint"]["type"] == "default" assert "ttl" not in result_no_ttl["cachePoint"] @@ -3040,28 +2918,6 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) -def test_add_cache_point_tool_block_stands_down_for_model_without_prompt_caching(monkeypatch): - """A tool carrying cache_control must not become a cachePoint for a Bedrock model - whose cost-map entry lacks prompt caching support, since Bedrock rejects the whole - request. An unmapped id keeps emitting so ARN deployments do not lose caching.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - add_cache_point_tool_block, - ) - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - tool = {"cache_control": {"type": "ephemeral"}} - - assert add_cache_point_tool_block(tool, model="nvidia.nemotron-super-3-120b") is None - assert add_cache_point_tool_block(tool, model="us.nvidia.nemotron-super-3-120b") is None - assert add_cache_point_tool_block( - tool, model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" - ) == {"cachePoint": {"type": "default"}} - assert add_cache_point_tool_block(tool, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") == { - "cachePoint": {"type": "default"} - } - - def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(monkeypatch): """ End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl @@ -3101,9 +2957,7 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(monkeypatch): assert cache_blocks[0]["cachePoint"]["ttl"] == "1h" # Older model: cachePoint should not have ttl - result_old = _bedrock_tools_pt( - tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) + result_old = _bedrock_tools_pt(tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0") cache_blocks_old = [b for b in result_old if "cachePoint" in b] assert len(cache_blocks_old) == 1 assert "ttl" not in cache_blocks_old[0]["cachePoint"] @@ -3178,9 +3032,7 @@ def test_bedrock_converse_messages_pt_document_various_formats(): } ] - result = _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) + result = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") doc_block = result[0]["content"][0] assert doc_block["document"]["format"] == expected_format, ( @@ -3207,12 +3059,8 @@ def test_bedrock_converse_messages_pt_document_deterministic_name(): } ] - result1 = _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) - result2 = _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) + result1 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + result2 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") name1 = result1[0]["content"][0]["document"]["name"] name2 = result2[0]["content"][0]["document"]["name"] @@ -3246,34 +3094,18 @@ def test_bedrock_converse_messages_pt_renames_duplicate_document_names(): }, ] - result1 = _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) - result2 = _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) + result1 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") + result2 = _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") - names1 = [ - block["document"]["name"] - for message in result1 - for block in message["content"] - if "document" in block - ] - names2 = [ - block["document"]["name"] - for message in result2 - for block in message["content"] - if "document" in block - ] + names1 = [block["document"]["name"] for message in result1 for block in message["content"] if "document" in block] + names2 = [block["document"]["name"] for message in result2 for block in message["content"] if "document" in block] assert len(names1) == 2 assert len(set(names1)) == 2 assert names1[1] == f"{names1[0]}_2" assert names1 == names2 - single_turn = _bedrock_converse_messages_pt( - [messages[0]], "anthropic.claude-sonnet-4-6", "bedrock" - ) + single_turn = _bedrock_converse_messages_pt([messages[0]], "anthropic.claude-sonnet-4-6", "bedrock") assert names1[0] == single_turn[0]["content"][0]["document"]["name"] @@ -3295,14 +3127,10 @@ def test_rename_duplicate_bedrock_document_names_skips_organic_suffixes(): def _names(contents): return [block["document"]["name"] for block in contents[0]["content"]] - organic_first = _rename_duplicate_bedrock_document_names( - _contents(["report", "report_2", "report"]) - ) + organic_first = _rename_duplicate_bedrock_document_names(_contents(["report", "report_2", "report"])) assert _names(organic_first) == ["report", "report_2", "report_3"] - organic_last = _rename_duplicate_bedrock_document_names( - _contents(["report", "report", "report_2"]) - ) + organic_last = _rename_duplicate_bedrock_document_names(_contents(["report", "report", "report_2"])) assert _names(organic_last) == ["report", "report_3", "report_2"] @@ -3324,18 +3152,11 @@ def test_bedrock_converse_messages_pt_document_rejects_url_source(): ] with pytest.raises(ValueError, match="only supports base64-encoded"): - _bedrock_converse_messages_pt( - messages, "anthropic.claude-sonnet-4-6", "bedrock" - ) + _bedrock_converse_messages_pt(messages, "anthropic.claude-sonnet-4-6", "bedrock") def _collect_cache_points(blocks): - return [ - block["cachePoint"] - for message in blocks - for block in message["content"] - if "cachePoint" in block - ] + return [block["cachePoint"] for message in blocks for block in message["content"] if "cachePoint" in block] @pytest.mark.parametrize( @@ -3599,9 +3420,7 @@ def test_bedrock_converse_pdf_only_user_message_gets_text_block(): } ] - result = _bedrock_converse_messages_pt( - messages, "anthropic.claude-haiku-4-5", "bedrock" - ) + result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") assert len(result) == 1 assert any("document" in block for block in result[0]["content"]) @@ -3619,9 +3438,7 @@ def test_bedrock_converse_document_with_text_gets_no_extra_text_block(): } ] - result = _bedrock_converse_messages_pt( - messages, "anthropic.claude-haiku-4-5", "bedrock" - ) + result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") assert _text_blocks(result[0]) == ["summarize this"] @@ -3634,9 +3451,7 @@ def test_bedrock_converse_image_only_user_message_gets_no_text_block(): } ] - result = _bedrock_converse_messages_pt( - messages, "anthropic.claude-haiku-4-5", "bedrock" - ) + result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") assert any("image" in block for block in result[0]["content"]) assert _text_blocks(result[0]) == [] @@ -3679,9 +3494,7 @@ def test_bedrock_converse_tool_round_trip_document_injects_text_before_cache_poi }, ] - result = _bedrock_converse_messages_pt( - messages, "anthropic.claude-haiku-4-5", "bedrock" - ) + result = _bedrock_converse_messages_pt(messages, "anthropic.claude-haiku-4-5", "bedrock") assert _text_blocks(result[0]) == ["read the pdf"] document_message = result[-1] diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 91e43cba825..e17216b7b34 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -488,13 +488,6 @@ def test_shipped_gemini_chat_baseline_resolves_unmapped_ids(shipped_cost_map, mo assert not info.get("output_cost_per_token") -def test_shipped_gemini_chat_baseline_loses_to_perplexity_exact_entries(shipped_cost_map): - info = litellm.get_model_info("google/gemini-2.5-pro", custom_llm_provider="perplexity") - entry = litellm.model_cost["perplexity/google/gemini-2.5-pro"] - assert info["mode"] == "responses" - assert entry["supports_reasoning"] is False - - def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map): for model in ( "gemini/gemini-4-flash-image", @@ -809,24 +802,6 @@ def test_shipped_rules_flag_unmapped_wandb_ids_as_reasoning(shipped_cost_map): assert litellm.supports_reasoning(model="zai-org/GLM-6-Turbo", custom_llm_provider="wandb") is True -def test_shipped_wandb_rule_loses_to_mapped_non_reasoning_entries(shipped_cost_map): - """The whole point of a fallback is that it only fills gaps. A wandb model the map - describes as non-reasoning must stay non-reasoning, otherwise the rule silently - re-introduces the blanket supports_reasoning it exists to avoid.""" - for model in ( - "meta-llama/Llama-3.1-8B-Instruct", - "microsoft/Phi-4-mini-instruct", - "moonshotai/Kimi-K2-Instruct", - "Qwen/Qwen3-Coder-480B-A35B-Instruct", - ): - assert f"wandb/{model}" in litellm.model_cost, model - assert litellm.supports_reasoning(model=model, custom_llm_provider="wandb") is False, model - - -def test_shipped_wandb_rule_does_not_fill_missing_mapped_entries(shipped_cost_map): - assert match_fill_missing_generalizations("wandb/meta-llama/Llama-3.1-8B-Instruct", "wandb") is None - - def test_shipped_wandb_rule_is_anchored_to_the_wandb_namespace(shipped_cost_map): """``^wandb/`` is anchored, so it cannot leak onto another provider's ids.""" assert match_capability_generalizations("wandb/some-new-model") == {"supports_reasoning": True} @@ -880,27 +855,6 @@ def test_router_registration_does_not_shadow_shipped_rules(shipped_cost_map): assert litellm.supports_reasoning(model="claude-opus-9", custom_llm_provider="anthropic") is True -def test_deployment_model_info_beats_the_seeded_rule_defaults(shipped_cost_map): - """Seeding a registration from the rules is a floor, not an override: an explicit - model_info on the deployment still wins, so a non-reasoning model can be configured - under a reasoning-first namespace.""" - from litellm import Router - - model = "wandb/some-org/NoThink-1" - Router( - model_list=[ - { - "model_name": model, - "litellm_params": {"model": model, "api_key": "fake"}, - "model_info": {"supports_reasoning": False}, - } - ] - ) - - assert litellm.model_cost[model]["supports_reasoning"] is False - assert litellm.supports_reasoning(model="some-org/NoThink-1", custom_llm_provider="wandb") is False - - def test_shipped_rules_flag_unmapped_openai_reasoning_families(shipped_cost_map): for model in ( "gpt-5.7-nova", @@ -941,66 +895,11 @@ def test_shipped_openai_reasoning_rule_skips_non_reasoning_gpt_ids(shipped_cost_ assert match_capability_generalizations(model) is None, model -def test_shipped_openai_reasoning_rule_loses_to_mapped_entries(shipped_cost_map): - assert "gpt-5-search-api" in litellm.model_cost - assert litellm.supports_reasoning(model="gpt-5-search-api", custom_llm_provider="openai") is False - - -@pytest.mark.parametrize( - "model,provider,expected_supports_reasoning", - [ - ("azure/us/o1-2024-12-17", "azure", True), - ("github_copilot/gpt-5", "github_copilot", None), - ("perplexity/openai/gpt-5.4-mini", "perplexity", None), - ], -) -def test_shipped_openai_reasoning_rule_backfills_only_approved_providers( - shipped_cost_map, model, provider, expected_supports_reasoning -): - assert model in litellm.model_cost - raw_entry = litellm.model_cost[model] - assert "supports_reasoning" not in raw_entry - model_without_provider = model.removeprefix(f"{provider}/") - info = litellm.get_model_info(model=model_without_provider, custom_llm_provider=provider) - assert info.get("supports_reasoning") is expected_supports_reasoning - assert info["input_cost_per_token"] == raw_entry.get("input_cost_per_token", 0) - - def test_shipped_openai_reasoning_rule_matches_only_openai(shipped_cost_map): assert match_fill_missing_generalizations("gpt-5.4", "openai") == {"supports_reasoning": True} assert match_fill_missing_generalizations("gpt-5.4", "openrouter") is None -def test_shipped_openai_reasoning_rule_skips_non_text_modes(shipped_cost_map): - model = "gemini/deep-research-pro-preview-12-2025" - assert model in litellm.model_cost - raw_entry = litellm.model_cost[model] - assert "supports_reasoning" not in raw_entry - assert raw_entry["mode"] == "image_generation" - - info = litellm.get_model_info("deep-research-pro-preview-12-2025", custom_llm_provider="gemini") - assert info.get("supports_reasoning") is None - - -def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map): - model = "perplexity/anthropic/claude-sonnet-4-6" - assert model in litellm.model_cost - raw_entry = litellm.model_cost[model] - assert "supports_adaptive_thinking" not in raw_entry - assert "max_input_tokens" not in raw_entry - - info = litellm.get_model_info(model="anthropic/claude-sonnet-4-6", custom_llm_provider="perplexity") - assert info.get("supports_adaptive_thinking") is None - assert info.get("supports_legacy_thinking") is None - assert info.get("max_input_tokens") is None - assert match_fill_missing_generalizations("claude-sonnet-4-6", "anthropic") == { - "supports_adaptive_thinking": True, - "supports_legacy_thinking": True, - "supports_tool_search": True, - } - assert match_fill_missing_generalizations("claude-sonnet-4-6", "perplexity") is None - - @pytest.mark.parametrize( "model,provider,tool_search", [ @@ -1023,27 +922,3 @@ def test_shipped_tool_search_rule_version_boundaries(shipped_cost_map, model, pr assert model not in litellm.model_cost info = litellm.get_model_info(model, custom_llm_provider=provider) assert info.get("supports_tool_search") is tool_search, model - - -def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipped_cost_map): - """A mapped Claude 4.5+ entry with no supports_tool_search key gets it from the rule - on Anthropic direct, Vertex and Bedrock, a mapped pre-4.5 entry stays without one, - and Azure Foundry and reseller copies of the same model are not touched.""" - for key, model, provider in ( - ("claude-opus-4-7", "claude-opus-4-7", "anthropic"), - ("vertex_ai/claude-opus-5", "claude-opus-5", "vertex_ai"), - ): - assert "supports_tool_search" not in litellm.model_cost[key] - assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True - - assert "supports_tool_search" not in litellm.model_cost["claude-opus-4-1"] - opus_4_1_info = litellm.get_model_info("claude-opus-4-1", custom_llm_provider="anthropic") - assert opus_4_1_info.get("supports_tool_search") is None - - assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"] - azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai") - assert azure_opus_5_info.get("supports_tool_search") is None - - assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True - assert "supports_tool_search" not in match_fill_missing_generalizations("claude-opus-5", "azure_ai") - assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 9124a655840..28ba46a7e75 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -395,48 +395,6 @@ class TestGetRouterDeploymentModelInfo: logging_obj.litellm_params = {"api_base": ""} assert logging_obj.get_router_deployment_model_info() is None - - def test_a_published_batch_rate_never_displaces_a_declared_standard_rate(self) -> None: - """Ownership is per token direction, not per field. - - Filling the batch field from the published entry let that rate win, so a - deployment configuring only its standard rate had batches billed at the - published batch price instead of half the rate it configured. - """ - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - - model = "ft:gpt-3.5-turbo" - published = litellm.get_model_info(model=model) - assert published["input_cost_per_token_batches"] is not None - - deployment_id = "deploy-standard-input-only-1" - litellm.model_cost[deployment_id] = { - "id": deployment_id, - "input_cost_per_token": 1e-06, - "litellm_provider": "openai", - "mode": "chat", - } - obj = LiteLLMLoggingObj( - model=model, - messages=[], - stream=False, - call_type="aretrieve_batch", - start_time=time.time(), - litellm_call_id="direction-ownership", - function_id="f", - ) - obj.litellm_params = {"litellm_metadata": {"model_info": {"id": deployment_id}}, "model": model} - obj.model_call_details["model"] = model - try: - info = obj.get_router_deployment_model_info() - assert info is not None - assert info["input_cost_per_token"] == 1e-06 - assert info["input_cost_per_token_batches"] is None - assert info["output_cost_per_token"] == published["output_cost_per_token"] - assert info["output_cost_per_token_batches"] == published["output_cost_per_token_batches"] - finally: - litellm.model_cost.pop(deployment_id, None) - def test_merging_does_not_mutate_the_cached_model_info(self) -> None: """The published-rate merge must not write into get_model_info's lru-cached dict. @@ -2378,7 +2336,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() Test that _generate_cold_storage_object_key uses s3_path from custom logger instance. """ from datetime import datetime, timezone - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import MagicMock, patch from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -2425,7 +2383,7 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path(): Test that _generate_cold_storage_object_key falls back to empty s3_path when logger has no s3_path. """ from datetime import datetime, timezone - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import MagicMock, patch from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -3929,9 +3887,7 @@ def test_get_standard_logging_object_payload_carries_matched_access_groups(loggi "model": "gpt-4o", "messages": [], "litellm_params": { - "metadata": { - "user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"] - }, + "metadata": {"user_api_key_matched_model_access_groups": ["premium-pool", "shared-pool"]}, "proxy_server_request": {"body": {}}, }, }, @@ -4015,9 +3971,7 @@ def _model_router_response(selected_model: str, stamp: bool): from litellm.types.utils import ModelResponse response = ModelResponse(model=selected_model) - response._hidden_params = ( - {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {} - ) + response._hidden_params = {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {} return response @@ -4041,9 +3995,7 @@ def test_standard_logging_payload_uses_stamped_model_router_model(logging_obj): "messages": [], "litellm_params": {"metadata": {}}, }, - init_response_obj=_model_router_response( - "azure_ai/grok-4-1-fast-reasoning", stamp=True - ), + init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=True), start_time=now, end_time=now, logging_obj=logging_obj, @@ -4075,9 +4027,7 @@ def test_standard_logging_payload_keeps_requested_model_without_router_stamp( "messages": [], "litellm_params": {"metadata": {}}, }, - init_response_obj=_model_router_response( - "azure_ai/grok-4-1-fast-reasoning", stamp=False - ), + init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=False), start_time=now, end_time=now, logging_obj=logging_obj, @@ -5565,9 +5515,7 @@ class TestNonInferenceCallTypesAreNotBilled: init_response_obj=self._retrieved_response(), start_time=now, end_time=now, - logging_obj=self._logging_obj( - "aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA - ), + logging_obj=self._logging_obj("aget_responses", litellm_metadata=self.BACKGROUND_POLL_METADATA), status="success", ) @@ -5813,9 +5761,7 @@ async def test_streaming_success_callbacks_survive_cost_calculation_failure(): releasing.async_log_success_event = AsyncMock() patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing]) - with patcher, patch.object( - logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block") - ): + with patcher, patch.object(logging_obj, "_response_cost_calculator", side_effect=ValueError("bad usage block")): await logging_obj.async_success_handler(result=_assembled_stream_result()) assert logging_obj.model_call_details["response_cost"] is None @@ -5828,8 +5774,9 @@ async def test_streaming_success_callbacks_survive_standard_logging_payload_fail releasing.async_log_success_event = AsyncMock() patcher, logging_obj = _streaming_logging_obj_with_callbacks([releasing]) - with patcher, patch.object( - logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream") + with ( + patcher, + patch.object(logging_obj, "_build_standard_logging_payload", side_effect=ValueError("incomplete stream")), ): await logging_obj.async_success_handler(result=_assembled_stream_result()) @@ -6177,6 +6124,8 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa ) for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) + + def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch): """With LITELLM_OTEL_V2 on and operator credentials present, the "newrelic" callback builds the OTel v2 logger (per-team credential routing); with the @@ -6332,7 +6281,9 @@ def test_get_error_information_skips_traceback_for_budget_rejection_with_provide from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup assert litellm.log_client_error_tracebacks is False - over_budget = _raise_and_catch(litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic")) + over_budget = _raise_and_catch( + litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic") + ) result = StandardLoggingPayloadSetup.get_error_information(over_budget) assert result["error_code"] == "429" assert result["llm_provider"] == "anthropic" @@ -6905,9 +6856,7 @@ def test_passthrough_embeddings_result_swapped_for_callbacks(): ], "model": "EmbeddingsGigaR", }, - request=httpx.Request( - "POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings" - ), + request=httpx.Request("POST", "https://gigachat.devices.sberbank.ru/api/v1/embeddings"), ) _, _, swapped_result = logging_obj._success_handler_helper_fn( @@ -6926,12 +6875,14 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene request-level guardrail_status but never mask an intervention.""" flagged = {"guardrail_status": "guardrail_flagged"} - assert _get_status_fields( - "success", [{"guardrail_status": "success"}, flagged], None - )["guardrail_status"] == "guardrail_flagged" - assert _get_status_fields( - "success", [flagged, {"guardrail_status": "guardrail_intervened"}], None - )["guardrail_status"] == "guardrail_intervened" + assert ( + _get_status_fields("success", [{"guardrail_status": "success"}, flagged], None)["guardrail_status"] + == "guardrail_flagged" + ) + assert ( + _get_status_fields("success", [flagged, {"guardrail_status": "guardrail_intervened"}], None)["guardrail_status"] + == "guardrail_intervened" + ) def test_get_error_information_redacts_provider_key_from_upstream_url(): @@ -6984,22 +6935,41 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non return httpx.Response(200, json=mock_responses_api_response(content).model_dump()) if provider == "anthropic": - return httpx.Response(200, json={ - "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", - "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", - "usage": {"input_tokens": 10, "output_tokens": 5}, - }) + return httpx.Response( + 200, + json={ + "id": "msg-audit", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) if provider == "bedrock": - return httpx.Response(200, json={ - "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, - "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, - "metrics": {"latencyMs": 1}, - }) - return httpx.Response(200, json={ - "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", - "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - }) + return httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }, + ) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-audit", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) async def capture(kwargs, response_obj, start_time, end_time): logs.put_nowait(kwargs["standard_logging_object"]) @@ -7010,11 +6980,15 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler.client = http_client client: Final = ( AsyncAzureOpenAI( - api_key="transport-only", azure_endpoint="https://azure.invalid", - api_version="2025-04-01-preview", http_client=http_client, + api_key="transport-only", + azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", + http_client=http_client, ) - if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) - if provider == "openai" else handler + if provider == "azure" + else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" + else handler ) model: Final = { "openai": "openai/gpt-5.6", @@ -7027,23 +7001,44 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non async def run(marker: str) -> None: if provider == "responses": await litellm.aresponses( - model=model, api_key="transport-only", client=client, max_output_tokens=128, - instructions="classifier-rubric", input=marker, + model=model, + api_key="transport-only", + client=client, + max_output_tokens=128, + instructions="classifier-rubric", + input=marker, metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, + success_callback=[capture], + num_retries=0, ) return await litellm.acompletion( - model=model, api_key="transport-only", client=client, max_tokens=128, - aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + model=model, + api_key="transport-only", + client=client, + max_tokens=128, + aws_access_key_id="transport-only", + aws_secret_access_key="transport-only", + aws_region_name="us-east-1", messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, - **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), - **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} - if provider in ("openai", "azure") else {}), + success_callback=[capture], + num_retries=0, + **( + {"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} + if provider == "azure" + else {} + ), + **( + { + "extra_body": {"audit_context": "provider-extra"}, + "extra_headers": {"X-Audit": "header-only-secret"}, + } + if provider in ("openai", "azure") + else {} + ), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -7067,14 +7062,17 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non @pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("call_type", ["completion", "acompletion", "responses", "aresponses"]) -def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status, call_type): +def test_classifier_audit_obeys_message_logging_before_payload_emission( + logging_obj, monkeypatch, redaction, status, call_type +): from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") params: Final = { - "metadata": {"internal_call_origin": "autorouter_classifier", **( - {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} - )}, + "metadata": { + "internal_call_origin": "autorouter_classifier", + **({"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}), + }, "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, } logging_obj.call_type = call_type @@ -7087,8 +7085,12 @@ def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_ ) now: Final = datetime.datetime.now() payload: Final = get_standard_logging_object_payload( - kwargs={**logging_obj.model_call_details, "call_type": call_type}, init_response_obj={}, - start_time=now, end_time=now, logging_obj=logging_obj, status=status, + kwargs={**logging_obj.model_call_details, "call_type": call_type}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status=status, ) assert payload is not None if redaction == "none": diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index fe73bdba9cb..7a70a146667 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,4 +1,3 @@ -import json from collections.abc import Mapping, Sequence from typing import Final @@ -188,11 +187,7 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk( - thinking_blocks=[ - {"type": "thinking", "thinking": None, "signature": "sig_block1"} - ] - ), + make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block1"}]), make_chunk( thinking_blocks=[ { @@ -210,16 +205,10 @@ def test_get_combined_thinking_content_preserves_interleaved_blocks(): } ] ), - make_chunk( - thinking_blocks=[ - {"type": "thinking", "thinking": None, "signature": "sig_block2"} - ] - ), + make_chunk(thinking_blocks=[{"type": "thinking", "thinking": None, "signature": "sig_block2"}]), ] - thinking_chunks = [ - chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks") - ] + thinking_chunks = [chunk for chunk in chunks if chunk["choices"][0]["delta"].get("thinking_blocks")] processor = ChunkProcessor(chunks=chunks) result = processor.get_combined_thinking_content(thinking_chunks) @@ -264,9 +253,7 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=11779, total_tokens=11784, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=11775 - ), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=11775), cache_creation_input_tokens=4, cache_read_input_tokens=11775, ), @@ -300,9 +287,7 @@ def test_cache_read_input_tokens_retained(): prompt_tokens=0, total_tokens=214, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=0 - ), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0), cache_creation_input_tokens=0, cache_read_input_tokens=0, ), @@ -362,10 +347,7 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown(): ) # Sanity: the delta event genuinely lacks the breakdown - this is the input # condition that used to defeat cost calc. - assert ( - getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(message_delta_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def _usage_chunk(usage, finish_reason): return ModelResponseStream( @@ -400,7 +382,6 @@ def test_streaming_preserves_anthropic_1hr_cache_creation_breakdown(): assert usage.cache_read_input_tokens == 8728 - def test_streaming_keeps_cache_creation_breakdown_from_final_chunk(): """When the final usage chunk itself carries the cache-creation breakdown, aggregation must keep that breakdown instead of re-attaching a stale one @@ -485,9 +466,7 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): prompt_tokens=1234, total_tokens=1239, completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetails( - audio_tokens=None, cached_tokens=543 - ).model_dump(), + prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=543).model_dump(), ), index=2, ) @@ -504,6 +483,7 @@ def test_cache_read_input_tokens_retained_genericstreamingchunk(): assert usage.prompt_tokens_details.cached_tokens == 543 + def test_stream_chunk_builder_litellm_usage_chunks(): """ Validate ChunkProcessor.calculate_usage uses provided usage fields from streaming chunks @@ -577,9 +557,7 @@ def test_stream_chunk_builder_litellm_usage_chunks(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage( - chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="" - ) + usage = processor.calculate_usage(chunks=chunks, model="gemini/gemini-2.5-flash-lite", completion_output="") assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -623,15 +601,11 @@ def test_calculate_usage_honors_openai_sdk_completion_usage_chunks(): provider_specific_fields=None, stream_options={"include_usage": True}, ) - usage_chunk.usage = CompletionUsage( - prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704 - ) + usage_chunk.usage = CompletionUsage(prompt_tokens=20, completion_tokens=60, total_tokens=80, cost=0.000704) assert type(usage_chunk.usage) is CompletionUsage chunks = [content_chunk, usage_chunk] - usage = ChunkProcessor(chunks=chunks).calculate_usage( - chunks=chunks, model="mantle-claude", completion_output="" - ) + usage = ChunkProcessor(chunks=chunks).calculate_usage(chunks=chunks, model="mantle-claude", completion_output="") assert usage.prompt_tokens == 20 assert usage.completion_tokens == 60 @@ -654,9 +628,7 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4.1-nano-2025-04-14", "id": "chatcmpl-123", "choices": []}, ] - result = ChunkProcessor._get_model_from_chunks( - chunks=chunks, first_chunk_model="azure-model-router" - ) + result = ChunkProcessor._get_model_from_chunks(chunks=chunks, first_chunk_model="azure-model-router") # Should return the actual model, not the request model assert result == "gpt-4.1-nano-2025-04-14" @@ -667,9 +639,7 @@ def test_get_model_from_chunks_azure_model_router(): {"model": "gpt-4", "id": "chatcmpl-456", "choices": []}, ] - result_same = ChunkProcessor._get_model_from_chunks( - chunks=chunks_same_model, first_chunk_model="gpt-4" - ) + result_same = ChunkProcessor._get_model_from_chunks(chunks=chunks_same_model, first_chunk_model="gpt-4") # Should return the first chunk's model when all are the same assert result_same == "gpt-4" @@ -745,9 +715,7 @@ def test_stream_chunk_builder_anthropic_web_search(): chunks = [chunk1, chunk2] processor = ChunkProcessor(chunks=chunks) - usage = processor.calculate_usage( - chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" - ) + usage = processor.calculate_usage(chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="") assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 @@ -899,15 +867,11 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields(): ], ) chunk_dict = chunk.model_dump() - chunk_dict["_hidden_params"] = { - "provider_specific_fields": {"traffic_type": "default"} - } + chunk_dict["_hidden_params"] = {"provider_specific_fields": {"traffic_type": "default"}} response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert ( - response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" - ) + assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): @@ -952,10 +916,7 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): assert getattr(response, "vertex_ai_grounding_metadata") == grounding_metadata assert getattr(response, "vertex_ai_url_context_metadata") == url_context_metadata assert response._hidden_params["vertex_ai_grounding_metadata"] == grounding_metadata - assert ( - response._hidden_params["vertex_ai_url_context_metadata"] - == url_context_metadata - ) + assert response._hidden_params["vertex_ai_url_context_metadata"] == url_context_metadata dumped = response.model_dump() assert dumped["vertex_ai_grounding_metadata"] == grounding_metadata @@ -1002,9 +963,7 @@ def test_stream_chunk_builder_uses_assembled_model_for_provider_metadata(): def test_stream_chunk_builder_propagates_vertex_ai_safety_results(): """Assembled response must expose safety data under the non-streaming field name.""" - safety_ratings = [ - [{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}] - ] + safety_ratings = [[{"category": "HARM_CATEGORY_HATE_SPEECH", "probability": "NEGLIGIBLE"}]] chunk = ModelResponseStream( id="chatcmpl-vertex-safety", @@ -1046,18 +1005,12 @@ def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_dict_chunks(): ) ], ).model_dump() - chunk_dict["_hidden_params"] = { - "vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}] - } + chunk_dict["_hidden_params"] = {"vertex_ai_grounding_metadata": [{"webSearchQueries": ["test query"]}]} response = stream_chunk_builder(chunks=[chunk_dict]) assert response is not None - assert getattr(response, "vertex_ai_grounding_metadata") == [ - {"webSearchQueries": ["test query"]} - ] - assert response.model_dump()["vertex_ai_grounding_metadata"] == [ - {"webSearchQueries": ["test query"]} - ] + assert getattr(response, "vertex_ai_grounding_metadata") == [{"webSearchQueries": ["test query"]}] + assert response.model_dump()["vertex_ai_grounding_metadata"] == [{"webSearchQueries": ["test query"]}] def test_cost_field_in_usage_chunks(): @@ -1066,29 +1019,21 @@ def test_cost_field_in_usage_chunks(): id="chatcmpl-1", created=1745513206, model="openrouter/claude", - choices=[ - StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) - ], + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], usage=chunk1_usage, ) - chunk2_usage = Usage( - completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025 - ) + chunk2_usage = Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.00025) chunk2 = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openrouter/claude", - choices=[ - StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) - ], + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], usage=chunk2_usage, ) processor = ChunkProcessor(chunks=[chunk1, chunk2]) - usage = processor.calculate_usage( - chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi" - ) + usage = processor.calculate_usage(chunks=[chunk1, chunk2], model="openrouter/claude", completion_output="Hi") assert hasattr(usage, "cost") assert usage.cost == 0.00025 @@ -1122,45 +1067,6 @@ def test_stream_chunk_builder_tolerates_trailing_chunk_without_choices(): assert response.choices[0].message.content == "Hello world" -def test_anthropic_speed_and_geo_survive_stream_assembly(): - """Anthropic prices fast mode and non-global regions with a multiplier read off - ``usage.speed`` / ``usage.inference_geo``. Dropping them while reassembling a stream - bills streamed fast-mode calls at the standard rate.""" - from litellm.llms.anthropic.cost_calculation import cost_per_token - - def _usage(**extra): - usage = Usage(completion_tokens=100, prompt_tokens=1000, total_tokens=1100) - for key, value in extra.items(): - setattr(usage, key, value) - return usage - - def _chunk(usage): - return ModelResponseStream( - id="chatcmpl-1", - created=1745513206, - model="claude-opus-4-8", - choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="Hi"))], - usage=usage, - ) - - fast_chunk = _chunk(_usage(speed="fast", inference_geo="global")) - fast_usage = ChunkProcessor(chunks=[fast_chunk]).calculate_usage( - chunks=[fast_chunk], model="claude-opus-4-8", completion_output="Hi" - ) - standard_chunk = _chunk(_usage(inference_geo="global")) - standard_usage = ChunkProcessor(chunks=[standard_chunk]).calculate_usage( - chunks=[standard_chunk], model="claude-opus-4-8", completion_output="Hi" - ) - - assert fast_usage.speed == "fast" - assert fast_usage.inference_geo == "global" - assert getattr(standard_usage, "speed", None) is None - - fast_cost = sum(cost_per_token(model="claude-opus-4-8", usage=fast_usage)) - standard_cost = sum(cost_per_token(model="claude-opus-4-8", usage=standard_usage)) - assert fast_cost == pytest.approx(standard_cost * 2.0) - - def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): """Regression for #34801: a trailing usage chunk that omits `prompt_tokens_details` must not wipe the OpenAI cache-read/cache-write split, @@ -1171,25 +1077,19 @@ def test_prompt_tokens_details_survive_later_usage_chunk_without_details(): id="chatcmpl-1", created=1745513206, model="openai/gpt-5.6-sol", - choices=[ - StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi")) - ], + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="Hi"))], usage=Usage( prompt_tokens=6017, completion_tokens=4, total_tokens=6021, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=6004, cache_write_tokens=10 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=6004, cache_write_tokens=10), ), ) chunk_without_details = ModelResponseStream( id="chatcmpl-1", created=1745513207, model="openai/gpt-5.6-sol", - choices=[ - StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) - ], + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))], usage=Usage(prompt_tokens=6017, completion_tokens=4, total_tokens=6021), ) @@ -1472,9 +1372,7 @@ def test_calculate_usage_fills_unknown_split_from_reasoning_estimate( assert usage.completion_tokens_details.text_tokens == expected_text_tokens -def _openai_chunk( - choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None -) -> dict[str, object]: +def _openai_chunk(choices: Sequence[Mapping[str, object]], usage: Mapping[str, int] | None = None) -> dict[str, object]: base: Final = { "id": "chatcmpl-lit6552", "object": "chat.completion.chunk", diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 269c351f866..a4b05da7023 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,4 +1,3 @@ - import pytest from unittest.mock import MagicMock, patch @@ -33,13 +32,9 @@ def test_response_format_transformation_unit_test(): "additionalProperties": False, } - result = config._create_json_tool_call_for_response_format( - json_schema=response_format_json_schema - ) + result = config._create_json_tool_call_for_response_format(json_schema=response_format_json_schema) - assert result["input_schema"]["properties"] == { - "agent_doing": {"title": "Agent Doing", "type": "string"} - } + assert result["input_schema"]["properties"] == {"agent_doing": {"title": "Agent Doing", "type": "string"}} print(result) @@ -550,9 +545,7 @@ def test_extract_response_content_with_citations(): }, } - _, citations, _, _, _, _, _, _ = config.extract_response_content( - completion_response - ) + _, citations, _, _, _, _, _, _ = config.extract_response_content(completion_response) assert citations == [ [ { @@ -625,12 +618,8 @@ def test_web_search_tool_transformation(): assert anthropic_web_search_tool["user_location"]["city"] == "San Francisco" -@pytest.mark.parametrize( - "search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)] -) -def test_web_search_tool_transformation_with_search_context_size( - search_context_size, expected_max_uses -): +@pytest.mark.parametrize("search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)]) +def test_web_search_tool_transformation_with_search_context_size(search_context_size, expected_max_uses): from litellm.types.llms.openai import OpenAIWebSearchOptions config = AnthropicConfig() @@ -805,10 +794,7 @@ def test_web_search_tool_result_in_provider_specific_fields(): assert "web_search_results" in provider_fields assert len(provider_fields["web_search_results"]) == 1 assert provider_fields["web_search_results"][0]["type"] == "web_search_tool_result" - assert ( - provider_fields["web_search_results"][0]["tool_use_id"] - == "srvtoolu_provider_test" - ) + assert provider_fields["web_search_results"][0]["tool_use_id"] == "srvtoolu_provider_test" def test_multiple_web_search_tool_results(): @@ -1032,10 +1018,7 @@ def test_transform_response_with_prefix_prompt(): ) assert result is not None - assert ( - result.choices[0].message.content - == "You are a helpful assistant. The grass is green." - ) + assert result.choices[0].message.content == "You are a helpful assistant. The grass is green." def test_get_supported_params_thinking(): @@ -1150,18 +1133,12 @@ def test_anthropic_beta_header_merging_with_output_format(): } } - result_headers = config.update_headers_with_optional_anthropic_beta( - headers, optional_params - ) + result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params) # Both beta headers should be present beta_value = result_headers["anthropic-beta"] - assert ( - "context-1m-2025-08-07" in beta_value - ), f"User's context-1m beta header missing from: {beta_value}" - assert ( - "structured-outputs-2025-11-13" in beta_value - ), f"Structured output beta header missing from: {beta_value}" + assert "context-1m-2025-08-07" in beta_value, f"User's context-1m beta header missing from: {beta_value}" + assert "structured-outputs-2025-11-13" in beta_value, f"Structured output beta header missing from: {beta_value}" def test_anthropic_beta_header_merging_with_multiple_features(): @@ -1183,9 +1160,7 @@ def test_anthropic_beta_header_merging_with_multiple_features(): "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}], } - result_headers = config.update_headers_with_optional_anthropic_beta( - headers, optional_params - ) + result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params) beta_value = result_headers["anthropic-beta"] @@ -1228,9 +1203,7 @@ def test_anthropic_structured_output_beta_header(): "strict": True, "schema": { "description": 'Progress report for the thinking process\n\nThis model represents a snapshot of the agent\'s current progress during\nthe thinking process, providing a brief description of the current activity.\n\nAttributes:\n agent_doing: Brief description of what the agent is currently doing.\n Should be kept under 10 words. Example: "Learning about home automation"', - "properties": { - "agent_doing": {"title": "Agent Doing", "type": "string"} - }, + "properties": {"agent_doing": {"title": "Agent Doing", "type": "string"}}, "required": ["agent_doing"], "title": "ThinkingStep", "type": "object", @@ -1244,10 +1217,7 @@ def test_anthropic_structured_output_beta_header(): assert response is not None print(f"response: {response}") print(f"raw_request_headers: {response['raw_request_headers']}") - assert ( - "structured-outputs-2025-11-13" - in response["raw_request_headers"]["anthropic-beta"] - ) + assert "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] @pytest.mark.parametrize( @@ -1383,9 +1353,7 @@ def test_tool_search_regex_detection(): config = AnthropicModelInfo() # Test with tool search regex tool - tools = [ - {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} - ] + tools = [{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}] assert config.is_tool_search_used(tools) is True # Test without tool search @@ -1400,9 +1368,7 @@ def test_tool_search_bm25_detection(): config = AnthropicModelInfo() # Test with tool search BM25 tool - tools = [ - {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} - ] + tools = [{"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}] assert config.is_tool_search_used(tools) is True @@ -1594,9 +1560,7 @@ def test_tool_search_complete_response_parsing(): "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "content": { "type": "tool_search_tool_search_result", - "tool_references": [ - {"type": "tool_reference", "tool_name": "get_weather"} - ], + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}], }, }, {"type": "text", "text": "Great! I found a weather tool."}, @@ -1647,9 +1611,7 @@ def test_tool_search_complete_response_parsing(): assert usage.server_tool_use is not None assert usage.server_tool_use.web_search_requests == 0 - assert ( - usage.server_tool_use.tool_search_requests == 1 - ) # Counted from server_tool_use blocks + assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks def test_allowed_callers_field_preservation(): @@ -1701,9 +1663,7 @@ def test_programmatic_tool_calling_beta_header(): assert is_programmatic is True # Test header generation - headers = model_info.get_anthropic_headers( - api_key="test-key", programmatic_tool_calling_used=True - ) + headers = model_info.get_anthropic_headers(api_key="test-key", programmatic_tool_calling_used=True) assert "anthropic-beta" in headers assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] @@ -1847,9 +1807,7 @@ def test_input_examples_beta_header(): assert is_examples_used is True # Test header generation - headers = model_info.get_anthropic_headers( - api_key="test-key", input_examples_used=True - ) + headers = model_info.get_anthropic_headers(api_key="test-key", input_examples_used=True) assert "anthropic-beta" in headers assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] @@ -1935,10 +1893,7 @@ def test_input_examples_empty_list_not_added(): transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None # Empty list should not be added - assert ( - "input_examples" not in transformed_tool - or len(transformed_tool.get("input_examples", [])) == 0 - ) + assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 # ============ Effort Parameter Tests ============ @@ -1998,9 +1953,7 @@ def test_effort_beta_header_injection(): effort_used = model_info.is_effort_used(optional_params=optional_params, custom_llm_provider="anthropic") assert effort_used is True - headers = model_info.get_anthropic_headers( - api_key="test-key", effort_used=effort_used - ) + headers = model_info.get_anthropic_headers(api_key="test-key", effort_used=effort_used) assert "anthropic-beta" in headers assert "effort-2025-11-24" in headers["anthropic-beta"] @@ -2026,9 +1979,7 @@ def test_effort_validation(): optional_params = {"output_config": {"effort": "invalid"}} - with pytest.raises( - litellm.exceptions.BadRequestError, match="Invalid effort value" - ): + with pytest.raises(litellm.exceptions.BadRequestError, match="Invalid effort value"): config.transform_request( model="claude-opus-4-5-20251101", messages=messages, @@ -2264,16 +2215,8 @@ def test_anthropic_model_supports_speed_param_rejects_non_anthropic_providers( ): """Fast mode is direct-Anthropic-only. Vertex/Azure/Bedrock strip their prefix before the shared transform runs, so the bare Opus id must still be rejected.""" - assert ( - AnthropicConfig._model_supports_speed_param( - "claude-opus-4-8", custom_llm_provider - ) - is False - ) - assert ( - AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") - is True - ) + assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", custom_llm_provider) is False + assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") is True def test_vertex_anthropic_drops_speed_for_opus_with_drop_params(monkeypatch): @@ -2464,42 +2407,6 @@ def test_get_max_tokens_for_model_none(): assert max_tokens == 4096 -def test_get_config_with_model_uses_dynamic_max_tokens(): - """ - Test that get_config returns dynamic max_tokens based on model. - - Fixes: https://github.com/BerriAI/litellm/issues/8835 - """ - - def _mock_get_max_tokens(model): - """Return expected max_output_tokens for each model.""" - model_map = { - "claude-3-sonnet-20240229": 4096, - "claude-3-5-sonnet-20241022": 8192, - "claude-3-7-sonnet-20250219": 64000, - } - result = model_map.get(model) - if result is None: - raise Exception(f"Model {model} not found") - return result - - with patch( - "litellm.llms.anthropic.chat.transformation.get_max_tokens", - side_effect=_mock_get_max_tokens, - ): - # Claude 3 model should get 4096 - config_claude3 = AnthropicConfig.get_config(model="claude-3-sonnet-20240229") - assert config_claude3["max_tokens"] == 4096 - - # Claude 3.5 model should get 8192 - config_claude35 = AnthropicConfig.get_config(model="claude-3-5-sonnet-20241022") - assert config_claude35["max_tokens"] == 8192 - - # Claude 3.7 model should get 64000 (64K default, 128K requires beta header) - config_claude37 = AnthropicConfig.get_config(model="claude-3-7-sonnet-20250219") - assert config_claude37["max_tokens"] == 64000 - - def test_get_config_without_model_uses_fallback(): """ Test that get_config without model parameter uses 4096 fallback. @@ -2557,9 +2464,7 @@ def test_supports_effort_level_handles_provider_prefixes(model, level, expected) ("claude-opus-4-5-20251101", None, False), ], ) -def test_validate_effort_for_model_centralises_per_model_gating( - model, effort, expect_error -): +def test_validate_effort_for_model_centralises_per_model_gating(model, effort, expect_error): err = AnthropicConfig._validate_effort_for_model(model, effort, "anthropic") if expect_error: assert err is not None @@ -2608,11 +2513,7 @@ def test_transform_request_injects_dummy_tool_without_tools_param(): litellm.modify_params = prev_modify_params assert "tools" in result - names = [ - t.get("name") - for t in result["tools"] - if isinstance(t, dict) and t.get("name") is not None - ] + names = [t.get("name") for t in result["tools"] if isinstance(t, dict) and t.get("name") is not None] assert "dummy_tool" in names @@ -2678,13 +2579,9 @@ def test_calculate_usage_completion_tokens_details_with_reasoning(): "output_tokens": 500, } # Simulating reasoning content that would count as ~50 tokens - reasoning_content = ( - "Let me think about this step by step. " * 10 - ) # Roughly 50 tokens + reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens - usage = config.calculate_usage( - usage_object=usage_object, reasoning_content=reasoning_content - ) + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=reasoning_content) # completion_tokens_details should be populated with both reasoning and text tokens assert usage.completion_tokens_details is not None @@ -2735,9 +2632,7 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): # reasoning_effort should not be in the result (it's transformed to thinking) assert "reasoning_effort" not in result # Should set output_config with the mapped effort value - assert ( - "output_config" in result - ), f"output_config missing for {model} with effort={effort}" + assert "output_config" in result, f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort_map[effort] @@ -2838,9 +2733,7 @@ def test_raw_adaptive_thinking_untouched_for_46_plus_model(): ("gpt-4o", False), ], ) -def test_is_adaptive_thinking_model_is_sourced_from_cost_map( - local_model_cost_map, model, expected -): +def test_is_adaptive_thinking_model_is_sourced_from_cost_map(local_model_cost_map, model, expected): """Adaptive thinking resolves from the cost map first (an explicit supports_adaptive_thinking entry, or the anthropic-claude fallback rule for unmapped future Claudes), then from a date-safe opus/sonnet/haiku >= 4.6 name version as a @@ -2956,9 +2849,7 @@ def test_reasoning_effort_sets_output_config_for_46_models(): drop_params=False, ) - assert ( - "output_config" in result - ), f"output_config missing for {model} with effort={effort}" + assert "output_config" in result, f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort @@ -2997,9 +2888,7 @@ def test_reasoning_effort_does_not_set_output_config_for_older_models(): drop_params=False, ) - assert ( - "output_config" not in result - ), f"output_config should not be set for {model}" + assert "output_config" not in result, f"output_config should not be set for {model}" @pytest.mark.parametrize( @@ -3039,14 +2928,10 @@ def test_reasoning_effort_accepts_dict_shape_for_adaptive_model(reasoning_effort ) # thinking must be set (adaptive for 4.6+) - assert ( - "thinking" in result - ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" + assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "adaptive" # output_config must carry the mapped effort - assert ( - "output_config" in result - ), f"output_config missing for reasoning_effort={reasoning_effort_value!r}" + assert "output_config" in result, f"output_config missing for reasoning_effort={reasoning_effort_value!r}" assert result["output_config"]["effort"] == "low" @@ -3075,16 +2960,13 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( drop_params=False, ) - assert ( - "thinking" in result - ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}" + assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}" assert result["thinking"]["type"] == "enabled" assert "budget_tokens" in result["thinking"] assert result["thinking"]["budget_tokens"] > 0 # Older models must not get adaptive-thinking output_config assert "output_config" not in result, ( - f"output_config should not be set for non-adaptive model " - f"(reasoning_effort={reasoning_effort_value!r})" + f"output_config should not be set for non-adaptive model (reasoning_effort={reasoning_effort_value!r})" ) @@ -3135,12 +3017,8 @@ def test_reasoning_effort_unparseable_dict_is_dropped(bad_value): model="claude-sonnet-4-6-20260219", drop_params=False, ) - assert ( - "thinking" not in result - ), f"thinking should not be set for bad value {bad_value!r}" - assert ( - "output_config" not in result - ), f"output_config should not be set for bad value {bad_value!r}" + assert "thinking" not in result, f"thinking should not be set for bad value {bad_value!r}" + assert "output_config" not in result, f"output_config should not be set for bad value {bad_value!r}" @pytest.mark.parametrize( @@ -3202,27 +3080,6 @@ def test_max_effort_accepted_for_opus_47(): assert result["output_config"]["effort"] == "max" -def test_effort_beta_header_not_injected_for_46_models(): - """ - Test that is_effort_used returns False for Claude 4.6 models. - - Claude 4.6 models use output_config as a stable API feature — - no beta header should be injected. - """ - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - model_info = AnthropicModelInfo() - - for model in ["claude-opus-4-6-20250514", "claude-sonnet-4-6-20260219"]: - # Even with output_config present, should return False for 4.6 models - result = model_info.is_effort_used( - optional_params={"output_config": {"effort": "high"}}, - model=model, - custom_llm_provider="anthropic", - ) - assert result is False, f"is_effort_used should return False for {model}" - - @pytest.mark.parametrize( "model", [ @@ -3271,9 +3128,7 @@ def test_reasoning_effort_garbage_raises_bad_request(effort): ("max", DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET), ], ) -def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model( - effort, expected_budget -): +def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model(effort, expected_budget): """``xhigh`` / ``max`` extend the budget_tokens progression on budget-mode models.""" config = AnthropicConfig() @@ -3318,23 +3173,6 @@ def test_reasoning_effort_minimal_floors_at_anthropic_provider_minimum(): assert result["thinking"]["budget_tokens"] >= 1024 -def test_effort_beta_header_still_injected_for_older_models(): - """ - Test that is_effort_used still returns True for pre-4.6 models - when output_config is present. - """ - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - model_info = AnthropicModelInfo() - - result = model_info.is_effort_used( - optional_params={"output_config": {"effort": "low"}}, - model="claude-opus-4-5-20251101", - custom_llm_provider="anthropic", - ) - assert result is True - - def test_code_execution_tool_results_extraction(): """ Test that code execution tool results (bash_code_execution_tool_result, @@ -3420,17 +3258,11 @@ def test_code_execution_tool_results_extraction(): # Verify first tool call assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert ( - transformed_response.choices[0].message.tool_calls[0].function.name - == "bash_code_execution" - ) + assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" # Verify second tool call assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert ( - transformed_response.choices[0].message.tool_calls[1].function.name - == "text_editor_code_execution" - ) + assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields @@ -3453,10 +3285,7 @@ def test_code_execution_tool_results_extraction(): assert editor_result["content"]["is_file_update"] is False # Verify text content is properly concatenated - assert ( - "I'll calculate that for you." - in transformed_response.choices[0].message.content - ) + assert "I'll calculate that for you." in transformed_response.choices[0].message.content assert "Done!" in transformed_response.choices[0].message.content @@ -3524,10 +3353,7 @@ def test_code_execution_tool_results_in_hidden_params(): assert "provider_specific_fields" in hidden assert "tool_results" in hidden["provider_specific_fields"] assert len(hidden["provider_specific_fields"]["tool_results"]) == 1 - assert ( - hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] - == "hello\n" - ) + assert hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] == "hello\n" def test_tool_search_tool_result_not_in_tool_results(): @@ -3723,10 +3549,7 @@ def test_compaction_block_in_provider_specific_fields(): assert "compaction_blocks" in provider_fields assert len(provider_fields["compaction_blocks"]) == 1 assert provider_fields["compaction_blocks"][0]["type"] == "compaction" - assert ( - "Summary of the conversation" - in provider_fields["compaction_blocks"][0]["content"] - ) + assert "Summary of the conversation" in provider_fields["compaction_blocks"][0]["content"] def test_multiple_compaction_blocks(): @@ -3774,9 +3597,7 @@ def test_compaction_block_request_transformation(): {"role": "user", "content": "What is the weather in San Francisco?"}, { "role": "assistant", - "content": [ - {"type": "text", "text": "I don't have access to real-time data."} - ], + "content": [{"type": "text", "text": "I don't have access to real-time data."}], "provider_specific_fields": { "compaction_blocks": [ { @@ -3789,9 +3610,7 @@ def test_compaction_block_request_transformation(): {"role": "user", "content": "What about New York?"}, ] - result = anthropic_messages_pt( - messages=messages, model="claude-opus-4-6", llm_provider="anthropic" - ) + result = anthropic_messages_pt(messages=messages, model="claude-opus-4-6", llm_provider="anthropic") # Find the assistant message assistant_message = None @@ -3905,9 +3724,7 @@ def test_map_openai_context_management_to_anthropic(): "instructions": "Focus on preserving code snippets", } ] - result = config.map_openai_context_management_to_anthropic( - openai_format_with_instructions - ) + result = config.map_openai_context_management_to_anthropic(openai_format_with_instructions) assert result is not None assert result["edits"][0]["trigger"]["value"] == 150000 @@ -3934,9 +3751,7 @@ def test_map_openai_params_with_context_management(): config = AnthropicConfig() # Test with OpenAI list format - non_default_params = { - "context_management": [{"type": "compaction", "compact_threshold": 200000}] - } + non_default_params = {"context_management": [{"type": "compaction", "compact_threshold": 200000}]} optional_params = {} result = config.map_openai_params( @@ -3973,10 +3788,7 @@ def test_map_openai_params_with_context_management(): ) assert "context_management" in result - assert ( - result["context_management"] - == non_default_params_anthropic["context_management"] - ) + assert result["context_management"] == non_default_params_anthropic["context_management"] def test_cache_control_in_supported_params(): @@ -4087,10 +3899,7 @@ def test_compaction_block_empty_list_not_added(): # Verify compaction_blocks is not in provider_specific_fields when there are none provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: - assert ( - "compaction_blocks" not in provider_fields - or provider_fields.get("compaction_blocks") is None - ) + assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None def test_fast_mode_beta_header(): @@ -4139,9 +3948,7 @@ def test_fast_mode_usage_calculation(): "output_tokens": 500, } - usage = config.calculate_usage( - usage_object=usage_object, reasoning_content=None, speed="fast" - ) + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None, speed="fast") assert usage.prompt_tokens == 1000 assert usage.completion_tokens == 500 @@ -4149,48 +3956,6 @@ def test_fast_mode_usage_calculation(): assert usage.speed == "fast" -def test_fast_mode_cost_calculation(): - """ - Test that fast mode applies the 'fast' multiplier from provider_specific_entry - on top of the base model cost (1.1x for claude-opus-4-6). - """ - - from litellm.llms.anthropic.cost_calculation import cost_per_token - from litellm.types.utils import Usage - - base_prompt = 0.005 - base_completion = 0.025 - - with ( - patch( - "litellm.llms.anthropic.cost_calculation.generic_cost_per_token" - ) as mock_cost, - patch("litellm.get_model_info") as mock_info, - ): - mock_cost.return_value = (base_prompt, base_completion) - mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} - - usage_fast = Usage( - prompt_tokens=1000, - completion_tokens=1000, - speed="fast", - ) - - prompt_cost, completion_cost = cost_per_token( - model="claude-opus-4-6", - usage=usage_fast, - ) - - # generic_cost_per_token called with the plain base model name - mock_cost.assert_called_once() - assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" - assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" - - # 1.1x multiplier applied - assert abs(prompt_cost - base_prompt * 1.1) < 1e-10 - assert abs(completion_cost - base_completion * 1.1) < 1e-10 - - def test_fast_mode_with_inference_geo(): """ Test that fast mode + inference_geo both apply their multipliers from @@ -4204,9 +3969,7 @@ def test_fast_mode_with_inference_geo(): base_completion = 0.025 with ( - patch( - "litellm.llms.anthropic.cost_calculation.generic_cost_per_token" - ) as mock_cost, + patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, patch("litellm.get_model_info") as mock_info, ): mock_cost.return_value = (base_prompt, base_completion) @@ -4397,9 +4160,7 @@ def test_map_tool_helper_enforces_object_type_when_missing(): "name": "search_code", "description": "Search for code patterns", "parameters": { - "properties": { - "query": {"type": "string", "description": "Search query"} - }, + "properties": {"query": {"type": "string", "description": "Search query"}}, "required": ["query"], }, }, @@ -4412,9 +4173,9 @@ def test_map_tool_helper_enforces_object_type_when_missing(): assert "properties" in result["input_schema"] assert "query" in result["input_schema"]["properties"] # Original parameters dict must not be modified in place - assert ( - tool["function"]["parameters"] == original_params - ), "parameters dict was mutated; _map_tool_helper should not modify caller data" + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) def test_map_tool_helper_enforces_object_type_when_wrong_type(): @@ -4440,13 +4201,13 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type(): result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" - assert ( - result["input_schema"].get("properties") == {} - ), "properties should be injected as {} when schema has non-object type and no properties key" + assert result["input_schema"].get("properties") == {}, ( + "properties should be injected as {} when schema has non-object type and no properties key" + ) # Original parameters dict must not be modified in place - assert ( - tool["function"]["parameters"] == original_params - ), "parameters dict was mutated; _map_tool_helper should not modify caller data" + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) def test_map_tool_helper_preserves_valid_object_schema(): @@ -4513,12 +4274,8 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "Hello"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( - completion_response_null - ) - assert ( - thinking_blocks is not None - ), "thinking blocks should not be None when thinking=null" + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_null) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking=null" assert len(thinking_blocks) == 1 assert "Hello" in text @@ -4529,12 +4286,8 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "World"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( - completion_response_missing - ) - assert ( - thinking_blocks is not None - ), "thinking blocks should not be None when thinking key is absent" + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_missing) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking key is absent" assert len(thinking_blocks) == 1 assert "World" in text @@ -4545,9 +4298,7 @@ def test_extract_response_content_thinking_block_null_thinking(): {"type": "text", "text": "Done"}, ] } - text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( - completion_response_text - ) + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_text) assert thinking_blocks is not None assert len(thinking_blocks) == 1 assert thinking_blocks[0]["thinking"] == "Let me think..." @@ -4606,12 +4357,8 @@ def test_advisor_beta_header_injected(): } ] } - result = config.update_headers_with_optional_anthropic_beta( - headers, optional_params - ) - assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get( - "anthropic-beta", "" - ) + result = config.update_headers_with_optional_anthropic_beta(headers, optional_params) + assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get("anthropic-beta", "") def test_advisor_beta_header_not_injected_without_tool(): @@ -4619,9 +4366,7 @@ def test_advisor_beta_header_not_injected_without_tool(): config = AnthropicConfig() headers: dict = {} optional_params: dict = {"tools": []} - result = config.update_headers_with_optional_anthropic_beta( - headers, optional_params - ) + result = config.update_headers_with_optional_anthropic_beta(headers, optional_params) assert "advisor-tool-2026-03-01" not in result.get("anthropic-beta", "") @@ -4648,9 +4393,7 @@ def test_advisor_tool_result_preserved_in_response(): {"type": "text", "text": "Here is the implementation."}, ] } - text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content( - completion_response - ) + text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content(completion_response) assert "Consulting advisor." in text assert "Here is the implementation." in text # server_tool_use (advisor) should be a tool_call @@ -4765,9 +4508,7 @@ def test_basic_sanitize_anthropic_tool_name_replaces_invalid_chars(): ) assert ( - _basic_sanitize_anthropic_tool_name( - "github_openapi_mcp-actions/download-job-logs-for-workflow-run" - ) + _basic_sanitize_anthropic_tool_name("github_openapi_mcp-actions/download-job-logs-for-workflow-run") == "github_openapi_mcp-actions_download-job-logs-for-workflow-run" ) # other punctuation @@ -4796,9 +4537,7 @@ def test_build_anthropic_tool_name_maps_no_collisions(): ] ) assert forward == { - "actions/download-job-logs-for-workflow-run": ( - "actions_download-job-logs-for-workflow-run" - ), + "actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run"), "pulls/list-files": "pulls_list-files", } assert reverse == {v: k for k, v in forward.items()} @@ -4849,9 +4588,7 @@ def test_build_anthropic_tool_name_maps_three_way_collision(): _build_anthropic_tool_name_maps, ) - forward, reverse = _build_anthropic_tool_name_maps( - ["foo_bar", "foo/bar", "foo.bar"] - ) + forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar", "foo.bar"]) assert "foo_bar" not in forward # untouched assert forward["foo/bar"] == "foo_bar_2" assert forward["foo.bar"] == "foo_bar_3" @@ -4924,16 +4661,13 @@ def test_map_openai_params_does_not_pollute_optional_params_with_internal_keys() ) # No internal keys may appear in optional_params for ANY input. for key in optional_params: - assert not key.startswith( - "_anthropic_tool_name" - ), f"optional_params leaked internal key {key!r}: {optional_params}" + assert not key.startswith("_anthropic_tool_name"), ( + f"optional_params leaked internal key {key!r}: {optional_params}" + ) # And no key starting with `_` either; optional_params should only # contain documented Anthropic Messages API parameters. for key in optional_params: - assert not key.startswith("_"), ( - f"optional_params leaked underscore-prefixed key {key!r}: " - f"{optional_params}" - ) + assert not key.startswith("_"), f"optional_params leaked underscore-prefixed key {key!r}: {optional_params}" def test_map_openai_params_no_maps_when_all_names_already_valid(): @@ -4962,11 +4696,7 @@ def test_map_openai_params_no_maps_when_all_names_already_valid(): def test_rewrite_tool_names_in_messages_uses_forward_map(): config = AnthropicConfig() - forward_map = { - "actions/download-job-logs-for-workflow-run": ( - "actions_download-job-logs-for-workflow-run" - ) - } + forward_map = {"actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run")} messages = [ {"role": "user", "content": "go"}, { @@ -4989,15 +4719,9 @@ def test_rewrite_tool_names_in_messages_uses_forward_map(): out = config._rewrite_tool_names_in_messages(messages, forward_map) # input list must not be mutated - assert ( - messages[1]["tool_calls"][0]["function"]["name"] - == "actions/download-job-logs-for-workflow-run" - ) + assert messages[1]["tool_calls"][0]["function"]["name"] == "actions/download-job-logs-for-workflow-run" # output rewritten according to forward map - assert ( - out[1]["tool_calls"][0]["function"]["name"] - == "actions_download-job-logs-for-workflow-run" - ) + assert out[1]["tool_calls"][0]["function"]["name"] == "actions_download-job-logs-for-workflow-run" # non-tool-call messages pass through unchanged (same object) assert out[0] is messages[0] assert out[2] is messages[2] @@ -5073,9 +4797,7 @@ def test_sanitize_tool_names_in_request_does_not_mutate_caller_tool_dicts(): caller_tools = [caller_tool] optional_params: dict = {"tools": caller_tools} - forward, reverse = config._sanitize_tool_names_in_request( - optional_params=optional_params - ) + forward, reverse = config._sanitize_tool_names_in_request(optional_params=optional_params) assert forward.get(original_name) sanitized = forward[original_name] @@ -5224,10 +4946,7 @@ def test_streaming_iterator_reverse_maps_tool_use_name(): parsed = iterator.chunk_parser(chunk=chunk) tool_calls = parsed.choices[0].delta.tool_calls assert tool_calls is not None and len(tool_calls) == 1 - assert ( - tool_calls[0]["function"]["name"] - == "actions/download-job-logs-for-workflow-run" - ) + assert tool_calls[0]["function"]["name"] == "actions/download-job-logs-for-workflow-run" def test_streaming_iterator_passthrough_when_name_not_in_map(): @@ -5323,9 +5042,9 @@ def test_transform_request_does_not_leak_internal_keys_into_body(): for tool in data.get("tools", []): name = tool.get("name") assert isinstance(name, str) - assert _re.fullmatch( - r"[a-zA-Z0-9_-]{1,128}", name - ), f"sanitized tool name {name!r} still violates Anthropic regex" + assert _re.fullmatch(r"[a-zA-Z0-9_-]{1,128}", name), ( + f"sanitized tool name {name!r} still violates Anthropic regex" + ) # Sent name for the bad tool is the disambiguated form, valid name passes through. sent_names = {t["name"] for t in data["tools"]} @@ -5461,9 +5180,7 @@ def test_transform_request_rewrites_tool_names_in_history(): for block in content: if isinstance(block, dict) and block.get("type") == "tool_use": tool_use_names.append(block.get("name")) - assert ( - tool_use_names - ), "expected at least one tool_use block in transformed messages" + assert tool_use_names, "expected at least one tool_use block in transformed messages" for name in tool_use_names: assert name == "actions_download-job-logs-for-workflow-run", ( f"history tool_use.name {name!r} not rewritten -- Anthropic will " @@ -5487,19 +5204,12 @@ def test_sanitize_tool_names_in_request_skips_hosted_tools(): } forward, reverse = AnthropicConfig._sanitize_tool_names_in_request(optional_params) # Only the custom tool was rewritten. - assert forward == { - "actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run" - } - assert reverse == { - "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run" - } + assert forward == {"actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run"} + assert reverse == {"actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run"} # Hosted tool's name unchanged. assert optional_params["tools"][0]["name"] == "web_search" # Custom tool's name updated in place. - assert ( - optional_params["tools"][1]["name"] - == "actions_download-job-logs-for-workflow-run" - ) + assert optional_params["tools"][1]["name"] == "actions_download-job-logs-for-workflow-run" def test_sanitize_tool_names_in_request_no_tools_is_noop(): @@ -5733,9 +5443,7 @@ def test_translate_system_message_keeps_billing_header_for_first_party_anthropic assert config.should_strip_billing_metadata() is False result = config.translate_system_message( - messages=_system_with_billing_header( - "You are Claude Code, Anthropic's official CLI for Claude." - ) + messages=_system_with_billing_header("You are Claude Code, Anthropic's official CLI for Claude.") ) texts = [block["text"] for block in result] @@ -5751,9 +5459,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock(): config = BedrockClaudePlatformConfig() assert config.should_strip_billing_metadata() is True - result = config.translate_system_message( - messages=_system_with_billing_header("real system prompt") - ) + result = config.translate_system_message(messages=_system_with_billing_header("real system prompt")) texts = [block["text"] for block in result] assert all(not t.startswith("x-anthropic-billing-header:") for t in texts) @@ -5819,9 +5525,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): config = AmazonAnthropicClaudeConfig() assert config.should_strip_billing_metadata() is True - result = config.translate_system_message( - messages=_system_with_billing_header("real system prompt") - ) + result = config.translate_system_message(messages=_system_with_billing_header("real system prompt")) texts = [block["text"] for block in result] assert all(not t.startswith("x-anthropic-billing-header:") for t in texts) @@ -5875,9 +5579,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): ), ], ) -def test_should_strip_billing_metadata_by_provider( - module_path, class_name, expected_strip -): +def test_should_strip_billing_metadata_by_provider(module_path, class_name, expected_strip): import importlib config_cls = getattr(importlib.import_module(module_path), class_name) @@ -6045,35 +5747,6 @@ def test_sampling_params_forwarded_on_models_that_accept_them(model): assert result["top_p"] == 0.9 -def test_sampling_param_gating_driven_by_model_map_flag(monkeypatch): - """The drop/raise decision must come from ``supports_sampling_params`` in - the model map, not just name matching: a flagged entry gates a model whose - name says nothing, and an explicit ``true`` overrides the name fallback.""" - monkeypatch.setitem( - litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False} - ) - monkeypatch.setitem( - litellm.model_cost, "claude-fable-5-test", {"supports_sampling_params": True} - ) - config = AnthropicConfig() - - flagged_off = config.map_openai_params( - non_default_params={"top_p": 0.9}, - optional_params={}, - model="claude-zeta-9", - drop_params=True, - ) - assert "top_p" not in flagged_off - - flagged_on = config.map_openai_params( - non_default_params={"top_p": 0.9}, - optional_params={}, - model="claude-fable-5-test", - drop_params=True, - ) - assert flagged_on["top_p"] == 0.9 - - def test_top_k_dropped_at_transform_for_models_that_removed_it(): """``top_k`` is a provider-specific kwarg that bypasses ``map_openai_params``, so it must be stripped at the transform_request @@ -6174,9 +5847,7 @@ def test_is_anthropic_usage_object_rejects_responses_api_usage(): ("claude-sonnet-4-5-20250929", False), ], ) -def test_disabled_thinking_omitted_only_for_always_on_models( - local_model_cost_map, model, expected_dropped -): +def test_disabled_thinking_omitted_only_for_always_on_models(local_model_cost_map, model, expected_dropped): """``thinking={"type": "disabled"}`` is omitted for always-on-thinking models (Fable/Mythos, which 400 on it: the API remedy is to omit the param) and is forwarded verbatim for every model that accepts it.""" @@ -6222,9 +5893,7 @@ def test_forced_tool_choice_raises_clean_error_on_fable_5_1_without_drop_params( "tool_choice", ["required", {"type": "required"}, {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params( - local_model_cost_map, tool_choice -): +def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params(local_model_cost_map, tool_choice): config = AnthropicConfig() result = config.map_openai_params( @@ -6251,9 +5920,7 @@ def test_forced_tool_choice_downgrade_keeps_parallel_tool_calls_flag(local_model @pytest.mark.parametrize("tool_choice, expected_type", [("auto", "auto"), ("none", "none")]) -def test_unforced_tool_choice_forwarded_on_fable_5_1( - local_model_cost_map, tool_choice, expected_type, monkeypatch -): +def test_unforced_tool_choice_forwarded_on_fable_5_1(local_model_cost_map, tool_choice, expected_type, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AnthropicConfig() @@ -6268,9 +5935,7 @@ def test_unforced_tool_choice_forwarded_on_fable_5_1( @pytest.mark.parametrize("model", ["claude-fable-5", "claude-opus-5", "claude-sonnet-5"]) -def test_forced_tool_choice_forwarded_on_models_that_support_it( - local_model_cost_map, model, monkeypatch -): +def test_forced_tool_choice_forwarded_on_models_that_support_it(local_model_cost_map, model, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AnthropicConfig() diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py index 788f1b465d7..03cbc98dcb9 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/test_reasoning_effort_fields.py @@ -9,7 +9,7 @@ Covers: import json import os -from typing import Any, Dict, Optional +from typing import Any, Dict import pytest @@ -42,22 +42,6 @@ class TestGetModelInfoReasoningEffortFields: """get_model_info should expose supports_minimal_reasoning_effort and supports_max_reasoning_effort from the model registry.""" - def test_opus_4_6_has_supports_minimal(self): - info = get_model_info("claude-opus-4-6") - assert "supports_minimal_reasoning_effort" in info - - def test_opus_4_6_has_supports_max(self): - info = get_model_info("claude-opus-4-6") - assert "supports_max_reasoning_effort" in info - - def test_opus_4_7_has_supports_minimal(self): - info = get_model_info("claude-opus-4-7") - assert "supports_minimal_reasoning_effort" in info - - def test_opus_4_7_has_supports_max(self): - info = get_model_info("claude-opus-4-7") - assert "supports_max_reasoning_effort" in info - # --------------------------------------------------------------------------- # Commit 2: JSON registry has correct reasoning effort fields @@ -177,9 +161,7 @@ class TestAdapterAdaptiveThinking: ) adapter = LiteLLMAnthropicMessagesAdapter() - result = adapter.translate_anthropic_thinking_to_reasoning_effort( - {"type": "adaptive"} - ) + result = adapter.translate_anthropic_thinking_to_reasoning_effort({"type": "adaptive"}) assert result == "medium" def test_messages_adapter_adaptive_overridden_by_output_config(self): diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index e1b39c4ba13..3c4bf91fc97 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1974,21 +1974,6 @@ class TestClaudeOpus48AdaptiveThinking: assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True - def test_resolver_reads_flag_through_bedrock_invoke_prefix(self, local_model_cost_map): - """The resolver fix: ``bedrock/invoke/...`` resolves to the flagged - Bedrock entry. Pure ``_supports_factory`` without prefix-stripping - returns False here, which is why the data-only fix alone was not enough.""" - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - assert ( - AnthropicModelInfo._supports_model_capability( - "bedrock/invoke/us.anthropic.claude-opus-4-8", - "supports_adaptive_thinking", - "anthropic", - ) - is True - ) - @pytest.mark.parametrize( "model", [ @@ -2172,15 +2157,6 @@ class TestCapabilityProbeUsesCallerProvider: assert AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") is False - def test_native_anthropic_probe_still_reads_anthropic_entry(self, local_model_cost_map, monkeypatch): - import litellm - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - monkeypatch.setitem(litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False) - litellm.get_model_info.cache_clear() - - assert AnthropicModelInfo._is_adaptive_thinking_model("claude-opus-4-8", "anthropic") is True - def test_create_anthropic_model_list_response_shape(): from litellm.llms.anthropic.common_utils import ( diff --git a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py index 6ed6be6f34f..f929c97ba39 100644 --- a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py +++ b/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py @@ -1,6 +1,4 @@ import io -import json -from pathlib import Path from unittest.mock import MagicMock import httpx @@ -68,11 +66,7 @@ def test_azure_speech_audio_transcription_uses_dedicated_api_base_env(monkeypatc monkeypatch.setattr( "litellm.llms.azure.audio_transcription.transformation.get_secret_str", - lambda key: ( - "https://centralus.api.cognitive.microsoft.com" - if key == "AZURE_SPEECH_API_BASE" - else None - ), + lambda key: "https://centralus.api.cognitive.microsoft.com" if key == "AZURE_SPEECH_API_BASE" else None, ) url = config.get_complete_url( @@ -226,14 +220,3 @@ def test_azure_speech_transcription_routes_through_provider_config(monkeypatch): AzureSpeechAudioTranscriptionConfig, ) assert audio_handler.call_args.kwargs["custom_llm_provider"] == "azure" - - -def test_azure_speech_stt_has_non_zero_input_pricing(): - pricing_path = Path(__file__).parents[4] / "model_prices_and_context_window.json" - pricing = json.loads(pricing_path.read_text()) - - assert pricing["azure/speech/azure-stt"]["input_cost_per_second"] > 0 - assert ( - pricing["azure/speech/azure-stt"]["audio_transcription_config"] - == "azure_speech" - ) diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index f8cc0b5071e..f128954a338 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -180,32 +180,6 @@ def test_a_gpt_5_name_without_a_foundry_row_keeps_reading_its_own_entry( assert optional_params["logprobs"] is True -def test_azure_ai_grok_stop_parameter_handling(): - """ - Test that Grok models properly handle stop parameter filtering in Azure AI Studio. - """ - config = AzureAIStudioConfig() - - # Test Grok model detection - assert config._supports_stop_reason("grok-4-fast") is False - assert config._supports_stop_reason("grok-4.3") is False - assert config._supports_stop_reason("grok-4") is False - assert config._supports_stop_reason("grok-3-mini") is False - assert config._supports_stop_reason("grok-code-fast") is False - assert config._supports_stop_reason("gpt-4") is True - - # Test supported parameters for Grok models - for model in ("grok-4-fast", "grok-4.3"): - grok_params = config.get_supported_openai_params(model) - assert ( - "stop" not in grok_params - ), "Grok models should not support stop parameter" - - # Test supported parameters for non-Grok models - gpt_params = config.get_supported_openai_params("gpt-4") - assert "stop" in gpt_params, "GPT models should support stop parameter" - - def test_azure_model_router_response_shows_actual_model(): """ Test that Azure Model Router returns the actual model used in the response, @@ -278,8 +252,7 @@ def test_azure_model_router_response_shows_actual_model(): # Verify that the response contains the actual model used, not the router model assert result.model == "azure_ai/gpt-5-nano-2025-08-07", ( - f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), " - f"but got '{result.model}'" + f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), but got '{result.model}'" ) @@ -337,19 +310,11 @@ def test_azure_model_router_stamps_selected_model_on_hidden_params(): ) assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == result.model - assert ( - result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] - == "azure_ai/grok-4-1-fast-reasoning" - ) - assert AzureFoundryModelInfo.get_model_router_selected_model( - result._hidden_params - ) == ("azure_ai/grok-4-1-fast-reasoning") - assert ( - AzureFoundryModelInfo.is_model_router_call( - model="smart-pick", hidden_params=result._hidden_params - ) - is True + assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == "azure_ai/grok-4-1-fast-reasoning" + assert AzureFoundryModelInfo.get_model_router_selected_model(result._hidden_params) == ( + "azure_ai/grok-4-1-fast-reasoning" ) + assert AzureFoundryModelInfo.is_model_router_call(model="smart-pick", hidden_params=result._hidden_params) is True def test_azure_model_router_stamp_does_not_leak_across_responses(): @@ -387,14 +352,10 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): mock_response.text = error_text mock_response.json.return_value = json.loads(error_text) mock_response.status_code = 400 - e = httpx.HTTPStatusError( - message="400", request=MagicMock(), response=mock_response - ) + e = httpx.HTTPStatusError(message="400", request=MagicMock(), response=mock_response) assert config._error_has_tool_level_extra_fields(error_text) is True - assert ( - config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True - ) + assert config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True request_data = { "model": "FW-Kimi-K2.6", @@ -517,9 +478,7 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): { "role": "assistant", "content": "I can help.", - "thinking_blocks": [ - {"type": "thinking", "thinking": "Reading the file.", "signature": "sig"} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "Reading the file.", "signature": "sig"}], "provider_specific_fields": {"thought_signature": "sig-top"}, "tool_calls": [ { diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 326edde743d..87b9fb8b307 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -3,9 +3,7 @@ import json import os import sys -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) -) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) from unittest.mock import patch @@ -39,9 +37,7 @@ class TestAzureAnthropicMessagesConfig: litellm_params = {"api_key": "test-api-key"} api_key = "test-api-key" - with patch( - "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" - ) as mock_validate: + with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -72,9 +68,7 @@ class TestAzureAnthropicMessagesConfig: optional_params = {} litellm_params = {"api_key": "test-api-key"} - with patch( - "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" - ) as mock_validate: + with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -98,9 +92,7 @@ class TestAzureAnthropicMessagesConfig: optional_params = {} litellm_params = {"api_key": "test-api-key"} - with patch( - "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" - ) as mock_validate: + with patch("litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment") as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} result, api_base = config.validate_anthropic_messages_environment( headers=headers, @@ -173,7 +165,6 @@ class TestAzureAnthropicMessagesConfig: assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" - def test_get_complete_url_with_base_url_without_anthropic(self): """Test get_complete_url with base URL without /anthropic""" config = AzureAnthropicMessagesConfig() @@ -267,9 +258,7 @@ class TestAzureAnthropicMessagesConfig: assert "scope" not in result["system"][0]["cache_control"] assert result["system"][0]["cache_control"]["type"] == "ephemeral" assert "scope" not in result["messages"][0]["content"][0]["cache_control"] - assert ( - result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" - ) + assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" class TestProviderConfigManagerAzureAnthropicMessages: @@ -317,47 +306,6 @@ class TestProviderConfigManagerAzureAnthropicMessages: assert config is None - -def test_messages_thinking_shape_follows_exact_azure_entry_flag(local_model_cost_map, monkeypatch): - """The Azure messages config must probe capabilities under ``azure_ai`` so an - operator setting ``supports_adaptive_thinking: false`` on the exact - ``azure_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry. - With the inherited ``"anthropic"`` provider default the flip was ignored and - the transform kept emitting ``thinking.type='adaptive'``.""" - import litellm - - config = AzureAnthropicMessagesConfig() - - def transform(): - return config.transform_anthropic_messages_request( - model="claude-opus-4-8", - messages=[{"role": "user", "content": "Hello"}], - anthropic_messages_optional_request_params={ - "max_tokens": 4096, - "reasoning_effort": "medium", - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - result = transform() - assert result.get("thinking") == {"type": "adaptive", "display": "summarized"} - assert result.get("output_config") == {"effort": "medium"} - - monkeypatch.setitem( - litellm.model_cost["azure_ai/claude-opus-4-8"], "supports_adaptive_thinking", False - ) - litellm.get_model_info.cache_clear() - assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True - - flipped = transform() - thinking = flipped.get("thinking") - assert isinstance(thinking, dict) - assert thinking.get("type") == "enabled" - assert isinstance(thinking.get("budget_tokens"), int) - assert "output_config" not in flipped - - def _azure_transform(model, messages, system=None): config = AzureAnthropicMessagesConfig() params = {"max_tokens": 256} @@ -417,9 +365,7 @@ class TestAzureAnthropicMidConversationSystem: {"role": "assistant", "content": "reading"}, {"role": "user", "content": "continue"}, ] - result = _azure_transform( - "claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}] - ) + result = _azure_transform("claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}]) assert result["messages"] == [ {"role": "user", "content": "read the file"}, { @@ -450,9 +396,7 @@ def test_azure_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_fl import litellm - cost_map_path = os.path.join( - os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" - ) + cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") with open(cost_map_path) as f: cost_map = json.load(f) rules = cost_map["fallback_generalizations"]["rules"] diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 9b28e42f93b..d8c3a458082 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1,15 +1,13 @@ -import asyncio import json import os import httpx import pytest -from fastapi.testclient import TestClient from unittest.mock import MagicMock, patch import litellm -from litellm import ModelResponse, RateLimitError, completion +from litellm import ModelResponse from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ConverseTokenUsageBlock @@ -30,16 +28,11 @@ def test_transform_usage(): openai_usage = config.transform_usage(usage) assert ( openai_usage.prompt_tokens - == usage["inputTokens"] - + usage["cacheReadInputTokens"] - + usage["cacheWriteInputTokens"] + == usage["inputTokens"] + usage["cacheReadInputTokens"] + usage["cacheWriteInputTokens"] ) assert openai_usage.completion_tokens == usage["outputTokens"] assert openai_usage.total_tokens == usage["totalTokens"] - assert ( - openai_usage.prompt_tokens_details.cached_tokens - == usage["cacheReadInputTokens"] - ) + assert openai_usage.prompt_tokens_details.cached_tokens == usage["cacheReadInputTokens"] assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] # completion_tokens_details should always be populated @@ -87,10 +80,7 @@ def test_transform_usage_with_mismatched_cache_details_falls_back(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert ( - getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def test_transform_usage_without_cache_details_stays_none(): @@ -106,10 +96,7 @@ def test_transform_usage_without_cache_details_stays_none(): ) config = AmazonConverseConfig() openai_usage = config.transform_usage(usage) - assert ( - getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) - is None - ) + assert getattr(openai_usage.prompt_tokens_details, "cache_creation_token_details", None) is None def test_bedrock_converse_1h_cache_write_billed_at_1h_rate(monkeypatch): @@ -196,61 +183,6 @@ def test_transform_usage_reads_invoke_model_count_suffixed_cache_keys( assert openai_usage.total_tokens == 12270 -def test_bedrock_invoke_nova_cache_read_billed_at_discounted_rate(monkeypatch): - """Nova cache reads are billed at the entry's discounted cache read rate; without a - ``cache_read_input_token_cost`` entry the cached tokens were billed at nothing.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - usage = ConverseTokenUsageBlock( - **{ - "inputTokens": 5, - "outputTokens": 3, - "totalTokens": 12270, - "cacheReadInputTokenCount": 12262, - "cacheWriteInputTokenCount": 0, - } - ) - openai_usage = AmazonConverseConfig().transform_usage(usage) - model = "bedrock/invoke/us.amazon.nova-pro-v1:0" - prompt_cost, completion_cost = litellm.cost_calculator.cost_per_token(model=model, usage_object=openai_usage) - model_info = litellm.get_model_info(model=model) - assert 0 < model_info["cache_read_input_token_cost"] < model_info["input_cost_per_token"] - assert prompt_cost == pytest.approx( - 5 * model_info["input_cost_per_token"] + 12262 * model_info["cache_read_input_token_cost"] - ) - assert prompt_cost > 5 * model_info["input_cost_per_token"] - assert completion_cost == pytest.approx(3 * model_info["output_cost_per_token"]) - - -@pytest.mark.parametrize( - "model", - [ - "amazon.nova-micro-v1:0", - "amazon.nova-lite-v1:0", - "amazon.nova-pro-v1:0", - "us.amazon.nova-micro-v1:0", - "us.amazon.nova-lite-v1:0", - "us.amazon.nova-pro-v1:0", - "eu.amazon.nova-micro-v1:0", - "eu.amazon.nova-lite-v1:0", - "eu.amazon.nova-pro-v1:0", - "apac.amazon.nova-micro-v1:0", - "apac.amazon.nova-lite-v1:0", - "apac.amazon.nova-pro-v1:0", - "bedrock/us-gov-west-1/amazon.nova-micro-v1:0", - "bedrock/us-gov-west-1/amazon.nova-lite-v1:0", - "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", - "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", - ], -) -def test_nova_prompt_caching_models_price_cache_reads_below_the_input_rate(model, monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - entry = litellm.model_cost[model] - assert entry["supports_prompt_caching"] is True - assert 0 < entry["cache_read_input_token_cost"] < entry["input_cost_per_token"] - - def test_transform_usage_with_reasoning_content(): """Test that completion_tokens_details correctly tracks reasoning vs text tokens.""" usage = ConverseTokenUsageBlock( @@ -396,14 +328,10 @@ def test_apply_tool_call_transformation_if_needed(): role="user", content=json.dumps(tool_response), ) - transformed_message, _ = config.apply_tool_call_transformation_if_needed( - message, tool_calls - ) + transformed_message, _ = config.apply_tool_call_transformation_if_needed(message, tool_calls) assert len(transformed_message.tool_calls) == 1 assert transformed_message.tool_calls[0].function.name == "test_function" - assert transformed_message.tool_calls[0].function.arguments == json.dumps( - tool_response["parameters"] - ) + assert transformed_message.tool_calls[0].function.arguments == json.dumps(tool_response["parameters"]) def test_transform_tool_call_with_cache_control(): @@ -452,12 +380,7 @@ def test_transform_tool_call_with_cache_control(): print(function_out_msg) assert function_out_msg["toolSpec"]["name"] == "get_location" assert function_out_msg["toolSpec"]["description"] == "Get the user's location" - assert ( - function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"][ - "type" - ] - == "string" - ) + assert function_out_msg["toolSpec"]["inputSchema"]["json"]["properties"]["location"]["type"] == "string" transformed_cache_msg = result["toolConfig"]["tools"][1] assert "cachePoint" in transformed_cache_msg @@ -592,9 +515,7 @@ def test_reasoning_effort_none_omits_thinking_for_anthropic_converse(model): ("bedrock/converse/us.anthropic.claude-sonnet-4-6", "minimal", "low"), ], ) -def test_reasoning_effort_sets_output_config_for_adaptive_models_converse( - model, effort, expected_effort -): +def test_reasoning_effort_sets_output_config_for_adaptive_models_converse(model, effort, expected_effort): """Adaptive Claude 4.6 / 4.7 on Bedrock Converse routes the tier via ``output_config.effort``.""" config = AmazonConverseConfig() @@ -822,9 +743,7 @@ def test_output_config_format_translated_to_native_output_config_converse(): assert additional.get("output_config") == {"effort": "xhigh"} assert "format" not in additional["output_config"] assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - parsed_schema = json.loads( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] - ) + parsed_schema = json.loads(result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"]) assert parsed_schema == {**schema, "additionalProperties": False} @@ -860,10 +779,7 @@ def test_output_config_format_dropped_on_unsupported_converse_model_warns(caplog ) assert "outputConfig" not in result - assert any( - "dropping `output_config.format`" in record.getMessage() - for record in caplog.records - ) + assert any("dropping `output_config.format`" in record.getMessage() for record in caplog.records) def test_output_config_normalized_marker_does_not_leak_into_optional_params(): @@ -899,9 +815,7 @@ def test_output_config_normalized_marker_does_not_leak_into_optional_params(): ("bedrock/converse/us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_output_config_effort_normalized_for_bedrock_converse_opus( - model, expected_effort -): +def test_output_config_effort_normalized_for_bedrock_converse_opus(model, expected_effort): """Bedrock Converse accepts ``xhigh`` and forwards the provider-safe effort.""" config = AmazonConverseConfig() @@ -1174,17 +1088,13 @@ def test_get_supported_openai_params_bedrock_converse(): for model in litellm.BEDROCK_CONVERSE_MODELS: print(f"Testing model: {model}") config = AmazonConverseConfig() - supported_params_without_prefix = config.get_supported_openai_params( - model=model - ) + supported_params_without_prefix = config.get_supported_openai_params(model=model) - supported_params_with_prefix = config.get_supported_openai_params( - model=f"bedrock/converse/{model}" - ) + supported_params_with_prefix = config.get_supported_openai_params(model=f"bedrock/converse/{model}") - assert set(supported_params_without_prefix) == set( - supported_params_with_prefix - ), f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" + assert set(supported_params_without_prefix) == set(supported_params_with_prefix), ( + f"Supported params mismatch for model: {model}. Without prefix: {supported_params_without_prefix}, With prefix: {supported_params_with_prefix}" + ) print(f"✅ Passed for model: {model}") @@ -1377,13 +1287,8 @@ def test_parallel_tool_calls_config_dropped_for_ttl_only_model( def test_transform_response_with_computer_use_tool(): """Test response transformation with computer use tool call.""" - import httpx from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig - from litellm.types.llms.bedrock import ( - ConverseResponseBlock, - ConverseTokenUsageBlock, - ) from litellm.types.utils import ModelResponse # Simulate a Bedrock Converse response with a computer-use tool call @@ -1472,13 +1377,8 @@ def test_transform_response_with_computer_use_tool(): def test_transform_response_with_bash_tool(): """Test response transformation with bash tool call.""" - import httpx from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig - from litellm.types.llms.bedrock import ( - ConverseResponseBlock, - ConverseTokenUsageBlock, - ) from litellm.types.utils import ModelResponse # Simulate a Bedrock Converse response with a bash tool call @@ -1686,9 +1586,7 @@ def test_transform_response_with_structured_response_calling_tool(): "output": { "message": { "content": [ - { - "text": "I'll check the current weather in San Francisco for you." - }, + {"text": "I'll check the current weather in San Francisco for you."}, { "toolUse": { "input": { @@ -2198,9 +2096,7 @@ def test_transform_request_with_function_tool(): } ] - messages = [ - {"role": "user", "content": "What's the weather like in San Francisco?"} - ] + messages = [{"role": "user", "content": "What's the weather like in San Francisco?"}] # Transform request request_data = config.transform_request( @@ -2308,22 +2204,18 @@ async def test_assistant_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2369,12 +2261,10 @@ async def test_assistant_message_list_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2427,12 +2317,10 @@ async def test_tool_message_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2446,10 +2334,7 @@ async def test_tool_message_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert ( - tool_message_content[0]["toolResult"]["content"][0]["text"] - == "Weather data: sunny, 25°C" - ) + assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather data: sunny, 25°C" # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -2491,12 +2376,10 @@ async def test_tool_message_string_content_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2507,10 +2390,7 @@ async def test_tool_message_string_content_cache_control(): # First should be tool result assert "toolResult" in tool_message_content[0] - assert ( - tool_message_content[0]["toolResult"]["content"][0]["text"] - == "Weather: sunny, 25°C" - ) + assert tool_message_content[0]["toolResult"]["content"][0]["text"] == "Weather: sunny, 25°C" # Second should be cachePoint assert "cachePoint" in tool_message_content[1] @@ -2550,9 +2430,7 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() "source": "Great Source of Information About Apptio", "title": "12adbd74-46bd-4a88-88b2-0048755f6eb5", "content": [ - { - "text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM" - } + {"text": "Apptio is a company that makes calls to Bedrock using passthrough APIs via LiteLLM"} ], "citations": {"enabled": True}, } @@ -2565,12 +2443,10 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2579,10 +2455,7 @@ async def test_tool_message_search_results_maps_to_bedrock_search_result_block() assert tool_result["status"] == "success" assert len(tool_result["content"]) == 1 assert "searchResult" in tool_result["content"][0] - assert ( - tool_result["content"][0]["searchResult"]["title"] - == "12adbd74-46bd-4a88-88b2-0048755f6eb5" - ) + assert tool_result["content"][0]["searchResult"]["title"] == "12adbd74-46bd-4a88-88b2-0048755f6eb5" @pytest.mark.asyncio @@ -2619,12 +2492,10 @@ async def test_tool_message_empty_search_results_falls_back_to_content(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2786,12 +2657,10 @@ async def test_assistant_tool_calls_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2846,12 +2715,10 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -2897,12 +2764,10 @@ async def test_no_cache_control_no_cache_point(): llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -3072,10 +2937,7 @@ def test_guarded_text_with_mixed_content_types(): # Third should be guardContent assert "guardContent" in content[2] - assert ( - content[2]["guardContent"]["text"]["text"] - == "This sensitive content should be guarded" - ) + assert content[2]["guardContent"]["text"]["text"] == "This sensitive content should be guarded" @pytest.mark.asyncio @@ -3170,10 +3032,7 @@ def test_guarded_text_with_tool_calls(): # Second should be guardContent assert "guardContent" in content[1] - assert ( - content[1]["guardContent"]["text"]["text"] - == "Please be careful with sensitive information" - ) + assert content[1]["guardContent"]["text"]["text"] == "Please be careful with sensitive information" # Other messages should not have guardContent for i in range(1, 3): @@ -3234,52 +3093,36 @@ def test_auto_convert_last_user_message_to_guarded_text(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" def test_auto_convert_last_user_message_string_content(): """Test that last user message with string content is automatically converted to guarded_text when guardrailConfig is present.""" config = AmazonConverseConfig() - messages = [ - {"role": "user", "content": "What is the main topic of this legal document?"} - ] + messages = [{"role": "user", "content": "What is the main topic of this legal document?"}] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 assert converted_messages[0]["role"] == "user" assert len(converted_messages[0]["content"]) == 1 assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" def test_no_conversion_when_no_guardrail_config(): @@ -3301,9 +3144,7 @@ def test_no_conversion_when_no_guardrail_config(): optional_params = {} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify no conversion happened assert converted_messages == messages @@ -3320,14 +3161,10 @@ def test_no_conversion_when_guarded_text_already_present(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify no conversion happened assert converted_messages == messages @@ -3353,14 +3190,10 @@ def test_auto_convert_with_mixed_content(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 1 @@ -3369,17 +3202,11 @@ def test_auto_convert_with_mixed_content(): # First element should be converted to guarded_text assert converted_messages[0]["content"][0]["type"] == "guarded_text" - assert ( - converted_messages[0]["content"][0]["text"] - == "What is the main topic of this legal document?" - ) + assert converted_messages[0]["content"][0]["text"] == "What is the main topic of this legal document?" # Second element should remain unchanged assert converted_messages[0]["content"][1]["type"] == "image_url" - assert ( - converted_messages[0]["content"][1]["image_url"]["url"] - == "https://example.com/image.jpg" - ) + assert converted_messages[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg" def test_auto_convert_in_full_transformation(): @@ -3398,9 +3225,7 @@ def test_auto_convert_in_full_transformation(): } ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the full transformation result = config._transform_request( @@ -3420,10 +3245,7 @@ def test_auto_convert_in_full_transformation(): assert "content" in message assert len(message["content"]) == 1 assert "guardContent" in message["content"][0] - assert ( - message["content"][0]["guardContent"]["text"]["text"] - == "What is the main topic of this legal document?" - ) + assert message["content"][0]["guardContent"]["text"]["text"] == "What is the main topic of this legal document?" def test_convert_consecutive_user_messages_to_guarded_text(): @@ -3437,14 +3259,10 @@ def test_convert_consecutive_user_messages_to_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion - only the last two user messages should be converted assert len(converted_messages) == 4 @@ -3479,14 +3297,10 @@ def test_convert_all_user_messages_when_all_consecutive(): {"role": "user", "content": [{"type": "text", "text": "Third user message"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify all three user messages are converted assert len(converted_messages) == 3 @@ -3510,14 +3324,10 @@ def test_convert_consecutive_user_messages_with_string_content(): {"role": "user", "content": "Second user message"}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 3 @@ -3550,14 +3360,10 @@ def test_skip_consecutive_user_messages_with_existing_guarded_text(): {"role": "user", "content": [{"type": "text", "text": "Should be converted"}]}, ] - optional_params = { - "guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"} - } + optional_params = {"guardrailConfig": {"guardrailIdentifier": "gr-abc123", "guardrailVersion": "1"}} # Test the helper method directly - converted_messages = config._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + converted_messages = config._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) # Verify the conversion assert len(converted_messages) == 2 @@ -4116,24 +3922,22 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): optional_params = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Verify the condition is detected - assert last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ), "Should detect missing thinking_blocks" + assert last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks), ( + "Should detect missing thinking_blocks" + ) # Simulate what _transform_request_helper does if ( optional_params.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) ): if litellm.modify_params: optional_params.pop("thinking", None) - assert ( - "thinking" not in optional_params - ), "thinking param should be dropped when modify_params=True and thinking_blocks are missing" + assert "thinking" not in optional_params, ( + "thinking param should be dropped when modify_params=True and thinking_blocks are missing" + ) # Test case 2: thinking should NOT be dropped when thinking_blocks are present messages_with_thinking_blocks = [ @@ -4148,137 +3952,52 @@ def test_drop_thinking_param_when_thinking_blocks_missing(): "function": {"name": "search", "arguments": "{}"}, } ], - "thinking_blocks": [ - {"type": "thinking", "thinking": "Let me search for weather..."} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "Let me search for weather..."}], }, {"role": "tool", "content": "Weather is sunny", "tool_call_id": "call_123"}, ] - optional_params_with_thinking = { - "thinking": {"type": "enabled", "budget_tokens": 1000} - } + optional_params_with_thinking = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Verify the condition is NOT detected when thinking_blocks are present - assert not last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_with_thinking_blocks - ), "Should NOT detect missing thinking_blocks when they are present" + assert not last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks), ( + "Should NOT detect missing thinking_blocks when they are present" + ) # Simulate what _transform_request_helper does if ( optional_params_with_thinking.get("thinking") is not None and messages_with_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_with_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_with_thinking_blocks) ): if litellm.modify_params: optional_params_with_thinking.pop("thinking", None) - assert ( - "thinking" in optional_params_with_thinking - ), "thinking param should NOT be dropped when thinking_blocks are present" + assert "thinking" in optional_params_with_thinking, ( + "thinking param should NOT be dropped when thinking_blocks are present" + ) # Test case 3: thinking should NOT be dropped when modify_params=False litellm.modify_params = False - optional_params_no_modify = { - "thinking": {"type": "enabled", "budget_tokens": 1000} - } + optional_params_no_modify = {"thinking": {"type": "enabled", "budget_tokens": 1000}} # Simulate what _transform_request_helper does if ( optional_params_no_modify.get("thinking") is not None and messages_without_thinking_blocks is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks( - messages_without_thinking_blocks - ) + and last_assistant_with_tool_calls_has_no_thinking_blocks(messages_without_thinking_blocks) ): if litellm.modify_params: optional_params_no_modify.pop("thinking", None) - assert ( - "thinking" in optional_params_no_modify - ), "thinking param should NOT be dropped when modify_params=False" + assert "thinking" in optional_params_no_modify, "thinking param should NOT be dropped when modify_params=False" finally: # Restore original modify_params setting litellm.modify_params = original_modify_params -def test_supports_native_structured_outputs(monkeypatch): - """Test model detection for native structured outputs support. - - Support is driven by the ``supports_native_structured_output`` flag in the - cost JSON (litellm.model_cost), not a hardcoded model set. - """ - old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - old_cost = litellm.model_cost - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - try: - config = AmazonConverseConfig() - - # Supported models (have supports_native_structured_output=true in cost JSON) - assert config._supports_native_structured_outputs( - "anthropic.claude-sonnet-4-5-20250929-v1:0" - ) - assert config._supports_native_structured_outputs( - "anthropic.claude-haiku-4-5-20251001-v1:0" - ) - assert config._supports_native_structured_outputs( - "anthropic.claude-opus-4-6-v1" - ) - # Regional prefix is stripped by get_bedrock_base_model - assert config._supports_native_structured_outputs( - "eu.anthropic.claude-opus-4-5-20251101-v1:0" - ) - # Claude 4.6 Sonnet - assert config._supports_native_structured_outputs("anthropic.claude-sonnet-4-6") - assert config._supports_native_structured_outputs( - "us.anthropic.claude-sonnet-4-6" - ) - # Non-Anthropic models - assert config._supports_native_structured_outputs( - "qwen.qwen3-235b-a22b-2507-v1:0" - ) - assert config._supports_native_structured_outputs( - "mistral.mistral-large-3-675b-instruct" - ) - assert config._supports_native_structured_outputs("minimax.minimax-m2") - assert config._supports_native_structured_outputs("moonshot.kimi-k2-thinking") - assert config._supports_native_structured_outputs("nvidia.nemotron-nano-3-30b") - # DeepSeek: old substring "deepseek-v3.1" didn't match real ID - assert config._supports_native_structured_outputs("deepseek.v3-v1:0") - assert config._supports_native_structured_outputs("deepseek.v3.2") - assert config._supports_native_structured_outputs("zai.glm-5") - - # Unsupported models -- should fall back to tool-call approach - assert not config._supports_native_structured_outputs( - "anthropic.claude-sonnet-4-20250514-v1:0" - ) - assert not config._supports_native_structured_outputs( - "meta.llama3-3-70b-instruct-v1:0" - ) - assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0") - # Excluded: broken constrained decoding on Bedrock - assert not config._supports_native_structured_outputs("openai.gpt-oss-120b-1:0") - assert not config._supports_native_structured_outputs( - "mistral.magistral-small-2509" - ) - # Excluded: ignores schema or broken on Bedrock - assert not config._supports_native_structured_outputs("google.gemma-3-27b-it") - assert not config._supports_native_structured_outputs( - "nvidia.nemotron-nano-12b-v2" - ) - finally: - litellm.model_cost = old_cost - if old_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) - - def test_create_output_config_for_response_format(): """Test outputConfig dict creation from JSON schema.""" config = AmazonConverseConfig() @@ -4356,19 +4075,14 @@ def test_translate_response_format_native_output_config(monkeypatch): assert "fake_stream" not in result # Verify the schema content (additionalProperties: false is added by normalization) - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ - "schema" - ] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] parsed_schema = json.loads(schema_str) expected_schema = { **response_format["json_schema"]["schema"], "additionalProperties": False, } assert parsed_schema == expected_schema - assert ( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] - == "WeatherResult" - ) + assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "WeatherResult" finally: litellm.model_cost = old_cost if old_env is None: @@ -4446,9 +4160,7 @@ def test_native_structured_output_no_fake_stream(monkeypatch): assert "fake_stream" not in result # Verify the schema content - schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"][ - "schema" - ] + schema_str = result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["schema"] assert json.loads(schema_str) == { "type": "object", "properties": {"answer": {"type": "string"}}, @@ -4501,10 +4213,7 @@ def test_transform_request_with_output_config(): assert "outputConfig" in result assert result["outputConfig"]["textFormat"]["type"] == "json_schema" - assert ( - result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] - == "TestSchema" - ) + assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" def test_transform_request_strips_anthropic_output_config(): @@ -4625,10 +4334,7 @@ def test_transform_response_native_structured_output(): ) # Content should be the JSON text directly - assert ( - result.choices[0].message.content - == '{"temp": 62, "description": "Mild and foggy"}' - ) + assert result.choices[0].message.content == '{"temp": 62, "description": "Mild and foggy"}' # Should NOT have tool_calls assert result.choices[0].message.tool_calls is None assert result.choices[0].finish_reason == "stop" @@ -4741,10 +4447,7 @@ def test_add_additional_properties_definitions(): # definitions object assert result["definitions"]["Item"]["additionalProperties"] is False # Nested object inside definitions - assert ( - result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] - is False - ) + assert result["definitions"]["Item"]["properties"]["details"]["additionalProperties"] is False def test_json_object_no_schema_skips_tool_injection(monkeypatch): @@ -4801,9 +4504,7 @@ def test_output_config_applies_additional_properties(): output_config = AmazonConverseConfig._create_output_config_for_response_format( json_schema=schema, name="test_schema" ) - parsed = json.loads( - output_config["textFormat"]["structure"]["jsonSchema"]["schema"] - ) + parsed = json.loads(output_config["textFormat"]["structure"]["jsonSchema"]["schema"]) assert parsed["additionalProperties"] is False assert parsed["properties"]["nested"]["additionalProperties"] is False @@ -4852,12 +4553,7 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert ( - request_data["additionalModelRequestFields"]["tool_choice"][ - "disable_parallel_tool_use" - ] - is True - ) + assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] @@ -4889,12 +4585,7 @@ def test_parallel_tool_calls_flag_decoupled_from_ttl_pricing(monkeypatch): headers={}, ) - assert ( - request_data["additionalModelRequestFields"]["tool_choice"][ - "disable_parallel_tool_use" - ] - is True - ) + assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True def test_parallel_tool_calls_older_model_drops_disable_flag(): @@ -5041,9 +4732,7 @@ def test_parallel_tool_use_merge_preserves_user_tool_choice_type(): class TestBedrockMinThinkingBudgetTokens: """Test that thinking.budget_tokens is clamped to the Bedrock minimum (1024).""" - def _map_params( - self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0" - ): + def _map_params(self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0"): """Helper to call map_openai_params with the given thinking value.""" config = AmazonConverseConfig() non_default_params = {"thinking": thinking_value} @@ -5270,9 +4959,7 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 2: json_tool_call delta — should become text, not tool_use json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"temp": 62}'}) - text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event( - json_delta, index=0 - ) + text_2, tool_use_2, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) assert text_2 == '{"temp": 62}' assert tool_use_2 is None @@ -5296,9 +4983,7 @@ def test_streaming_filters_json_tool_call_with_real_tools(): # Chunk 5: real tool delta real_delta = ContentBlockDeltaEvent(toolUse={"input": '{"location": "SF"}'}) - text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event( - real_delta, index=1 - ) + text_5, tool_use_5, _, _, _ = decoder._handle_converse_delta_event(real_delta, index=1) assert text_5 == "" assert tool_use_5 is not None assert tool_use_5["function"]["arguments"] == '{"location": "SF"}' @@ -5331,9 +5016,7 @@ def test_streaming_without_json_mode_passes_all_tools(): # json_tool_call delta — should be a tool_use, not text json_delta = ContentBlockDeltaEvent(toolUse={"input": '{"data": 1}'}) - text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event( - json_delta, index=0 - ) + text, tool_use_delta, _, _, _ = decoder._handle_converse_delta_event(json_delta, index=0) assert text == "" assert tool_use_delta is not None assert tool_use_delta["function"]["arguments"] == '{"data": 1}' @@ -5584,87 +5267,6 @@ def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model(): assert tools[-1] == {"cachePoint": {"type": "default"}} -@pytest.mark.parametrize( - ("model", "expects_cache_points"), - [ - pytest.param("nvidia.nemotron-super-3-120b", False, id="mapped-model-without-prompt-caching"), - pytest.param("us.nvidia.nemotron-super-3-120b", False, id="regional-prefix-resolves-through-base-model"), - pytest.param( - "us.anthropic.claude-3-5-sonnet-20240620-v1:0", False, id="claude-named-but-not-caching-on-bedrock" - ), - pytest.param("us.anthropic.claude-sonnet-4-5-20250929-v1:0", True, id="mapped-model-with-prompt-caching"), - pytest.param( - "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", - True, - id="unmapped-arn-keeps-emitting", - ), - pytest.param("global.openai.gpt-6-astra", False, id="openai-family-implicit-caching-only"), - pytest.param("openai.gpt-oss-120b-1:0", False, id="openai-gpt-oss"), - pytest.param("us.openai.gpt-99-unmapped", False, id="unmapped-openai-family-still-suppressed"), - ], -) -def test_cache_points_emitted_only_for_models_that_support_prompt_caching(model, expects_cache_points, monkeypatch): - """Bedrock rejects cachePoint blocks for models without prompt caching support - ("You invoked an unsupported model or your request did not allow prompt caching"), - and clients like Claude Code attach cache_control to every request, so a map-known - model without the capability must not receive them. Unmapped ids (application - inference profile ARNs, models newer than the map) keep emitting so existing - caching setups never silently degrade.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - - body = AmazonConverseConfig().transform_request( - model=model, - messages=[ - {"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]}, - {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}, - ], - optional_params={}, - litellm_params={}, - headers={}, - ) - - assert ("cachePoint" in json.dumps(body)) is expects_cache_points - assert body["system"][0]["text"] == "sys" - assert body["messages"][0]["content"][0]["text"] == "hi" - - -def test_tool_config_cachepoint_not_placed_or_credited_for_model_without_prompt_caching(monkeypatch): - """The tool_config injection point must stand down with the rest of the cachePoint - emission when the model cannot cache, and spend attribution must not credit the - gateway for a breakpoint that was never placed.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - - bucket: dict = {"user_api_key": "sk-test"} - data = AmazonConverseConfig()._transform_request_helper( - model="nvidia.nemotron-super-3-120b", - system_content_blocks=[], - optional_params={ - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, - }, - } - ], - "cache_control_injection_points": [{"location": "tool_config"}], - }, - messages=[{"role": "user", "content": "hi"}], - litellm_params={"metadata": bucket, "litellm_metadata": None, "model_info": {"id": "dep-bedrock"}}, - ) - - assert "cachePoint" not in json.dumps(data.get("toolConfig", {})) - assert "litellm_gateway_injected_cache" not in bucket - - def test_translate_response_format_json_schema_still_injects_tool(): """ response_format with an explicit json_schema should still use the @@ -5898,11 +5500,7 @@ def test_transform_response_citation_null_source_title_become_empty_strings(): "content": [ { "citationsContent": { - "content": [ - { - "text": "Apptio is a company that makes calls to Bedrock" - } - ], + "content": [{"text": "Apptio is a company that makes calls to Bedrock"}], "citations": [ { "location": { @@ -6037,15 +5635,11 @@ def test_transform_response_citations_offset_tracks_text_only_blocks(): message = result.choices[0].message expected_start = len(leading_text) assert message.content == leading_text + cited_text - assert ( - message.content[expected_start : expected_start + len(cited_text)] == cited_text - ) + assert message.content[expected_start : expected_start + len(cited_text)] == cited_text assert message.annotations is not None assert len(message.annotations) == 1 assert message.annotations[0]["url_citation"]["start_index"] == expected_start - assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len( - cited_text - ) + assert message.annotations[0]["url_citation"]["end_index"] == expected_start + len(cited_text) def test_transform_response_stitches_citations_for_whitespace_punctuation_text(): @@ -6155,9 +5749,7 @@ def test_bedrock_tool_message_openai_file_pdf_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_1" @@ -6199,9 +5791,7 @@ def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert tool_result["toolUseId"] == "tooluse_pdf_img_1" @@ -6258,9 +5848,7 @@ def test_bedrock_tool_message_file_id_http_url_becomes_document(): "process_image_sync", return_value=fake_document_block, ) as mock_proc: - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") mock_proc.assert_called_once() assert mock_proc.call_args.kwargs["image_url"] == pdf_url @@ -6331,9 +5919,7 @@ def test_bedrock_tool_message_image_url_png_still_becomes_image(): }, ] - translated_msg = _bedrock_converse_messages_pt( - messages=messages, model="", llm_provider="" - ) + translated_msg = _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") tool_result = translated_msg[-1]["content"][-1]["toolResult"] assert len(tool_result["content"]) == 1 @@ -6528,12 +6114,10 @@ async def test_grounding_source_and_query_rendered_as_text(): model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + llm_provider="bedrock_converse", ) assert result == async_result @@ -6577,9 +6161,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_no_tools(): (#24158, #27138).""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -6600,9 +6182,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_when_tools_empty(tools_value): structured tool blocks with no toolConfig.""" messages = _orphaned_tool_history_messages() - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={"tools": tools_value} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={"tools": tools_value}) serialized = json.dumps(result) assert "tool_calls" not in serialized @@ -6618,9 +6198,7 @@ def test_neutralize_orphaned_tool_blocks_rewrites_tool_result_only_history(): {"role": "tool", "tool_call_id": "call_xyz", "content": "lookup result"}, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) assert not any(m.get("role") in ("tool", "function") for m in result) serialized = json.dumps(result) @@ -6657,13 +6235,9 @@ def test_neutralize_orphaned_tool_blocks_non_text_result_marked_not_empty(): }, ] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) - rewritten = next( - m for m in result if m.get("role") == "user" and m is not messages[0] - ) + rewritten = next(m for m in result if m.get("role") == "user" and m is not messages[0]) text = rewritten["content"] assert text.strip() # never empty assert "non-text tool result omitted" in text @@ -6686,9 +6260,7 @@ def test_neutralize_orphaned_tool_blocks_noop_when_no_tool_history(): """Plain conversation with no tool blocks is returned unchanged.""" messages = [{"role": "user", "content": "hi"}] - result = AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + result = AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) assert result is messages @@ -6699,14 +6271,9 @@ def test_neutralize_orphaned_tool_blocks_logs_warning(caplog): messages = _orphaned_tool_history_messages() with caplog.at_level("WARNING"): - AmazonConverseConfig._neutralize_orphaned_tool_blocks( - messages, optional_params={} - ) + AmazonConverseConfig._neutralize_orphaned_tool_blocks(messages, optional_params={}) - assert any( - "neutralizing orphaned tool blocks" in record.getMessage() - for record in caplog.records - ) + assert any("neutralizing orphaned tool blocks" in record.getMessage() for record in caplog.records) def _assert_no_structured_tool_blocks(result): @@ -6824,9 +6391,7 @@ def test_transform_request_neutralized_tool_output_is_guarded(monkeypatch): }, {"role": "tool", "tool_call_id": "c1", "content": "secret tool output"}, ], - optional_params={ - "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} - }, + optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, litellm_params={}, headers={}, ) @@ -6866,23 +6431,19 @@ def test_transform_request_neutralized_tool_output_guarded_mid_history(monkeypat {"role": "assistant", "content": "Here is the summary."}, {"role": "user", "content": "thanks"}, ], - optional_params={ - "guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"} - }, + optional_params={"guardrailConfig": {"guardrailIdentifier": "gid", "guardrailVersion": "1"}}, litellm_params={}, headers={}, ) _assert_no_structured_tool_blocks(result) blocks = [block for message in result["messages"] for block in message["content"]] - guarded_texts = [ - block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block - ] + guarded_texts = [block["guardContent"]["text"]["text"] for block in blocks if "guardContent" in block] plain_texts = [block["text"] for block in blocks if "text" in block and "guardContent" not in block] assert any("malware" in text for text in guarded_texts), "mid-history tool output must be guarded" - assert not any( - "malware" in text for text in plain_texts - ), "mid-history tool output must not reach the model as unguarded text" + assert not any("malware" in text for text in plain_texts), ( + "mid-history tool output must not reach the model as unguarded text" + ) @pytest.mark.asyncio @@ -7018,10 +6579,7 @@ def _agentic_messages_with_ttl(ttl_target: str): def _collect_cache_points(result): return [ - block["cachePoint"] - for message in result - for block in message.get("content") or [] - if "cachePoint" in block + block["cachePoint"] for message in result for block in message.get("content") or [] if "cachePoint" in block ] @@ -7047,12 +6605,10 @@ async def test_message_level_cache_control_honors_ttl_for_supported_model( model="global.anthropic.claude-opus-4-7", llm_provider="bedrock_converse", ) - async_result = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model="global.anthropic.claude-opus-4-7", - llm_provider="bedrock_converse", - ) + async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="global.anthropic.claude-opus-4-7", + llm_provider="bedrock_converse", ) assert result == async_result @@ -7356,7 +6912,6 @@ def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_cras assert "maxTokens" not in optional_params - @pytest.mark.parametrize( "model, expected_dropped", [ @@ -7365,9 +6920,7 @@ def test_update_optional_params_with_thinking_tokens_bool_thinking_does_not_cras ("us.anthropic.claude-opus-4-8", False), ], ) -def test_disabled_thinking_omitted_for_always_on_models_converse( - local_model_cost_map, model, expected_dropped -): +def test_disabled_thinking_omitted_for_always_on_models_converse(local_model_cost_map, model, expected_dropped): """Bedrock Converse: ``thinking={"type": "disabled"}`` is omitted for always-on-thinking models and forwarded verbatim for models that accept it.""" config = AmazonConverseConfig() @@ -7386,6 +6939,7 @@ def test_disabled_thinking_omitted_for_always_on_models_converse( else: assert additional.get("thinking") == {"type": "disabled"} + @pytest.mark.parametrize( "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], @@ -7394,14 +6948,10 @@ def test_disabled_thinking_omitted_for_always_on_models_converse( "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse( - local_model_cost_map, model, tool_choice -): +def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse(local_model_cost_map, model, tool_choice): config = AmazonConverseConfig() - result = config.map_tool_choice_values( - model=model, tool_choice=tool_choice, drop_params=True - ) + result = config.map_tool_choice_values(model=model, tool_choice=tool_choice, drop_params=True) assert result == {"auto": {}} @@ -7410,16 +6960,12 @@ def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_converse( "tool_choice", ["required", {"type": "function", "function": {"name": "get_weather"}}], ) -def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse( - local_model_cost_map, tool_choice, monkeypatch -): +def test_forced_tool_choice_raises_clean_error_on_fable_5_1_converse(local_model_cost_map, tool_choice, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() with pytest.raises(litellm.utils.UnsupportedParamsError, match="forced tool use"): - config.map_tool_choice_values( - model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False - ) + config.map_tool_choice_values(model="anthropic.claude-fable-5-1", tool_choice=tool_choice, drop_params=False) @pytest.mark.parametrize("tool_choice", ["auto", "none"]) @@ -7437,9 +6983,7 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ "model", ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], ) -def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( - local_model_cost_map, model -): +def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(local_model_cost_map, model): """Regression: Bedrock rejects both ``outputConfig`` structured output and forced tool_choice for Fable 5.1, so response_format must map to a tool without a forced tool_choice.""" @@ -7466,15 +7010,11 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve assert result.get("json_mode") is True -def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( - local_model_cost_map, monkeypatch -): +def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) config = AmazonConverseConfig() - result = config.map_tool_choice_values( - model="anthropic.claude-fable-5", tool_choice="required", drop_params=False - ) + result = config.map_tool_choice_values(model="anthropic.claude-fable-5", tool_choice="required", drop_params=False) assert result == {"any": {}} diff --git a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py index 58411a9ae18..ebecd615605 100644 --- a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py +++ b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py @@ -3,7 +3,7 @@ import base64 import io from typing import cast -from unittest.mock import Mock, patch +from unittest.mock import Mock import httpx import pytest @@ -203,9 +203,7 @@ def test_transform_request_image_pathlike_input(tmp_path): ) assert body["taskType"] == "IMAGE_VARIATION" - assert body["imageVariationParams"]["images"][0] == base64.b64encode( - image_bytes - ).decode("utf-8") + assert body["imageVariationParams"]["images"][0] == base64.b64encode(image_bytes).decode("utf-8") def test_transform_request_inpainting_with_mask(): @@ -366,9 +364,7 @@ def test_transform_request_inpainting_explicit_task_without_mask_raises(): """INPAINTING taskType without mask or maskPrompt must fail fast.""" config = BedrockAmazonNovaCanvasImageEditConfig() img = io.BytesIO(b"img") - with pytest.raises( - ValueError, match="INPAINTING requires either maskPrompt or maskImage" - ): + with pytest.raises(ValueError, match="INPAINTING requires either maskPrompt or maskImage"): config.transform_image_edit_request( model="amazon.nova-canvas-v1:0", prompt="fix it", @@ -483,55 +479,6 @@ def test_transform_request_unknown_quality_reaches_image_generation_config(): assert body["imageGenerationConfig"]["quality"] == "auto" -def test_is_nova_canvas_image_edit_model_uses_model_cost_flag(monkeypatch): - """Routing uses supports_nova_canvas_image_edit in model_cost, not a hardcoded name substring.""" - fake_id = "amazon.custom-bedrock-image-edit-v99:0" - monkeypatch.setitem( - litellm.model_cost, - fake_id, - { - "litellm_provider": "bedrock", - "mode": "image_generation", - "supports_nova_canvas_image_edit": True, - }, - ) - assert ( - BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(fake_id) - is True - ) - - monkeypatch.setitem( - litellm.model_cost, - "amazon.not-nova-canvas-v1:0", - { - "litellm_provider": "bedrock", - "mode": "image_generation", - }, - ) - assert ( - BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( - "amazon.not-nova-canvas-v1:0" - ) - is False - ) - - # Name-shaped ids do not route without supports_nova_canvas_image_edit (no substring heuristic). - monkeypatch.setitem( - litellm.model_cost, - "amazon.nova-canvas-v2:0", - { - "litellm_provider": "bedrock", - "mode": "image_generation", - }, - ) - assert ( - BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( - "amazon.nova-canvas-v2:0" - ) - is False - ) - - def test_transform_response_to_openai_format(): """Response maps images[] to ImageResponse.data b64_json.""" config = BedrockAmazonNovaCanvasImageEditConfig() diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index ddf184abed3..7d243594cb3 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -32,7 +32,6 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran ) - @pytest.mark.asyncio async def test_bedrock_sse_wrapper_encodes_dict_chunks(): """Verify that `bedrock_sse_wrapper` converts dictionary chunks to properly formatted Server-Sent Events and forwards non-dict chunks unchanged.""" @@ -49,9 +48,7 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): _dummy_stream(), litellm_logging_obj=LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", - messages=[ - {"role": "user", "content": "Hello, can you tell me a short joke?"} - ], + messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], stream=True, call_type="chat", start_time=datetime.now(), @@ -228,9 +225,7 @@ async def test_bedrock_sse_wrapper_keeps_usage_in_message_start_and_message_delt def test_chunk_parser_usage_transformation(): """Ensure Bedrock invocation metrics are transformed to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0") chunk = { "type": "message_delta", @@ -259,9 +254,7 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): fields and cache tokens end up billed at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -287,9 +280,7 @@ def test_chunk_parser_preserves_cache_usage_fields_with_invocation_metrics(): def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): """Cache itemization inside invocationMetrics maps to Anthropic usage keys.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -312,9 +303,7 @@ def test_chunk_parser_maps_cache_token_counts_from_invocation_metrics(): def test_chunk_parser_keeps_existing_token_counts_over_invocation_metrics(): """Token counts reported in the chunk's own usage block win over invocationMetrics.""" - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") chunk = { "type": "message_stop", @@ -349,9 +338,7 @@ async def test_bedrock_sse_wrapper_preserves_cache_usage_with_invocation_metrics final usage billed cache reads and writes at $0. """ - decoder = AmazonAnthropicClaudeMessagesStreamDecoder( - model="bedrock/invoke/anthropic.claude-sonnet-4-6" - ) + decoder = AmazonAnthropicClaudeMessagesStreamDecoder(model="bedrock/invoke/anthropic.claude-sonnet-4-6") cfg = AmazonAnthropicClaudeMessagesConfig() raw_chunks = [ @@ -561,11 +548,7 @@ def test_normalize_custom_field_on_tools(): assert request4["tools"] is None # Case 5: an explicit top-level flag wins over a conflicting wrapped one - request5 = { - "tools": [ - {"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}} - ] - } + request5 = {"tools": [{"name": "Read", "defer_loading": False, "custom": {"defer_loading": True}}]} normalize_custom_field_on_tools(request5) assert request5["tools"][0] == {"name": "Read", "defer_loading": False} @@ -586,9 +569,7 @@ def test_normalize_custom_field_on_tools(): assert request7["tools"] == [{"name": "Read"}, {"name": "Write"}] -@pytest.mark.parametrize( - "deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}] -) +@pytest.mark.parametrize("deferred_marker", [{"custom": {"defer_loading": True}}, {"defer_loading": True}]) def test_bedrock_invoke_messages_transform_emits_top_level_defer_loading( deferred_marker, ): @@ -721,9 +702,7 @@ def test_bedrock_invoke_messages_skips_thinking_injection_when_already_enabled( "max_tokens": 32000, "stream": False, "thinking": {"type": "enabled", "budget_tokens": 2048}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( model="global.anthropic.claude-sonnet-4-6-v1:0", @@ -825,9 +804,7 @@ def test_remove_ttl_from_cache_control_processes_tools(local_model_cost_map): "messages": [], } - cfg._remove_ttl_from_cache_control( - request, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) + cfg._remove_ttl_from_cache_control(request, model="anthropic.claude-3-5-sonnet-20241022-v2:0") # Tool ttl should be stripped assert "ttl" not in request["tools"][0]["cache_control"] @@ -863,9 +840,7 @@ def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(local_ ], } - cfg._remove_ttl_from_cache_control( - request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + cfg._remove_ttl_from_cache_control(request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") # Both tools and system should preserve ttl for Claude 4.5 assert request["tools"][0]["cache_control"]["ttl"] == "1h" @@ -949,9 +924,7 @@ def test_bedrock_messages_strips_output_config(): headers={}, ) - assert "output_config" not in result, ( - "output_config should be stripped for models that don't support it" - ) + assert "output_config" not in result, "output_config should be stripped for models that don't support it" assert result.get("max_tokens") == 4096 @@ -984,9 +957,7 @@ def test_bedrock_messages_preserves_output_config_for_claude_4_6(): headers={}, ) - assert "output_config" in result, ( - "output_config should be preserved for supported models" - ) + assert "output_config" in result, "output_config should be preserved for supported models" assert result["output_config"] == {"effort": "high"} assert result.get("max_tokens") == 4096 @@ -1138,9 +1109,7 @@ def test_bedrock_messages_converts_output_config_format_to_inline_schema(): ("anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bedrock_messages_normalizes_output_config_effort_for_opus( - model, expected_effort -): +def test_bedrock_messages_normalizes_output_config_effort_for_opus(model, expected_effort): """Bedrock /v1/messages accepts ``xhigh`` and forwards the provider-safe effort.""" from unittest.mock import patch @@ -1198,9 +1167,7 @@ def test_bedrock_messages_does_not_mutate_callers_messages_when_embedding_schema headers={}, ) - assert caller_messages == [ - {"role": "user", "content": [{"type": "text", "text": "Hello"}]} - ] + assert caller_messages == [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] assert caller_message == { "role": "user", "content": [{"type": "text", "text": "Hello"}], @@ -1516,9 +1483,7 @@ def test_bedrock_messages_strips_context_management(): messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}] optional_params = { "max_tokens": 4096, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( @@ -1529,9 +1494,7 @@ def test_bedrock_messages_strips_context_management(): headers={}, ) - assert "context_management" not in result, ( - "context_management should be stripped — Bedrock Invoke rejects it" - ) + assert "context_management" not in result, "context_management should be stripped — Bedrock Invoke rejects it" assert result.get("max_tokens") == 4096 @@ -1678,12 +1641,8 @@ def test_bedrock_messages_filters_user_provided_unsupported_beta_header(): ) betas = result.get("anthropic_beta") or [] - assert "advisor-tool-2026-03-01" not in betas, ( - "user-provided beta not in the Bedrock mapping must be dropped" - ) - assert "context-1m-2025-08-07" in betas, ( - "user-provided beta that IS in the Bedrock mapping should survive" - ) + assert "advisor-tool-2026-03-01" not in betas, "user-provided beta not in the Bedrock mapping must be dropped" + assert "context-1m-2025-08-07" in betas, "user-provided beta that IS in the Bedrock mapping should survive" def test_bedrock_messages_renames_user_provided_aliased_beta_header(): @@ -1711,9 +1670,7 @@ def test_bedrock_messages_renames_user_provided_aliased_beta_header(): assert "advanced-tool-use-2025-11-20" not in betas, ( "Anthropic-direct spelling should be rewritten, not forwarded verbatim" ) - assert "tool-search-tool-2025-10-19" in betas, ( - "user-provided beta should be renamed to the Bedrock-side spelling" - ) + assert "tool-search-tool-2025-10-19" in betas, "user-provided beta should be renamed to the Bedrock-side spelling" @pytest.mark.asyncio @@ -1913,7 +1870,6 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46(): same logging reconstruction as Anthropic /messages. Ensures token counts and completion_cost match model_prices for us.anthropic.claude-sonnet-4-6. """ - from litellm import completion_cost from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) @@ -1976,9 +1932,7 @@ async def test_unified_bedrock_messages_sse_usage_and_cost_claude_sonnet_46(): "global.anthropic.claude-fable-5", ], ) -def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models( - local_model_cost_map, model -): +def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models(local_model_cost_map, model): """clear_thinking_20251015 without a top-level ``thinking`` field must inject ``thinking.type=adaptive`` plus ``output_config.effort`` on adaptive-thinking models (Opus 4.7/4.8, Fable 5). The legacy ``thinking.type=enabled`` shape is @@ -1988,9 +1942,7 @@ def test_bedrock_clear_thinking_injects_adaptive_with_effort_for_adaptive_models cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2013,9 +1965,7 @@ def test_bedrock_clear_thinking_converts_legacy_enabled_budget_to_effort(): "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, }, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2033,10 +1983,7 @@ def test_resolve_clear_thinking_budget_tokens_honors_explicit_zero(): and only fall back to the minimum when the caller omits the budget.""" cfg = AmazonAnthropicClaudeMessagesConfig() assert cfg._resolve_clear_thinking_budget_tokens(0) == 0 - assert ( - cfg._resolve_clear_thinking_budget_tokens(None) - == BEDROCK_MIN_THINKING_BUDGET_TOKENS - ) + assert cfg._resolve_clear_thinking_budget_tokens(None) == BEDROCK_MIN_THINKING_BUDGET_TOKENS assert cfg._resolve_clear_thinking_budget_tokens(12000) == 12000 @@ -2046,9 +1993,7 @@ def test_bedrock_clear_thinking_keeps_enabled_for_non_adaptive_models(): cfg = AmazonAnthropicClaudeMessagesConfig() request = { "max_tokens": 32000, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2073,9 +2018,7 @@ def test_bedrock_invoke_transform_emits_adaptive_thinking_for_opus_4_8(): optional_params = { "max_tokens": 32000, "stream": False, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } result = cfg.transform_anthropic_messages_request( @@ -2112,9 +2055,7 @@ def test_bedrock_invoke_transform_normalizes_system_role_message_into_system(): assert all(m.get("role") != "system" for m in result["messages"]) assert result["messages"] == [{"role": "user", "content": "hi"}] - assert result["system"] == [ - {"type": "text", "text": "You are a careful assistant."} - ] + assert result["system"] == [{"type": "text", "text": "You are a careful assistant."}] def test_bedrock_invoke_transform_merges_system_role_into_existing_system(): @@ -2229,9 +2170,7 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(lo ) assert result["messages"] == messages - assert result["system"] == [ - {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} - ] + assert result["system"] == [{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}] def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): @@ -2414,13 +2353,13 @@ def test_bedrock_invoke_transform_converted_system_carries_only_its_content(loca assert result["messages"][2] == { "role": "user", "content": [ - { - "type": "text", - "text": ( - "Operator note (not from the user): the following was " - "originally a mid-conversation system-role reminder." - ), - }, + { + "type": "text", + "text": ( + "Operator note (not from the user): the following was " + "originally a mid-conversation system-role reminder." + ), + }, {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, ], } @@ -2556,10 +2495,7 @@ def test_as_system_content_blocks_handles_each_shape(): def test_effort_from_thinking_budget_tiers(budget_tokens, expected_effort): """The budget -> effort mapping pins each tier boundary so a shifted threshold is caught.""" - assert ( - AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) - == expected_effort - ) + assert AmazonAnthropicClaudeMessagesConfig._effort_from_thinking_budget(budget_tokens) == expected_effort def test_inject_adaptive_thinking_preserves_existing_effort(): @@ -2568,9 +2504,7 @@ def test_inject_adaptive_thinking_preserves_existing_effort(): cfg = AmazonAnthropicClaudeMessagesConfig() request = {"output_config": {"effort": "max", "other": "keep"}} - cfg._inject_adaptive_thinking_for_clear_thinking( - request, budget_tokens=24000, model="us.anthropic.claude-fable-5" - ) + cfg._inject_adaptive_thinking_for_clear_thinking(request, budget_tokens=24000, model="us.anthropic.claude-fable-5") assert request["thinking"] == {"type": "adaptive"} assert request["output_config"] == {"effort": "max", "other": "keep"} @@ -2583,9 +2517,7 @@ def test_bedrock_clear_thinking_noops_when_thinking_already_adaptive(): request = { "max_tokens": 32000, "thinking": {"type": "adaptive"}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2605,9 +2537,7 @@ def test_bedrock_clear_thinking_replaces_disabled_thinking_on_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "disabled"}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2627,9 +2557,7 @@ def test_bedrock_clear_thinking_leaves_enabled_thinking_on_non_adaptive_model(): request = { "max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 8000}, - "context_management": { - "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] - }, + "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}, } changed = cfg._ensure_thinking_for_clear_thinking_context_management( @@ -2664,9 +2592,7 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] optional_params = { "max_tokens": 4096, - "context_management": { - "edits": [{"type": "clear_tool_uses_20250919"}] - }, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, } result = cfg.transform_anthropic_messages_request( @@ -2677,12 +2603,11 @@ def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_ headers={}, ) - assert result.get("context_management") == { - "edits": [{"type": "clear_tool_uses_20250919"}] - }, "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" + assert result.get("context_management") == {"edits": [{"type": "clear_tool_uses_20250919"}]}, ( + "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" + ) assert "context-management-2025-06-27" in result.get("anthropic_beta", []), ( - "context-management-2025-06-27 beta must reach the InvokeModel body so " - "the tool-call-clearing edit is accepted" + "context-management-2025-06-27 beta must reach the InvokeModel body so the tool-call-clearing edit is accepted" ) @@ -2759,9 +2684,9 @@ def test_bedrock_messages_filters_clear_thinking_keeps_clear_tool_uses( cm = result.get("context_management") assert cm is not None - assert [e.get("type") for e in cm["edits"]] == [ - "clear_tool_uses_20250919" - ], "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" + assert [e.get("type") for e in cm["edits"]] == ["clear_tool_uses_20250919"], ( + "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" + ) betas = result.get("anthropic_beta", []) assert "context-management-2025-06-27" in betas @@ -2902,65 +2827,6 @@ def test_bedrock_messages_tool_search_follows_claude_tool_search_rule(local_mode assert cfg._supports_tool_search_on_bedrock(model) is expected -def test_bedrock_messages_tool_search_rule_fills_mapped_entry_without_flag(local_model_cost_map, monkeypatch): - """LIT-5851: a Bedrock entry that is in the map but carries no ``supports_tool_search`` - key, the state Opus 4.8, Opus 5 and Sonnet 5 shipped in, is filled by the - ``claude-tool-search`` rule instead of resolving to ``None`` and losing the beta.""" - import litellm - - model = "us.anthropic.claude-opus-5" - cfg = AmazonAnthropicClaudeMessagesConfig() - - monkeypatch.delitem(litellm.model_cost[model], "supports_tool_search") - litellm.get_model_info.cache_clear() - - assert litellm.get_model_info(model, custom_llm_provider="bedrock")["supports_tool_search"] is True - assert cfg._supports_tool_search_on_bedrock(model) is True - - -def test_bedrock_messages_thinking_shape_follows_exact_bedrock_entry_flag( - local_model_cost_map, monkeypatch -): - """The outbound thinking payload must follow the exact Bedrock cost-map entry. - Before threading the caller's provider through the capability probes, the probe - was pinned to ``"anthropic"``: the exact ``global.anthropic.claude-opus-4-8`` - entry was rejected by the provider match and the anthropic-scoped fallback rule - forced ``thinking.type='adaptive'`` even with ``supports_adaptive_thinking`` - explicitly set to ``false`` on the entry.""" - import litellm - - from litellm.types.router import GenericLiteLLMParams - - model = "global.anthropic.claude-opus-4-8" - cfg = AmazonAnthropicClaudeMessagesConfig() - - def transform(): - return cfg.transform_anthropic_messages_request( - model=model, - messages=[{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], - anthropic_messages_optional_request_params={ - "max_tokens": 4096, - "reasoning_effort": "medium", - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - result = transform() - assert result.get("thinking") == {"type": "adaptive", "display": "summarized"} - assert result.get("output_config") == {"effort": "medium"} - - monkeypatch.setitem(litellm.model_cost[model], "supports_adaptive_thinking", False) - litellm.get_model_info.cache_clear() - - flipped = transform() - thinking = flipped.get("thinking") - assert isinstance(thinking, dict) - assert thinking.get("type") == "enabled" - assert isinstance(thinking.get("budget_tokens"), int) - assert "output_config" not in flipped - - @pytest.mark.parametrize( "search_results, expected_evidence", [ diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index a8a21e2cd37..8deb16bceb2 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -1,8 +1,6 @@ - import pytest - from litellm.llms.bedrock.common_utils import BedrockModelInfo # --------------------------------------------------------------------------- # @@ -31,9 +29,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object( - mod, "_load_bedrock_response_stream_shape", return_value=sentinel - ) as mock_load: + with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -80,9 +76,7 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert ( - loaded_shape is not None - ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -147,9 +141,7 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route( - model="bedrock/us.deepseek.r1-v1:0" - ) + bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") assert bedrock_route == "converse" @@ -222,27 +214,19 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -256,23 +240,14 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") - == "anthropic.claude-opus-4-6-v1" - ) - assert ( - get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") - == "anthropic.claude-sonnet-4-6" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" + assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") - == "anthropic.claude-opus-4-6-v1" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -312,9 +287,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( - model, expected_ceiling -): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -333,54 +306,24 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - ) - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "invoke" - ) - assert ( - BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") - is False - ) + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "invoke" + assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False # A genuine mantle route still resolves, via the startswith branch... - assert ( - BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert ( - BedrockModelInfo.get_bedrock_route( - "bedrock/mantle/anthropic.claude-mythos-preview" - ) - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True # f"/{prefix}" boundary branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True # neither branch: the token only appears glued to another segment - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock_mantle/openai.gpt-5.5", "mantle/" - ) - is False - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False @pytest.mark.parametrize( @@ -430,44 +373,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert ( - BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") - is False - ) + assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert ( - BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") - is True - ) - - -def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): - """ - Regression test: a regional model_cost entry without the capability field - must not shadow a base entry that has it (`get(model) or get(base)` used to - short-circuit on the truthy regional dict and drop the capability). - """ - import litellm - from litellm.llms.bedrock.common_utils import ( - bedrock_converse_supports_parallel_tool_use_config, - is_claude_4_5_on_bedrock, - ) - - base = "anthropic.claude-fallback-test" - regional = f"eu.{base}" - monkeypatch.setitem(litellm.model_cost, regional, {"input_cost_per_token": 1e-06}) - monkeypatch.setitem( - litellm.model_cost, - base, - { - "cache_creation_input_token_cost_above_1hr": 1e-05, - "supports_parallel_tool_use_config": True, - }, - ) - - assert is_claude_4_5_on_bedrock(regional) is True - assert bedrock_converse_supports_parallel_tool_use_config(regional) is True + assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 6758a333b35..df67ee7d5ae 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -52,10 +52,7 @@ class TestBedrockMantleResponsesURL: api_base="https://bedrock-mantle.us-east-2.api.aws/v1/", litellm_params={}, ) - assert ( - url_trailing - == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" - ) + assert url_trailing == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses" def test_url_does_not_double_openai_v1(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) @@ -115,9 +112,7 @@ class TestBedrockMantleResponsesURL: with pytest.raises(ValueError, match="api\\.aws\\.attacker\\.example/'\\. Region names must contain only"): cfg.get_complete_url( api_base=None, - litellm_params={ - "aws_region_name": "us-east-1.api.aws.attacker.example/" - }, + litellm_params={"aws_region_name": "us-east-1.api.aws.attacker.example/"}, ) def test_url_region_default_us_east_1(self, monkeypatch): @@ -170,9 +165,7 @@ class TestBedrockMantleResponsesURL: class TestBedrockMantleGetLlmProviderRegion: - def test_get_llm_provider_uses_supplemental_litellm_params( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_supplemental_litellm_params(self, monkeypatch, local_cost_map): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -189,9 +182,7 @@ class TestBedrockMantleGetLlmProviderRegion: # the resolved chat base) is on the /openai/v1 base per the AWS card. assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_get_llm_provider_uses_aws_region_from_litellm_params( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_aws_region_from_litellm_params(self, monkeypatch, local_cost_map): monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) monkeypatch.delenv("AWS_REGION", raising=False) @@ -225,18 +216,14 @@ class TestBedrockMantleResponsesAuth: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert headers["Authorization"] == "Bearer env-key" def test_bedrock_bearer_token_fallback(self, monkeypatch): monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "bearer-key") cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert headers["Authorization"] == "Bearer bearer-key" def test_missing_bearer_does_not_raise_in_validate_environment(self, monkeypatch): @@ -244,9 +231,7 @@ class TestBedrockMantleResponsesAuth: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) cfg = BedrockMantleResponsesAPIConfig() - headers = cfg.validate_environment( - headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams() - ) + headers = cfg.validate_environment(headers={}, model="openai.gpt-5.5", litellm_params=GenericLiteLLMParams()) assert "Authorization" not in headers def test_project_id_sets_openai_project_header(self): @@ -254,9 +239,7 @@ class TestBedrockMantleResponsesAuth: headers = cfg.validate_environment( headers={}, model="openai.gpt-5.5", - litellm_params=GenericLiteLLMParams( - api_key="fake-key", aws_bedrock_project_id="proj_abc123def456" - ), + litellm_params=GenericLiteLLMParams(api_key="fake-key", aws_bedrock_project_id="proj_abc123def456"), ) assert headers["OpenAI-Project"] == "proj_abc123def456" @@ -357,9 +340,7 @@ class TestBedrockMantleResponsesTools: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" - ) as mock_warning: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: cfg.map_openai_params( response_api_optional_params={"tools": [{"type": "file_search", "vector_store_ids": ["vs_123"]}]}, model="openai.gpt-5.5", @@ -484,19 +465,6 @@ class TestBedrockMantleResponsesWebSearch: ) assert body["tools"] == [self._WEB_SEARCH_TOOL] - @pytest.mark.parametrize( - "model", - [ - "bedrock_mantle/openai.gpt-5.6-sol", - "bedrock_mantle/openai.gpt-5.6-terra", - "bedrock_mantle/openai.gpt-5.6-luna", - "bedrock_mantle/openai.gpt-5.5", - "bedrock_mantle/openai.gpt-5.4", - ], - ) - def test_cost_map_advertises_web_search_support(self, model): - assert litellm.supports_web_search(model=model) is True - def _codex_exec_tool(): return { @@ -573,9 +541,7 @@ class TestBedrockMantleServiceTier: from unittest.mock import patch cfg = BedrockMantleResponsesAPIConfig() - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning" - ) as mock_warning: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.warning") as mock_warning: cfg.map_openai_params( response_api_optional_params={"service_tier": "priority"}, model="openai.gpt-5.5", @@ -664,7 +630,9 @@ class TestBedrockMantleReasoningSummary: model="openai.gpt-5.6-sol", drop_params=True, ) - warnings = [record for record in caplog.records if "dropping unsupported reasoning.summary" in record.getMessage()] + warnings = [ + record for record in caplog.records if "dropping unsupported reasoning.summary" in record.getMessage() + ] assert len(warnings) == 1 assert "detailed" in warnings[0].getMessage() @@ -838,9 +806,7 @@ class TestBedrockMantleCodexAdditionalTools: def test_hoist_is_logged_at_debug_level(self): from unittest.mock import patch - with patch( - "litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug" - ) as mock_debug: + with patch("litellm.llms.bedrock_mantle.responses.transformation.verbose_logger.debug") as mock_debug: self._transform( input=[ {"type": "additional_tools", "role": "developer", "tools": self._CODEX_TOOLS}, @@ -997,7 +963,13 @@ class TestBedrockMantleCodexInputItemNormalization: {"type": "function_call", "name": "shell", "arguments": "{}", "call_id": "call_2"}, {"type": "function_call_output", "call_id": "call_2", "output": "ok"}, {"type": "tool_search_call", "call_id": "call_3", "execution": "server", "arguments": {"query": "x"}}, - {"type": "tool_search_output", "call_id": "call_3", "status": "completed", "execution": "server", "tools": []}, + { + "type": "tool_search_output", + "call_id": "call_3", + "status": "completed", + "execution": "server", + "tools": [], + }, {"type": "compaction_trigger"}, ] body = self._transform(input=copy.deepcopy(supported_items)) @@ -1011,7 +983,12 @@ class TestBedrockMantleCodexInputItemNormalization: with caplog.at_level(logging.WARNING, logger="LiteLLM"): body = self._transform( input=[ - {"type": "agent_message", "author": "a", "recipient": "b", "content": [{"type": "input_text", "text": "hi"}]}, + { + "type": "agent_message", + "author": "a", + "recipient": "b", + "content": [{"type": "input_text", "text": "hi"}], + }, self._USER_MESSAGE, ] ) @@ -1150,9 +1127,7 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_price_map_flag_routes_non_gpt_name_to_openai_path( - self, restore_model_cost - ): + def test_price_map_flag_routes_non_gpt_name_to_openai_path(self, restore_model_cost): # Data-driven onboarding: a frontier model whose name does NOT match the # openai.gpt- convention can still be routed to /openai/v1/responses by # declaring use_openai_responses_path in its price-map entry, with no code @@ -1175,22 +1150,6 @@ class TestBedrockMantleResponsesRegistry: assert isinstance(cfg, BedrockMantleResponsesAPIConfig) assert cfg.use_openai_path is True - def test_gpt_5_5_price_map_declares_openai_responses_path(self, local_cost_map): - # The gpt-5.x entries must carry the data-driven flag so frontier routing - # does not rely on the name-string fallback alone. - assert ( - litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get( - "use_openai_responses_path" - ) - is True - ) - assert ( - litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get( - "use_openai_responses_path" - ) - is True - ) - @pytest.mark.parametrize( "model", [ @@ -1224,9 +1183,7 @@ class TestBedrockMantleResponsesRegistry: ) assert cfg is None - def test_declared_responses_non_openai_routes_to_standard_path( - self, restore_model_cost - ): + def test_declared_responses_non_openai_routes_to_standard_path(self, restore_model_cost): # New feature: a non-OpenAI model declared mode=responses (e.g. via a # user's proxy model_info block) must route to the STANDARD /v1/responses # path, not the frontier /openai/v1/responses path. Fails before the @@ -1324,88 +1281,12 @@ class TestMantleBaseSegment: the /openai/v1 base, everything else on /v1. An unmapped model defaults to /v1. """ - @pytest.mark.parametrize( - "model,model_cost,expected", - [ - ( - "openai.gpt-5.5", - {"bedrock_mantle/openai.gpt-5.5": {"use_openai_responses_path": True}}, - "openai/v1", - ), - ( - "google.gemma-4-31b", - { - "bedrock_mantle/google.gemma-4-31b": { - "use_openai_responses_path": True - } - }, - "openai/v1", - ), - ( - "openai.gpt-oss-120b", - {"bedrock_mantle/openai.gpt-oss-120b": {}}, - "v1", - ), - ("openai.gpt-oss-120b", {}, "v1"), - (None, {}, "v1"), - ], - ) - def test_base_segment(self, model, model_cost, expected): - from litellm.llms.bedrock_mantle.common_utils import mantle_base_segment - - assert mantle_base_segment(model, model_cost) == expected - class TestMantleSupportsResponses: """The capability helper is data-driven (supported_endpoints / mode), with no model-name match: per-model, so gpt-oss-120b is supported but the safeguard variant is not despite the shared substring.""" - @pytest.mark.parametrize( - "model,model_cost,expected", - [ - # supported_endpoints lists responses -> supported - ( - "openai.gpt-oss-120b", - { - "bedrock_mantle/openai.gpt-oss-120b": { - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] - } - }, - True, - ), - # chat-only supported_endpoints -> not supported (the discriminator) - ( - "openai.gpt-oss-safeguard-120b", - { - "bedrock_mantle/openai.gpt-oss-safeguard-120b": { - "supported_endpoints": ["/v1/chat/completions"] - } - }, - False, - ), - # mode=responses (no supported_endpoints) -> supported - ( - "somelab.future-model", - {"bedrock_mantle/somelab.future-model": {"mode": "responses"}}, - True, - ), - # mode=chat, no responses endpoint -> not supported - ( - "google.gemma-3-27b-it", - {"bedrock_mantle/google.gemma-3-27b-it": {"mode": "chat"}}, - False, - ), - # absent from model_cost -> no signal -> not supported - ("somelab.unmapped", {}, False), - (None, {}, False), - ], - ) - def test_supports_responses(self, model, model_cost, expected): - from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses - - assert mantle_supports_responses(model, model_cost) is expected - class TestBedrockMantlePerModelResponsesURL: """End-to-end: the registry-selected config must build the correct wire URL @@ -1420,9 +1301,7 @@ class TestBedrockMantlePerModelResponsesURL: model=model, ) assert isinstance(cfg, BedrockMantleResponsesAPIConfig) - return cfg.get_complete_url( - api_base=None, litellm_params={"aws_region_name": region} - ) + return cfg.get_complete_url(api_base=None, litellm_params={"aws_region_name": region}) def test_gpt_oss_uses_standard_responses_path(self, local_cost_map): url = self._url_for("openai.gpt-oss-120b") @@ -1521,9 +1400,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.delenv("BEDROCK_MANTLE_API_KEY", raising=False) signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, signed_body = cfg.sign_request( @@ -1545,9 +1422,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1569,9 +1444,7 @@ class TestBedrockMantleResponsesSigV4: monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-bearer") signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("get_credentials must not run for bearer auth") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("get_credentials must not run for bearer auth")) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) headers, _ = cfg.sign_request( @@ -1717,9 +1590,7 @@ class TestBedrockMantleResponsesSigV4: } cfg = BedrockMantleResponsesAPIConfig(aws_signer=BaseAWSLLM()) url = cfg.get_complete_url(api_base=None, litellm_params=params) - assert ( - url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" - ) + assert url == "https://bedrock-mantle.ap-southeast-2.api.aws/openai/v1/responses" headers, _ = cfg.sign_request( headers={}, @@ -1730,9 +1601,7 @@ class TestBedrockMantleResponsesSigV4: ) assert "/ap-southeast-2/bedrock/aws4_request" in headers["Authorization"] - def test_injected_default_region_base_does_not_override_aws_region_name( - self, monkeypatch - ): + def test_injected_default_region_base_does_not_override_aws_region_name(self, monkeypatch): """2nd-round adversarial regression: responses/main.py auto-injects litellm_params.api_base = https://bedrock-mantle..api.aws/v1 (default region, ignoring aws_region_name). The config must still pin BOTH the URL host @@ -1835,7 +1704,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1865,7 +1734,7 @@ class TestBedrockMantleResponsesSigV4: signer.get_credentials = MagicMock(side_effect=cred_error) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -1890,9 +1759,7 @@ class TestBedrockMantleResponsesSigV4: signer = BaseAWSLLM() signer.get_credentials = MagicMock( - side_effect=ConnectTimeoutError( - endpoint_url="https://sts.us-east-2.amazonaws.com" - ) + side_effect=ConnectTimeoutError(endpoint_url="https://sts.us-east-2.amazonaws.com") ) cfg = BedrockMantleResponsesAPIConfig(aws_signer=signer) @@ -1910,8 +1777,6 @@ class TestBedrockMantleResponsesSigV4: class TestBedrockMantleResponsesPricing: - - def test_models_registered(self, local_cost_map): assert "bedrock_mantle/openai.gpt-5.5" in litellm.bedrock_mantle_models assert "bedrock_mantle/openai.gpt-5.4" in litellm.bedrock_mantle_models diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 15570eaec4d..97465d8c49e 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -46,21 +46,6 @@ class TestBedrockMantleProviderRegistration: def test_provider_in_provider_list(self): assert "bedrock_mantle" in litellm.provider_list - def test_models_loaded(self, monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") - litellm.add_known_models() - assert len(litellm.bedrock_mantle_models) > 0 - assert "bedrock_mantle/openai.gpt-oss-120b" in litellm.bedrock_mantle_models - assert "bedrock_mantle/openai.gpt-oss-20b" in litellm.bedrock_mantle_models - assert ( - "bedrock_mantle/openai.gpt-oss-safeguard-120b" - in litellm.bedrock_mantle_models - ) - assert ( - "bedrock_mantle/openai.gpt-oss-safeguard-20b" - in litellm.bedrock_mantle_models - ) - class TestBedrockMantleConfig: def test_custom_llm_provider(self): @@ -113,9 +98,7 @@ class TestBedrockMantleConfig: cfg._get_openai_compatible_provider_info( None, None, - litellm_params=GenericLiteLLMParams( - aws_region_name="us-east-1.api.aws.attacker.example/" - ), + litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), ) def test_get_llm_provider_rejects_malicious_aws_region_name(self, monkeypatch): @@ -128,14 +111,10 @@ class TestBedrockMantleConfig: litellm.get_llm_provider( model="openai.gpt-5.5", custom_llm_provider="bedrock_mantle", - litellm_params=GenericLiteLLMParams( - aws_region_name="us-east-1.api.aws.attacker.example/" - ), + litellm_params=GenericLiteLLMParams(aws_region_name="us-east-1.api.aws.attacker.example/"), ) - def test_get_llm_provider_uses_aws_region_name_for_responses( - self, monkeypatch, local_cost_map - ): + def test_get_llm_provider_uses_aws_region_name_for_responses(self, monkeypatch, local_cost_map): from litellm.types.router import GenericLiteLLMParams monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) @@ -193,18 +172,14 @@ class TestBedrockMantleConfig: monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - None, None, model="openai.gpt-oss-120b" - ) + api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model="openai.gpt-oss-120b") assert api_base == "https://bedrock-mantle.us-east-2.api.aws/v1" @pytest.mark.parametrize( "model_id", ["google.gemma-4-31b", "google.gemma-4-26b-a4b", "google.gemma-4-e2b"], ) - def test_chat_base_for_gemma_4_uses_openai_v1( - self, monkeypatch, local_cost_map, model_id - ): + def test_chat_base_for_gemma_4_uses_openai_v1(self, monkeypatch, local_cost_map, model_id): # The chat-config bug the Gemma 4 cards exposed: gemma-4-* is served on the # /openai/v1 base, not the hardcoded /v1. Driven by the price-map # use_openai_responses_path flag (loaded by local_cost_map). Fails before @@ -212,22 +187,16 @@ class TestBedrockMantleConfig: monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2") monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - None, None, model=model_id - ) + api_base, _ = cfg._get_openai_compatible_provider_info(None, None, model=model_id) assert api_base == "https://bedrock-mantle.us-east-2.api.aws/openai/v1" - def test_chat_base_explicit_api_base_wins_over_derived( - self, monkeypatch, local_cost_map - ): + def test_chat_base_explicit_api_base_wins_over_derived(self, monkeypatch, local_cost_map): # An explicit api_base must not be overridden by the data-driven default, # even for a model whose default differs (gemma-4 -> openai/v1). monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) custom_base = "https://bedrock-mantle.us-west-2.api.aws/v1" cfg = BedrockMantleChatConfig() - api_base, _ = cfg._get_openai_compatible_provider_info( - custom_base, None, model="google.gemma-4-31b" - ) + api_base, _ = cfg._get_openai_compatible_provider_info(custom_base, None, model="google.gemma-4-31b") assert api_base == custom_base def test_api_key_from_env(self, monkeypatch): @@ -282,9 +251,7 @@ class TestBedrockMantleChatAuth: from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM signer = BaseAWSLLM() - signer.get_credentials = MagicMock( - side_effect=AssertionError("SigV4 must not run when a Bearer token exists") - ) + signer.get_credentials = MagicMock(side_effect=AssertionError("SigV4 must not run when a Bearer token exists")) return signer def test_bearer_token_skips_sigv4(self, monkeypatch): @@ -401,9 +368,7 @@ class TestBedrockMantleChatAuth: assert "/eu-west-1/bedrock/aws4_request" in headers["Authorization"] - def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees( - self, monkeypatch - ): + def test_sigv4_scope_matches_api_base_when_aws_region_name_disagrees(self, monkeypatch): # If a caller (e.g. proxy) passes a stale api_base in one region and an # aws_region_name in a different region, the SigV4 credential scope must # match the URL host or Bedrock rejects the request with 401. Without the @@ -491,7 +456,7 @@ class TestBedrockMantleChatAuth: signer.get_credentials = MagicMock(side_effect=NoCredentialsError()) cfg = BedrockMantleChatConfig(aws_signer=signer) - with pytest.raises(ValueError, match='Bedrock Mantle auth failed: no Bearer token and no usable') as exc: + with pytest.raises(ValueError, match="Bedrock Mantle auth failed: no Bearer token and no usable") as exc: cfg.sign_request( headers={}, optional_params={"aws_region_name": "us-east-2"}, @@ -517,9 +482,7 @@ class TestBedrockMantleChatAuth: ): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE") - monkeypatch.setenv( - "AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0" - ) + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0") monkeypatch.setenv("AWS_REGION", "us-east-2") requests = [] @@ -549,9 +512,7 @@ class TestBedrockMantleChatAuth: request=httpx.Request("POST", url), ) - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post - ): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -595,7 +556,9 @@ class TestBedrockMantleChatAuth: "object": "chat.completion", "created": 1733529600, "model": "google.gemma-4-31b", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, }, request=httpx.Request("POST", url), @@ -661,9 +624,7 @@ class TestBedrockMantleProjectHeader: def mock_post(self, url, data=None, headers=None, **kwargs): raw_body = data.decode("utf-8") if isinstance(data, bytes) else data - requests.append( - {"headers": headers or {}, "body": json.loads(raw_body or "{}")} - ) + requests.append({"headers": headers or {}, "body": json.loads(raw_body or "{}")}) return httpx.Response( status_code=200, json={ @@ -687,9 +648,7 @@ class TestBedrockMantleProjectHeader: request=httpx.Request("POST", url), ) - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post - ): + with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): response = litellm.completion( model="bedrock_mantle/openai.gpt-oss-120b", messages=[{"role": "user", "content": "hello"}], @@ -705,20 +664,15 @@ class TestBedrockMantleProjectHeader: class TestBedrockMantleProviderResolution: def test_get_llm_provider_resolves_correctly(self): - model, provider, _, _ = litellm.get_llm_provider( - "bedrock_mantle/openai.gpt-oss-120b" - ) + model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-120b") assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-120b" def test_get_llm_provider_20b(self): - model, provider, _, _ = litellm.get_llm_provider( - "bedrock_mantle/openai.gpt-oss-20b" - ) + model, provider, _, _ = litellm.get_llm_provider("bedrock_mantle/openai.gpt-oss-20b") assert provider == "bedrock_mantle" assert model == "openai.gpt-oss-20b" - def test_get_llm_provider_strips_region_prefix(self, monkeypatch, local_cost_map): for var in ("BEDROCK_MANTLE_REGION", "BEDROCK_MANTLE_API_BASE", "AWS_REGION", "AWS_REGION_NAME"): monkeypatch.delenv(var, raising=False) @@ -751,7 +705,9 @@ class TestBedrockMantleProviderResolution: "object": "chat.completion", "created": 1733529600, "model": "xai.grok-4.3", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 38, "completion_tokens": 20, "total_tokens": 58}, }, request=request, @@ -836,15 +792,6 @@ class TestBedrockMantleProviderResolution: class TestBedrockMantlePricing: """Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing.""" - def test_safeguard_models_have_larger_output_tokens(self, monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") - litellm.add_known_models() - info_120b = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") - info_safeguard = litellm.get_model_info( - "bedrock_mantle/openai.gpt-oss-safeguard-120b" - ) - assert info_safeguard["max_output_tokens"] > info_120b["max_output_tokens"] - @pytest.mark.parametrize( "model_id", diff --git a/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py b/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py deleted file mode 100644 index 7ee34c6c55a..00000000000 --- a/tests/test_litellm/llms/cohere/ocr/test_cohere_parse_cost.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path - -import pytest - -import litellm -from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo - -REPO_ROOT = Path(__file__).parents[5] -COST_MAPS = [ - REPO_ROOT / "model_prices_and_context_window.json", - REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json", -] -MODELS = [("cohere/parse-v5.0", "cohere"), ("azure_ai/Cohere-parse-v5", "azure_ai")] - - -def _ocr_response(model: str, pages_processed: int) -> OCRResponse: - return OCRResponse( - pages=[OCRPage(index=i, markdown=f"page {i}") for i in range(pages_processed)], - model=model, - usage_info=OCRUsageInfo(pages_processed=pages_processed), - ) - - -@pytest.mark.parametrize("model, provider", MODELS) -def test_model_info_resolves_ocr_mode_and_price(local_model_cost_map, model: str, provider: str) -> None: - info = litellm.get_model_info(model=model, custom_llm_provider=provider) - - assert info["mode"] == "ocr" diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 34a6d37663b..80372418026 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -103,33 +103,3 @@ def test_crusoe_provider_detection_by_prefix(): model, provider, _, _ = get_llm_provider("crusoe/meta-llama/Llama-3.3-70B-Instruct") assert provider == "crusoe" assert model == "meta-llama/Llama-3.3-70B-Instruct" - - -def test_crusoe_model_list_populated(monkeypatch): - """Test Crusoe models are present in model_prices_and_context_window.json""" - import litellm - - original_model_cost = litellm.model_cost - original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - try: - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - - expected = [ - "crusoe/meta-llama/Llama-3.3-70B-Instruct", - "crusoe/deepseek-ai/DeepSeek-R1-0528", - "crusoe/deepseek-ai/DeepSeek-V3-0324", - "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", - "crusoe/moonshotai/Kimi-K2-Thinking", - "crusoe/openai/gpt-oss-120b", - "crusoe/google/gemma-3-12b-it", - ] - for model in expected: - assert model in litellm.model_cost, f"{model} not found in model_cost" - assert litellm.model_cost[model].get("litellm_provider") == "crusoe" - finally: - litellm.model_cost = original_model_cost - if original_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env) diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py index a30d35d46f2..17bbf9852e7 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py @@ -42,9 +42,7 @@ class TestDashscopeCostCalculator: """ usage = Usage(prompt_tokens=1000, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-max", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-max", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-max") expected_prompt_cost = 1000 * model_info["input_cost_per_token"] @@ -60,9 +58,7 @@ class TestDashscopeCostCalculator: """ # Tier 1 for qwen-flash is [0, 256,000] tokens usage = Usage(prompt_tokens=100000, completion_tokens=50000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1_pricing = model_info["tiered_pricing"][0] @@ -80,9 +76,7 @@ class TestDashscopeCostCalculator: """ # Tiering for qwen-flash: Tier 1: [0, 256k], Tier 2: [256k, 1M] usage = Usage(prompt_tokens=300000, completion_tokens=300000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1 = model_info["tiered_pricing"][0] @@ -94,9 +88,7 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) - graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + ( - 44000 * tier_2["input_cost_per_token"] - ) + graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (44000 * tier_2["input_cost_per_token"]) assert prompt_cost > graduated_prompt_cost def test_dashscope_tiered_pricing_boundary_stays_in_lower_tier(self): @@ -105,18 +97,12 @@ class TestDashscopeCostCalculator: official `0 < Token <= 256K` phrasing. """ usage = Usage(prompt_tokens=256000, completion_tokens=1000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose( - prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10 - ) - assert math.isclose( - completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10) + assert math.isclose(completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10) def test_dashscope_tiered_pricing_output_uses_input_selected_tier(self): """ @@ -128,9 +114,7 @@ class TestDashscopeCostCalculator: tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose( - completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10) def test_dashscope_tiered_pricing_with_caching(self): """ @@ -159,17 +143,13 @@ class TestDashscopeCostCalculator: """ Requests above the highest declared range bill entirely at the last tier's rate. """ - usage = Usage( - prompt_tokens=1200000, completion_tokens=1000 - ) # Max defined range for qwen-flash is 1M + usage = Usage(prompt_tokens=1200000, completion_tokens=1000) # Max defined range for qwen-flash is 1M prompt_cost, _ = dashscope_cost_per_token(model="qwen-flash", usage=usage) tier_2 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][1] - assert math.isclose( - prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10) def _register_tiered_model(self, model_key: str, tiered_pricing: list[dict]) -> None: litellm.model_cost[model_key] = { @@ -204,9 +184,7 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-str-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) assert math.isclose(prompt_cost, 500 * float("4e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 200 * float("1.6e-06"), rel_tol=1e-10) @@ -219,9 +197,7 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=2500, completion_tokens=3000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-str-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) assert math.isclose(prompt_cost, 2500 * float("8e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 3000 * float("3.2e-06"), rel_tol=1e-10) @@ -254,18 +230,12 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=300000, # 200k new + 60k cache creation + 40k cache read completion_tokens=1000, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=40000, cache_creation_tokens=60000 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-write-test", usage=usage) - expected_prompt_cost = ( - (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) - ) + expected_prompt_cost = (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -302,13 +272,9 @@ class TestDashscopeCostCalculator: completion_tokens_details={"reasoning_tokens": 170}, ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-nested-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-nested-cache-write-test", usage=usage) - assert math.isclose( - prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10) def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self): """ @@ -332,9 +298,7 @@ class TestDashscopeCostCalculator: prompt_tokens_details=PromptTokensDetailsWrapper(cache_creation_tokens=4000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-no-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-no-cache-write-test", usage=usage) assert math.isclose(prompt_cost, 10000 * 3.25e-07, rel_tol=1e-10) @@ -352,18 +316,12 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=10000, completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=2000, cache_creation_tokens=3000 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=2000, cache_creation_tokens=3000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-flat-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-flat-cache-write-test", usage=usage) - expected_prompt_cost = ( - (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) - ) + expected_prompt_cost = (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -380,9 +338,7 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-input-only-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-input-only-tier-test", usage=usage) assert math.isclose(prompt_cost, 500 * 4e-07, rel_tol=1e-10) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) @@ -405,13 +361,9 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-input-only-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-input-only-reasoning-test", usage=usage) - assert math.isclose( - completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10 - ) + assert math.isclose(completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10) def test_dashscope_tier_output_rate_wins_over_the_model_reasoning_rate(self): """ @@ -436,36 +388,10 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-tier-output-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-tier-output-reasoning-test", usage=usage) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) - def test_dashscope_model_zero_reasoning_rate_bills_reasoning_free(self): - """ - Regression: a model declaring an explicit zero reasoning rate had it treated as - missing, billing reasoning tokens at the plain output rate instead of free. - """ - litellm.model_cost["dashscope/qwen-zero-reasoning-test"] = { - "litellm_provider": "dashscope", - "mode": "chat", - "input_cost_per_token": 4e-07, - "output_cost_per_token": 1.6e-06, - "output_cost_per_reasoning_token": 0, - } - - usage = Usage( - prompt_tokens=500, - completion_tokens=200, - completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), - ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-zero-reasoning-test", usage=usage - ) - - assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10) - def test_dashscope_tier_zero_reasoning_rate_bills_reasoning_free(self): """ Regression: a tier declaring an explicit zero reasoning rate had it treated as @@ -489,9 +415,7 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-tier-zero-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-tier-zero-reasoning-test", usage=usage) assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10) @@ -520,9 +444,7 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=0, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-zero-input-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-zero-input-test", usage=usage) assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index db25c4307d2..ea55980a558 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch import pytest import litellm -from litellm import supports_reasoning, supports_vision +from litellm import supports_vision from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id @@ -216,9 +216,7 @@ def test_validate_environment_raises_without_api_key(monkeypatch): def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): assert ( - get_fireworks_session_id( - {"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"} - ) + get_fireworks_session_id({"litellm_session_id": "session-123", "litellm_trace_id": "trace-123"}) == "session-123" ) @@ -270,59 +268,18 @@ def test_handle_message_content_with_tool_calls(): }, } ] - updated_message = config._handle_message_content_with_tool_calls( - message, tool_calls - ) + updated_message = config._handle_message_content_with_tool_calls(message, tool_calls) assert updated_message.tool_calls is not None assert len(updated_message.tool_calls) == 1 assert updated_message.tool_calls[0].function.name == "get_current_weather" - assert ( - updated_message.tool_calls[0].function.arguments - == expected_tool_call.function.arguments - ) - - -def test_supports_reasoning_effort(): - """Test that reasoning_effort is only supported for specific Fireworks AI models.""" - supported_models = [ - "fireworks_ai/accounts/fireworks/models/qwen3-8b", - "fireworks_ai/accounts/fireworks/models/qwen3-32b", - "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct", - "fireworks_ai/accounts/fireworks/models/deepseek-v3p1", - "fireworks_ai/accounts/fireworks/models/deepseek-v3p2", - "fireworks_ai/accounts/fireworks/models/glm-4p5", - "fireworks_ai/accounts/fireworks/models/glm-4p5-air", - "fireworks_ai/accounts/fireworks/models/glm-4p6", - "fireworks_ai/accounts/fireworks/models/glm-4p7", - "fireworks_ai/accounts/fireworks/models/glm-5p1", - "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", - "fireworks_ai/accounts/fireworks/models/gpt-oss-20b", - "fireworks_ai/glm-5p1", - ] - - unsupported_models = [ - "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct", - "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct", - ] - - for model in supported_models: - assert ( - supports_reasoning(model=model, custom_llm_provider="fireworks_ai") is True - ), f"{model} should support reasoning_effort" - - for model in unsupported_models: - assert ( - supports_reasoning(model=model, custom_llm_provider="fireworks_ai") is False - ), f"{model} should not support reasoning_effort" + assert updated_message.tool_calls[0].function.arguments == expected_tool_call.function.arguments def test_get_supported_openai_params_reasoning_effort(): """Test that reasoning_effort is only included in supported params for models that support it.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "reasoning_effort" in supported_params assert "thinking" in supported_params @@ -337,9 +294,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): """Test that parallel_tool_calls is included for models that support function calling.""" config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p1" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p1") assert "parallel_tool_calls" in supported_params assert "tools" in supported_params assert "tool_choice" in supported_params @@ -353,9 +308,7 @@ def test_get_supported_openai_params_parallel_tool_calls(): def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_entry(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/deepseek-v4-pro-0813" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/deepseek-v4-pro-0813") assert "tool_choice" in supported_params assert "reasoning_effort" in supported_params @@ -364,46 +317,11 @@ def test_get_supported_openai_params_short_model_name_resolves_account_prefixed_ def test_get_supported_openai_params_preserves_generic_reasoning_fallback(): config = FireworksAIConfig() - supported_params = config.get_supported_openai_params( - "fireworks_ai/accounts/fireworks/models/glm-5p3-flash" - ) + supported_params = config.get_supported_openai_params("fireworks_ai/accounts/fireworks/models/glm-5p3-flash") assert "reasoning_effort" in supported_params -def test_get_supported_openai_params_parallel_tool_calls_without_tool_choice( - monkeypatch, -): - """Test that parallel_tool_calls is gated on tools, not tool_choice.""" - config = FireworksAIConfig() - model = "fireworks_ai/test-tools-without-tool-choice" - monkeypatch.setitem( - litellm.model_cost, - model, - { - "supports_function_calling": True, - "supports_tool_choice": False, - }, - ) - - supported_params = config.get_supported_openai_params(model) - - assert "tools" in supported_params - assert "parallel_tool_calls" in supported_params - assert "tool_choice" not in supported_params - - -def test_get_provider_info_omits_false_supports_reasoning(monkeypatch): - """Test that Fireworks only overrides supports_reasoning for supported models.""" - config = FireworksAIConfig() - model = "fireworks_ai/test-reasoning-false" - monkeypatch.setitem(litellm.model_cost, model, {"supports_reasoning": False}) - - info = config.get_provider_info(model) - - assert "supports_reasoning" not in info - - @pytest.mark.parametrize( "api_base, expected_url_prefix", [ @@ -433,14 +351,10 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): mock_response = MagicMock() mock_response.status_code = 200 - mock_response.json.return_value = { - "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] - } + mock_response.json.return_value = {"models": [{"name": "accounts/fireworks/models/llama-v3-70b"}]} with ( - patch( - "litellm.module_level_client.get", return_value=mock_response - ) as mock_get, + patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, patch( "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", side_effect=lambda key: { @@ -452,13 +366,9 @@ def test_get_models_url_no_double_v1(api_base, expected_url_prefix): ): result = config.get_models(api_key="test-key", api_base=api_base) - called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get( - "url", "" - ) + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" - assert called_url.startswith( - expected_url_prefix - ), f"URL {called_url} does not start with {expected_url_prefix}" + assert called_url.startswith(expected_url_prefix), f"URL {called_url} does not start with {expected_url_prefix}" assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] @@ -486,21 +396,11 @@ def test_transform_messages_helper_removes_provider_specific_fields(): }, ] # Call helper - out = config._transform_messages_helper( - messages, model="fireworks/test", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="fireworks/test", litellm_params={}) for msg in out: assert "provider_specific_fields" not in msg -def test_unmapped_model_fallback_function_calling(): - """Test that a model not in model_cost still defaults to supporting function calling for Fireworks.""" - config = FireworksAIConfig() - model = "fireworks_ai/unmapped-future-model" - info = config.get_provider_info(model) - assert info["supports_function_calling"] is True - - def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_content(): """Fireworks rejects thinking_blocks but requires reasoning_content to be replayed for reasoning_history.""" config = FireworksAIConfig() @@ -509,15 +409,11 @@ def test_transform_messages_helper_strips_thinking_blocks_but_keeps_reasoning_co { "role": "assistant", "content": "I can help.", - "thinking_blocks": [ - {"type": "thinking", "thinking": "internal", "signature": ""} - ], + "thinking_blocks": [{"type": "thinking", "thinking": "internal", "signature": ""}], "reasoning_content": "internal", }, ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p1", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p1", litellm_params={}) assert "thinking_blocks" not in out[1] assert out[1]["reasoning_content"] == "internal" assert out[1]["content"] == "I can help." @@ -1007,9 +903,7 @@ def test_transform_messages_helper_rejects_file_blocks(): litellm.BadRequestError, match="Fireworks AI chat completions does not support file content blocks", ): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/kimi-k2p6", litellm_params={}) def test_transform_messages_helper_rejects_non_vision_image_inputs(): @@ -1021,18 +915,14 @@ def test_transform_messages_helper_rejects_non_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] with pytest.raises(litellm.BadRequestError, match="does not support image inputs"): - config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) def test_transform_messages_helper_allows_vision_image_inputs(): @@ -1044,9 +934,7 @@ def test_transform_messages_helper_allows_vision_image_inputs(): {"type": "text", "text": "Describe this"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } @@ -1070,9 +958,7 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): custom_model = "accounts/myorg/models/custom-glm-5p2" assert config._get_model_cost_capability(custom_model, "supports_vision") is False - assert ( - config._get_model_cost_capability_exact(custom_model, "supports_vision") is None - ) + assert config._get_model_cost_capability_exact(custom_model, "supports_vision") is None messages = [ { @@ -1080,16 +966,12 @@ def test_image_inputs_not_rejected_for_fuzzy_non_vision_match(): "content": [ { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE=" - }, + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE="}, }, ], } ] - out = config._transform_messages_helper( - messages, model=custom_model, litellm_params={} - ) + out = config._transform_messages_helper(messages, model=custom_model, litellm_params={}) assert out == messages @@ -1102,9 +984,7 @@ def test_transform_messages_helper_skips_non_dict_content(): } ] - out = config._transform_messages_helper( - messages, model="accounts/fireworks/models/glm-5p2", litellm_params={} - ) + out = config._transform_messages_helper(messages, model="accounts/fireworks/models/glm-5p2", litellm_params={}) assert out == messages @@ -1125,26 +1005,6 @@ def test_transform_messages_helper_no_transform_inline(): assert "#transform=inline" not in block["image_url"] -def test_get_provider_info_vision_from_model_cost(monkeypatch): - config = FireworksAIConfig() - - vision_model = "fireworks_ai/test-vision-from-cost" - monkeypatch.setitem( - litellm.model_cost, - vision_model, - {"supports_vision": True, "supports_pdf_input": True}, - ) - info = config.get_provider_info(vision_model) - assert info["supports_vision"] is True - assert info["supports_pdf_input"] is True - - no_vision_model = "fireworks_ai/test-no-vision-from-cost" - monkeypatch.setitem(litellm.model_cost, no_vision_model, {}) - info_no_vision = config.get_provider_info(no_vision_model) - assert info_no_vision.get("supports_vision") is not True - assert "supports_pdf_input" not in info_no_vision - - def test_reasoning_effort_boolean_true_to_medium(): config = FireworksAIConfig() result = config.map_openai_params( @@ -1344,9 +1204,7 @@ def test_streaming_surfaces_fireworks_response_fields(): surfaced: dict = {} for chunk in stream: fields = getattr(chunk, "provider_specific_fields", None) or {} - surfaced.update( - {k: v for k, v in fields.items() if k.startswith("fireworks_")} - ) + surfaced.update({k: v for k, v in fields.items() if k.startswith("fireworks_")}) assert surfaced["fireworks_token_ids"] == [[123]] assert surfaced["fireworks_raw_outputs"] == [raw_output] @@ -1399,9 +1257,7 @@ def test_transform_request_direct_route_passthrough(): def test_map_extra_body_params_translates_truncate_prompt_tokens(): config = FireworksAIConfig() - result = config.map_extra_body_params( - {"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"truncate_prompt_tokens": 4096}}, _REASONING_MODEL) assert result == {"prompt_truncate_len": 4096} @@ -1560,9 +1416,7 @@ def test_map_extra_body_params_non_dict_chat_template_kwargs_dropped(): def test_map_extra_body_params_guided_json(): config = FireworksAIConfig() schema = {"type": "object", "properties": {"x": {"type": "string"}}} - result = config.map_extra_body_params( - {"extra_body": {"guided_json": schema}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {"guided_json": schema}}, _REASONING_MODEL) assert result == { "response_format": { "type": "json_schema", @@ -1573,16 +1427,10 @@ def test_map_extra_body_params_guided_json(): def test_map_extra_body_params_guided_grammar_and_choice(): config = FireworksAIConfig() - grammar = config.map_extra_body_params( - {"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL - ) - assert grammar == { - "response_format": {"type": "grammar", "grammar": "root ::= 'hello'"} - } + grammar = config.map_extra_body_params({"extra_body": {"guided_grammar": "root ::= 'hello'"}}, _REASONING_MODEL) + assert grammar == {"response_format": {"type": "grammar", "grammar": "root ::= 'hello'"}} - choice = config.map_extra_body_params( - {"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL - ) + choice = config.map_extra_body_params({"extra_body": {"guided_choice": ["yes", "no"]}}, _REASONING_MODEL) assert choice == { "response_format": { "type": "json_schema", @@ -1668,9 +1516,7 @@ def test_map_extra_body_params_strips_unsupported_nim_vllm_params(param, value, config = FireworksAIConfig() with caplog.at_level(logging.DEBUG): - result = config.map_extra_body_params( - {"extra_body": {param: value}}, _REASONING_MODEL - ) + result = config.map_extra_body_params({"extra_body": {param: value}}, _REASONING_MODEL) assert result == {} assert param in caplog.text @@ -1762,10 +1608,7 @@ def test_in_schema_unsupported_params_still_raise(): def test_streaming_preserves_selected_model_for_private_accounting(): from litellm.llms.custom_httpx.http_handler import HTTPHandler - requested_route = ( - "accounts/fireworks/routers/firerouter/" - "kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" - ) + requested_route = "accounts/fireworks/routers/firerouter/kimi-k3/deepseek-v4-pro-0813/deepseek-v4-flash-0731" selected_model = "deepseek-v4-flash-0731" sse_lines = [ "data: " @@ -1819,19 +1662,14 @@ def test_streaming_preserves_selected_model_for_private_accounting(): assert chunks assert {chunk.model for chunk in chunks} == {requested_route} - assert { - chunk._hidden_params.get("provider_response_model") for chunk in chunks - } == {selected_model} + assert {chunk._hidden_params.get("provider_response_model") for chunk in chunks} == {selected_model} assembled = litellm.stream_chunk_builder(chunks=chunks) assert assembled is not None assert assembled.model == requested_route assert assembled._hidden_params["provider_response_model"] == selected_model selected_model_info = litellm.model_cost[f"fireworks_ai/{selected_model}"] - expected_cost = ( - 5 * selected_model_info["input_cost_per_token"] - + selected_model_info["output_cost_per_token"] - ) + expected_cost = 5 * selected_model_info["input_cost_per_token"] + selected_model_info["output_cost_per_token"] assert litellm.completion_cost( completion_response=assembled, custom_llm_provider="fireworks_ai", diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 1bee310d9d3..c162415b53f 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -122,20 +122,6 @@ def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_ assert math.isclose(peak_prompt_cost, 1000 * STANDARD_INPUT_COST, rel_tol=1e-10) -def test_off_peak_defaults_to_the_current_time(): - """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the - default current time.""" - _register_off_peak_model( - {"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08} - ) - usage = _usage(prompt_tokens=1000, cached_tokens=0, completion_tokens=200) - - prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage) - - assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10) - assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) - - COMPONENT_MODEL = "accounts/fireworks/models/cost-components-test" COMPONENT_INPUT_COST = 1e-06 COMPONENT_OUTPUT_COST = 2e-06 diff --git a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py index 1a0340a0a67..c4c023077fc 100644 --- a/tests/test_litellm/llms/inception/test_inception_chat_transformation.py +++ b/tests/test_litellm/llms/inception/test_inception_chat_transformation.py @@ -7,7 +7,6 @@ import os from unittest import mock import httpx -import pytest import litellm from litellm.llms.inception.chat.transformation import InceptionChatConfig @@ -189,21 +188,15 @@ def test_inception_does_not_leak_key_to_caller_api_base(): caller also supplies their own key. """ config = InceptionChatConfig() - with mock.patch.dict( - os.environ, {"INCEPTION_API_KEY": "server-secret"}, clear=True - ): + with mock.patch.dict(os.environ, {"INCEPTION_API_KEY": "server-secret"}, clear=True): with mock.patch.object(litellm, "inception_key", "module-secret"): # caller overrides api_base without a key -> server key withheld - api_base, api_key = config._get_openai_compatible_provider_info( - "https://attacker.example/v1", None - ) + api_base, api_key = config._get_openai_compatible_provider_info("https://attacker.example/v1", None) assert api_base == "https://attacker.example/v1" assert api_key is None # caller overrides api_base AND supplies their own key -> used as-is - _, api_key = config._get_openai_compatible_provider_info( - "https://attacker.example/v1", "caller-key" - ) + _, api_key = config._get_openai_compatible_provider_info("https://attacker.example/v1", "caller-key") assert api_key == "caller-key" # default/server base -> server-managed key resolved @@ -218,9 +211,7 @@ def test_get_llm_provider_inception(): assert model == "mercury-2" assert provider == "inception" - model, provider, _, api_base = get_llm_provider( - "mercury-2", api_base="https://api.inceptionlabs.ai/v1" - ) + model, provider, _, api_base = get_llm_provider("mercury-2", api_base="https://api.inceptionlabs.ai/v1") assert model == "mercury-2" assert provider == "inception" assert api_base == "https://api.inceptionlabs.ai/v1" @@ -232,18 +223,6 @@ def test_inception_in_provider_lists(): assert "https://api.inceptionlabs.ai/v1" in litellm.openai_compatible_endpoints -def test_inception_model_list_populated(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.inception_models = set() - litellm.add_known_models() - - assert "inception/mercury-2" in litellm.inception_models - assert "inception/mercury-2.5" in litellm.inception_models - for model in litellm.inception_models: - assert model.startswith("inception/") - - def test_inception_completion_targets_inception_endpoint(): """ End-to-end: a completion routed through the inception provider must hit @@ -306,5 +285,3 @@ def test_inception_completion_targets_inception_endpoint(): assert captured["body"]["model"] == "mercury-2" assert captured["body"]["tool_choice"] == "auto" assert response.choices[0].message.content == "hi" - - diff --git a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py index d484fa437ae..f94ea5e3db2 100644 --- a/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/test_litellm/llms/moonshot/test_moonshot_chat_transformation.py @@ -730,10 +730,6 @@ class TestMoonshotResponseSchemaSupport: def model_cost_map(self): return GetModelCostMap.load_local_model_cost_map() - def test_supports_response_schema_utility_reports_true(self, model_cost_map, monkeypatch): - monkeypatch.setattr(litellm, "model_cost", model_cost_map) - assert litellm.utils.supports_response_schema(model="moonshot/kimi-k2.5") is True - class TestMoonshotReasoningEffort: """Moonshot documents reasoning_effort as a top-level chat completions field for its reasoning diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py index 46a91520ab0..a75883d7846 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -1,5 +1,3 @@ -import json -import os from unittest.mock import MagicMock, patch import httpx @@ -307,73 +305,3 @@ class TestOCIEmbeddingConfig: optional_params={}, litellm_params={}, ) - - def test_model_prices_embedding_models(self): - """test all 8 OCI embedding models exist in model_prices_and_context_window.json with mode=embedding.""" - model_prices_path = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "..", - "..", - "model_prices_and_context_window.json", - ) - with open(model_prices_path) as f: - model_prices = json.load(f) - - expected_embedding_models = [ - "oci/cohere.embed-english-v3.0", - "oci/cohere.embed-english-light-v3.0", - "oci/cohere.embed-multilingual-v3.0", - "oci/cohere.embed-multilingual-light-v3.0", - "oci/cohere.embed-english-image-v3.0", - "oci/cohere.embed-english-light-image-v3.0", - "oci/cohere.embed-multilingual-light-image-v3.0", - "oci/cohere.embed-v4.0", - ] - - for model_key in expected_embedding_models: - assert model_key in model_prices, f"Missing model: {model_key}" - assert ( - model_prices[model_key].get("mode") == "embedding" - ), f"Model {model_key} does not have mode='embedding'" - - def test_model_prices_new_chat_models(self): - """test the 16 new OCI chat models exist in model_prices_and_context_window.json with mode=chat.""" - model_prices_path = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "..", - "..", - "model_prices_and_context_window.json", - ) - with open(model_prices_path) as f: - model_prices = json.load(f) - - expected_chat_models = [ - "oci/xai.grok-3", - "oci/xai.grok-3-fast", - "oci/xai.grok-3-mini", - "oci/xai.grok-3-mini-fast", - "oci/xai.grok-4", - "oci/xai.grok-4-fast", - "oci/xai.grok-4.1-fast", - "oci/xai.grok-4.20", - "oci/xai.grok-4.20-multi-agent", - "oci/xai.grok-code-fast-1", - "oci/cohere.command-a-03-2025", - "oci/cohere.command-a-reasoning-08-2025", - "oci/cohere.command-a-vision-07-2025", - "oci/cohere.command-a-translate-08-2025", - "oci/google.gemini-2.5-pro", - "oci/google.gemini-2.5-flash", - ] - - for model_key in expected_chat_models: - assert model_key in model_prices, f"Missing model: {model_key}" - assert ( - model_prices[model_key].get("mode") == "chat" - ), f"Model {model_key} does not have mode='chat'" diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 2bc8d74e82c..1df47223f06 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1,9 +1,8 @@ import json from types import SimpleNamespace from typing import Final -from unittest.mock import AsyncMock, MagicMock, Mock, patch +from unittest.mock import MagicMock, Mock, patch -import httpx import pytest @@ -15,7 +14,6 @@ from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, OutputTextDeltaEvent, ResponseCompletedEvent, - ResponsesAPIRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) @@ -111,9 +109,7 @@ class TestOpenAIResponsesAPIConfig: # Check expected fields have correct values for field, value in expected_fields.items(): assert field in params, f"Missing expected field: {field}" - assert ( - params[field] == value - ), f"Field {field} has value {params[field]}, expected {value}" + assert params[field] == value, f"Field {field} has value {params[field]}, expected {value}" def test_transform_responses_api_request(self): """Test request transformation""" @@ -461,9 +457,7 @@ class TestOpenAIResponsesAPIConfig: } # Mock the get_event_model_class to avoid validation issues in tests - with patch.object( - OpenAIResponsesAPIConfig, "get_event_model_class" - ) as mock_get_class: + with patch.object(OpenAIResponsesAPIConfig, "get_event_model_class") as mock_get_class: mock_get_class.return_value = ResponseCompletedEvent result = self.config.transform_streaming_response( @@ -482,9 +476,7 @@ class TestOpenAIResponsesAPIConfig: headers = {} api_key = "test_api_key" litellm_params = GenericLiteLLMParams(api_key=api_key) - result = self.config.validate_environment( - headers=headers, model=self.model, litellm_params=litellm_params - ) + result = self.config.validate_environment(headers=headers, model=self.model, litellm_params=litellm_params) assert "Authorization" in result assert result["Authorization"] == f"Bearer {api_key}" @@ -495,9 +487,7 @@ class TestOpenAIResponsesAPIConfig: with patch("litellm.api_key", "litellm_api_key"): litellm_params = GenericLiteLLMParams() - result = self.config.validate_environment( - headers=headers, model=self.model, litellm_params=litellm_params - ) + result = self.config.validate_environment(headers=headers, model=self.model, litellm_params=litellm_params) assert "Authorization" in result assert result["Authorization"] == "Bearer litellm_api_key" @@ -603,10 +593,7 @@ class TestOpenAIResponsesAPIConfig: headers={}, ) - assert ( - url - == "https://custom-openai.example.com/v1/responses/..%2F..%2Ffiles%3Fx%3D1%23frag/input_items" - ) + assert url == "https://custom-openai.example.com/v1/responses/..%2F..%2Ffiles%3Fx%3D1%23frag/input_items" assert data["limit"] == 20 def test_get_event_model_class_generic_event(self): @@ -681,9 +668,7 @@ class TestOpenAIResponsesAPIConfig: ) assert isinstance(result, ImageGenerationPartialImageEvent) - assert ( - result.type == ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE - ) + assert result.type == ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE assert result.partial_image_index == idx assert result.b64_json == chunk["b64_json"] @@ -898,9 +883,7 @@ class TestOpenAIResponsesAPIConfig: "namespace": "drop", }, ] - out = BaseResponsesAPIConfig.strip_custom_tool_call_namespace_from_responses_input( - inp - ) + out = BaseResponsesAPIConfig.strip_custom_tool_call_namespace_from_responses_input(inp) assert out[0]["namespace"] == "keep" assert "namespace" not in out[1] @@ -973,30 +956,21 @@ class TestAzureResponsesAPIConfig: api_base=base_url, litellm_params={"api_version": "preview"}, ) - assert ( - result_preview - == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" - ) + assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview" # Test with latest version - should use openai/v1/responses result_latest = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "latest"}, ) - assert ( - result_latest - == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" - ) + assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest" # Test with date-based version - should use openai/responses result_date = self.config.get_complete_url( api_base=base_url, litellm_params={"api_version": "2025-01-01"}, ) - assert ( - result_date - == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" - ) + assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01" def test_azure_transform_then_normalize_strips_custom_tool_call_namespace(self): """Same as OpenAI path: ``normalize_responses_api_request_dict`` strips custom_tool_call only.""" @@ -1163,10 +1137,7 @@ class TestTransformListInputItemsRequest: ) # Assert - assert ( - url - == "https://test.openai.azure.com/openai/responses/compact?api-version=2024-05-01-preview" - ) + assert url == "https://test.openai.azure.com/openai/responses/compact?api-version=2024-05-01-preview" assert data["model"] == "gpt-5.2-codex" assert data["input"] == "hello" @@ -1253,9 +1224,7 @@ class TestTransformListInputItemsRequest: assert params == expected_params @patch("litellm.router.Router") - def test_mock_litellm_router_with_transform_list_input_items_request( - self, mock_router - ): + def test_mock_litellm_router_with_transform_list_input_items_request(self, mock_router): """Mock test using litellm.router for transform_list_input_items_request""" # Setup mock router mock_router_instance = Mock() @@ -1269,9 +1238,7 @@ class TestTransformListInputItemsRequest: ) # Setup router mock - mock_router_instance.get_provider_responses_api_config.return_value = ( - mock_provider_config - ) + mock_router_instance.get_provider_responses_api_config.return_value = mock_provider_config # Test parameters response_id = "resp_test123" @@ -1587,9 +1554,7 @@ class TestPhaseParameter: phase = getattr(output_item, "phase", None) expected = "commentary" if idx == 0 else "final_answer" - assert ( - phase == expected - ), f"output[{idx}] phase={phase!r}, expected {expected!r}" + assert phase == expected, f"output[{idx}] phase={phase!r}, expected {expected!r}" def test_streaming_output_item_done_preserves_phase(self): """OutputItemDoneEvent must preserve phase on its item.""" @@ -1723,9 +1688,7 @@ class TestPhaseParameter: if isinstance(item, dict): input_items.append(item) else: - input_items.append( - item.model_dump() if hasattr(item, "model_dump") else dict(item) - ) + input_items.append(item.model_dump() if hasattr(item, "model_dump") else dict(item)) input_items.append( { @@ -1822,9 +1785,7 @@ class TestResponsesSurfaceSharesTheEffortRule: ("gpt-6-astra", "low", False), ], ) - def test_temperature_follows_the_resolved_effort( - self, local_model_cost_map, model, effort, temperature_survives - ): + def test_temperature_follows_the_resolved_effort(self, local_model_cost_map, model, effort, temperature_survives): params = {"temperature": 0} if effort is not None: params["reasoning"] = {"effort": effort} @@ -2228,19 +2189,6 @@ class TestReasoningFollowsModelSupport: ) assert mapped["reasoning"] == reasoning - def test_an_explicit_supports_reasoning_false_beats_the_bundled_floor(self, local_model_cost_map, monkeypatch): - overridden = { - name: ({**entry, "supports_reasoning": False} if name == "o3" else entry) - for name, entry in litellm.model_cost.items() - } - monkeypatch.setattr(litellm, "model_cost", overridden) - mapped = OpenAIResponsesAPIConfig().map_openai_params( - response_api_optional_params={"reasoning": {"effort": "medium"}}, - model="o3", - drop_params=True, - ) - assert "reasoning" not in mapped - def test_azure_deployments_keep_reasoning_even_on_a_non_reasoning_model_name(self, local_model_cost_map): mapped = AzureOpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params={"reasoning": {"effort": "medium"}}, diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index ba51209e0d5..63bd5f6e1ed 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -27,9 +27,7 @@ def gpt5_config() -> OpenAIGPT5Config: @pytest.fixture(autouse=True) def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr( - litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url) - ) + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) litellm.add_known_models(model_cost_map=litellm.model_cost) @@ -39,9 +37,7 @@ def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): def test_gpt5_chat_does_not_support_reasoning_effort(config: OpenAIConfig): - assert "reasoning_effort" not in config.get_supported_openai_params( - model="gpt-5-chat-latest" - ) + assert "reasoning_effort" not in config.get_supported_openai_params(model="gpt-5-chat-latest") def test_gpt5_chat_supports_temperature(config: OpenAIConfig): @@ -288,24 +284,6 @@ def test_gpt5_1_gpt5_2_gpt5_4_drop_minimal_reasoning_effort(config: OpenAIConfig # GPT-5.1 temperature handling tests -def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): - """Test that models supporting reasoning_effort='none' are correctly detected via model map.""" - # gpt-5.1 and gpt-5.2 chat variants support none - assert gpt5_config._supports_reasoning_effort_level("gpt-5.1", "none") - assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-2025-11-13", "none") - assert gpt5_config._supports_reasoning_effort_level("gpt-5.1-chat-latest", "none") - assert gpt5_config._supports_reasoning_effort_level("gpt-5.2", "none") - assert gpt5_config._supports_reasoning_effort_level("gpt-5.2-2025-12-11", "none") - # codex/pro/chat variants do not support none - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex", "none") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.1-codex-max", "none") - assert not gpt5_config._supports_reasoning_effort_level( - "gpt-5.2-chat-latest", "none" - ) - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.2-pro", "none") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5", "none") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5-mini", "none") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5-codex", "none") def test_gpt5_1_temperature_with_reasoning_effort_none(config: OpenAIConfig): @@ -469,9 +447,7 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): """Dict with effort='minimal' triggers minimal model-support validation.""" with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "minimal", "summary": "detailed"} - }, + non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, optional_params={}, model="gpt-5.4-mini", drop_params=False, @@ -481,9 +457,7 @@ def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='minimal' passes through for gpt-5.""" params = config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "minimal", "summary": "detailed"} - }, + non_default_params={"reasoning_effort": {"effort": "minimal", "summary": "detailed"}}, optional_params={}, model="gpt-5", drop_params=False, @@ -491,14 +465,6 @@ def test_gpt5_minimal_dict_accepted_for_supported_model(config: OpenAIConfig): assert params["reasoning_effort"] == "minimal" -def test_gpt5_supports_reasoning_effort_level_minimal(gpt5_config: OpenAIGPT5Config): - """Test that _supports_reasoning_effort_level correctly identifies minimal support.""" - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4", "minimal") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-pro", "minimal") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-mini", "minimal") - assert not gpt5_config._supports_reasoning_effort_level("gpt-5.4-nano", "minimal") - - def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): """_is_reasoning_effort_level_explicitly_disabled returns True only for explicit False entries. @@ -506,21 +472,11 @@ def test_gpt5_minimal_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): Models with supports_minimal_reasoning_effort=true (or missing) → not disabled. Provider-prefixed models (openai/gpt-5.4-mini) are normalized before lookup. """ - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.4-mini", "minimal" - ) - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.4-nano", "minimal" - ) - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "openai/gpt-5.4-mini", "minimal" - ) - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.4", "minimal" - ) - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.4-pro", "minimal" - ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-mini", "minimal") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-nano", "minimal") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("openai/gpt-5.4-mini", "minimal") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4", "minimal") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4-pro", "minimal") def test_is_explicitly_disabled_factory_minimal(): @@ -615,26 +571,16 @@ def test_gpt5_unknown_model_passes_through_low(config: OpenAIConfig): def test_gpt5_low_explicitly_disabled_check(gpt5_config: OpenAIGPT5Config): """supports_low_reasoning_effort=false → disabled; missing/true → not disabled.""" - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.5-pro", "low" - ) - assert gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.5-pro-2026-04-23", "low" - ) - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.5", "low" - ) - assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled( - "gpt-5.4", "low" - ) + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5-pro", "low") + assert gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5-pro-2026-04-23", "low") + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.5", "low") + assert not gpt5_config._is_reasoning_effort_level_explicitly_disabled("gpt-5.4", "low") def test_gpt5_normalizes_reasoning_effort_dict_with_summary(config: OpenAIConfig): """Dict with summary/generate_summary is normalized for chat completions.""" params = config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "high", "summary": "detailed"} - }, + non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, optional_params={}, model="gpt-5.4", drop_params=False, @@ -650,9 +596,7 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): """ with pytest.raises(litellm.utils.UnsupportedParamsError): config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} - }, + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, optional_params={}, model="gpt-5.1", drop_params=False, @@ -662,9 +606,7 @@ def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): """Dict with effort='xhigh' passes through for gpt-5.4+.""" params = config.map_openai_params( - non_default_params={ - "reasoning_effort": {"effort": "xhigh", "summary": "detailed"} - }, + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, optional_params={}, model="gpt-5.4", drop_params=False, @@ -719,9 +661,7 @@ def test_gpt5_normalizes_reasoning_effort_dict_with_summary_from_optional_params """reasoning_effort dict with summary in optional_params is normalized.""" params = config.map_openai_params( non_default_params={}, - optional_params={ - "reasoning_effort": {"effort": "medium", "summary": "detailed"} - }, + optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, model="gpt-5.4", drop_params=False, ) @@ -971,9 +911,7 @@ def test_gpt5_search_supported_params(gpt5_config: OpenAIGPT5Config): "reasoning_effort", ] for param in rejected: - assert ( - param not in supported - ), f"{param} should not be supported for search models" + assert param not in supported, f"{param} should not be supported for search models" def test_gpt5_search_has_expected_params(gpt5_config: OpenAIGPT5Config): @@ -1059,21 +997,15 @@ def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): optional_params = {"reasoningSummary": False, "reasoning_summary": "ignored"} assert peek_reasoning_summary_aliases(optional_params) is False - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( - optional_params - ) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) assert rs_val is False assert stripped == {} - optional_params = { - "extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"} - } + optional_params = {"extra_body": {"reasoningSummary": False, "reasoning_summary": "ignored"}} assert peek_reasoning_summary_aliases(optional_params) is False - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( - optional_params - ) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) assert rs_val is False assert stripped == {} @@ -1087,9 +1019,7 @@ def test_reasoning_summary_alias_helpers_preserve_falsy_and_strip_all_aliases(): } assert peek_reasoning_summary_aliases(optional_params) == "auto" - stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params( - optional_params - ) + stripped, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params) assert rs_val == "auto" assert stripped == {"extra_body": {"metadata": "ok"}} @@ -1108,9 +1038,7 @@ def test_gpt5_rejects_params_unsupported_by_openai(config: OpenAIConfig): for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex", "gpt-5.1", "gpt-5.2"]: supported = config.get_supported_openai_params(model=model) for param in rejected_params: - assert ( - param not in supported - ), f"{param} should not be supported for {model}" + assert param not in supported, f"{param} should not be supported for {model}" def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): @@ -1119,22 +1047,16 @@ def test_gpt5_1_supports_logprobs_top_p(config: OpenAIConfig): supported = config.get_supported_openai_params(model=model) assert "logprobs" in supported, f"logprobs should be supported for {model}" assert "top_p" in supported, f"top_p should be supported for {model}" - assert ( - "top_logprobs" in supported - ), f"top_logprobs should be supported for {model}" + assert "top_logprobs" in supported, f"top_logprobs should be supported for {model}" def test_gpt5_base_does_not_support_logprobs_top_p(config: OpenAIConfig): """Base gpt-5/gpt-5-mini do NOT support logprobs, top_p, top_logprobs.""" for model in ["gpt-5", "gpt-5-mini", "gpt-5-codex"]: supported = config.get_supported_openai_params(model=model) - assert ( - "logprobs" not in supported - ), f"logprobs should not be supported for {model}" + assert "logprobs" not in supported, f"logprobs should not be supported for {model}" assert "top_p" not in supported, f"top_p should not be supported for {model}" - assert ( - "top_logprobs" not in supported - ), f"top_logprobs should not be supported for {model}" + assert "top_logprobs" not in supported, f"top_logprobs should not be supported for {model}" def test_gpt5_1_logprobs_passthrough(config: OpenAIConfig): @@ -1340,19 +1262,6 @@ def test_gpt5_6_forwards_reasoning_effort_max_for_the_responses_bridge(config: O assert params["reasoning_effort"] == "max" -@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) -def test_gpt5_6_never_advertises_reasoning_effort_max(model: str): - """/v1/chat/completions answers max with "Unsupported value: 'reasoning_effort' does not support - 'max' with this model. Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'", so no - gpt-5.6 entry asserts supports_max_reasoning_effort and the advertised set stops at xhigh.""" - from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - - resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model), deployment_is_mapped=True) - assert resolved is not None - assert "max" not in resolved - assert "xhigh" in resolved - - def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api( responses_config: OpenAIResponsesAPIConfig, ): diff --git a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py index 1402a8fa7b5..68cd33bf745 100644 --- a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py +++ b/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py @@ -6,11 +6,8 @@ import os import sys from unittest.mock import patch -import pytest -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) -) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) class TestSimpleProviderConfigSupportedEndpoints: @@ -20,9 +17,7 @@ class TestSimpleProviderConfigSupportedEndpoints: """supported_endpoints defaults to [] (chat always enabled, nothing else)""" from litellm.llms.openai_like.json_loader import SimpleProviderConfig - config = SimpleProviderConfig( - "test", {"base_url": "https://example.com", "api_key_env": "TEST_KEY"} - ) + config = SimpleProviderConfig("test", {"base_url": "https://example.com", "api_key_env": "TEST_KEY"}) assert config.supported_endpoints == [] def test_custom_supported_endpoints(self): @@ -58,46 +53,11 @@ class TestSimpleProviderConfigSupportedEndpoints: class TestJSONProviderRegistryResponsesAPI: """Test supports_responses_api on JSONProviderRegistry.""" - def test_existing_provider_no_responses(self): - """Existing providers without supported_endpoints don't support responses""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - # publicai has no supported_endpoints in JSON, defaults to [] - assert JSONProviderRegistry.supports_responses_api("publicai") is False - def test_nonexistent_provider(self): """Non-existent provider returns False""" from litellm.llms.openai_like.json_loader import JSONProviderRegistry - assert ( - JSONProviderRegistry.supports_responses_api("nonexistent_provider_xyz") - is False - ) - - def test_provider_with_responses_endpoint(self): - """A provider with /v1/responses in supported_endpoints returns True""" - from litellm.llms.openai_like.json_loader import ( - JSONProviderRegistry, - SimpleProviderConfig, - ) - - # Temporarily inject a test provider - test_config = SimpleProviderConfig( - "test_responses_provider", - { - "base_url": "https://test.example.com", - "api_key_env": "TEST_API_KEY", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], - }, - ) - JSONProviderRegistry._providers["test_responses_provider"] = test_config - try: - assert ( - JSONProviderRegistry.supports_responses_api("test_responses_provider") - is True - ) - finally: - del JSONProviderRegistry._providers["test_responses_provider"] + assert JSONProviderRegistry.supports_responses_api("nonexistent_provider_xyz") is False class TestCreateResponsesConfigClass: @@ -150,9 +110,7 @@ class TestCreateResponsesConfigClass: config_cls = create_responses_config_class(provider) config = config_cls() - url = config.get_complete_url( - api_base="https://custom.api.com/v1", litellm_params={} - ) + url = config.get_complete_url(api_base="https://custom.api.com/v1", litellm_params={}) assert url == "https://custom.api.com/v1/responses" def test_generated_class_get_complete_url_strips_trailing_slash(self): @@ -165,9 +123,7 @@ class TestCreateResponsesConfigClass: config_cls = create_responses_config_class(provider) config = config_cls() - url = config.get_complete_url( - api_base="https://custom.api.com/v1/", litellm_params={} - ) + url = config.get_complete_url(api_base="https://custom.api.com/v1/", litellm_params={}) assert url == "https://custom.api.com/v1/responses" def test_generated_class_validate_environment(self): @@ -184,9 +140,7 @@ class TestCreateResponsesConfigClass: "litellm.llms.openai_like.dynamic_config.get_secret_str", return_value="sk-test-key-123", ): - headers = config.validate_environment( - headers={}, model="test-model", litellm_params=None - ) + headers = config.validate_environment(headers={}, model="test-model", litellm_params=None) assert headers["Authorization"] == "Bearer sk-test-key-123" def test_generated_class_validate_environment_litellm_params_override(self): @@ -201,9 +155,7 @@ class TestCreateResponsesConfigClass: config = config_cls() litellm_params = GenericLiteLLMParams(api_key="sk-override-key") - headers = config.validate_environment( - headers={}, model="test-model", litellm_params=litellm_params - ) + headers = config.validate_environment(headers={}, model="test-model", litellm_params=litellm_params) assert headers["Authorization"] == "Bearer sk-override-key" def test_generated_class_inherits_openai_responses_methods(self): diff --git a/tests/test_litellm/llms/openai_like/test_cognition_provider.py b/tests/test_litellm/llms/openai_like/test_cognition_provider.py index 9bbbb3b88f2..9a38456da16 100644 --- a/tests/test_litellm/llms/openai_like/test_cognition_provider.py +++ b/tests/test_litellm/llms/openai_like/test_cognition_provider.py @@ -110,15 +110,6 @@ class TestCognitionProviderIdentity: class TestCognitionCostTracking: - - - def test_lightning_is_five_times_the_standard_tier(self): - standard = litellm.get_model_info(model="cognition/swe-1.7") - lightning = litellm.get_model_info(model="cognition/swe-1.7-lightning") - - assert lightning["input_cost_per_token"] == pytest.approx(standard["input_cost_per_token"] * 5) - assert lightning["output_cost_per_token"] == pytest.approx(standard["output_cost_per_token"] * 5) - def test_supported_endpoints_matrix(self): matrix = json.loads((Path(litellm.__file__).parent / "provider_endpoints_support_backup.json").read_text()) @@ -127,6 +118,3 @@ class TestCognitionCostTracking: assert endpoints["messages"] is True assert endpoints["responses"] is True assert endpoints["embeddings"] is False - - - diff --git a/tests/test_litellm/llms/openai_like/test_meta_provider.py b/tests/test_litellm/llms/openai_like/test_meta_provider.py index 0a0ba369e71..359416b581c 100644 --- a/tests/test_litellm/llms/openai_like/test_meta_provider.py +++ b/tests/test_litellm/llms/openai_like/test_meta_provider.py @@ -24,11 +24,6 @@ class TestMetaProviderConfig: assert meta.api_key_env == "META_API_KEY" assert meta.api_base_env == "META_API_BASE" - def test_meta_supports_responses_api(self): - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - assert JSONProviderRegistry.supports_responses_api("meta") - def test_meta_in_openai_compatible_providers(self): from litellm.constants import openai_compatible_providers @@ -95,9 +90,7 @@ class TestMetaProviderConfig: class TestMetaReasoningParams: def test_muse_spark_supports_reasoning_effort(self): - params = litellm.get_supported_openai_params( - model="muse-spark-1.1", custom_llm_provider="meta" - ) + params = litellm.get_supported_openai_params(model="muse-spark-1.1", custom_llm_provider="meta") assert params is not None assert "reasoning_effort" in params @@ -116,9 +109,7 @@ class TestMetaReasoningParams: def test_reasoning_effort_gated_on_capability(self): """A meta model without reasoning metadata must not advertise reasoning_effort.""" - params = litellm.get_supported_openai_params( - model="some-non-reasoning-model", custom_llm_provider="meta" - ) + params = litellm.get_supported_openai_params(model="some-non-reasoning-model", custom_llm_provider="meta") assert params is not None assert "reasoning_effort" not in params @@ -190,6 +181,3 @@ class TestMetaAnthropicMessages: ) assert headers["authorization"] == "Bearer sk-env-key" assert headers["anthropic-version"] == "2023-06-01" - - - diff --git a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py index 947d9b73e1a..76e818bfc49 100644 --- a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py +++ b/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py @@ -154,27 +154,6 @@ class TestSCXAIModelMetadata: with open(json_path) as f: return json.load(f) - def test_scx_ai_models_registered_with_correct_metadata(self): - model_cost = self._load(("model_prices_and_context_window.json",)) - for model in self.SCX_MODELS: - info = model_cost.get(model) - assert info is not None, f"{model} missing from model_prices_and_context_window.json" - assert info["litellm_provider"] == "scx-ai" - assert info["mode"] == "chat" - assert info["input_cost_per_token"] > 0 - assert info["output_cost_per_token"] > 0 - assert info["supports_function_calling"] is True - assert info["supports_tool_choice"] is True - assert info["supports_reasoning"] is True - assert info["supports_response_schema"] is True - assert info.get("supports_vision", False) is (model in self.VISION_MODELS) - - assert info["supports_prompt_caching"] is True - assert 0 < info["cache_read_input_token_cost"] < info["input_cost_per_token"] - - assert info["max_tokens"] == info["max_output_tokens"] - assert info["max_input_tokens"] >= 1_000_000 - def test_scx_ai_models_synced_to_backup(self): model_cost = self._load(("model_prices_and_context_window.json",)) backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) diff --git a/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py b/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py index 66dd18fc8d7..1ff70142719 100644 --- a/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py +++ b/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py @@ -79,20 +79,6 @@ class TestTensormeshProviderConfig: matching the text_completion flag in provider_endpoints_support.json.""" assert "tensormesh" in litellm.openai_text_completion_compatible_providers - def test_tensormesh_responses_api_enabled(self): - """Tensormesh declares /v1/responses in supported_endpoints, so litellm - resolves a responses config for it.""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - from litellm.utils import ProviderConfigManager - - assert JSONProviderRegistry.supports_responses_api("tensormesh") is True - config = ProviderConfigManager.get_provider_responses_api_config( - provider="tensormesh", - model="tensormesh/openai/gpt-oss-120b", - ) - assert config is not None - assert config.custom_llm_provider == "tensormesh" - def test_tensormesh_router_config(self): """Test that tensormesh can be used in Router configuration""" from litellm import Router @@ -129,16 +115,6 @@ class TestTensormeshCostMap: litellm.model_cost = original_model_cost litellm.get_model_info.cache_clear() - def test_models_registered_with_capabilities(self): - for model in TENSORMESH_MODELS: - info = litellm.get_model_info(model) - assert info["litellm_provider"] == "tensormesh" - assert info["mode"] == "chat" - assert litellm.supports_function_calling(model) is True, model - assert litellm.supports_response_schema(model) is True, model - assert litellm.model_cost[model]["supports_tool_choice"] is True, model - assert litellm.model_cost[model]["supports_prompt_caching"] is True, model - def test_reasoning_flag_matches_expected_set(self): reasoning_models = { "tensormesh/deepseek-ai/DeepSeek-V4-Flash", @@ -153,4 +129,3 @@ class TestTensormeshCostMap: } for model in TENSORMESH_MODELS: assert litellm.supports_reasoning(model) is (model in reasoning_models), model - diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..f4828a19fc1 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -204,19 +204,6 @@ class TestPerplexityCostCalculator: assert math.isclose(prompt_cost, (1000 * 1e-07) + (100 * 2e-06), rel_tol=1e-10) assert math.isclose(completion_cost, (150 * 2e-07) + (50 * 3e-06) + 0.005, rel_tol=1e-10) - def test_off_peak_defaults_to_the_current_time(self): - """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the - default current time.""" - self._register_off_peak_model( - {"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-07, "output_cost_per_token": 2e-07} - ) - usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) - - prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) - - assert math.isclose(prompt_cost, 1000 * 1e-07, rel_tol=1e-10) - assert math.isclose(completion_cost, 200 * 2e-07, rel_tol=1e-10) - def test_provider_stated_cost_still_wins_inside_an_off_peak_window(self): """A response that carries Perplexity's own metered cost bills that cost whatever the window says; the caller strips it when the deployment carries custom pricing.""" diff --git a/tests/test_litellm/llms/reducto/test_model_info.py b/tests/test_litellm/llms/reducto/test_model_info.py index de7a3ccba64..548e5a308d4 100644 --- a/tests/test_litellm/llms/reducto/test_model_info.py +++ b/tests/test_litellm/llms/reducto/test_model_info.py @@ -1,44 +1,8 @@ -import uuid - import litellm -from litellm.utils import _invalidate_model_cost_lowercase_map - def test_reducto_provider_registration(): - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model="reducto/parse-v3" - ) + model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="reducto/parse-v3") assert model == "parse-v3" assert custom_llm_provider == "reducto" - - -def test_get_model_info_preserves_ocr_cost_per_credit(): - test_model_name = f"reducto/test-cost-propagation-{uuid.uuid4().hex[:12]}" - previous_model_entry = litellm.model_cost.get(test_model_name) - _invalidate_model_cost_lowercase_map() - - try: - litellm.register_model( - { - test_model_name: { - "litellm_provider": "reducto", - "mode": "ocr", - "ocr_cost_per_credit": 0.003, - } - } - ) - - model_info = litellm.get_model_info( - model=test_model_name, - custom_llm_provider="reducto", - ) - - assert model_info.get("ocr_cost_per_credit") == 0.003 - finally: - if previous_model_entry is None: - litellm.model_cost.pop(test_model_name, None) - else: - litellm.model_cost[test_model_name] = previous_model_entry - _invalidate_model_cost_lowercase_map() diff --git a/tests/test_litellm/llms/tencent/chat/test_tencent_chat_transformation.py b/tests/test_litellm/llms/tencent/chat/test_tencent_chat_transformation.py index 9f510786d50..4d6d252ae6e 100644 --- a/tests/test_litellm/llms/tencent/chat/test_tencent_chat_transformation.py +++ b/tests/test_litellm/llms/tencent/chat/test_tencent_chat_transformation.py @@ -247,23 +247,6 @@ class TestAdaptiveThinkingCoercion: assert config._is_adaptive_thinking_model("tencent/no-such-model") is False -def test_minimax_m3_cost_map_entry_marks_adaptive_thinking(): - """The capability flag driving the coercion must exist in the cost map - (and its backup, which is shipped with the package).""" - import json - from pathlib import Path - - repo_root = Path(__file__).parents[5] - for filename in ("model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json"): - with open(repo_root / filename) as f: - entry = json.load(f).get("tencent/minimax-m3") - - assert entry is not None, f"tencent/minimax-m3 not found in {filename}" - assert entry["litellm_provider"] == "tencent" - assert entry.get("supports_adaptive_thinking") is True - assert entry.get("supports_reasoning") is True - - def test_get_complete_url_default(): config = TencentChatConfig() diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 7d2dfbb962e..11b08081568 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -143,24 +143,6 @@ def test_anyof_with_excessive_nesting(): convert_anyof_null_to_nullable(schema) -@pytest.mark.asyncio -async def test_get_supports_system_message(): - """Test get_supports_system_message with different models""" - from litellm.llms.vertex_ai.common_utils import get_supports_system_message - - # fine-tuned vertex gemini models will specifiy they are in the /gemini spec format - result = get_supports_system_message( - model="gemini/1234567890", custom_llm_provider="vertex_ai" - ) - assert result == True - - # non-fine-tuned vertex gemini models will not specifiy they are in the /gemini spec format - result = get_supports_system_message( - model="random-model-name", custom_llm_provider="vertex_ai" - ) - assert result == False - - @pytest.mark.parametrize( "model, expected", [ @@ -230,13 +212,9 @@ def test_build_vertex_schema(): "properties": { "tags": {"items": {"type": "string"}, "type": "array"}, "metadata": {"type": "object"}, - "callbacks": { - "anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}] - }, + "callbacks": {"anyOf": [{"items": {}, "type": "array"}, {}, {"type": "null"}]}, "run_name": {"type": "string"}, - "max_concurrency": { - "anyOf": [{"type": "integer"}, {"type": "null"}] - }, + "max_concurrency": {"anyOf": [{"type": "integer"}, {"type": "null"}]}, "recursion_limit": {"type": "integer"}, "configurable": {"type": "object"}, "run_id": { @@ -280,9 +258,7 @@ def test_build_vertex_schema(): ] }, "run_name": {"type": "string"}, - "max_concurrency": { - "anyOf": [{"type": "integer", "nullable": True}] - }, + "max_concurrency": {"anyOf": [{"type": "integer", "nullable": True}]}, "recursion_limit": {"type": "integer"}, "configurable": {"type": "object"}, "run_id": {"anyOf": [{"type": "string", "nullable": True}]}, @@ -383,13 +359,10 @@ def test_build_vertex_schema_array_branch_missing_items_in_anyof(): array_branches = [b for b in callbacks_anyof if b.get("type") == "array"] assert array_branches, "expected an array branch to remain after transform" for branch in array_branches: - assert branch.get("items") == { - "type": "object" - }, f"array branch must have items synthesized; got {branch}" + assert branch.get("items") == {"type": "object"}, f"array branch must have items synthesized; got {branch}" def test_vertex_ai_complex_response_schema(): - import json from copy import deepcopy from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -659,58 +632,6 @@ def test_get_vertex_url_global_region(stream, expected_endpoint_suffix): assert url == expected_url -@pytest.mark.parametrize( - "model_cost_entry, vertex_region, expected_region", - [ - # Model with supported_regions=["global"], no user region -> use "global" - ({"supported_regions": ["global"]}, None, "global"), - # Model with supported_regions=["global"], user passes unsupported region -> override to "global" - ({"supported_regions": ["global"]}, "us-central1", "global"), - # Model with supported_regions=["global"], user passes unsupported region -> override to "global" - ({"supported_regions": ["global"]}, "europe-west1", "global"), - # Model with supported_regions=["us-west2"], no user region -> use "us-west2" - ({"supported_regions": ["us-west2"]}, None, "us-west2"), - # Model with supported_regions=["us-west2", "us-central1"], user passes supported region -> respect it - ( - {"supported_regions": ["us-west2", "us-central1"]}, - "us-central1", - "us-central1", - ), - # Model with supported_regions=["us-west2", "us-central1"], user passes unsupported region -> override - ( - {"supported_regions": ["us-west2", "us-central1"]}, - "europe-west1", - "us-west2", - ), - # No model_cost entry, no user region -> default us-central1 - ({}, None, "us-central1"), - # No model_cost entry, user specifies region -> use specified region - ({}, "europe-west1", "europe-west1"), - # No model_cost entry, user specifies region -> use specified region - ({}, "us-east1", "us-east1"), - ], -) -def test_get_vertex_region_global_only_model( - model_cost_entry, vertex_region, expected_region -): - """Test get_vertex_region resolves region from model_cost supported_regions""" - import litellm - from litellm.llms.vertex_ai.vertex_llm_base import VertexBase - - vertex_base = VertexBase() - - with patch.dict( - litellm.model_cost, - {"vertex_ai/test-model": model_cost_entry}, - clear=False, - ): - result = vertex_base.get_vertex_region( - vertex_region=vertex_region, model="test-model" - ) - - assert result == expected_region - - def test_vertex_filter_format_uri(): import json @@ -824,9 +745,7 @@ def test_convert_schema_types_type_array_conversion(): assert anyof_types[1]["type"] == "number" # 4. Other properties preserved - assert ( - input_schema["properties"]["studio"]["description"] == "The studio ID or name" - ) + assert input_schema["properties"]["studio"]["description"] == "The studio ID or name" assert input_schema["required"] == ["studio"] @@ -993,7 +912,9 @@ def test_construct_target_url_with_version_prefix(): ), ], ) -def test_construct_target_url_versionless_project_route_gets_api_version(requested_route: str, expected_url: str) -> None: +def test_construct_target_url_versionless_project_route_gets_api_version( + requested_route: str, expected_url: str +) -> None: from litellm.llms.vertex_ai.common_utils import construct_target_url target_url = construct_target_url( @@ -1126,10 +1047,7 @@ def test_fix_enum_types(): # 2. Non-string enums are removed assert "enum" not in input_schema["properties"]["maxLength"] assert "enum" not in input_schema["properties"]["enabled"] - assert ( - "enum" - not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] - ) + assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] # 3. anyOf with string type keeps enum, non-string removes it assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0] @@ -1192,7 +1110,7 @@ async def test_vertex_ai_token_counter_routes_partner_models(): Test that VertexAITokenCounter correctly routes partner models (Claude, Mistral, etc.) to the partner models token counter instead of the Gemini token counter. """ - from unittest.mock import AsyncMock, patch + from unittest.mock import patch from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter from litellm.types.utils import TokenCountResponse @@ -1242,7 +1160,6 @@ async def test_vertex_ai_token_counter_uses_count_tokens_location(): from unittest.mock import patch from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter - from litellm.types.utils import TokenCountResponse token_counter = VertexAITokenCounter() @@ -1283,7 +1200,7 @@ async def test_vertex_ai_token_counter_routes_gemini_models(): Test that VertexAITokenCounter correctly routes Gemini models to the Gemini token counter (not partner models). """ - from unittest.mock import AsyncMock, patch + from unittest.mock import patch from litellm.llms.vertex_ai.common_utils import VertexAITokenCounter from litellm.types.utils import TokenCountResponse @@ -1334,9 +1251,7 @@ async def test_vertex_ai_token_counter_converts_messages_to_contents_for_gemini( token_counter = VertexAITokenCounter() - with patch( - "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens" - ) as mock_acount_tokens: + with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens: mock_acount_tokens.return_value = { "totalTokens": 42, "tokenizer_used": "gemini", @@ -1378,9 +1293,7 @@ async def test_vertex_ai_token_counter_returns_none_when_api_omits_total_tokens( token_counter = VertexAITokenCounter() - with patch( - "litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens" - ) as mock_acount_tokens: + with patch("litellm.llms.vertex_ai.count_tokens.handler.VertexAITokenCounter.acount_tokens") as mock_acount_tokens: mock_acount_tokens.return_value = {"tokenizer_used": "gemini"} result = await token_counter.count_tokens( @@ -1423,9 +1336,7 @@ async def test_vertex_ai_partner_model_detection(): # Test Minimax models assert VertexAIPartnerModels.is_vertex_partner_model("minimaxai/minimax-m2-maas") # Test Moonshot models - assert VertexAIPartnerModels.is_vertex_partner_model( - "moonshotai/kimi-k2-thinking-maas" - ) + assert VertexAIPartnerModels.is_vertex_partner_model("moonshotai/kimi-k2-thinking-maas") # Test Gemini models (should NOT be detected as partner model) assert not VertexAIPartnerModels.is_vertex_partner_model("gemini-1.5-pro") @@ -1456,9 +1367,7 @@ def test_vertex_ai_moonshot_uses_openai_handler(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.should_use_openai_handler( - "moonshotai/kimi-k2-thinking-maas" - ) + assert VertexAIPartnerModels.should_use_openai_handler("moonshotai/kimi-k2-thinking-maas") def test_vertex_ai_zai_uses_openai_handler(): @@ -1493,9 +1402,7 @@ def test_vertex_ai_gemma_maas_is_partner_model(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.is_vertex_partner_model( - "google/gemma-4-26b-a4b-it-maas" - ) + assert VertexAIPartnerModels.is_vertex_partner_model("google/gemma-4-26b-a4b-it-maas") def test_vertex_ai_gemma_maas_uses_openai_handler(): @@ -1506,9 +1413,7 @@ def test_vertex_ai_gemma_maas_uses_openai_handler(): VertexAIPartnerModels, ) - assert VertexAIPartnerModels.should_use_openai_handler( - "google/gemma-4-26b-a4b-it-maas" - ) + assert VertexAIPartnerModels.should_use_openai_handler("google/gemma-4-26b-a4b-it-maas") def test_vertex_ai_gemma_maas_routes_to_partner_models(): @@ -1590,36 +1495,24 @@ def test_build_vertex_schema_empty_properties(): # Verify the transformation removed empty properties # Navigate to the go_back schema - go_back_schema = result["properties"]["action"]["items"]["anyOf"][0]["properties"][ - "go_back" - ] + go_back_schema = result["properties"]["action"]["items"]["anyOf"][0]["properties"]["go_back"] # Verify empty properties was removed assert "properties" not in go_back_schema, "Empty properties should be removed" # Verify type is kept as object (Gemini requires type: object even without properties) - assert ( - go_back_schema.get("type") == "object" - ), "Type should be kept as object when properties is empty" + assert go_back_schema.get("type") == "object", "Type should be kept as object when properties is empty" # Verify required was also removed - assert ( - "required" not in go_back_schema - ), "Required should be removed when properties is empty" + assert "required" not in go_back_schema, "Required should be removed when properties is empty" # Verify description is preserved - assert ( - go_back_schema.get("description") == "Go back" - ), "Description should be preserved" + assert go_back_schema.get("description") == "Go back", "Description should be preserved" # Verify parent schema still has proper structure parent_schema = result["properties"]["action"]["items"]["anyOf"][0] - assert ( - parent_schema["type"] == "object" - ), "Parent schema should still have object type" - assert ( - "go_back" in parent_schema["properties"] - ), "go_back should still be in parent properties" + assert parent_schema["type"] == "object", "Parent schema should still have object type" + assert "go_back" in parent_schema["properties"], "go_back should still be in parent properties" def test_add_object_type_schema_with_no_properties_and_no_type(): @@ -1710,12 +1603,8 @@ def test_pop_vertex_request_labels_prefers_explicit_labels_then_metadata(): def test_pop_vertex_request_labels_uses_litellm_metadata_when_metadata_absent(): optional: dict = {} - litellm_params = { - "litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}} - } - assert pop_vertex_request_labels(optional, litellm_params) == { - "team": "from_litellm_meta" - } + litellm_params = {"litellm_metadata": {"requester_metadata": {"team": "from_litellm_meta"}}} + assert pop_vertex_request_labels(optional, litellm_params) == {"team": "from_litellm_meta"} def test_vertex_text_embedding_request_includes_labels_from_metadata(): @@ -1725,9 +1614,7 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata(): input="hi", optional_params={}, model="text-embedding-004", - litellm_params={ - "metadata": {"requester_metadata": {"project_id": "cost-center-1"}} - }, + litellm_params={"metadata": {"requester_metadata": {"project_id": "cost-center-1"}}}, ) assert req.get("labels") == {"project_id": "cost-center-1"} @@ -1755,19 +1642,3 @@ def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(mode from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info assert get_vertex_ai_lyria_model_info(model=model) is None - - -def test_get_vertex_ai_lyria_model_info_falls_back_to_bundled_map(monkeypatch): - import litellm - from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info - - stale_runtime_model_cost = { - key: value for key, value in litellm.model_cost.items() if not key.startswith("vertex_ai/lyria") - } - monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost) - - model_info = get_vertex_ai_lyria_model_info(model="lyria-3-pro-preview") - - assert model_info is not None - assert model_info["vertex_ai_audio_api"] == "lyria_interactions" - assert model_info["supported_audio_formats"] == ("mp3", "wav") diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index b5eec42b569..387cc405f02 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -181,59 +181,6 @@ class TestVertexAILyriaTextToSpeechConfig: assert isinstance(config, VertexAILyriaTextToSpeechConfig) - @pytest.mark.parametrize( - ("model", "vertex_ai_audio_api", "supported_audio_formats", "expected_url"), - [ - ( - "future-lyria-predict", - "lyria_predict", - ["wav"], - "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/" - "us-central1/publishers/google/models/future-lyria-predict:predict", - ), - ( - "future-music-interactions", - "lyria_interactions", - ["mp3", "wav"], - "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", - ), - ], - ) - def test_dispatches_from_model_metadata( - self, - monkeypatch, - model, - vertex_ai_audio_api, - supported_audio_formats, - expected_url, - ): - monkeypatch.setitem( - litellm.model_cost, - f"vertex_ai/{model}", - { - "vertex_ai_audio_api": vertex_ai_audio_api, - "supported_audio_formats": supported_audio_formats, - }, - ) - - config = ProviderConfigManager.get_provider_text_to_speech_config( - model=model, - provider=LlmProviders.VERTEX_AI, - ) - - assert isinstance(config, VertexAILyriaTextToSpeechConfig) - assert ( - config.get_complete_url( - model=model, - api_base=None, - litellm_params={ - "vertex_project": "music-project", - "vertex_location": "us-central1", - }, - ) - == expected_url - ) - def test_vertex_chirp_does_not_select_lyria_config(self): config = ProviderConfigManager.get_provider_text_to_speech_config( model="chirp", @@ -261,9 +208,7 @@ class TestVertexAILyriaTextToSpeechConfig: ) def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None: - injected: Final = ( - "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" - ) + injected: Final = "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" encoded: Final = ( "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle" "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D" diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index f6da1bbcd0e..8471c9c99bc 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -452,44 +452,6 @@ def test_vertex_claude_completion_does_not_mutate_shared_extra_headers(): assert shared_extra_headers == {}, "extra_headers must not be mutated by completion()" -def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cost_map, monkeypatch): - """The Vertex messages config must probe capabilities under ``vertex_ai`` so an - operator setting ``supports_adaptive_thinking: false`` on the exact - ``vertex_ai/claude-opus-4-8`` entry beats the unmodified ``anthropic`` entry. - With the inherited ``"anthropic"`` provider default the flip was ignored and - the transform kept emitting ``thinking.type='adaptive'``.""" - import litellm - - config = VertexAIPartnerModelsAnthropicMessagesConfig() - - def transform(): - return config.transform_anthropic_messages_request( - model="claude-opus-4-8", - messages=[{"role": "user", "content": "Hello"}], - anthropic_messages_optional_request_params={ - "max_tokens": 4096, - "reasoning_effort": "medium", - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - result = transform() - assert result.get("thinking") == {"type": "adaptive", "display": "summarized"} - assert result.get("output_config") == {"effort": "medium"} - - monkeypatch.setitem(litellm.model_cost["vertex_ai/claude-opus-4-8"], "supports_adaptive_thinking", False) - litellm.get_model_info.cache_clear() - assert litellm.model_cost["claude-opus-4-8"]["supports_adaptive_thinking"] is True - - flipped = transform() - thinking = flipped.get("thinking") - assert isinstance(thinking, dict) - assert thinking.get("type") == "enabled" - assert isinstance(thinking.get("budget_tokens"), int) - assert "output_config" not in flipped - - def _vertex_transform(model, messages, system=None): config = VertexAIPartnerModelsAnthropicMessagesConfig() params = {"max_tokens": 256} diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index a57672cfbfb..6b50dadbb38 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1,4 +1,3 @@ - import pytest from litellm.anthropic_beta_headers_manager import ( @@ -16,9 +15,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im ], ) def test_vertex_ai_anthropic_thinking_param(model, expected_thinking): - supported_openai_params = VertexAIAnthropicConfig().get_supported_openai_params( - model=model - ) + supported_openai_params = VertexAIAnthropicConfig().get_supported_openai_params(model=model) if expected_thinking: assert "thinking" in supported_openai_params @@ -32,50 +29,6 @@ def test_get_supported_params_thinking(): assert "thinking" in params -def test_vertex_ai_anthropic_web_search_header_in_completion(): - """Test that web search tool adds the required beta header for Vertex AI completion requests""" - from unittest.mock import MagicMock, patch - - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - # Create the config instance - model_info = AnthropicModelInfo() - - # Test the header generation directly - tools = [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] - - # Check if web search tool is detected - web_search_detected = model_info.is_web_search_tool_used(tools=tools) - assert web_search_detected is True, "Web search tool should be detected" - - # Generate headers with is_vertex_request=True - headers = model_info.get_anthropic_headers( - api_key="test-key", - web_search_tool_used=web_search_detected, - is_vertex_request=True, - ) - - # Assert that the anthropic-beta header with web-search is present - assert "anthropic-beta" in headers, "anthropic-beta header should be present" - assert ( - headers["anthropic-beta"] == "web-search-2025-03-05" - ), f"anthropic-beta should be 'web-search-2025-03-05', got: {headers['anthropic-beta']}" - - # Test that header is NOT added for non-Vertex requests - headers_non_vertex = model_info.get_anthropic_headers( - api_key="test-key", - web_search_tool_used=web_search_detected, - is_vertex_request=False, - ) - - # For non-Vertex (Anthropic-hosted), the web search header should NOT be in anthropic-beta - # because Anthropic doesn't require it - assert ( - "anthropic-beta" not in headers_non_vertex - or "web-search" not in headers_non_vertex.get("anthropic-beta", "") - ), "anthropic-beta with web-search should not be present for non-Vertex requests" - - def test_vertex_ai_anthropic_context_management_compact_beta_header(): """Test that context_management with compact adds the correct beta header for Vertex AI""" config = VertexAIAnthropicConfig() @@ -163,13 +116,11 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): }, "is_vertex_request": True, } - result_vertex = config.update_headers_with_optional_anthropic_beta( - headers_vertex, optional_params_vertex - ) + result_vertex = config.update_headers_with_optional_anthropic_beta(headers_vertex, optional_params_vertex) - assert ( - "anthropic-beta" not in result_vertex - ), f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + assert "anthropic-beta" not in result_vertex, ( + f"Vertex request should NOT have anthropic-beta header for structured output, got: {result_vertex.get('anthropic-beta')}" + ) # Test case 2: Non-Vertex request with output_format SHOULD add beta header headers_non_vertex = {} @@ -187,12 +138,12 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): headers_non_vertex, optional_params_non_vertex ) - assert ( - "anthropic-beta" in result_non_vertex - ), "Non-Vertex request SHOULD have anthropic-beta header for structured output" - assert ( - result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13" - ), f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" + assert "anthropic-beta" in result_non_vertex, ( + "Non-Vertex request SHOULD have anthropic-beta header for structured output" + ) + assert result_non_vertex["anthropic-beta"] == "structured-outputs-2025-11-13", ( + f"Expected 'structured-outputs-2025-11-13', got: {result_non_vertex.get('anthropic-beta')}" + ) def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): @@ -247,9 +198,7 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # Should have tools and tool_choice (tool-based approach) assert "tools" in result_params, "Tools should be present for structured output" - assert ( - "tool_choice" in result_params - ), "Tool choice should be present for structured output" + assert "tool_choice" in result_params, "Tool choice should be present for structured output" assert "json_mode" in result_params, "JSON mode should be enabled" # Verify the tool is the response format tool @@ -274,9 +223,7 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # Mock the parent transform_request to return data with output_format original_transform = config.__class__.__bases__[0].transform_request - def mock_transform_request( - self, model, messages, optional_params, litellm_params, headers - ): + def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): # Return test data that includes output_format return test_data.copy() @@ -298,9 +245,7 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): # callers who explicitly requested them. assert "output_format" in final_data assert final_data["output_format"]["type"] == "json_schema" - assert ( - "model" not in final_data - ), "model is still stripped (Vertex routes by URL)" + assert "model" not in final_data, "model is still stripped (Vertex routes by URL)" assert "tools" in final_data, "tools should still be present" assert "tool_choice" in final_data, "tool_choice should still be present" @@ -336,9 +281,7 @@ def test_vertex_ai_anthropic_other_models_still_use_tools(): ) # Should still use tool-based approach - assert ( - "tools" in result_params - ), "Claude 3 Sonnet should also use tool-based structured output" + assert "tools" in result_params, "Claude 3 Sonnet should also use tool-based structured output" assert "tool_choice" in result_params, "Tool choice should be present" assert "json_mode" in result_params, "JSON mode should be enabled" @@ -463,34 +406,21 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05 from the anthropic-beta headers. """ - from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( - VertexAIPartnerModelsAnthropicMessagesConfig, - ) # This beta header should be removed PROMPT_CACHING_BETA_HEADER = "prompt-caching-scope-2026-01-05" - headers = { - "anthropic-beta": f"other-feature,{PROMPT_CACHING_BETA_HEADER},web-search-2025-03-05" - } + headers = {"anthropic-beta": f"other-feature,{PROMPT_CACHING_BETA_HEADER},web-search-2025-03-05"} headers = update_headers_with_filtered_beta(headers, "vertex_ai") beta_header = headers.get("anthropic-beta") - assert PROMPT_CACHING_BETA_HEADER not in ( - beta_header or "" - ), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" - assert "other-feature" not in ( - beta_header or "" - ), "Other non-excluded beta headers should remain" - assert "web-search-2025-03-05" in ( - beta_header or "" - ), "Other non-excluded beta headers should remain" + assert PROMPT_CACHING_BETA_HEADER not in (beta_header or ""), f"{PROMPT_CACHING_BETA_HEADER} should be filtered out" + assert "other-feature" not in (beta_header or ""), "Other non-excluded beta headers should remain" + assert "web-search-2025-03-05" in (beta_header or ""), "Other non-excluded beta headers should remain" # If prompt-caching was the only value, header should be removed completely headers2 = {"anthropic-beta": PROMPT_CACHING_BETA_HEADER} headers2 = update_headers_with_filtered_beta(headers2, "vertex_ai") - assert ( - "anthropic-beta" not in headers2 - ), "Header should be removed if no supported values remain" + assert "anthropic-beta" not in headers2, "Header should be removed if no supported values remain" def test_vertex_ai_anthropic_output_config_effort_only_forwarded(): @@ -636,9 +566,7 @@ def test_vertex_ai_anthropic_output_format_and_output_config_effort_preserved(): original_transform = config.__class__.__bases__[0].transform_request - def mock_transform_request( - self, model, messages, optional_params, litellm_params, headers - ): + def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): return test_data.copy() config.__class__.__bases__[0].transform_request = mock_transform_request diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py index 957d7475d91..a8da13e2f36 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py @@ -48,37 +48,6 @@ _GEMMA_MODEL_COST_ENTRY = { # --------------------------------------------------------------------------- -@pytest.fixture(autouse=True) -def _reset_litellm_http_client_cache(): - """Ensure each test gets a fresh async HTTP client mock.""" - from litellm import in_memory_llm_clients_cache - - in_memory_llm_clients_cache.flush_cache() - - -@pytest.fixture(autouse=True) -def clean_vertex_env(): - """Clear Google/Vertex AI environment variables before each test to prevent test isolation issues.""" - saved_env = {} - env_vars_to_clear = [ - "GOOGLE_APPLICATION_CREDENTIALS", - "GOOGLE_CLOUD_PROJECT", - "VERTEXAI_PROJECT", - "VERTEX_PROJECT", - "VERTEX_LOCATION", - "VERTEX_AI_PROJECT", - ] - for var in env_vars_to_clear: - if var in os.environ: - saved_env[var] = os.environ[var] - del os.environ[var] - - yield - - for var, value in saved_env.items(): - os.environ[var] = value - - # --------------------------------------------------------------------------- # Unit tests: region and URL construction # --------------------------------------------------------------------------- @@ -92,11 +61,7 @@ class TestVertexBaseGetVertexRegionGemma: with patch.dict( litellm.model_cost, - { - "vertex_ai/google/gemma-4-26b-a4b-it-maas": { - "supported_regions": ["global"] - } - }, + {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, clear=False, ): result = vertex_base.get_vertex_region( @@ -110,11 +75,7 @@ class TestVertexBaseGetVertexRegionGemma: with patch.dict( litellm.model_cost, - { - "vertex_ai/google/gemma-4-26b-a4b-it-maas": { - "supported_regions": ["global"] - } - }, + {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, clear=False, ): result = vertex_base.get_vertex_region( @@ -140,9 +101,9 @@ class TestCreateVertexURLGemma: which in turn generates the /endpoints/openapi URL shape. If this mapping ever changes, the URL-shape tests below become misleading. """ - assert VertexAIPartnerModels.should_use_openai_handler( - "google/gemma-4-26b-a4b-it-maas" - ), "Gemma MaaS must use the OpenAI-compatible handler (VertexPartnerProvider.llama path)" + assert VertexAIPartnerModels.should_use_openai_handler("google/gemma-4-26b-a4b-it-maas"), ( + "Gemma MaaS must use the OpenAI-compatible handler (VertexPartnerProvider.llama path)" + ) def test_global_location_url_format(self): # VertexPartnerProvider.llama is correct: Gemma MaaS reaches create_vertex_url @@ -180,28 +141,6 @@ class TestCreateVertexURLGemma: # --------------------------------------------------------------------------- -def test_gemma_maas_supports_function_calling(): - """supports_function_calling=true in model_cost must be surfaced by the utility.""" - with patch.dict(litellm.model_cost, _GEMMA_MODEL_COST_ENTRY, clear=False): - assert ( - litellm.utils.supports_function_calling( - model="vertex_ai/google/gemma-4-26b-a4b-it-maas" - ) - is True - ) - - -def test_gemma_maas_supports_vision(): - """supports_vision=true in model_cost must be surfaced by the utility.""" - with patch.dict(litellm.model_cost, _GEMMA_MODEL_COST_ENTRY, clear=False): - assert ( - litellm.utils.supports_vision( - model="vertex_ai/google/gemma-4-26b-a4b-it-maas" - ) - is True - ) - - # --------------------------------------------------------------------------- # Integration tests: verify payloads reach the global OpenAI endpoint # @@ -235,6 +174,37 @@ _MOCK_RESPONSE_JSON = { } +@pytest.fixture(autouse=True) +def _reset_litellm_http_client_cache(): + """Ensure each test gets a fresh async HTTP client mock.""" + from litellm import in_memory_llm_clients_cache + + in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture(autouse=True) +def clean_vertex_env(): + """Clear Google/Vertex AI environment variables before each test to prevent test isolation issues.""" + saved_env = {} + env_vars_to_clear = [ + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "VERTEXAI_PROJECT", + "VERTEX_PROJECT", + "VERTEX_LOCATION", + "VERTEX_AI_PROJECT", + ] + for var in env_vars_to_clear: + if var in os.environ: + saved_env[var] = os.environ[var] + del os.environ[var] + + yield + + for var, value in saved_env.items(): + os.environ[var] = value + + @pytest.mark.asyncio async def test_vertex_ai_gemma_global_endpoint_url(): """ @@ -250,9 +220,7 @@ async def test_vertex_ai_gemma_global_endpoint_url(): mock_vertexai.preview = MagicMock() with ( - patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" - ) as mock_http_handler, + patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), @@ -263,11 +231,7 @@ async def test_vertex_ai_gemma_global_endpoint_url(): ), patch.dict( litellm.model_cost, - { - "vertex_ai/google/gemma-4-26b-a4b-it-maas": { - "supported_regions": ["global"] - } - }, + {"vertex_ai/google/gemma-4-26b-a4b-it-maas": {"supported_regions": ["global"]}}, clear=False, ), ): @@ -326,9 +290,7 @@ async def test_vertex_ai_gemma_function_calling_passthrough(): mock_vertexai.preview = MagicMock() with ( - patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" - ) as mock_http_handler, + patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), @@ -399,9 +361,7 @@ async def test_vertex_ai_gemma_vision_passthrough(): mock_vertexai.preview = MagicMock() with ( - patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler" - ) as mock_http_handler, + patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler, patch( "litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token", return_value=("fake-token", "test-project"), diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py index b6b638c6dbe..ae2c60c1781 100644 --- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py @@ -13,8 +13,6 @@ import httpx import pytest import litellm -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.llms.openai.cost_calculation import video_generation_cost from litellm.llms.vertex_ai.videos.transformation import ( VertexAIVideoConfig, _convert_image_to_vertex_format, @@ -23,14 +21,8 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject VEO_31_LITE_VERTEX_MODEL = "vertex_ai/veo-3.1-lite-generate-001" -ROOT_MODEL_COST_PATH = ( - Path(__file__).parents[5] / "model_prices_and_context_window.json" -) -BACKUP_MODEL_COST_PATH = ( - Path(__file__).parents[5] - / "litellm" - / "model_prices_and_context_window_backup.json" -) +ROOT_MODEL_COST_PATH = Path(__file__).parents[5] / "model_prices_and_context_window.json" +BACKUP_MODEL_COST_PATH = Path(__file__).parents[5] / "litellm" / "model_prices_and_context_window_backup.json" ModelCostMap = Mapping[str, Mapping[str, object]] @@ -84,9 +76,7 @@ class TestVertexAIVideoConfig: "vertex_location": "us-central1", } - url = self.config.get_complete_url( - model="vertex_ai/veo-002", api_base=None, litellm_params=litellm_params - ) + url = self.config.get_complete_url(model="vertex_ai/veo-002", api_base=None, litellm_params=litellm_params) expected = "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/veo-002" assert url == expected @@ -119,29 +109,7 @@ class TestVertexAIVideoConfig: monkeypatch.setattr(litellm, "vertex_project", None) with pytest.raises(ValueError, match="vertex_project is required"): - self.config.get_complete_url( - model="veo-002", api_base=None, litellm_params={} - ) - - - def test_veo_31_lite_provider_routing_from_local_model_map( - self, monkeypatch: pytest.MonkeyPatch - ): - model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) - vertex_video_models = { - model_name.removeprefix("vertex_ai/") - for model_name, info in model_cost.items() - if info.get("litellm_provider") == "vertex_ai-video-models" - } - monkeypatch.setattr(litellm, "vertex_ai_video_models", vertex_video_models) - - model, custom_llm_provider, _, _ = get_llm_provider( - model="veo-3.1-lite-generate-001" - ) - - assert model == "veo-3.1-lite-generate-001" - assert custom_llm_provider == "vertex_ai" - + self.config.get_complete_url(model="veo-002", api_base=None, litellm_params={}) def test_transform_video_create_request(self): """Test transformation of video creation request.""" @@ -282,9 +250,7 @@ class TestVertexAIVideoConfig: assert mapped["aspectRatio"] == "16:9" assert "resolution" not in mapped - def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3( - self, monkeypatch: pytest.MonkeyPatch - ): + def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3(self, monkeypatch: pytest.MonkeyPatch): model = "veo-3.1-generate-001" model_key = f"vertex_ai/{model}" model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) @@ -457,9 +423,7 @@ class TestVertexAIVideoConfig: "raiMediaFilteredCount": 0, "videos": [ { - "bytesBase64Encoded": base64.b64encode( - b"fake_video_data" - ).decode(), + "bytesBase64Encoded": base64.b64encode(b"fake_video_data").decode(), "mimeType": "video/mp4", } ], @@ -525,9 +489,7 @@ class TestVertexAIVideoConfig: "done": True, "response": { "@type": "type.googleapis.com/cloud.ai.large_models.vision.GenerateVideoResponse", - "videos": [ - {"bytesBase64Encoded": encoded_video, "mimeType": "video/mp4"} - ], + "videos": [{"bytesBase64Encoded": encoded_video, "mimeType": "video/mp4"}], }, } @@ -547,9 +509,7 @@ class TestVertexAIVideoConfig: } with pytest.raises(ValueError, match="Video generation is not complete yet"): - self.config.transform_video_content_response( - raw_response=mock_response, logging_obj=self.mock_logging_obj - ) + self.config.transform_video_content_response(raw_response=mock_response, logging_obj=self.mock_logging_obj) def test_transform_video_content_response_missing_video_data(self): """Test that missing video data raises error.""" @@ -561,9 +521,7 @@ class TestVertexAIVideoConfig: } with pytest.raises(ValueError, match="No video data found"): - self.config.transform_video_content_response( - raw_response=mock_response, logging_obj=self.mock_logging_obj - ) + self.config.transform_video_content_response(raw_response=mock_response, logging_obj=self.mock_logging_obj) def test_get_video_edit_prefetch_params(self): """Test that prefetch params returns the fetchPredictOperation URL and body.""" @@ -589,9 +547,7 @@ class TestVertexAIVideoConfig: prefetched = { "done": True, - "response": { - "videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}] - }, + "response": {"videos": [{"bytesBase64Encoded": fake_bytes, "mimeType": "video/mp4"}]}, } url, data, files = self.config.transform_video_edit_request( @@ -618,9 +574,7 @@ class TestVertexAIVideoConfig: prefetched = { "done": True, - "response": { - "videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}] - }, + "response": {"videos": [{"gcsUri": "gs://bucket/video.mp4", "mimeType": "video/mp4"}]}, } _, data, _ = self.config.transform_video_edit_request( @@ -746,9 +700,7 @@ class TestVertexAIVideoConfig: def test_get_error_class(self): """Test error class generation.""" - error = self.config.get_error_class( - error_message="Test error", status_code=500, headers={} - ) + error = self.config.get_error_class(error_message="Test error", status_code=500, headers={}) # Should return VertexAIError from litellm.llms.vertex_ai.common_utils import VertexAIError @@ -960,10 +912,7 @@ class TestImageAndParametersPassthrough: # instances contains prompt + image assert len(data["instances"]) == 1 instance = data["instances"][0] - assert ( - instance["prompt"] - == "Cinematic drone shot moving forward along the beach boardwalk" - ) + assert instance["prompt"] == "Cinematic drone shot moving forward along the beach boardwalk" assert instance["image"] == image # parameters block is correct and not double-nested diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index dd0d1bdbb9d..02f22a4135d 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -75,22 +75,6 @@ def wandb_request_mock(respx_mock: respx.MockRouter) -> respx.Route: class TestWandbConfig: """Test class for WandB Inference functionality""" - @pytest.mark.parametrize("model", WANDB_REASONING_MODELS) - def test_map_openai_params_preserves_reasoning_effort(self, wandb_test_config, model: str): - assert litellm.model_cost[f"wandb/{model}"].get("supports_reasoning") is True - supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") - assert supported_params is not None - assert "reasoning_effort" in supported_params - - result = WandbConfig().map_openai_params( - non_default_params={"reasoning_effort": "medium", "max_completion_tokens": 64}, - optional_params={}, - model=model, - drop_params=True, - ) - - assert result == {"reasoning_effort": "medium", "max_tokens": 64} - def test_default_api_base(self): """Test that default API base is used when none is provided""" config = WandbConfig() @@ -123,9 +107,7 @@ class TestWandbConfig: This test mocks the actual HTTP request to test the integration properly. """ - litellm.disable_aiohttp_transport = ( - True # since this uses respx, we need to set use_aiohttp_transport to False - ) + litellm.disable_aiohttp_transport = True # since this uses respx, we need to set use_aiohttp_transport to False # Set up environment variables for the test api_key = "fake-wandb-key" @@ -162,9 +144,7 @@ class TestWandbConfig: # Make the actual API call through LiteLLM response = completion( model=model, - messages=[ - {"role": "user", "content": "write code for saying hey from LiteLLM"} - ], + messages=[{"role": "user", "content": "write code for saying hey from LiteLLM"}], api_key=api_key, api_base=api_base, ) @@ -243,53 +223,6 @@ class TestWandbConfig: assert request_body["max_tokens"] == 64 assert "max_completion_tokens" not in request_body - @pytest.mark.respx(assert_all_called=False) - @pytest.mark.parametrize("drop_params", [True, False]) - @pytest.mark.parametrize( - "model,explicit_false", - [ - ("meta-llama/Llama-3.1-8B-Instruct", False), - ("openai/gpt-oss-20b", True), - ], - ) - def test_wandb_completion_without_reasoning_support( - self, - wandb_test_config, - wandb_request_mock: respx.Route, - respx_mock: respx.MockRouter, - monkeypatch: pytest.MonkeyPatch, - model: str, - explicit_false: bool, - drop_params: bool, - ): - with monkeypatch.context() as context: - if explicit_false: - context.setitem(litellm.model_cost[f"wandb/{model}"], "supports_reasoning", False) - - kwargs = { - "model": f"wandb/{model}", - "messages": [{"role": "user", "content": "Hello"}], - "api_key": "fake-wandb-key", - "api_base": "https://api.inference.wandb.ai/v1", - "reasoning_effort": "medium", - "drop_params": drop_params, - } - if not drop_params: - with pytest.raises(litellm.UnsupportedParamsError, match="reasoning_effort"): - completion(**kwargs) - assert len(respx_mock.calls) == 0 - return - - completion(**kwargs) - assert wandb_request_mock.call_count == 1 - request_body = json.loads(wandb_request_mock.calls[0].request.content) - assert request_body["model"] == model - assert "reasoning_effort" not in request_body - - supported_params = litellm.get_supported_openai_params(model=f"wandb/{model}") - assert supported_params is not None - assert "reasoning_effort" not in supported_params - @pytest.mark.respx() def test_wandb_completion_keeps_reasoning_effort_for_an_unregistered_model( self, wandb_test_config, wandb_request_mock: respx.Route diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/test_litellm/llms/xai/test_xai_model_registry.py index a455d1fb233..47c91e24f14 100644 --- a/tests/test_litellm/llms/xai/test_xai_model_registry.py +++ b/tests/test_litellm/llms/xai/test_xai_model_registry.py @@ -7,7 +7,6 @@ from __future__ import annotations import json from pathlib import Path -import pytest REPO_ROOT = Path(__file__).parents[4] PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -23,30 +22,6 @@ RESPONSES_ONLY_MODELS = ( MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH) -@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS]) -def cost_map(request: pytest.FixtureRequest) -> dict: - path = next(p for p in MAP_PATHS if p.name == request.param) - return json.loads(path.read_text(encoding="utf-8")) - - -@pytest.mark.parametrize("model", RESPONSES_ONLY_MODELS) -def test_multi_agent_models_are_responses_only(cost_map: dict, model: str): - entry = cost_map[model] - assert entry["supported_endpoints"] == ["/v1/responses"] - assert entry["mode"] == "responses" - - -def test_surviving_xai_chat_models_still_serve_chat_completions(cost_map: dict): - """Guard against the removal above over-reaching into live models.""" - chat_models = [ - key - for key, value in cost_map.items() - if isinstance(value, dict) and value.get("litellm_provider") == "xai" and value.get("mode") == "chat" - ] - assert "xai/grok-4.3" in chat_models - assert "xai/grok-4.6" in chat_models - - def test_both_cost_maps_agree_on_xai_entries(): prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) diff --git a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py index bbbcfb1b9dc..83e8925f70b 100644 --- a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py +++ b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py @@ -85,11 +85,6 @@ def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str): assert entry[field] == target[field], field -def test_a_live_xai_model_is_untouched(cost_map: dict): - """Guard against the repricing leaking onto models xAI still serves directly.""" - assert cost_map["xai/grok-4.6"]["input_cost_per_token"] != cost_map[REDIRECT_TARGET]["input_cost_per_token"] - - @pytest.mark.parametrize("slug", REDIRECTED_SLUGS) def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str): """The request executes as grok-4.3, so it is tiered at grok-4.3's 200k boundary.""" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 36bfc4c5dd3..10873c4772a 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -1,10 +1,7 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import patch import pytest -from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, Member -from litellm.proxy.auth.handle_jwt import JWTAuthManager - def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models @@ -14,9 +11,7 @@ def test_get_team_models_for_all_models_and_team_only_models(): model_access_groups = {} include_model_access_groups = False - result = get_team_models( - team_models, proxy_model_list, model_access_groups, include_model_access_groups - ) + result = get_team_models(team_models, proxy_model_list, model_access_groups, include_model_access_groups) combined_models = team_models + proxy_model_list assert set(result) == set(combined_models) @@ -249,9 +244,7 @@ def test_get_key_models_does_not_mutate_input(): ), ], ) -def test_get_complete_model_list_order( - key_models, team_models, proxy_model_list, model_list, expected -): +def test_get_complete_model_list_order(key_models, team_models, proxy_model_list, model_list, expected): """ Test that get_complete_model_list preserves order """ @@ -404,9 +397,7 @@ def test_wildcard_credential_hydration_preserves_deployment_params( captured_params["api_key"] = litellm_params.api_key captured_params["api_version"] = litellm_params.api_version captured_params["credential_name"] = litellm_params.litellm_credential_name - captured_params["has_unexpected_field"] = hasattr( - litellm_params, "unexpected_field" - ) + captured_params["has_unexpected_field"] = hasattr(litellm_params, "unexpected_field") return ["gpt-4o"] monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) @@ -451,9 +442,7 @@ def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): result = get_known_models_from_wildcard( wildcard_model="ollama_server1/*", - litellm_params=LiteLLM_Params( - model="ollama_chat/*", custom_llm_provider="ollama_chat" - ), + litellm_params=LiteLLM_Params(model="ollama_chat/*", custom_llm_provider="ollama_chat"), ) assert result == ["ollama_server1/gemma3:1b", "ollama_server1/llama3:8b"] @@ -480,9 +469,7 @@ def test_wildcard_custom_prefix_keeps_org_segment_for_non_provider_first_segment result = get_known_models_from_wildcard( wildcard_model="my_hf/*", - litellm_params=LiteLLM_Params( - model="huggingface/*", custom_llm_provider="huggingface" - ), + litellm_params=LiteLLM_Params(model="huggingface/*", custom_llm_provider="huggingface"), ) assert result == ["my_hf/meta-llama/Llama-3-8B"] @@ -844,9 +831,7 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] try: litellm.add_known_models( - model_cost_map={ - fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"} - } + model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} ) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference @@ -858,23 +843,6 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): assert fake_model not in litellm.models_by_provider["vertex_ai"] -def test_azure_ai_wildcard_lists_the_foundry_gpt_6_astra_entry(monkeypatch): - import litellm - from litellm.proxy.auth.model_checks import get_known_models_from_wildcard - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - foundry_key = "azure_ai/gpt-6-astra" - local_entry = litellm.get_model_cost_map(url="")[foundry_key] - registered_before = foundry_key in litellm.azure_ai_models - try: - litellm.add_known_models(model_cost_map={foundry_key: local_entry}) - assert foundry_key in get_known_models_from_wildcard("azure_ai/*") - finally: - if not registered_before: - litellm.azure_ai_models.discard(foundry_key) - litellm.add_known_models(model_cost_map={}) - - def test_get_complete_model_list_drops_no_default_models_sentinel(): from litellm.proxy.auth.model_checks import get_complete_model_list diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 615938f2e33..94ce8019b1b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -7,7 +7,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_toke from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token from litellm.proxy.spend_tracking.savings import ( _baseline_usage, - _resolve_model, compute_autorouter_savings, compute_savings_spend, marks_gateway_injection, @@ -29,7 +28,11 @@ def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], c assert usage.prompt_tokens_details.cached_tokens == 0 selected_cost: Final = 0.013 assert compute_autorouter_savings( - "claude-opus-5", "claude-sonnet-5", "anthropic", usage, conversation_continuing=continuing, + "claude-opus-5", + "claude-sonnet-5", + "anthropic", + usage, + conversation_continuing=continuing, cost_breakdown={"input_cost": 0.01, "output_cost": 0.003}, ) == pytest.approx(sum(anthropic_cost_per_token("claude-opus-5", expected)) - selected_cost) @@ -37,11 +40,17 @@ def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], c def test_anthropic_baseline_keeps_negotiated_prices_with_provider_multiplier() -> None: info: Final = { **litellm.get_model_info("claude-opus-5", "anthropic"), - "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "cache_read_input_token_cost": 3e-7, + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 3e-7, } usage: Final = _usage(1000, 1000, 0, 100).model_copy(update={"speed": "fast"}) assert compute_autorouter_savings( - "claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info, + "claude-opus-5", + "claude-sonnet-5", + "anthropic", + usage, + baseline_info=info, cost_breakdown={"input_cost": 0.01, "output_cost": 0.003}, ) == pytest.approx(0.0015 * 2 - 0.013) @@ -758,84 +767,6 @@ def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" -def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): - """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, - because those providers cache implicitly and charge nothing to write. Leaving this - request's written tokens in the creation bucket priced them at the 0.0 the cost - resolver falls back to, so the baseline carried a 20k prompt for free and a first - turn that saved money reported a loss. Those tokens are plain input on such a model. - """ - first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) - reported = compute_autorouter_savings( - baseline_model="gpt-5", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=first_turn, - conversation_continuing=False, - ) - - gpt5 = litellm.get_model_info("gpt-5", "openai") - assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" - haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") - baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] - actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] - assert reported == pytest.approx(baseline_pays_input - actually_paid) - assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" - - -def _priced_chat_model_without_cache_read_rate() -> tuple[str, str, str]: - """A chat model the bundled map prices per token for input and output but not for cache - reads, derived from the map itself: a hardcoded pick goes stale the moment the registry - prices that model's cache reads, which is exactly how this test's premise last broke. - Candidates go through the savings module's own resolver, so the pick is one the code - under test can actually price.""" - for key in sorted(litellm.model_cost): - entry = litellm.model_cost[key] - provider = entry.get("litellm_provider") - if not isinstance(provider, str) or not key.startswith(f"{provider}/"): - continue - if entry.get("mode") != "chat" or entry.get("cache_read_input_token_cost") is not None: - continue - if not entry.get("input_cost_per_token") or not entry.get("output_cost_per_token"): - continue - if _resolve_model(key, None) is None: - continue - priced = compute_autorouter_savings( - baseline_model=key, - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=_usage(fresh=1_000, cached=0, written=0, out=100), - conversation_continuing=True, - ) - if priced == 0.0: - continue - return key, key.removeprefix(f"{provider}/"), provider - raise AssertionError("the bundled map has no per-token chat model without a cache-read rate") - - -def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): - """The same hole on the other bucket. A baseline whose entry has no - `cache_read_input_token_cost` reads for 0.0, so a continuing turn priced the whole - prompt at nothing and every switch away from it reported a loss. - """ - baseline_key, baseline_name, baseline_provider = _priced_chat_model_without_cache_read_rate() - continuing = _usage(fresh=0, cached=0, written=20_000, out=1_000) - reported = compute_autorouter_savings( - baseline_model=baseline_key, - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=continuing, - conversation_continuing=True, - ) - - baseline = litellm.get_model_info(baseline_name, baseline_provider) - assert baseline.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" - haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") - baseline_pays_input = 20_000 * baseline["input_cost_per_token"] + 1_000 * baseline["output_cost_per_token"] - actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] - assert reported == pytest.approx(baseline_pays_input - actually_paid) - - def _breakdown(input_cost: float, output_cost: float = 0.0, **extra: object) -> dict: """A `cost_breakdown` as the cost calculator records it on the spend log.""" return {"input_cost": input_cost, "output_cost": output_cost, **extra} @@ -875,51 +806,6 @@ def test_the_served_arm_is_read_from_the_record_not_repriced(): assert reported == pytest.approx(public - (negotiated_input + negotiated_output)) -@pytest.mark.parametrize( - "basis, expected_multiplier", - [ - pytest.param({"service_tier": "priority"}, 2.5, id="priority tier uplifts the baseline"), - pytest.param({"data_residency": "eu"}, 1.1, id="eu residency uplifts the baseline"), - pytest.param({}, 1.0, id="no basis recorded prices at standard"), - pytest.param(None, 1.0, id="row predating the field prices at standard"), - pytest.param({"service_tier": True, "data_residency": 17}, 1.0, id="a non-string basis is dropped"), - ], -) -def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, expected_multiplier): - """A request billed at a priority tier, or through a regional host, would have been - billed the same way on the single model an operator ran instead of the router, so the - counterfactual carries that basis too. Dropping it prices the two arms from different - books; neither multiplier cancels out of the difference, because both are per-model. - - The served model has no tiered rates and no uplift of its own, so only the baseline - can move: a fix that forwards the basis to the served arm alone leaves these numbers - unchanged. The non-string case guards the JSON round trip, where `.lower()` inside - the pricer would raise and be swallowed into a silent $0.00 for the whole row. - """ - gpt = litellm.get_model_info("gpt-5.5", "openai") - haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") - assert gpt.get("input_cost_per_token_priority") == pytest.approx(2.5 * gpt["input_cost_per_token"]) - assert gpt.get("output_cost_per_token_priority") == pytest.approx(2.5 * gpt["output_cost_per_token"]) - assert gpt.get("regional_processing_uplift_multiplier_eu") == 1.1 - assert haiku.get("input_cost_per_token_priority") is None, "served model must not move with the basis" - assert haiku.get("regional_processing_uplift_multiplier_eu") is None - - usage = _usage(fresh=20_000, cached=0, written=0, out=1_000) - served = 20_000 * haiku["input_cost_per_token"] + 1_000 * haiku["output_cost_per_token"] - - reported = compute_autorouter_savings( - baseline_model="openai/gpt-5.5", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=usage, - conversation_continuing=False, - cost_breakdown=None if basis is None else _breakdown(served, **basis), - ) - - baseline = 20_000 * gpt["input_cost_per_token"] + 1_000 * gpt["output_cost_per_token"] - assert reported == pytest.approx(expected_multiplier * baseline - served) - - def test_the_baseline_is_priced_on_the_vertex_location_the_request_was_billed_at(monkeypatch): """A request served from a regional Vertex endpoint was billed with the regional-endpoint uplift, so the counterfactual single-model operator would diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index b2f3c6e7c0e..9b8c55d51bb 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -2151,32 +2151,6 @@ async def test_proxy_only_error_5xx_keeps_traceback_and_runs_sync_callbacks(monk assert "test_proxy_utils" in captured["async_traceback"] -def test_create_model_info_response_resolves_mode_through_deployment_model(): - """`mode` is derived from the same lookup, so an aliased embedding deployment - currently reports no mode at all; it must report `embedding`.""" - from litellm import Router - - saved_model_cost = dict(litellm.model_cost) - try: - router = Router( - model_list=[ - { - "model_name": "my-embeddings", - "litellm_params": {"model": "openai/text-embedding-3-small"}, - } - ] - ) - - response = create_model_info_response( - model_id="my-embeddings", provider="openai", llm_router=router - ) - finally: - litellm.model_cost.clear() - litellm.model_cost.update(saved_model_cost) - - assert response["mode"] == "embedding" - - @pytest.mark.parametrize( "key_metadata, team_metadata, expected_to_run", [ @@ -2252,7 +2226,9 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data={"metadata": {}}, - original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"), + original_exception=HTTPException( + status_code=400, detail="Upstream passthrough request failed with status 400" + ), user_api_key_dict=UserAPIKeyAuth(), traceback_str=upstream_traceback, ) @@ -2316,9 +2292,13 @@ def test_mcp_conversion_preserves_request_policy_and_isolates_guardrail_data(buc parent = { "model": "parent-model", bucket: { - "guardrails": ["policy-rule"], "guardrail_config": {"language": "en"}, - "applied_policies": ["parent-policy"], "policy_sources": {"parent-policy": "model"}, - "_guardrail_pipelines": [], "_pipeline_managed_guardrails": ["pipeline-rule"], "tags": ["review"], + "guardrails": ["policy-rule"], + "guardrail_config": {"language": "en"}, + "applied_policies": ["parent-policy"], + "policy_sources": {"parent-policy": "model"}, + "_guardrail_pipelines": [], + "_pipeline_managed_guardrails": ["pipeline-rule"], + "tags": ["review"], }, "guardrails": [{"request-rule": {"extra_body": {"threshold": 0.9}}}], "guardrail_config": {"entities": ["EMAIL_ADDRESS"]}, @@ -2348,13 +2328,26 @@ def test_mcp_conversion_honors_only_authenticated_global_guardrail_opt_outs(opt_ from litellm.responses.mcp.request_context import MCPRequestContext auth = UserAPIKeyAuth(metadata={"opted_out_global_guardrails": ["global-rule"] if opt_out else []}) - context = MCPRequestContext.resolve(kwargs={"metadata": { - "user_api_key_auth": auth, "disable_global_guardrails": True, - "user_api_key_metadata": {"disable_global_guardrails": True}, - }}, tools=None) + context = MCPRequestContext.resolve( + kwargs={ + "metadata": { + "user_api_key_auth": auth, + "disable_global_guardrails": True, + "user_api_key_metadata": {"disable_global_guardrails": True}, + } + }, + tools=None, + ) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) - kwargs = {"name": "execute", "arguments": {}, "user_api_key_auth": auth, "guardrail_context": context.guardrail_context} - synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) + kwargs = { + "name": "execute", + "arguments": {}, + "user_api_key_auth": auth, + "guardrail_context": context.guardrail_context, + } + synthetic = proxy_logging._convert_mcp_to_llm_format( + proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs + ) guardrail = CustomGuardrail(guardrail_name="global-rule", event_hook="pre_mcp_call", default_on=True) assert guardrail.should_run_guardrail(synthetic, GuardrailEventHooks.pre_mcp_call) is (not opt_out) synthetic["metadata"]["user_api_key_metadata"]["opted_out_global_guardrails"].append("unrelated") @@ -2368,18 +2361,25 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyGuardrails registry = policy_registry.PolicyRegistry() - registry._policies = {"model-policy": Policy( - condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) - )} + registry._policies = { + "model-policy": Policy( + condition=PolicyCondition(model="parent-model"), guardrails=PolicyGuardrails(add=["model-rule"]) + ) + } registry._initialized = True monkeypatch.setattr(policy_registry, "_policy_registry", registry) monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) kwargs = { - "name": "execute", "arguments": {}, + "name": "execute", + "arguments": {}, "user_api_key_auth": UserAPIKeyAuth(metadata={"policies": ["model-policy"]}), - "guardrail_context": MCPRequestContext.resolve_guardrail_context({"model": model, "guardrails": ["request-rule"]}), + "guardrail_context": MCPRequestContext.resolve_guardrail_context( + {"model": model, "guardrails": ["request-rule"]} + ), } - synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) + synthetic = proxy_logging._convert_mcp_to_llm_format( + proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs + ) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"] diff --git a/tests/test_litellm/router_strategy/complexity_router/test_jev_classifier.py b/tests/test_litellm/router_strategy/complexity_router/test_jev_classifier.py index f27729d29e8..e8edb69ea6f 100644 --- a/tests/test_litellm/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/test_litellm/router_strategy/complexity_router/test_jev_classifier.py @@ -105,20 +105,6 @@ def test_build_jev_request_includes_system_prompt_and_criteria() -> None: assert request.questions["tier"].criteria == criteria -def test_jev_classifier_cost_uses_registry_pricing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setitem( - litellm.model_cost, - "typesafe/jev-1.13.0", - {"input_cost_per_token": 0.0001, "output_cost_per_token": 0.0002}, - ) - response: Final = JevSystemOneResponse( - model="jev-1.13.0", - answers={"tier": _answer()}, - usage=JevUsage(input_tokens=3, output_tokens=4), - ) - assert jev_classifier_cost(response, "jev-latest") == pytest.approx(0.0011) - - def test_jev_classifier_cost_is_none_without_registry_pricing() -> None: assert "typesafe/jev-unpriced" not in litellm.model_cost response: Final = JevSystemOneResponse( diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index ccd6766b13a..9a1cbe73ae8 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -325,29 +325,6 @@ KIMI_K3_PERPLEXITY_KEY = "perplexity/perplexity/kimi-k3" class TestKimiK3AdvertisesItsDocumentedLevels: - @pytest.mark.parametrize("model_key", KIMI_K3_PASSTHROUGH_KEYS) - def test_a_passthrough_entry_advertises_the_models_own_levels(self, local_model_cost_map, model_key): - """platform.kimi.ai documents exactly low, high and max, and these providers forward the - level unchanged. Undeclared, each entry resolves to unknown and the dashboard falls back to - a capability-blind list that omits max.""" - entry = dict(litellm.model_cost[model_key], key=model_key) - - assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ("low", "high", "max") - - def test_the_perplexity_entry_advertises_the_wider_set_it_maps_down(self, local_model_cost_map): - """Perplexity's Agent API takes a six-value enum and maps it down internally, so this - deployment is legitimately wider than a passthrough. One blanket list could not say both.""" - entry = dict(litellm.model_cost[KIMI_K3_PERPLEXITY_KEY], key=KIMI_K3_PERPLEXITY_KEY) - - assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ( - "minimal", - "low", - "medium", - "high", - "xhigh", - "max", - ) - @pytest.mark.parametrize("model, provider", [("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai")]) def test_the_declaration_survives_model_info_hydration(self, local_model_cost_map, model, provider): """The hydration line is the load-bearing seam: without it the key the map carries never @@ -359,19 +336,6 @@ class TestKimiK3AdvertisesItsDocumentedLevels: assert model_info["reasoning_effort_levels"] == ["low", "high", "max"] assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ("low", "high", "max") - def test_a_kimi_k3_deployment_now_narrows_a_mixed_group(self, local_model_cost_map): - """kimi used to contribute unknown, which never narrows, so the group advertised whatever - its other deployments agreed on.""" - kimi = resolve_supported_reasoning_efforts( - dict(litellm.model_cost["fireworks_ai/kimi-k3"], key="fireworks_ai/kimi-k3"), - deployment_is_mapped=True, - ) - - assert intersect_supported_reasoning_efforts(("none", "minimal", "low", "medium", "high", "xhigh"), kimi) == ( - "low", - "high", - ) - class TestGpt6AstraAdvertisesItsDocumentedLevels: def test_the_entry_advertises_low_through_max_without_none(self, local_model_cost_map): diff --git a/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py b/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py index 43df9a648c2..e9ea8c066df 100644 --- a/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py +++ b/tests/test_litellm/test_azure_ai_grok_4_6_model_metadata.py @@ -1,11 +1,8 @@ from pathlib import Path from typing import Final -import pytest from pydantic import TypeAdapter -from litellm import cost_per_token, get_model_info -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider REPO_ROOT: Final = Path(__file__).parents[2] MODEL: Final = "azure_ai/grok-4.6" @@ -16,27 +13,6 @@ def _cost_map_entry(path: Path) -> dict[str, object]: return COST_MAP_ADAPTER.validate_json(path.read_bytes())[MODEL] -@pytest.mark.usefixtures("local_model_cost_map") -def test_azure_ai_grok_4_6_is_priced_and_routed() -> None: - routed_model, provider, _, _ = get_llm_provider(model=MODEL) - assert (routed_model, provider) == ("grok-4.6", "azure_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "azure_ai" - assert info["mode"] == "chat" - assert info["supports_function_calling"] is True - assert info["supports_prompt_caching"] is True - assert info["supports_reasoning"] is True - assert info["supports_response_schema"] is True - assert info["supports_tool_choice"] is True - assert info["supports_vision"] is True - assert info["supports_web_search"] is True - - prompt_cost, completion_cost = cost_per_token(model=MODEL, prompt_tokens=1_000_000, completion_tokens=1_000_000) - assert prompt_cost > 0 - assert completion_cost > 0 - - def test_azure_ai_grok_4_6_entry_source_and_backup_match() -> None: main_entry = _cost_map_entry(REPO_ROOT / "model_prices_and_context_window.json") backup_entry = _cost_map_entry(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") diff --git a/tests/test_litellm/test_azure_audio_price_aliases.py b/tests/test_litellm/test_azure_audio_price_aliases.py deleted file mode 100644 index b87744aeae1..00000000000 --- a/tests/test_litellm/test_azure_audio_price_aliases.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Undated azure aliases for the audio models must exist and match their dated -variants. Azure deployments are commonly created under an admin-chosen name, so -the served model name means nothing to the cost lookup and `base_model: -azure/gpt-audio-mini` is what prices the call. That key resolved to nothing, the -lookup raised "This model isn't mapped yet", and the proxy logged the request at -$0. Issue #33170.""" - -import json -from pathlib import Path - -import pytest - -import litellm - -pytestmark = pytest.mark.usefixtures("local_model_cost_map") - - -COST_FIELDS = ( - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token", -) - -ALIAS_PAIRS = ( - ("azure/gpt-audio-mini", "azure/gpt-audio-mini-2025-10-06"), - ("azure/gpt-realtime-mini", "azure/gpt-realtime-mini-2025-10-06"), -) - - -def _load_root_cost_map() -> dict: - root_map_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" - with open(root_map_path) as f: - return json.load(f) - - -@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) -def test_undated_azure_audio_alias_matches_dated_entry(undated, dated): - undated_info = litellm.get_model_info(undated) - dated_info = litellm.get_model_info(dated) - - for field in COST_FIELDS: - assert undated_info.get(field) == dated_info.get(field), field - assert (undated_info.get(field) or 0) > 0, f"{undated}.{field} must be non-zero" - - assert undated_info.get("litellm_provider") == "azure" - assert undated_info.get("mode") == dated_info.get("mode") - - -@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) -def test_undated_azure_audio_alias_is_exact_mirror(undated, dated): - """The undated alias must be a byte-for-byte mirror of its dated entry, covering - every field (incl. realtime-specific cache/audio cost keys) so any future drift - between the pair is caught, not just the core COST_FIELDS.""" - model_map = litellm.model_cost - assert undated in model_map, f"{undated} missing from model cost map" - assert model_map[undated] == model_map[dated], ( - f"{undated} must exactly mirror {dated}; " - f"diff keys: {[k for k in set(model_map[undated]) | set(model_map[dated]) if model_map[undated].get(k) != model_map[dated].get(k)]}" - ) - - -@pytest.mark.parametrize("undated, dated", ALIAS_PAIRS) -def test_undated_azure_audio_alias_is_in_the_root_cost_map(undated, dated): - """`local_model_cost_map` pins `litellm.model_cost` to the packaged backup, but a - proxy left on its defaults fetches the root map instead, and that is the copy - that ships to the CDN. An alias added to only one of the two files still bills - $0 for every proxy reading the other, which is the very bug this file guards, so - assert the root map directly and assert the two files agree.""" - root_map = _load_root_cost_map() - assert undated in root_map, f"{undated} missing from the root cost map" - assert root_map[undated] == root_map[dated], f"{undated} must exactly mirror {dated} in the root cost map" - assert root_map[undated] == litellm.model_cost[undated], ( - f"{undated} differs between the root cost map and the packaged backup" - ) diff --git a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py index 31f3a67beac..f573c79434a 100644 --- a/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py +++ b/tests/test_litellm/test_baseten_glm_5_3_model_metadata.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest import litellm -from litellm.utils import supports_function_calling, supports_prompt_caching REPO_ROOT = Path(__file__).parents[2] MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -33,17 +32,6 @@ def local_model_cost_map(monkeypatch): litellm.get_model_info.cache_clear() -def test_baseten_glm_5_3_capabilities_are_visible_to_callers(local_model_cost_map): - """The entry advertises prompt caching and tool calling, so the helpers every - caller checks before sending a request must say so too.""" - assert supports_prompt_caching(model=MODEL) is True - assert supports_function_calling(model=MODEL) is True - - info = litellm.get_model_info(model="zai-org/GLM-5.3", custom_llm_provider="baseten") - assert info["max_input_tokens"] > 0 - assert info["max_output_tokens"] > 0 - - def test_backup_matches_main(): """Ensure the bundled (backup) cost map stays in sync with the canonical file. diff --git a/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py b/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py index 1a0e1665556..21e9b26d996 100644 --- a/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py +++ b/tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py @@ -3,7 +3,6 @@ from pathlib import Path import pytest -import litellm from litellm.constants import bedrock_embedding_models REPO_ROOT = Path(__file__).parents[2] @@ -31,13 +30,6 @@ def _load(path): return json.load(f) -@pytest.mark.parametrize("model", ALL_MODELS) -def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map): - info = litellm.get_model_info(model=model, custom_llm_provider="bedrock") - assert info["mode"] == "embedding" - assert info["output_vector_size"] == 512 - - def test_marengo_embed_3_is_a_known_bedrock_embedding_model(): assert BASE_MODEL in bedrock_embedding_models diff --git a/tests/test_litellm/test_bedrock_usgov_pricing.py b/tests/test_litellm/test_bedrock_usgov_pricing.py deleted file mode 100644 index a3a7fc4ed7a..00000000000 --- a/tests/test_litellm/test_bedrock_usgov_pricing.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Validate AWS GovCloud (Bedrock us-gov-*) Anthropic pricing entries. - -AWS Bedrock pricing in GovCloud carries a +20% premium over the global -Anthropic prices (not the +10% commercial-US premium). Until 2026-05-22 -these entries silently mirrored commercial US, undercharging customers -by ~9%. - -Source: https://aws.amazon.com/bedrock/pricing/ - - Sonnet 4.5 in us-gov-* (per million tokens): - input = $3.60 - output = $18.00 - cache write 5m = $4.50 - cache write 1h = $7.20 - cache read = $0.36 - -Reference: https://github.com/BerriAI/litellm/issues/27120 -""" - -import json -import os - -import pytest - - -@pytest.fixture(scope="module") -def model_data(): - json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") - with open(json_path) as f: - return json.load(f) - - -def test_usgov_east_haiku_profile_mirrors_in_region_row(model_data): - """us-gov-east-1 serves claude-3-haiku through the us-gov. inference profile - only, so the profile row must bill exactly like the in-region gov row. - """ - profile = model_data["us-gov.anthropic.claude-3-haiku-20240307-v1:0"] - in_region = model_data["bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0"] - assert profile["litellm_provider"] == "bedrock_converse" - assert {k: v for k, v in profile.items() if k != "litellm_provider"} == { - k: v for k, v in in_region.items() if k != "litellm_provider" - } - - -GOV_ROW_SOURCES = { - "us-gov.anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1", - "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1", - "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": "anthropic.claude-fable-5-1", - "us-gov.nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2", - "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2", - "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": "nvidia.nemotron-nano-9b-v2", - "us-gov.xai.grok-4.6": "us.xai.grok-4.6", - "bedrock_mantle/us-gov-west-1/xai.grok-4.6": "bedrock_mantle/xai.grok-4.6", - "bedrock_mantle/us-gov-east-1/xai.grok-4.6": "bedrock_mantle/xai.grok-4.6", - "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": "amazon.nova-2-multimodal-embeddings-v1:0", - "bedrock/us-gov-west-1/amazon.nova-lite-v1:0": "amazon.nova-lite-v1:0", - "bedrock/us-gov-west-1/amazon.nova-micro-v1:0": "amazon.nova-micro-v1:0", - "bedrock_mantle/us-gov-west-1/google.gemma-4-e2b": "bedrock_mantle/google.gemma-4-e2b", - "bedrock_mantle/us-gov-west-1/google.gemma-4-26b-a4b": "bedrock_mantle/google.gemma-4-26b-a4b", - "bedrock_mantle/us-gov-west-1/google.gemma-4-31b": "bedrock_mantle/google.gemma-4-31b", - "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": "bedrock_mantle/openai.gpt-oss-20b", - "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": "bedrock_mantle/openai.gpt-oss-20b", - "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": "bedrock_mantle/openai.gpt-oss-120b", - "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": "bedrock_mantle/openai.gpt-oss-120b", -} - - -def _non_pricing_fields(info): - return {k: v for k, v in info.items() if "cost" not in k and k not in ("litellm_provider", "source")} - - -@pytest.mark.parametrize("gov_key", GOV_ROW_SOURCES) -def test_usgov_rows_keep_commercial_limits_and_capabilities(model_data, gov_key): - """Gov rows preserve the commercial row's non-pricing fields.""" - gov = model_data[gov_key] - assert _non_pricing_fields(gov) == _non_pricing_fields(model_data[GOV_ROW_SOURCES[gov_key]]) diff --git a/tests/test_litellm/test_claude_fable_5_config.py b/tests/test_litellm/test_claude_fable_5_config.py index 4b03848da2c..a0ed8d856dc 100644 --- a/tests/test_litellm/test_claude_fable_5_config.py +++ b/tests/test_litellm/test_claude_fable_5_config.py @@ -26,67 +26,10 @@ def _load_root_cost_map() -> dict: return json.load(f) -def test_fable_5_present_in_bundled_backup(): - """The bundled backup is the runtime fallback (and what tests load with - ``LITELLM_LOCAL_MODEL_COST_MAP=True``) — it must carry the same entries as - the root cost map, otherwise the model resolves on one path but not the - other.""" - backup = GetModelCostMap.load_local_model_cost_map() - root = _load_root_cost_map() - for model_name in ( - "claude-fable-5", - "anthropic.claude-fable-5", - "global.anthropic.claude-fable-5", - "us.anthropic.claude-fable-5", - "eu.anthropic.claude-fable-5", - "vertex_ai/claude-fable-5", - "vertex_ai/claude-fable-5@default", - "azure_ai/claude-fable-5", - ): - assert model_name in backup, f"Missing from backup cost map: {model_name}" - assert backup[model_name] == root[model_name], model_name - - def test_fable_5_registered_for_bedrock_converse(): assert "anthropic.claude-fable-5" in BEDROCK_CONVERSE_MODELS -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_fable_5_all_variants_carry_adaptive_thinking_flag(cost_map): - """Every Fable 5 entry must advertise ``supports_adaptive_thinking``. - - Adaptive-thinking detection is cost-map driven, so a single variant missing - the flag silently sends the legacy ``thinking.type='enabled'`` shape and the - provider 400s (issue #29188 for the Opus 4.8 equivalent). Fable 5 is even - stricter than Opus 4.8: an explicit ``thinking.type='disabled'`` also 400s, - so adaptive is the only valid thinking shape LiteLLM can emit for it.""" - variants = [k for k in cost_map if "claude-fable-5" in k] - assert variants, "no claude-fable-5 entries found in cost map" - missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] - assert not missing, f"missing supports_adaptive_thinking: {missing}" - - -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_fable_5_all_variants_carry_thinking_always_on_flag(cost_map): - """Every Fable 5 entry must advertise ``thinking_always_on``. - - The flag drives the Anthropic transformations to omit an explicit - ``thinking.type='disabled'``, which Fable 5 rejects with a 400; a variant - missing the flag forwards the param verbatim and the provider 400s.""" - variants = [k for k in cost_map if "claude-fable-5" in k] - assert variants, "no claude-fable-5 entries found in cost map" - missing = [k for k in variants if cost_map[k].get("thinking_always_on") is not True] - assert not missing, f"missing thinking_always_on: {missing}" - - @pytest.mark.parametrize( "model", [ @@ -149,24 +92,3 @@ def test_adaptive_thinking_detected_for_fable_5_1(local_model_cost_map, model): from litellm.llms.anthropic.common_utils import AnthropicModelInfo assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True - - -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_sampling_params_flag_on_all_models_that_removed_them(cost_map): - """Fable 5 and Opus 4.7/4.8 reject ``top_p``/``top_k``/``temperature != 1``; - the drop/raise gating is cost-map driven, so every variant must carry an - explicit ``supports_sampling_params: false``. The perplexity route is - exempt: it is OpenAI-compatible and maps sampling params upstream.""" - variants = [ - k - for k in cost_map - if any(v in k for v in ("claude-fable-5", "claude-opus-4-7", "claude-opus-4-8")) - and not k.startswith("perplexity/") - ] - assert variants, "no matching entries found in cost map" - missing = [k for k in variants if cost_map[k].get("supports_sampling_params") is not False] - assert not missing, f"missing supports_sampling_params=false: {missing}" diff --git a/tests/test_litellm/test_claude_haiku_4_5_config.py b/tests/test_litellm/test_claude_haiku_4_5_config.py deleted file mode 100644 index d0b7f4f8a2c..00000000000 --- a/tests/test_litellm/test_claude_haiku_4_5_config.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Test Claude Haiku 4.5 model configurations for Bedrock -https://github.com/BerriAI/litellm/issues/15818 -""" - -import json -import os - - -def test_bedrock_haiku_4_5_matches_sonnet_capabilities(): - """ - Test that Haiku 4.5 has same capabilities as Sonnet 4.5 - (including computer_use, vision, tools, etc.) - """ - # Load model configuration - json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json") - with open(json_path) as f: - model_data = json.load(f) - - haiku_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" - sonnet_model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" - - haiku_info = model_data[haiku_model] - sonnet_info = model_data[sonnet_model] - - # Both should use bedrock_converse - assert haiku_info["litellm_provider"] == "bedrock_converse" - assert sonnet_info["litellm_provider"] == "bedrock_converse" - - # Shared capabilities that should match - shared_capabilities = [ - "supports_vision", - "supports_computer_use", - "supports_function_calling", - "supports_tool_choice", - "supports_prompt_caching", - "supports_response_schema", - "supports_pdf_input", - "supports_assistant_prefill", - "supports_reasoning", - ] - - for capability in shared_capabilities: - assert haiku_info.get(capability) == sonnet_info.get(capability), ( - f"Capability {capability} mismatch: Haiku={haiku_info.get(capability)}, Sonnet={sonnet_info.get(capability)}" - ) diff --git a/tests/test_litellm/test_claude_opus_4_6_config.py b/tests/test_litellm/test_claude_opus_4_6_config.py index 9a8632924f2..f7d264ec5ae 100644 --- a/tests/test_litellm/test_claude_opus_4_6_config.py +++ b/tests/test_litellm/test_claude_opus_4_6_config.py @@ -2,100 +2,9 @@ Validate Claude Opus 4.6 model configuration entries. """ -import json -import os - import litellm -def test_claude_4_6_australia_region_uses_au_prefix_not_apac(): - """ - Test that Australia region Claude 4.6 models use 'au.' prefix instead of incorrect 'apac.' prefix. - - AWS Bedrock cross-region inference uses specific regional prefixes: - - 'us.' for United States - - 'eu.' for Europe - - 'au.' for Australia (ap-southeast-2) - - 'apac.' for Asia-Pacific (Singapore, ap-southeast-1) - - This test ensures the Claude 4.6 models correctly use 'au.' for Australia, - and that 'apac.' is NOT incorrectly used for Australia region. - - Related: The 'apac.' prefix is valid for Asia-Pacific (Singapore) region models, - but should not be used for Australia which has its own 'au.' prefix. - """ - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) - with open(json_path) as f: - model_data = json.load(f) - - # Verify au.anthropic.claude-opus-4-6-v1 exists (correct) - assert ( - "au.anthropic.claude-opus-4-6-v1" in model_data - ), "Missing Australia region model: au.anthropic.claude-opus-4-6-v1" - - # Verify apac.anthropic.claude-opus-4-6-v1 does NOT exist (incorrect) - assert ( - "apac.anthropic.claude-opus-4-6-v1" not in model_data - ), "Incorrect model entry exists: apac.anthropic.claude-opus-4-6-v1 should be au.anthropic.claude-opus-4-6-v1" - - # Verify au.anthropic.claude-sonnet-4-6 exists (correct) - assert ( - "au.anthropic.claude-sonnet-4-6" in model_data - ), "Missing Australia region model: au.anthropic.claude-sonnet-4-6" - - # Verify apac.anthropic.claude-sonnet-4-6 does NOT exist (incorrect) - assert ( - "apac.anthropic.claude-sonnet-4-6" not in model_data - ), "Incorrect model entry exists: apac.anthropic.claude-sonnet-4-6 should be au.anthropic.claude-sonnet-4-6" - - # Verify the au. model is registered in bedrock_converse_models - assert ( - "au.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models - ), "au.anthropic.claude-opus-4-6-v1 not registered in bedrock_converse_models" - - # Verify apac. is NOT registered for this model - assert ( - "apac.anthropic.claude-opus-4-6-v1" not in litellm.bedrock_converse_models - ), "apac.anthropic.claude-opus-4-6-v1 should not be in bedrock_converse_models" - - # Verify the au. model is registered in bedrock_converse_models - assert ( - "au.anthropic.claude-sonnet-4-6" in litellm.bedrock_converse_models - ), "au.anthropic.claude-sonnet-4-6 not registered in bedrock_converse_models" - - # Verify apac. is NOT registered for this model - assert ( - "apac.anthropic.claude-sonnet-4-6" not in litellm.bedrock_converse_models - ), "apac.anthropic.claude-sonnet-4-6 should not be in bedrock_converse_models" - - -def test_opus_4_6_alias_and_dated_metadata_match(): - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) - with open(json_path) as f: - model_data = json.load(f) - - alias = model_data["claude-opus-4-6"] - dated = model_data["claude-opus-4-6-20260205"] - - keys_to_match = [ - "max_input_tokens", - "max_output_tokens", - "max_tokens", - "input_cost_per_token", - "output_cost_per_token", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr", - "cache_read_input_token_cost", - "supports_assistant_prefill", - ] - for key in keys_to_match: - assert alias[key] == dated[key], f"Mismatch for {key}" - - def test_opus_4_6_bedrock_converse_registration(): assert "anthropic.claude-opus-4-6-v1" in litellm.BEDROCK_CONVERSE_MODELS assert "global.anthropic.claude-opus-4-6-v1" in litellm.bedrock_converse_models diff --git a/tests/test_litellm/test_claude_opus_4_8_config.py b/tests/test_litellm/test_claude_opus_4_8_config.py index 1a4bab249fd..f41e6616c83 100644 --- a/tests/test_litellm/test_claude_opus_4_8_config.py +++ b/tests/test_litellm/test_claude_opus_4_8_config.py @@ -11,43 +11,13 @@ for Anthropic, Bedrock, Vertex AI, and Azure AI; those entries are what populate in ``get_llm_provider`` consumes. """ -import json import os -import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") -def _load_root_cost_map() -> dict: - json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") - with open(json_path) as f: - return json.load(f) - - def test_opus_4_8_registered_for_bedrock_converse(): assert "anthropic.claude-opus-4-8" in BEDROCK_CONVERSE_MODELS - - -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_opus_4_8_all_variants_carry_adaptive_thinking_flag(cost_map): - """Every Opus 4.8 entry must advertise ``supports_adaptive_thinking``. - - Adaptive-thinking detection is cost-map driven, so a single variant missing - the flag silently sends the legacy ``thinking.type='enabled'`` shape and the - provider 400s (issue #29188, which the Bedrock/Vertex/Azure variants hit - because only the bare ``claude-opus-4-8`` entry carried the flag). This guards - against a future variant being added without it.""" - variants = [k for k in cost_map if "claude-opus-4-8" in k] - assert variants, "no claude-opus-4-8 entries found in cost map" - missing = [ - k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True - ] - assert not missing, f"missing supports_adaptive_thinking: {missing}" diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index 07e493af914..3327b2795ce 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -12,13 +12,11 @@ validator accepts the full effort ladder, so the entries must not carry the ``anthropic/*`` wildcard deployment). """ -import json import os import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -45,12 +43,6 @@ BEDROCK_OPUS_5_VARIANTS = ( ) -def _load_root_cost_map() -> dict: - json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") - with open(json_path) as f: - return json.load(f) - - @pytest.mark.parametrize("model_name", BEDROCK_OPUS_5_VARIANTS) def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map): """Bedrock Converse routes Opus through a validator that rejects @@ -62,31 +54,5 @@ def test_opus_5_bedrock_rejects_strict_tools(model_name, local_model_cost_map): assert bedrock_converse_supports_strict_tools(model_name) is False -def test_opus_5_present_in_bundled_backup(): - """The bundled backup is the runtime fallback (and what tests load with - ``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the - root cost map, otherwise the model resolves on one path but not the other.""" - backup = GetModelCostMap.load_local_model_cost_map() - for model_name in ALL_OPUS_5_VARIANTS: - assert model_name in backup, f"Missing from backup cost map: {model_name}" - - def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS - - -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_opus_5_all_variants_carry_adaptive_thinking_flag(cost_map): - """Every Opus 5 entry must advertise ``supports_adaptive_thinking``. - - Adaptive-thinking detection is cost-map driven, so a single variant missing - the flag silently sends the legacy ``thinking.type='enabled'`` shape, which - Opus 5 rejects with a 400.""" - variants = [k for k in cost_map if "claude-opus-5" in k] - assert variants, "no claude-opus-5 entries found in cost map" - missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] - assert not missing, f"missing supports_adaptive_thinking: {missing}" diff --git a/tests/test_litellm/test_claude_sonnet_4_6_config.py b/tests/test_litellm/test_claude_sonnet_4_6_config.py deleted file mode 100644 index a669c21be30..00000000000 --- a/tests/test_litellm/test_claude_sonnet_4_6_config.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Test Claude Sonnet 4.6 model configurations for Bedrock cross-region inference. - -Pins the set of region-prefixed entries in model_prices_and_context_window.json -so future drops of a region (or pricing drift between regions) is caught. - -https://github.com/BerriAI/litellm/issues/22972 -""" - -import json -import os - - -def test_bedrock_sonnet_4_6_jp_matches_other_regional_pricing(): - """The jp. cross-region inference profile shares pricing with the other - regional profiles (us./eu./au.), which carry a 10% premium over the - base/global entries. - """ - json_path = os.path.join( - os.path.dirname(__file__), "../../model_prices_and_context_window.json" - ) - with open(json_path) as f: - model_data = json.load(f) - - jp_info = model_data["jp.anthropic.claude-sonnet-4-6"] - au_info = model_data["au.anthropic.claude-sonnet-4-6"] - - pricing_fields = [ - "input_cost_per_token", - "output_cost_per_token", - "cache_creation_input_token_cost", - "cache_read_input_token_cost", - ] - for field in pricing_fields: - assert jp_info[field] == au_info[field], ( - f"{field} mismatch between jp. and au. variants: " - f"jp={jp_info[field]}, au={au_info[field]}" - ) diff --git a/tests/test_litellm/test_claude_sonnet_5_config.py b/tests/test_litellm/test_claude_sonnet_5_config.py index 8c6d2cd1851..702da61a438 100644 --- a/tests/test_litellm/test_claude_sonnet_5_config.py +++ b/tests/test_litellm/test_claude_sonnet_5_config.py @@ -10,13 +10,10 @@ populate ``litellm.anthropic_models`` at import, which is what lets a bare ``anthropic/*`` wildcard deployment). """ -import json import os -import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") @@ -34,37 +31,5 @@ ALL_SONNET_5_VARIANTS = ( ) -def _load_root_cost_map() -> dict: - json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") - with open(json_path) as f: - return json.load(f) - - -def test_sonnet_5_present_in_bundled_backup(): - """The bundled backup is the runtime fallback (and what tests load with - ``LITELLM_LOCAL_MODEL_COST_MAP=True``); it must carry the same entries as the - root cost map, otherwise the model resolves on one path but not the other.""" - backup = GetModelCostMap.load_local_model_cost_map() - for model_name in ALL_SONNET_5_VARIANTS: - assert model_name in backup, f"Missing from backup cost map: {model_name}" - - def test_sonnet_5_registered_for_bedrock_converse(): assert "anthropic.claude-sonnet-5" in BEDROCK_CONVERSE_MODELS - - -@pytest.mark.parametrize( - "cost_map", - [_load_root_cost_map(), GetModelCostMap.load_local_model_cost_map()], - ids=["root", "bundled_backup"], -) -def test_sonnet_5_all_variants_carry_adaptive_thinking_flag(cost_map): - """Every Sonnet 5 entry must advertise ``supports_adaptive_thinking``. - - Adaptive-thinking detection is cost-map driven, so a single variant missing - the flag silently sends the legacy ``thinking.type='enabled'`` shape and the - provider 400s. This guards against a future variant being added without it.""" - variants = [k for k in cost_map if "claude-sonnet-5" in k] - assert variants, "no claude-sonnet-5 entries found in cost map" - missing = [k for k in variants if cost_map[k].get("supports_adaptive_thinking") is not True] - assert not missing, f"missing supports_adaptive_thinking: {missing}" diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ff28e69a909..09e76ea331b 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -27,7 +27,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) -from litellm.utils import TranscriptionResponse @pytest.fixture @@ -428,74 +427,6 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 -def test_get_transcription_model_falls_back_to_session_model(monkeypatch): - """session.model is used when transcription-specific model fields are absent.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - - from litellm.cost_calculator import _get_transcription_model_name_from_results - - results: OpenAIRealtimeStreamList = [ - {"type": "session.created", "session": {"model": "gpt-realtime-whisper"}}, - ] - assert _get_transcription_model_name_from_results(results) == "gpt-realtime-whisper" - - from litellm import Router - - router = Router( - model_list=[ - { - "model_name": "prod/claude-3-5-sonnet-20240620", - "litellm_params": { - "model": "anthropic/claude-sonnet-4-5-20250929", - "api_key": "test_api_key", - }, - "model_info": { - "id": "my-unique-model-id", - "input_cost_per_token": 0.000006, - "output_cost_per_token": 0.00003, - "cache_creation_input_token_cost": 0.0000075, - "cache_read_input_token_cost": 0.0000006, - }, - }, - { - "model_name": "claude-3-5-sonnet-20240620", - "litellm_params": { - "model": "anthropic/claude-sonnet-4-5-20250929", - "api_key": "test_api_key", - }, - "model_info": { - "input_cost_per_token": 100, - "output_cost_per_token": 200, - }, - }, - ] - ) - - result = router.completion( - model="claude-3-5-sonnet-20240620", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response=True, - ) - - result_2 = router.completion( - model="prod/claude-3-5-sonnet-20240620", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response=True, - ) - - assert result._hidden_params["response_cost"] > result_2._hidden_params["response_cost"] - - model_info = router.get_deployment_model_info( - model_id="my-unique-model-id", model_name="anthropic/claude-sonnet-4-5-20250929" - ) - assert model_info is not None - assert model_info["input_cost_per_token"] == 0.000006 - assert model_info["output_cost_per_token"] == 0.00003 - assert model_info["cache_creation_input_token_cost"] == 0.0000075 - assert model_info["cache_read_input_token_cost"] == 0.0000006 - - def test_custom_pricing_cost_calc_uses_router_model_id_from_litellm_metadata(): """When custom pricing is in litellm_metadata.model_info, use_custom_pricing_for_model should return True and @@ -2339,64 +2270,6 @@ def test_anthropic_geo_multiplier_applies_to_cache_tokens(_local_model_cost_map, assert geo_completion_cost == pytest.approx(base_completion_cost * 1.1) -def test_anthropic_geo_and_fast_multipliers_compose(_local_model_cost_map, monkeypatch): - """ - Anthropic's fast-mode pricing doubles every token type, cache reads and - writes included, and the regional uplift stacks on top, so a fast + - regional row prices as ``(non_cache + cache) * fast * geo``. - """ - from litellm.llms.anthropic.cost_calculation import ( - cost_per_token as anthropic_cost_per_token, - ) - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - - model = "claude-test-geo-fast-cache-model" - _register_anthropic_geo_cache_model(model) - - usage = Usage( - prompt_tokens=10_000, - completion_tokens=500, - total_tokens=10_500, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=2_000, - cache_creation_tokens=6_000, - ), - ) - usage.inference_geo = "us" - usage.speed = "fast" - - prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=usage) - - cache_cost = 2_000 * 0.5e-6 + 6_000 * 6.25e-6 - non_cache_cost = 2_000 * 5e-6 - assert prompt_cost == pytest.approx((non_cache_cost + cache_cost) * 2.0 * 1.1) - assert completion_cost == pytest.approx(500 * 25e-6 * 2.0 * 1.1) - - -@pytest.mark.parametrize( - "model,expected_fast", - [ - ("claude-opus-5", 2.0), - ("claude-opus-4-8", 2.0), - ("claude-opus-4-6", None), - ("claude-opus-4-6-20260205", None), - ("claude-opus-4-7", None), - ("claude-opus-4-7-20260416", None), - ], -) -def test_anthropic_fast_multiplier_only_on_models_with_fast_mode(_local_model_cost_map, model, expected_fast): - """ - Anthropic serves fast mode on Opus 5 and Opus 4.8 only, at 2x. Opus 4.6 and - 4.7 accept the ``speed`` request param but are always served standard, so a - ``fast`` multiplier on their map entries overbills every request that asked - for fast and was served standard. - """ - entry = litellm.model_cost[model] - assert entry["provider_specific_entry"].get("fast") == expected_fast - - @pytest.mark.parametrize( "model", ["claude-sonnet-4-6", "claude-mythos-5", "claude-mythos-preview"], @@ -2933,60 +2806,6 @@ def test_custom_pricing_without_cache_keys_preserves_legacy_behavior(): assert cost == pytest.approx(expected) -def test_completion_cost_logs_the_rates_it_billed_at(monkeypatch): - """A caller reporting the cost lines beside their per-token rates reads both off this one call. - completion_cost infers the provider, and xai's inclusive tier thresholds put a request sitting - exactly on 200k at the tier rate, which a lookup made without that inferred provider would miss. - """ - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import Logging - - monkeypatch.setitem( - litellm.model_cost, - "xai/tiered-model", - { - "input_cost_per_token": 3e-6, - "output_cost_per_token": 15e-6, - "cache_read_input_token_cost": 3e-7, - "input_cost_per_token_above_200k_tokens": 6e-6, - "output_cost_per_token_above_200k_tokens": 3e-5, - "cache_read_input_token_cost_above_200k_tokens": 6e-7, - "litellm_provider": "xai", - "mode": "chat", - }, - ) - logging_obj = Logging( - model="xai/tiered-model", - messages=[{"role": "user", "content": "Hello"}], - stream=False, - call_type="completion", - start_time=datetime.now(), - litellm_call_id="billed-rates", - function_id="f", - ) - usage = Usage( - prompt_tokens=200_000, - completion_tokens=1_000, - total_tokens=201_000, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000), - ) - - litellm.completion_cost( - completion_response=ModelResponse(model="xai/tiered-model", usage=usage), - model="xai/tiered-model", - custom_llm_provider=None, - litellm_logging_obj=logging_obj, - ) - - rates = logging_obj.billed_token_rates - assert rates is not None - assert rates.input_cost_per_token == pytest.approx(6e-6) - assert rates.cache_read_input_token_cost == pytest.approx(6e-7) - assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100_000 * rates.cache_read_input_token_cost) - assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(1_000 * rates.output_cost_per_token) - - def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing(): """ A custom-priced deployment bills cache tokens at its custom cache rates, but the @@ -3246,35 +3065,6 @@ def test_completion_cost_bills_interactions_google_search_per_query(): assert cost > 3 * per_query_cost -def test_completion_cost_bills_interactions_video_output_at_video_rate(): - from litellm.types.interactions import InteractionsAPIResponse - - model_info = litellm.get_model_info(model="gemini-omni-flash-preview", custom_llm_provider="gemini") - video_tokens = 5792 * 8 - response = InteractionsAPIResponse( - id="interactions/video123", - model="gemini-omni-flash-preview", - status="completed", - steps=[], - usage={ - "total_tokens": 10 + video_tokens, - "total_input_tokens": 10, - "input_tokens_by_modality": [{"modality": "text", "tokens": 10}], - "total_cached_tokens": 0, - "total_output_tokens": video_tokens, - "output_tokens_by_modality": [{"modality": "video", "tokens": video_tokens}], - "total_tool_use_tokens": 0, - "total_thought_tokens": 0, - }, - ) - - cost = completion_cost(completion_response=response, custom_llm_provider="gemini") - - expected = 10 * model_info["input_cost_per_token"] + video_tokens * model_info["output_cost_per_video_token"] - assert model_info["output_cost_per_video_token"] != model_info["output_cost_per_token"] - assert cost == pytest.approx(expected) - - @pytest.mark.parametrize("video_count", [2, 3]) def test_completion_cost_multiplies_video_cost_by_generated_video_count(video_count: int) -> None: """Regression for LIT-6896: a Veo request for N samples generates N videos and must be billed N times.""" @@ -3376,24 +3166,6 @@ def test_combine_usage_objects_sums_mirrored_cache_write_fields_once(): assert combined_pair.prompt_tokens_details.cache_creation_tokens == 100 -def _together_chat_response( - model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int -) -> ModelResponse: - return ModelResponse( - id="chatcmpl-together-cache", - choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}], - created=1756164000, - model=model, - object="chat.completion", - usage=Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), - ), - ) - - def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map): """A router-facing model_name alias containing "/" whose leading segment is NOT a registered provider must not be double-prefixed into a non-existent cost key. diff --git a/tests/test_litellm/test_dashscope_image_generation.py b/tests/test_litellm/test_dashscope_image_generation.py index 119efa010e0..397cc9b313a 100644 --- a/tests/test_litellm/test_dashscope_image_generation.py +++ b/tests/test_litellm/test_dashscope_image_generation.py @@ -5,7 +5,6 @@ qwen-image-3.0, qwen-image-3.0-pro). Run in docker: pytest tests/test_litellm/test_dashscope_image_generation.py -v """ -import json from unittest.mock import MagicMock, patch import httpx @@ -16,7 +15,7 @@ from litellm.llms.dashscope.image_generation.transformation import ( DashScopeImageGenerationConfig, DEFAULT_API_BASE, ) -from litellm.types.utils import ImageObject, ImageResponse +from litellm.types.utils import ImageResponse from litellm.utils import get_llm_provider from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -46,40 +45,6 @@ def test_get_llm_provider_returns_dashscope(model_string: str): # --------------------------------------------------------------------------- -@pytest.mark.parametrize( - "model_string, custom_provider", - [ - ("dashscope/qwen-image-2.0", "dashscope"), - ("dashscope/qwen-image-2.0-pro", "dashscope"), - ("dashscope/qwen-image-3.0", "dashscope"), - ("dashscope/qwen-image-3.0-pro", "dashscope"), - ], -) -def test_get_model_info_mode_is_image_generation( - model_string: str, custom_provider: str -): - import os - - prev_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") - prev_model_cost = litellm.model_cost - try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - info = litellm.get_model_info( - model=model_string, custom_llm_provider=custom_provider - ) - assert ( - info["mode"] == "image_generation" - ), f"Expected mode='image_generation', got '{info['mode']}'" - finally: - if prev_env is None: - os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) - else: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = prev_env - litellm.model_cost = prev_model_cost - - # --------------------------------------------------------------------------- # 3. Request transformation # --------------------------------------------------------------------------- @@ -105,9 +70,7 @@ class TestDashScopeImageGenerationConfig: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", ], ) - def test_get_complete_url_ignores_chat_compatible_mode_base( - self, chat_api_base: str - ): + def test_get_complete_url_ignores_chat_compatible_mode_base(self, chat_api_base: str): url = self.cfg.get_complete_url(chat_api_base, None, "qwen-image-3.0", {}, {}) assert url == DEFAULT_API_BASE @@ -168,9 +131,7 @@ class TestDashScopeImageGenerationConfig: headers={}, ) assert req["model"] == model - assert req["input"]["messages"][0]["content"][0]["text"] == ( - "a poster with small multilingual text" - ) + assert req["input"]["messages"][0]["content"][0]["text"] == ("a poster with small multilingual text") assert req["parameters"]["size"] == "2048*2048" assert req["parameters"]["n"] == 6 @@ -435,11 +396,7 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): "finish_reason": "stop", "message": { "role": "assistant", - "content": [ - { - "image": "https://dashscope-result.oss.aliyuncs.com/test.png" - } - ], + "content": [{"image": "https://dashscope-result.oss.aliyuncs.com/test.png"}], }, } ] @@ -453,9 +410,7 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): }, } - with patch( - "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" - ) as mock_post: + with patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") as mock_post: mock_http_response = MagicMock() mock_http_response.json.return_value = mock_response_body mock_http_response.status_code = 200 @@ -472,15 +427,11 @@ def test_litellm_image_generation_dashscope_end_to_end(model: str): assert response is not None assert response.data is not None assert len(response.data) == 1 - assert ( - response.data[0].url == "https://dashscope-result.oss.aliyuncs.com/test.png" - ) + assert response.data[0].url == "https://dashscope-result.oss.aliyuncs.com/test.png" # Verify the HTTP call was made to the DashScope endpoint call_args = mock_post.call_args - called_url = ( - call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") - ) + called_url = call_args[0][0] if call_args[0] else call_args.kwargs.get("url", "") assert called_url == DEFAULT_API_BASE # Verify request body contains DashScope format diff --git a/tests/test_litellm/test_deepseek_model_metadata.py b/tests/test_litellm/test_deepseek_model_metadata.py index 264f5e65fc5..91ed54b826c 100644 --- a/tests/test_litellm/test_deepseek_model_metadata.py +++ b/tests/test_litellm/test_deepseek_model_metadata.py @@ -15,7 +15,6 @@ import os import litellm from litellm.utils import ( _supports_factory, - supports_response_schema, ) # --------------------------------------------------------------------------- @@ -59,18 +58,6 @@ class TestSupportsResponseSchemaDeepSeek: """All calling conventions for DeepSeek should return True for ``supports_response_schema``.""" - def test_provider_slash_model(self): - assert supports_response_schema(model="deepseek/deepseek-chat") is True - - def test_explicit_provider(self): - assert supports_response_schema(model="deepseek-chat", custom_llm_provider="deepseek") is True - - def test_reasoner_provider_slash_model(self): - assert supports_response_schema(model="deepseek/deepseek-reasoner") is True - - def test_reasoner_explicit_provider(self): - assert supports_response_schema(model="deepseek-reasoner", custom_llm_provider="deepseek") is True - # --------------------------------------------------------------------------- # Fallback-logic test – bare model entry used when prefixed is incomplete diff --git a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py index 8467cbd43b1..bc400bfa362 100644 --- a/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py +++ b/tests/test_litellm/test_mistral_zai_glm_5_2_model_metadata.py @@ -3,8 +3,6 @@ from pathlib import Path import pytest -import litellm -from litellm.utils import supports_prompt_caching, supports_reasoning REPO_ROOT = Path(__file__).parents[2] MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json" @@ -22,27 +20,6 @@ def _load(path): return json.load(f) -@pytest.fixture -def local_model_cost_map(monkeypatch): - """Force get_model_info to resolve against the in-repo cost map instead of the - remote one fetched at import time, which still carries the pre-merge pricing.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - litellm.get_model_info.cache_clear() - yield - litellm.get_model_info.cache_clear() - - -@pytest.mark.parametrize("model", GLM_5_2_MODELS) -def test_zai_glm_5_2_capabilities_are_visible_to_callers(local_model_cost_map, model): - """Mistral advertises reasoning and prompt caching on this model, so the helpers - every caller checks before sending a request must say so too.""" - assert supports_reasoning(model=model) is True - assert supports_prompt_caching(model=model) is True - - assert litellm.get_model_info(model=model) - - @pytest.mark.parametrize("model", GLM_5_2_MODELS) def test_backup_matches_main(model): """Ensure the bundled (backup) cost map stays in sync with the canonical file.""" diff --git a/tests/test_litellm/test_sambanova_model_metadata.py b/tests/test_litellm/test_sambanova_model_metadata.py deleted file mode 100644 index 20f34f9f3cc..00000000000 --- a/tests/test_litellm/test_sambanova_model_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -from pathlib import Path - -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - -def test_sambanova_minimax_m27_model_info(): - model = "sambanova/MiniMax-M2.7" - json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" - with open(json_path) as f: - model_cost = json.load(f) - - info = model_cost.get(model) - assert info is not None, f"{model} not found in model_prices_and_context_window.json" - assert info["litellm_provider"] == "sambanova" - assert info["mode"] == "chat" - assert info["input_cost_per_token"] > 0 - assert info["output_cost_per_token"] > 0 - assert info["supports_function_calling"] is True - assert info["supports_reasoning"] is True - assert info["supports_tool_choice"] is True - - routed_model, provider, _, _ = get_llm_provider(model=model) - assert routed_model == "MiniMax-M2.7" - assert provider == "sambanova" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 4618c156046..3d9534628cb 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -162,15 +162,6 @@ def test_prompt_tokens_details_cache_write_creation_stay_in_sync_on_assignment() assert details.cache_write_tokens == details.cache_creation_tokens == 375 -def test_get_model_info_surfaces_supported_endpoints(local_model_cost_map): - """supported_endpoints ships in the cost map and is declared on ModelInfoBase, - but the constructor never copied it, so get_model_info always returned None. - The realtime health check reads it to spot GA-only transcription models - (LIT-6240).""" - info = litellm.get_model_info(model="gpt-realtime-whisper", custom_llm_provider="azure") - assert info["supported_endpoints"] == ["/v1/realtime", "/v1/realtime/transcription_sessions"] - - def test_potential_model_names_keeps_provider_prefixed_candidate(): """A provider whose own model ids repeat the litellm provider name (Perplexity's Agent API serves `perplexity/glm-5.2`, mapped as `perplexity/perplexity/glm-5.2`) @@ -236,23 +227,6 @@ def test_check_provider_match_github_allows_upstream_provider_metadata(): ) -def test_supports_function_calling_github_openai_alias(): - assert litellm.utils.supports_function_calling(model="github/gpt-4o-mini") is True - assert litellm.utils.supports_function_calling(model="gpt-4o-mini", custom_llm_provider="github") is True - - -def test_supports_function_calling_github_anthropic_alias(): - assert litellm.utils.supports_function_calling(model="github/claude-3-7-sonnet-20250219") is True - - -def test_supports_function_calling_deepinfra_llama(): - """Test that deepinfra Llama models correctly report function calling support. - - Regression test for https://github.com/BerriAI/litellm/issues/22619 - """ - assert litellm.utils.supports_function_calling(model="deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo") is True - - def test_supports_function_calling_unknown_github_alias_returns_false(): assert litellm.utils.supports_function_calling(model="github/non-existent-model-for-capability-check") is False @@ -565,25 +539,6 @@ def test_all_model_configs(): ) == {"max_output_tokens": 10} -def test_anthropic_web_search_in_model_info(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - - supported_models = [ - "anthropic/claude-4-sonnet-20250514", - "anthropic/claude-sonnet-4-5-20250929", - ] - for model in supported_models: - from litellm.utils import get_model_info - - model_info = get_model_info(model) - assert model_info is not None - assert model_info["supports_web_search"] is True, f"Model {model} should support web search" - assert model_info["search_context_cost_per_query"] is not None, ( - f"Model {model} should have a search context cost per query" - ) - - def test_cohere_embedding_optional_params(): from litellm import get_optional_params_embeddings @@ -1129,13 +1084,6 @@ def test_get_model_info_bedrock_regional_inference_profile_pricing(local_model_c assert control["key"] == "au.anthropic.claude-opus-4-8" -def test_get_model_info_bedrock_double_provider_prefix_resolves(local_model_cost_map): - """A doubled bedrock/ prefix routes at runtime via strip_bedrock_routing_prefix, - so model info must resolve it to the same entry the request actually bills as.""" - info = litellm.get_model_info(model="bedrock/bedrock/us.anthropic.claude-sonnet-4-6") - assert info["key"] == "us.anthropic.claude-sonnet-4-6" - - def test_openai_models_in_model_info(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") @@ -1149,51 +1097,6 @@ def test_openai_models_in_model_info(monkeypatch): assert len(violated_models) == 0, f"The following models should support pdf input: {violated_models}" -def test_supports_tool_choice_simple_tests(): - """ - simple sanity checks - """ - assert litellm.utils.supports_tool_choice(model="gpt-4o") == True - assert litellm.utils.supports_tool_choice(model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0") == True - assert litellm.utils.supports_tool_choice(model="anthropic.claude-3-sonnet-20240229-v1:0") is True - - assert ( - litellm.utils.supports_tool_choice( - model="anthropic.claude-3-sonnet-20240229-v1:0", - custom_llm_provider="bedrock_converse", - ) - is True - ) - - assert litellm.utils.supports_tool_choice(model="perplexity/sonar") is False - - -@pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", - [ - "amazon.nova-lite-v1:0", - "amazon.nova-micro-v1:0", - "amazon.nova-pro-v1:0", - "apac.amazon.nova-lite-v1:0", - "apac.amazon.nova-micro-v1:0", - "apac.amazon.nova-pro-v1:0", - "bedrock/us-gov-east-1/amazon.nova-pro-v1:0", - "bedrock/us-gov-west-1/amazon.nova-lite-v1:0", - "bedrock/us-gov-west-1/amazon.nova-micro-v1:0", - "bedrock/us-gov-west-1/amazon.nova-pro-v1:0", - "eu.amazon.nova-lite-v1:0", - "eu.amazon.nova-micro-v1:0", - "eu.amazon.nova-pro-v1:0", - "us.amazon.nova-lite-v1:0", - "us.amazon.nova-micro-v1:0", - "us.amazon.nova-pro-v1:0", - ], -) -def test_amazon_nova_v1_understanding_models_support_tool_choice(model: str) -> None: - assert litellm.utils.supports_tool_choice(model=model) is True - - def test_check_provider_match(): """ Test the _check_provider_match function for various provider scenarios @@ -1303,42 +1206,6 @@ for commitment in BEDROCK_COMMITMENTS: print("block_list", block_list) -def test_supports_computer_use_utility(monkeypatch): - """ - Tests the litellm.utils.supports_computer_use utility function. - """ - from litellm.utils import supports_computer_use - - # Ensure LITELLM_LOCAL_MODEL_COST_MAP is set for consistent test behavior, - # as supports_computer_use relies on get_model_info. - # This also requires litellm.model_cost to be populated. - original_env_var = os.getenv("LITELLM_LOCAL_MODEL_COST_MAP") - original_model_cost = getattr(litellm, "model_cost", None) - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") # Load with local/backup - - try: - # Test a model known to support computer_use from backup JSON - supports_cu_anthropic = supports_computer_use(model="anthropic/claude-4-sonnet-20250514") - assert supports_cu_anthropic is True - - # Test a model known not to have the flag or set to false (defaults to False via get_model_info) - supports_cu_gpt = supports_computer_use(model="gpt-3.5-turbo") - assert supports_cu_gpt is False - finally: - # Restore original environment and model_cost to avoid side effects - if original_env_var is None: - del os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] - else: - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", original_env_var) - - if original_model_cost is not None: - litellm.model_cost = original_model_cost - elif hasattr(litellm, "model_cost"): - delattr(litellm, "model_cost") - - @pytest.mark.parametrize( "model, custom_llm_provider", [ @@ -1658,33 +1525,6 @@ class TestProxyFunctionCalling: # For now, we expect False (current behavior), but document the limitation assert proxy_result is False, f"Current limitation: {proxy_model_with_hints} returns False without inference" - @pytest.mark.parametrize( - "proxy_model,expected_result", - [ - # Test specific proxy models that should support function calling - ("litellm_proxy/gpt-3.5-turbo", True), - ("litellm_proxy/gpt-4", True), - ("litellm_proxy/gpt-4o", True), - ("litellm_proxy/claude-sonnet-4-6", True), - ("litellm_proxy/gemini/gemini-2.5-pro", True), - # Test proxy models that should not support function calling - ("litellm_proxy/command-nightly", False), - ("litellm_proxy/anthropic.claude-instant-v1", False), - ], - ) - def test_proxy_only_function_calling_support(self, proxy_model, expected_result): - """ - Test proxy models independently to ensure they report correct function calling support. - - This test focuses on proxy models without comparing to direct models, - useful for cases where we only care about the proxy behavior. - """ - try: - result = supports_function_calling(model=proxy_model) - assert result == expected_result, f"Proxy model {proxy_model} returned {result}, expected {expected_result}" - except Exception as e: - pytest.fail(f"Error testing proxy model {proxy_model}: {e}") - def test_litellm_utils_supports_function_calling_import(self): """Test that supports_function_calling can be imported from litellm.utils.""" try: @@ -1704,29 +1544,6 @@ class TestProxyFunctionCalling: except Exception as e: pytest.fail(f"Failed to access litellm.supports_function_calling: {e}") - @pytest.mark.parametrize( - "model_name", - [ - "litellm_proxy/gpt-3.5-turbo", - "litellm_proxy/gpt-4", - "litellm_proxy/claude-sonnet-4-6", - "litellm_proxy/gemini/gemini-2.5-pro", - ], - ) - def test_proxy_model_with_custom_llm_provider_none(self, model_name): - """ - Test proxy models with custom_llm_provider=None parameter. - - This tests the supports_function_calling function with the custom_llm_provider - parameter explicitly set to None, which is a common usage pattern. - """ - try: - result = supports_function_calling(model=model_name, custom_llm_provider=None) - # All the models in this test should support function calling - assert result is True, f"Model {model_name} should support function calling but returned {result}" - except Exception as e: - pytest.fail(f"Error testing {model_name} with custom_llm_provider=None: {e}") - def test_edge_cases_and_malformed_proxy_models(self): """Test edge cases and malformed proxy model names.""" test_cases = [ @@ -1963,84 +1780,6 @@ class TestProxyFunctionCalling: f"(without config context). Description: {description}" ) - def test_real_world_proxy_config_documentation(self): - """ - Document how real-world proxy configurations would handle model mappings. - - This test provides documentation on how the proxy server configuration - would typically map custom model names to underlying models. - """ - print(""" - - REAL-WORLD PROXY SERVER CONFIGURATION EXAMPLE: - =============================================== - - In a proxy_server_config.yaml file, you would define: - - model_list: - - model_name: bedrock-claude-3-haiku - litellm_params: - model: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_region_name: us-east-1 - - - model_name: bedrock-claude-3-sonnet - litellm_params: - model: bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_region_name: us-east-1 - - - model_name: prod-claude-haiku - litellm_params: - model: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0 - aws_access_key_id: os.environ/PROD_AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/PROD_AWS_SECRET_ACCESS_KEY - aws_region_name: us-west-2 - - - FUNCTION CALLING WITH PROXY SERVER: - =================================== - - When using the proxy server with this configuration: - - 1. Client calls: supports_function_calling("bedrock-claude-3-haiku") - 2. Proxy server resolves to: bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0 - 3. LiteLLM evaluates the underlying model's capabilities - 4. Returns: True (because Claude 3 Haiku supports function calling) - - Without the proxy server configuration context, LiteLLM cannot resolve - the custom model name and returns False. - - - BEDROCK CONVERSE API BENEFITS: - ============================== - - The Bedrock Converse API provides: - - Standardized function calling interface across providers - - Better tool use capabilities compared to legacy APIs - - Consistent request/response format - - Enhanced streaming support for function calls - - """) - - # Verify that direct underlying models work as expected - bedrock_models = [ - "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", - "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0", - ] - - for model in bedrock_models: - try: - result = supports_function_calling(model) - print(f"Direct test - {model}: {result}") - # Claude 3 models should support function calling - assert result is True, f"Claude 3 model should support function calling: {model}" - except Exception as e: - print(f"Could not test {model}: {e}") - def test_register_model_with_scientific_notation(): """ @@ -3637,60 +3376,6 @@ _FIREWORKS_ROUTER_SHORT_FORMS = [ ] -def _assert_fireworks_entry( - model_cost, - model_path, - expected_max_input, - expected_max_output, - expected_vision, - expected_reasoning, -): - info = model_cost.get(f"fireworks_ai/{model_path}") - assert info is not None, f"fireworks_ai/{model_path} missing from model cost map" - assert info["litellm_provider"] == "fireworks_ai" - assert info["mode"] == "chat" - assert info["input_cost_per_token"] > 0 - assert info["output_cost_per_token"] > 0 - assert "cache_read_input_token_cost" in info - assert info["supports_function_calling"] is True - assert info["supports_tool_choice"] is True - assert info["supports_reasoning"] is expected_reasoning - assert info["supports_response_schema"] is True - assert info["supports_vision"] is expected_vision - - -@pytest.fixture -def fireworks_short_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: - monkeypatch.setattr( - litellm, - "model_cost", - { - "fireworks_ai/accounts/fireworks/models/glm-5p3": { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "litellm_provider": "fireworks_ai", - "mode": "chat", - "max_tokens": 100, - }, - "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast": { - "input_cost_per_token": 2.1e-6, - "output_cost_per_token": 6.6e-6, - "litellm_provider": "fireworks_ai", - "mode": "chat", - }, - "fireworks_ai/nomic-ai/nomic-embed-text-v1.5": { - "input_cost_per_token": 8e-9, - "output_cost_per_token": 0.0, - "litellm_provider": "fireworks_ai", - "mode": "embedding", - }, - }, - ) - litellm.get_model_info.cache_clear() - yield - litellm.get_model_info.cache_clear() - - class TestBedrockBaseModelLabelKeepsTools: """Regression for #29618: a Bedrock deployment whose ``base_model`` is a friendly label must not silently drop ``tools``/``tool_choice`` under ``drop_params``.""" @@ -3985,21 +3670,6 @@ def test_get_prompt_cache_min_tokens_resolves_per_model( assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens -def test_get_prompt_cache_min_tokens_uniform_for_fable_5_across_platforms(local_model_cost_map: None) -> None: - """Anthropic removed the Amazon Bedrock override for Claude Fable 5, so its 512-token minimum - now applies on every platform. The Bedrock entries carried the old 1024 and the re-export - entries carried nothing, so the router judged 512-1023-token prefixes uncacheable and skipped - prompt-cache-affinity routing for prompts the provider demonstrably caches (issue #35011).""" - wrong: Final = { - model: get_prompt_cache_min_tokens(model=model) - for model, info in litellm.model_cost.items() - if "fable-5" in model - and info.get("supports_prompt_caching") - and get_prompt_cache_min_tokens(model=model) != 512 - } - assert not wrong, f"every Claude Fable 5 entry must carry prompt_cache_min_tokens 512: {wrong}" - - ANTHROPIC_REEXPORT_CACHE_MIN: Final = { "azure_ai/claude-fable-5": 512, "azure_ai/claude-haiku-4-5": 4096, @@ -4048,21 +3718,6 @@ ANTHROPIC_REEXPORT_CACHE_MIN: Final = { } -def test_anthropic_reexport_entries_carry_explicit_prompt_cache_min_tokens(local_model_cost_map: None) -> None: - """Regression for issue #35011: these re-export entries carried no prompt_cache_min_tokens, so - they silently inherited the 1024 default. That skipped cache-affinity routing for Fable 5's - 512-1023-token prefixes and reported 1024-4095-token prompts as cacheable on the 2048/4096 - models. The entry must be explicit so a default change can never re-break them, which is why - this asserts the cost-map value itself and not just the resolver's answer.""" - wrong: Final = { - model: (litellm.model_cost[model].get("prompt_cache_min_tokens"), get_prompt_cache_min_tokens(model=model)) - for model, expected in ANTHROPIC_REEXPORT_CACHE_MIN.items() - if litellm.model_cost[model].get("prompt_cache_min_tokens") != expected - or get_prompt_cache_min_tokens(model=model) != expected - } - assert not wrong, f"(cost-map value, resolved value) diverge from Anthropic's published minimums: {wrong}" - - GEMINI_4096_CACHE_MIN_MODELS: Final = tuple( prefix + base for base in ( @@ -5981,82 +5636,6 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th assert snapshot["api_base"] -def test_fireworks_models_in_backup_cost_map(): - import json - from pathlib import Path - - json_path = Path(__file__).parents[2] / "litellm" / "model_prices_and_context_window_backup.json" - with open(json_path) as f: - model_cost = json.load(f) - - for entry in _FIREWORKS_MODELS: - _assert_fireworks_entry(model_cost, *entry) - - for short in _FIREWORKS_SHORT_FORMS: - long_key = f"fireworks_ai/accounts/fireworks/models/{short}" - short_key = f"fireworks_ai/{short}" - assert model_cost.get(short_key) == model_cost.get(long_key), ( - f"short-form {short_key} does not match long-form {long_key}" - ) - - for short in _FIREWORKS_ROUTER_SHORT_FORMS: - long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" - short_key = f"fireworks_ai/{short}" - assert model_cost.get(short_key) == model_cost.get(long_key), ( - f"short-form {short_key} does not match long-form {long_key}" - ) - - -def test_fireworks_models_in_cost_map(): - import json - from pathlib import Path - - json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" - with open(json_path) as f: - model_cost = json.load(f) - - for entry in _FIREWORKS_MODELS: - _assert_fireworks_entry(model_cost, *entry) - - for short in _FIREWORKS_SHORT_FORMS: - long_key = f"fireworks_ai/accounts/fireworks/models/{short}" - short_key = f"fireworks_ai/{short}" - assert model_cost.get(short_key) == model_cost.get(long_key), ( - f"short-form {short_key} does not match long-form {long_key}" - ) - - for short in _FIREWORKS_ROUTER_SHORT_FORMS: - long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" - short_key = f"fireworks_ai/{short}" - assert model_cost.get(short_key) == model_cost.get(long_key), ( - f"short-form {short_key} does not match long-form {long_key}" - ) - - -def test_fireworks_short_model_names_resolve_to_long_cost_map_keys(fireworks_short_model_cost_map: None) -> None: - model_info = litellm.get_model_info("fireworks_ai/glm-5p3") - assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3" - - model_info = litellm.get_model_info("glm-5p3", custom_llm_provider="fireworks_ai") - assert model_info["key"] == "fireworks_ai/accounts/fireworks/models/glm-5p3" - - model_info = litellm.get_model_info("fireworks_ai/glm-5p3-fast") - assert model_info["key"] == "fireworks_ai/accounts/fireworks/routers/glm-5p3-fast" - - model_info = litellm.get_model_info("fireworks_ai/nomic-ai/nomic-embed-text-v1.5") - assert model_info["key"] == "fireworks_ai/nomic-ai/nomic-embed-text-v1.5" - - with pytest.raises(Exception, match="isn't mapped"): - litellm.get_model_info("fireworks_ai/does-not-exist") - - -def test_get_model_info_bedrock_regional_profile_without_entry_falls_back_to_base(local_model_cost_map): - """A regional profile with no dedicated cost-map entry must still resolve to its - region-stripped base entry.""" - info = litellm.get_model_info(model="bedrock/apac.anthropic.claude-opus-4-8") - assert info["key"] == "anthropic.claude-opus-4-8" - - def test_get_model_info_gemini(monkeypatch): """ Tests if ALL gemini models have 'tpm' and 'rpm' in the model info @@ -6077,155 +5656,3 @@ def test_get_model_info_gemini(monkeypatch): ): assert info.get("tpm") is not None, f"{model} does not have tpm" assert info.get("rpm") is not None, f"{model} does not have rpm" - - -def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map): - """Perplexity's Agent API third-party models are keyed `perplexity/perplexity/` - because Perplexity's own id already starts with `perplexity/`. Callers run - `get_llm_provider` first, which hands `_get_potential_model_names` model - `perplexity/glm-5.2` with provider `perplexity`, and every candidate but the - provider-prefixed one strips that second `perplexity/` off. Regression: the - entries were unreachable from `supports_reasoning` and from the cost calculator's - per-token fallback, so a mapped model reported no reasoning support and raised - "This model isn't mapped yet" on the only path where its rates are ever used.""" - for model, reasoning in ( - ("perplexity/perplexity/glm-5.2", True), - ("perplexity/perplexity/kimi-k3", True), - ("perplexity/perplexity/deepseek-v4-flash-0731", True), - ("perplexity/perplexity/kimi-k2.7-code", False), - ("perplexity/perplexity/nemotron-3.5-lightning-30b-a3b", True), - ("perplexity/perplexity/nemotron-3-ultra-550b-a55b", True), - ): - assert litellm.supports_reasoning(model=model) is reasoning, model - - via_provider = litellm.get_model_info(model="perplexity/glm-5.2", custom_llm_provider="perplexity") - assert via_provider["key"] == "perplexity/perplexity/glm-5.2" - assert via_provider["mode"] == "responses" - - lightning = litellm.get_model_info( - model="perplexity/nemotron-3.5-lightning-30b-a3b", custom_llm_provider="perplexity" - ) - assert lightning["key"] == "perplexity/perplexity/nemotron-3.5-lightning-30b-a3b" - assert lightning["mode"] == "responses" - - ultra = litellm.get_model_info(model="perplexity/perplexity/nemotron-3-ultra-550b-a55b") - assert ultra["key"] == "perplexity/perplexity/nemotron-3-ultra-550b-a55b" - - -def test_get_model_info_shows_supports_computer_use(monkeypatch): - """ - Tests if 'supports_computer_use' is correctly retrieved by get_model_info. - We'll use 'claude-4-sonnet-20250514' as it's configured - in the backup JSON to have supports_computer_use: True. - """ - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - # Ensure litellm.model_cost is loaded, relying on the backup mechanism if primary fails - # as per previous debugging. - litellm.model_cost = litellm.get_model_cost_map(url="") - - # This model should have 'supports_computer_use': True in the backup JSON - model_known_to_support_computer_use = "claude-4-sonnet-20250514" - info = litellm.get_model_info(model_known_to_support_computer_use) - - # After the fix in utils.py, this should now be present and True - assert info.get("supports_computer_use") is True - - -def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map): - """supports_adaptive_thinking must flow through get_model_info like every other - capability flag: both from an explicit cost-map entry and from a - fallback-generalization rule for an unmapped model. Regression: the field shipped - in the JSON but was never declared on ModelInfo nor copied during construction, so - get_model_info (and _supports_factory) silently dropped it for any provider-prefixed - or unmapped name.""" - explicit = litellm.get_model_info(model="claude-opus-4-8") - assert explicit["supports_adaptive_thinking"] is True - - generalized = litellm.get_model_info(model="claude-opus-4-9", custom_llm_provider="anthropic") - assert generalized["supports_adaptive_thinking"] is True - - -def test_get_model_info_surfaces_supports_parallel_function_calling(local_model_cost_map): - """A registry entry's supports_parallel_function_calling must read back through get_model_info - and litellm.supports_parallel_function_calling. Regression: the key was never copied into - ModelInfo, so provider-prefixed entries read None / False even when the map said True, and an - explicit False was indistinguishable from unset.""" - declared_true = litellm.get_model_info(model="together_ai/zai-org/GLM-5.3-Flash") - assert declared_true["supports_parallel_function_calling"] is True - assert litellm.supports_parallel_function_calling(model="together_ai/zai-org/GLM-5.3-Flash") is True - - -def test_model_info_for_fireworks_short_form_models(): - """ - Test that fireworks_ai short-form model entries (fireworks_ai/) - are correctly configured in model_prices_and_context_window.json. - - These entries enable cost attribution for models called via short-form - names (e.g., fireworks_ai/glm-4p7 instead of - fireworks_ai/accounts/fireworks/models/glm-4p7). - """ - import json - from pathlib import Path - - json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" - with open(json_path) as f: - model_cost = json.load(f) - - # glm-4p7: short-form and long-form - for key in [ - "fireworks_ai/glm-4p7", - "fireworks_ai/accounts/fireworks/models/glm-4p7", - ]: - info = model_cost.get(key) - assert info is not None, f"{key} not found in model_prices_and_context_window.json" - assert info["litellm_provider"] == "fireworks_ai" - assert info["mode"] == "chat" - assert info["supports_reasoning"] is True - - # minimax-m2p1: short-form and long-form - for key in [ - "fireworks_ai/minimax-m2p1", - "fireworks_ai/accounts/fireworks/models/minimax-m2p1", - ]: - info = model_cost.get(key) - assert info is not None, f"{key} not found in model_prices_and_context_window.json" - assert info["litellm_provider"] == "fireworks_ai" - assert info["mode"] == "chat" - - # kimi-k2p5: short-form only (long-form already existed) - info = model_cost.get("fireworks_ai/kimi-k2p5") - assert info is not None, "fireworks_ai/kimi-k2p5 not found in model_prices_and_context_window.json" - assert info["litellm_provider"] == "fireworks_ai" - assert info["mode"] == "chat" - - -def test_model_info_for_vertex_ai_deepseek_model(): - model_info = litellm.get_model_info(model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas") - assert model_info is not None - assert model_info["litellm_provider"] == "vertex_ai-deepseek_models" - assert model_info["mode"] == "chat" - - assert model_info["input_cost_per_token"] is not None - assert model_info["output_cost_per_token"] is not None - - -def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map): - """The provider-prefixed candidate is tried last, after every candidate that - already existed, so no model that resolves today can change answer. `perplexity/sonar` - is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar` - are cost-map keys, and the shorter one must keep winning.""" - sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity") - assert sonar["key"] == "perplexity/sonar" - assert sonar["mode"] == "chat" - - still_sonar = litellm.get_model_info(model="perplexity/sonar", custom_llm_provider="perplexity") - assert still_sonar["key"] == "perplexity/sonar" - assert still_sonar["mode"] == "chat" - - for model, provider, expected_key in ( - ("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), - ("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), - ("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"), - ("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), - ): - assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key diff --git a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py index 72e98711f0c..4a9a429801c 100644 --- a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py +++ b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py @@ -3,8 +3,6 @@ from typing import Final import pytest import litellm -from litellm import get_model_info -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching MODEL: Final = "vertex_ai/xai/grok-4.6" @@ -24,15 +22,3 @@ def test_grok_models_with_cache_read_price_advertise_prompt_caching() -> None: assert missing_flag == (), ( f"grok models with cache_read_input_token_cost fail supports_prompt_caching: {missing_flag}" ) - - -@pytest.mark.usefixtures("local_model_cost_map") -def test_vertex_ai_grok_4_6_supports_prompt_caching_via_get_model_info() -> None: - routed_model, provider, _, _ = get_llm_provider(model=MODEL) - assert (routed_model, provider) == ("xai/grok-4.6", "vertex_ai") - - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "vertex_ai" - assert info.get("supports_prompt_caching") is True - - assert supports_prompt_caching(model=MODEL) is True