diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index fb8bc4f191f..13289ee890e 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, Optional, cast import httpx from httpx import Response +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig @@ -209,9 +210,24 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD all_translated_chunks: Final = [] if "invoke" in endpoint: - invoke_provider: Final = AmazonInvokeConfig.get_bedrock_invoke_provider(model) + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(model) if invoke_provider is None: - raise ValueError(f"Invalid invoke provider: {invoke_provider}, for model: {model}") + # Application Inference Profile ARNs don't encode provider info in the ARN + # itself. Try to derive the provider from the original LiteLLM model name + # (e.g. "global.anthropic.claude-opus-4-7") stored in logging context. + fallback_model: Final = litellm_logging_obj.model_call_details.get( + "litellm_params", + {}, # mutable-ok: read-only empty fallback + ).get("model", "") + if fallback_model: + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(fallback_model) + if invoke_provider is None: + verbose_logger.warning( + "Could not determine Bedrock invoke provider for model: %r. " + "Skipping streaming response logging for this passthrough request.", + model, + ) + return None obj = get_bedrock_event_stream_decoder( invoke_provider=invoke_provider, model=model, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 550e82fb5bb..d08dcb145af 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2763,6 +2763,13 @@ def test_bedrock_invoke_provider(): ) == "nova" ) + # Application Inference Profile ARNs have no provider info in the ARN + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + ) + is None + ) def test_bedrock_description_param(): diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index b005d77ac8b..b7833c86c43 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,5 +1,4 @@ -from unittest.mock import patch - +from unittest.mock import MagicMock, patch from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig @@ -500,3 +499,77 @@ def test_bedrock_passthrough_model_id_without_arn(): f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" ) assert url_str == expected_url + + +def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracefully(): + """ + Application Inference Profile ARNs don't embed provider info, so + get_bedrock_invoke_provider returns None. The logging path must not raise; + it should fall back to the original litellm model name stored in + model_call_details, and if that also yields nothing it must return None + silently rather than crashing the background logging task. + + Regression test for: https://github.com/BerriAI/litellm/issues/28105 + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + # Simulate the litellm logging object that carries model_call_details. + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = { + "litellm_params": {"model": "global.anthropic.claude-opus-4-7"}, + } + + # Intercept the decoder so we don't need real botocore event-stream bytes. + # The import is done inside handle_logging_collected_chunks, so patch the source module. + with patch( # test-quality-ok: the behavior under test is which invoke_provider this factory receives + "litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder" + ) as mock_decoder: + mock_chunk_obj = MagicMock() + mock_chunk_obj._chunk_parser.return_value = {} # not a valid GenericStreamingChunk + mock_decoder.return_value = mock_chunk_obj + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + # The decoder must have been called — this proves the fallback resolved + # "global.anthropic.claude-opus-4-7" to "anthropic" and did not hit the + # early-exit None return that fires when provider resolution fails entirely. + mock_decoder.assert_called_once() + call_kwargs = mock_decoder.call_args.kwargs + assert call_kwargs["invoke_provider"] == "anthropic" + + # With no chunks the assembled response is None, but no exception was raised. + assert result is None + + +def test_handle_logging_collected_chunks_inference_profile_arn_no_fallback_returns_none(): + """ + When neither the ARN nor the original model name yields a known provider, + handle_logging_collected_chunks must return None instead of raising. + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + mock_logging_obj = MagicMock() + # Empty litellm_params — no usable fallback model name. + mock_logging_obj.model_call_details = {"litellm_params": {}} + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + assert result is None