From 68840e02456d7ad4f04a97ee151637349b5a8718 Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Sun, 17 May 2026 20:55:41 +0545 Subject: [PATCH] fix(bedrock): handle Application Inference Profile ARNs in passthrough logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the model is an Application Inference Profile ARN (e.g. arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l), get_bedrock_invoke_provider returns None because the ARN contains no provider name. The logging path in handle_logging_collected_chunks then raised a ValueError, crashing the background task and preventing success_callback (s3_v2, langfuse, etc.) from ever firing — even though the actual LLM request succeeded. Fix: fall back to extracting the provider from the original LiteLLM model name stored in litellm_logging_obj.model_call_details (e.g. "global.anthropic.claude-opus-4-7"). If that also yields nothing, log a warning and return None gracefully instead of raising, so the logging task does not crash. Fixes #28105 --- .../bedrock/passthrough/transformation.py | 20 +++++- .../test_bedrock_completion.py | 7 ++ ...test_bedrock_passthrough_transformation.py | 71 ++++++++++++++++++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index d0a3c37ffb3..4eba0960232 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, Optional, cast from httpx import Response +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig @@ -200,9 +201,24 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD all_translated_chunks: Final = [] if "invoke" in endpoint: - invoke_provider: Final = AmazonInvokeConfig.get_bedrock_invoke_provider(model) + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(model) if invoke_provider is None: - raise ValueError(f"Invalid invoke provider: {invoke_provider}, for model: {model}") + # Application Inference Profile ARNs don't encode provider info in the ARN + # itself. Try to derive the provider from the original LiteLLM model name + # (e.g. "global.anthropic.claude-opus-4-7") stored in logging context. + fallback_model: Final = litellm_logging_obj.model_call_details.get( + "litellm_params", {} + ).get("model", "") + if fallback_model: + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider( + fallback_model + ) + if invoke_provider is None: + verbose_logger.warning( + f"Could not determine Bedrock invoke provider for model: {model!r}. " + "Skipping streaming response logging for this passthrough request." + ) + return None obj = get_bedrock_event_stream_decoder( invoke_provider=invoke_provider, model=model, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 550e82fb5bb..d08dcb145af 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2763,6 +2763,13 @@ def test_bedrock_invoke_provider(): ) == "nova" ) + # Application Inference Profile ARNs have no provider info in the ARN + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + ) + is None + ) def test_bedrock_description_param(): diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index b005d77ac8b..3752448b363 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig @@ -500,3 +500,72 @@ def test_bedrock_passthrough_model_id_without_arn(): f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" ) assert url_str == expected_url + + +def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracefully(): + """ + Application Inference Profile ARNs don't embed provider info, so + get_bedrock_invoke_provider returns None. The logging path must not raise; + it should fall back to the original litellm model name stored in + model_call_details, and if that also yields nothing it must return None + silently rather than crashing the background logging task. + + Regression test for: https://github.com/BerriAI/litellm/issues/28105 + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + # Simulate the litellm logging object that carries model_call_details. + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = { + "litellm_params": {"model": "global.anthropic.claude-opus-4-7"}, + } + + # Intercept the decoder so we don't need real botocore event-stream bytes. + # The import is done inside handle_logging_collected_chunks, so patch the source module. + with patch( + "litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder" + ) as mock_decoder: + mock_chunk_obj = MagicMock() + mock_chunk_obj._chunk_parser.return_value = ( + {} + ) # not a valid GenericStreamingChunk + mock_decoder.return_value = mock_chunk_obj + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + # With no chunks, the result is None (not an exception). + assert result is None + + +def test_handle_logging_collected_chunks_inference_profile_arn_no_fallback_returns_none(): + """ + When neither the ARN nor the original model name yields a known provider, + handle_logging_collected_chunks must return None instead of raising. + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + mock_logging_obj = MagicMock() + # Empty litellm_params — no usable fallback model name. + mock_logging_obj.model_call_details = {"litellm_params": {}} + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + assert result is None