From 68840e02456d7ad4f04a97ee151637349b5a8718 Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Sun, 17 May 2026 20:55:41 +0545 Subject: [PATCH 1/5] fix(bedrock): handle Application Inference Profile ARNs in passthrough logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the model is an Application Inference Profile ARN (e.g. arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l), get_bedrock_invoke_provider returns None because the ARN contains no provider name. The logging path in handle_logging_collected_chunks then raised a ValueError, crashing the background task and preventing success_callback (s3_v2, langfuse, etc.) from ever firing — even though the actual LLM request succeeded. Fix: fall back to extracting the provider from the original LiteLLM model name stored in litellm_logging_obj.model_call_details (e.g. "global.anthropic.claude-opus-4-7"). If that also yields nothing, log a warning and return None gracefully instead of raising, so the logging task does not crash. Fixes #28105 --- .../bedrock/passthrough/transformation.py | 20 +++++- .../test_bedrock_completion.py | 7 ++ ...test_bedrock_passthrough_transformation.py | 71 ++++++++++++++++++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index d0a3c37ffb3..4eba0960232 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Final, Optional, cast from httpx import Response +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig @@ -200,9 +201,24 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD all_translated_chunks: Final = [] if "invoke" in endpoint: - invoke_provider: Final = AmazonInvokeConfig.get_bedrock_invoke_provider(model) + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(model) if invoke_provider is None: - raise ValueError(f"Invalid invoke provider: {invoke_provider}, for model: {model}") + # Application Inference Profile ARNs don't encode provider info in the ARN + # itself. Try to derive the provider from the original LiteLLM model name + # (e.g. "global.anthropic.claude-opus-4-7") stored in logging context. + fallback_model: Final = litellm_logging_obj.model_call_details.get( + "litellm_params", {} + ).get("model", "") + if fallback_model: + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider( + fallback_model + ) + if invoke_provider is None: + verbose_logger.warning( + f"Could not determine Bedrock invoke provider for model: {model!r}. " + "Skipping streaming response logging for this passthrough request." + ) + return None obj = get_bedrock_event_stream_decoder( invoke_provider=invoke_provider, model=model, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 550e82fb5bb..d08dcb145af 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2763,6 +2763,13 @@ def test_bedrock_invoke_provider(): ) == "nova" ) + # Application Inference Profile ARNs have no provider info in the ARN + assert ( + litellm.AmazonInvokeConfig().get_bedrock_invoke_provider( + "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + ) + is None + ) def test_bedrock_description_param(): diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index b005d77ac8b..3752448b363 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig @@ -500,3 +500,72 @@ def test_bedrock_passthrough_model_id_without_arn(): f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" ) assert url_str == expected_url + + +def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracefully(): + """ + Application Inference Profile ARNs don't embed provider info, so + get_bedrock_invoke_provider returns None. The logging path must not raise; + it should fall back to the original litellm model name stored in + model_call_details, and if that also yields nothing it must return None + silently rather than crashing the background logging task. + + Regression test for: https://github.com/BerriAI/litellm/issues/28105 + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + # Simulate the litellm logging object that carries model_call_details. + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = { + "litellm_params": {"model": "global.anthropic.claude-opus-4-7"}, + } + + # Intercept the decoder so we don't need real botocore event-stream bytes. + # The import is done inside handle_logging_collected_chunks, so patch the source module. + with patch( + "litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder" + ) as mock_decoder: + mock_chunk_obj = MagicMock() + mock_chunk_obj._chunk_parser.return_value = ( + {} + ) # not a valid GenericStreamingChunk + mock_decoder.return_value = mock_chunk_obj + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + # With no chunks, the result is None (not an exception). + assert result is None + + +def test_handle_logging_collected_chunks_inference_profile_arn_no_fallback_returns_none(): + """ + When neither the ARN nor the original model name yields a known provider, + handle_logging_collected_chunks must return None instead of raising. + """ + config = BedrockPassthroughConfig() + + profile_arn = "arn:aws:bedrock:ap-northeast-2:123456789012:application-inference-profile/czu0ezc2tq2l" + + mock_logging_obj = MagicMock() + # Empty litellm_params — no usable fallback model name. + mock_logging_obj.model_call_details = {"litellm_params": {}} + + # Must not raise ValueError. + result = config.handle_logging_collected_chunks( + all_chunks=[], + litellm_logging_obj=mock_logging_obj, + model=profile_arn, + custom_llm_provider="bedrock", + endpoint="/model/some-model/invoke-with-response-stream", + ) + + assert result is None From 117254f67023fb11a14eafeacd28fe5674c083a1 Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Sun, 17 May 2026 21:03:16 +0545 Subject: [PATCH 2/5] test(bedrock): assert decoder is called to prove fallback resolved provider Without this assertion the fallback test passed whether the provider was resolved or not, because all_chunks=[] always returns None regardless. Now the test verifies the decoder was actually called with invoke_provider='anthropic', confirming the fallback from the ARN to the original model name worked. --- .../test_bedrock_passthrough_transformation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 3752448b363..d17d61a25ed 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -542,7 +542,14 @@ def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracef endpoint="/model/some-model/invoke-with-response-stream", ) - # With no chunks, the result is None (not an exception). + # The decoder must have been called — this proves the fallback resolved + # "global.anthropic.claude-opus-4-7" to "anthropic" and did not hit the + # early-exit None return that fires when provider resolution fails entirely. + mock_decoder.assert_called_once() + call_kwargs = mock_decoder.call_args.kwargs + assert call_kwargs["invoke_provider"] == "anthropic" + + # With no chunks the assembled response is None, but no exception was raised. assert result is None From a978a7963bca310eb38fe9be96aacccb8f065e03 Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Thu, 27 Aug 2026 10:34:38 +0545 Subject: [PATCH 3/5] style(bedrock): apply ruff format after rebase onto upstream/main Rebasing fix/bedrock-passthrough-inference-profile-logging onto upstream/main surfaced two merge conflicts (transformation.py and the passthrough test file) against unrelated refactors that landed since this branch diverged. This applies diff-scoped ruff format to the lines touched by the conflict resolution, matching CI's format check. --- litellm/llms/bedrock/passthrough/transformation.py | 10 ++++------ .../test_bedrock_passthrough_transformation.py | 9 ++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 4eba0960232..e0f69f29e8d 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -206,13 +206,11 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD # Application Inference Profile ARNs don't encode provider info in the ARN # itself. Try to derive the provider from the original LiteLLM model name # (e.g. "global.anthropic.claude-opus-4-7") stored in logging context. - fallback_model: Final = litellm_logging_obj.model_call_details.get( - "litellm_params", {} - ).get("model", "") + fallback_model: Final = litellm_logging_obj.model_call_details.get("litellm_params", {}).get( + "model", "" + ) if fallback_model: - invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider( - fallback_model - ) + invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(fallback_model) if invoke_provider is None: verbose_logger.warning( f"Could not determine Bedrock invoke provider for model: {model!r}. " diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index d17d61a25ed..d564bff0032 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,6 +1,5 @@ from unittest.mock import MagicMock, patch - from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig @@ -524,13 +523,9 @@ def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracef # Intercept the decoder so we don't need real botocore event-stream bytes. # The import is done inside handle_logging_collected_chunks, so patch the source module. - with patch( - "litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder" - ) as mock_decoder: + with patch("litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder") as mock_decoder: mock_chunk_obj = MagicMock() - mock_chunk_obj._chunk_parser.return_value = ( - {} - ) # not a valid GenericStreamingChunk + mock_chunk_obj._chunk_parser.return_value = {} # not a valid GenericStreamingChunk mock_decoder.return_value = mock_chunk_obj # Must not raise ValueError. From 6abbb79935a7db720269dbdfab62c14e8d2e906e Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Thu, 27 Aug 2026 11:31:00 +0545 Subject: [PATCH 4/5] fix(bedrock): satisfy CI lint gates on the passthrough fallback Two checks failed after the rebase: - type-discipline budget (LIT002): the {} default in model_call_details.get("litellm_params", {}) is a mutable-literal construction; mark it mutable-ok like the identical pattern used elsewhere in the codebase. - lazy-logging test: the warning built its message eagerly via an f-string; switch to %-style args so the interpolation is skipped when the warning is filtered out. --- litellm/llms/bedrock/passthrough/transformation.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index e0f69f29e8d..9fe50db66a7 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -206,15 +206,17 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD # Application Inference Profile ARNs don't encode provider info in the ARN # itself. Try to derive the provider from the original LiteLLM model name # (e.g. "global.anthropic.claude-opus-4-7") stored in logging context. - fallback_model: Final = litellm_logging_obj.model_call_details.get("litellm_params", {}).get( - "model", "" - ) + fallback_model: Final = litellm_logging_obj.model_call_details.get( + "litellm_params", + {}, # mutable-ok: read-only empty fallback + ).get("model", "") if fallback_model: invoke_provider = AmazonInvokeConfig.get_bedrock_invoke_provider(fallback_model) if invoke_provider is None: verbose_logger.warning( - f"Could not determine Bedrock invoke provider for model: {model!r}. " - "Skipping streaming response logging for this passthrough request." + "Could not determine Bedrock invoke provider for model: %r. " + "Skipping streaming response logging for this passthrough request.", + model, ) return None obj = get_bedrock_event_stream_decoder( From 1227ab8366e737e0634c72a8e16d5f3d53a8bf83 Mon Sep 17 00:00:00 2001 From: rsd-darshan Date: Thu, 27 Aug 2026 11:56:37 +0545 Subject: [PATCH 5/5] test(bedrock): justify the litellm-internal patch for CI's test-quality gate The test-quality budget (TQ008) flags any patch() targeting a litellm. internal, since that idiom usually pins wiring instead of behavior. Here the point of the test is exactly which invoke_provider the decoder factory receives, so the patch is unavoidable; annotate it. --- .../passthrough/test_bedrock_passthrough_transformation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index d564bff0032..b7833c86c43 100644 --- a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -523,7 +523,9 @@ def test_handle_logging_collected_chunks_inference_profile_arn_falls_back_gracef # Intercept the decoder so we don't need real botocore event-stream bytes. # The import is done inside handle_logging_collected_chunks, so patch the source module. - with patch("litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder") as mock_decoder: + with patch( # test-quality-ok: the behavior under test is which invoke_provider this factory receives + "litellm.llms.bedrock.chat.get_bedrock_event_stream_decoder" + ) as mock_decoder: mock_chunk_obj = MagicMock() mock_chunk_obj._chunk_parser.return_value = {} # not a valid GenericStreamingChunk mock_decoder.return_value = mock_chunk_obj