From 4e9c728d38e3121a786e4e44881bdbb43d61cfe3 Mon Sep 17 00:00:00 2001 From: kenneth-berri Date: Mon, 16 Nov 2026 16:10:52 -0800 Subject: [PATCH] fix(bedrock): log the prompt, settings and stream flag of passthrough requests (#52146) * fix(bedrock): log the prompt, settings and stream flag of passthrough requests * refactor: keep passthrough request logging within the lint gates * fix: keep the logging object deep-copyable * fix(bedrock): log Anthropic tool exchanges and skip guarded blocks without text --- litellm/litellm_core_utils/litellm_logging.py | 11 +- .../base_llm/passthrough/transformation.py | 12 +- .../bedrock/passthrough/transformation.py | 80 ++++++- litellm/passthrough/main.py | 8 +- ...test_bedrock_passthrough_transformation.py | 200 ++++++++++++++++++ 5 files changed, 301 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index bb66c29c177..443e4401333 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1043,6 +1043,8 @@ class Logging(LiteLLMLoggingBaseClass): self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000 self.callback_duration_ms: float = 0.0 self.stream = stream + self.passthrough_model_parameters: Mapping[str, object] | None = None + self.passthrough_stream: bool | None = None self.applied_timeout: float | Timeout | None = None self.start_time = start_time # log the call start time self.call_type = call_type @@ -8741,7 +8743,7 @@ def get_standard_logging_object_payload( kwargs=kwargs, ) - stream: bool | None = None + stream: bool | None = logging_obj.passthrough_stream if ( kwargs.get("complete_streaming_response") is not None or kwargs.get("async_complete_streaming_response") is not None @@ -8886,9 +8888,10 @@ def get_standard_logging_object_payload( ) ), response=final_response_obj, - model_parameters=ModelParamHelper.get_standard_logging_model_parameters( - kwargs.get("optional_params", None) or {} - ), + model_parameters={ + **ModelParamHelper.get_standard_logging_model_parameters(kwargs.get("optional_params", None) or {}), + **(logging_obj.passthrough_model_parameters or {}), + }, hidden_params=clean_hidden_params, model_map_information=model_cost_information, error_str=error_str, diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 898033bff8f..5670f782c2b 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from httpx import URL, Headers, Response from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesTerminalEvent + from litellm.types.llms.openai import AllMessageValues, ResponsesAPIResponse, ResponsesTerminalEvent from litellm.types.rerank import RerankResponse from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject @@ -135,6 +135,12 @@ class RawBytesStreamCollector: ) +@dataclass(frozen=True, slots=True) +class LoggedPassthroughRequest: + messages: tuple[AllMessageValues, ...] + model_parameters: Mapping[str, object] + + class BasePassthroughConfig(BaseLLMModelInfo): @abstractmethod def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: @@ -142,6 +148,10 @@ class BasePassthroughConfig(BaseLLMModelInfo): Check if the request is a streaming request """ + def logged_request(self, endpoint: str, request_data: Mapping[str, object]) -> LoggedPassthroughRequest | None: + """The prompt and settings of a provider-native request body, for providers whose body shape is known.""" + return None + def format_url( self, endpoint: str, diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 6e4868ab271..d758376bc0e 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping from litellm.llms.base_llm.passthrough.transformation import ( BasePassthroughConfig, + LoggedPassthroughRequest, PassthroughStreamCollector, relayed_json_object, ) @@ -22,7 +23,12 @@ from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( bedrock_invoke_guardrail_headers, ) -from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionSystemMessage, + ChatCompletionUserMessage, +) from litellm.types.utils import Embedding, EmbeddingResponse, ModelResponseStream, PromptTokensDetailsWrapper, Usage from ..base_aws_llm import BaseAWSLLM @@ -331,18 +337,27 @@ def _leaf_texts(block: Mapping[str, object]) -> Iterator[str]: yield text if "json" in block: yield json.dumps(block["json"], default=str) + guarded: Final = object_mapping((object_mapping(block.get("guardContent")) or {}).get("text")) or {} + if isinstance(guarded.get("text"), str): + yield str(guarded["text"]) if "toolUse" in block: yield json.dumps(_JSON_OBJECT.validate_python(block["toolUse"]).get("input"), default=str) + if block.get("type") == "tool_use": + yield json.dumps(block.get("input"), default=str) def _block_texts(block: Mapping[str, object]) -> Iterator[str]: """The prompt text a request content block carries, including the blocks of a tool result's content, which Bedrock does not nest further.""" yield from _leaf_texts(block) - if "toolResult" not in block: + tool_result: Final = block if block.get("type") == "tool_result" else object_mapping(block.get("toolResult")) + if tool_result is None: return - tool_result: Final = _JSON_OBJECT.validate_python(block["toolResult"]) - yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(tool_result.get("content", ())))) + result_content: Final = tool_result.get("content", ()) + if isinstance(result_content, str): + yield result_content + return + yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(result_content))) def _content_text(content: object) -> str: @@ -371,6 +386,60 @@ def _count_prompt_tokens(model: str, request_body: object) -> int: return token_counter(model=model, messages=messages) +def _logged_turn(turn: Mapping[str, object]) -> AllMessageValues: + content: Final = _content_text(turn.get("content", "")) + if turn.get("role") == "assistant": + return ChatCompletionAssistantMessage(role="assistant", content=content) + return ChatCompletionUserMessage(role="user", content=content) + + +_INVOKE_PROMPT_FIELDS: Final = ("prompt", "inputText") + + +def _logged_messages(body: Mapping[str, object]) -> Iterator[AllMessageValues]: + if body.get("system"): + yield ChatCompletionSystemMessage(role="system", content=_content_text(body["system"])) + if "messages" in body: + yield from map(_logged_turn, _JSON_OBJECTS.validate_python(body["messages"])) + yield from ( + ChatCompletionUserMessage(role="user", content=text) + for text in map(body.get, _INVOKE_PROMPT_FIELDS) + if isinstance(text, str) + ) + + +def _numeric_setting(value: object) -> object: + if isinstance(value, (bool, int, float)): + return value + if isinstance(value, Mapping): + return _numeric_settings(_JSON_OBJECT.validate_python(value)) or None + return None + + +def _is_set(setting: tuple[str, object]) -> bool: + return setting[1] is not None + + +def _numeric_settings(body: Mapping[str, object]) -> Mapping[str, object]: + """A native body's numbers and booleans, nested ones included. Model-specific bodies name their prompt fields + freely, so everything that could hold prompt text is left out of the logged settings.""" + settings: Final = ((name, _numeric_setting(value)) for name, value in body.items()) + return dict(filter(_is_set, settings)) + + +def _logged_request(endpoint: str, body: Mapping[str, object]) -> LoggedPassthroughRequest | None: + action: Final = _bedrock_endpoint_action(endpoint) + if action not in _BEDROCK_CONVERSE_ACTIONS | _BEDROCK_INVOKE_ACTIONS: + return None + try: + messages: Final = tuple(_logged_messages(body)) + except (ValidationError, KeyError): + return None + return LoggedPassthroughRequest( + messages=messages, model_parameters={**_numeric_settings(body), "operation": action} + ) + + def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]: reply: Final = relayed_json_object(httpx_response) or {} return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)} @@ -495,6 +564,9 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return "stream" in endpoint + def logged_request(self, endpoint: str, request_data: Mapping[str, object]) -> LoggedPassthroughRequest | None: + return _logged_request(endpoint, request_data) + def _encode_model_id_for_endpoint(self, model_id: str) -> str: """ Encode model_id (especially ARNs) for use in Bedrock endpoints. diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 77b2cf9b5b5..635cc1d5c22 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -587,8 +587,14 @@ def llm_passthrough_route( request_data=_streaming_request_data, ) - # Update logging object with streaming status litellm_logging_obj.stream = is_streaming_request + litellm_logging_obj.model_call_details["stream"] = is_streaming_request + litellm_logging_obj.passthrough_stream = is_streaming_request + logged_request: Final = provider_config.logged_request(endpoint=endpoint, request_data=_streaming_request_data) + if logged_request is not None: + litellm_logging_obj.passthrough_model_parameters = logged_request.model_parameters + if logged_request is not None and logged_request.messages: + litellm_logging_obj.update_messages(list(logged_request.messages)) ## LOGGING PRE-CALL request_data: Final = data if data else json diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index daa8139db4d..d3237856847 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,3 +1,4 @@ +import asyncio import base64 import gc import json @@ -13,6 +14,7 @@ from unittest.mock import patch import httpx import pytest +from pydantic import TypeAdapter import litellm from litellm.integrations.custom_logger import CustomLogger @@ -27,6 +29,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.passthrough.main import AsyncPassthroughStreamingResponse, PassthroughStreamingResponse from litellm.types.utils import EmbeddingResponse, ModelResponse +_STANDARD_LOGGING_OBJECT: Final = TypeAdapter(Mapping[str, object]) CONVERSE_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0" CONVERSE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/converse-stream" INVOKE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/invoke-with-response-stream" @@ -1894,3 +1897,200 @@ def test_invoke_titan_text_reply_is_still_logged_as_a_chat_completion() -> None: assert isinstance(logged, ModelResponse) assert logged.choices[0].message.content == "hi" + + +CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "OK"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 25, "outputTokens": 2, "totalTokens": 27}, +} +LLAMA_INVOKE_REPLY: Final = {"generation": "OK", "prompt_token_count": 25, "generation_token_count": 2} + + +class _AwaitableSuccessCallback(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged: Final[asyncio.Future[dict[str, object]]] = asyncio.get_running_loop().create_future() + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + if not self.logged.done(): + self.logged.set_result(kwargs) + + +async def _logged_passthrough_request( + action: str, body: dict[str, object], reply: httpx.Response, monkeypatch: pytest.MonkeyPatch +) -> Mapping[str, object]: + callback: Final = _AwaitableSuccessCallback() + monkeypatch.setattr(litellm, "callbacks", [callback]) + response: Final = await litellm.allm_passthrough_route( + model="bedrock/meta.llama3-1-8b-instruct-v1:0", + method="POST", + endpoint=f"/model/meta.llama3-1-8b-instruct-v1:0/{action}", + json=body, + client=AsyncHTTPHandler(transport=httpx.MockTransport(lambda request: reply)), + aws_access_key_id="AKIAROUTERKEY", + aws_secret_access_key="router-secret", + aws_region_name="us-west-2", + ) + if not isinstance(response, httpx.Response): + _ = [chunk async for chunk in response] + assert await response.wait_for_spend_flush() + return _STANDARD_LOGGING_OBJECT.validate_python((await callback.logged)["standard_logging_object"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("action", "body", "reply", "expected_messages", "expected_parameters"), + [ + pytest.param( + "invoke", + {"prompt": "actual llama prompt", "max_gen_len": 2, "temperature": 0.5}, + LLAMA_INVOKE_REPLY, + [{"role": "user", "content": "actual llama prompt"}], + {"operation": "invoke", "max_gen_len": 2, "temperature": 0.5}, + id="invoke prompt", + ), + pytest.param( + "invoke", + { + "inputText": "actual titan input", + "textGenerationConfig": {"maxTokenCount": 2, "stopSequences": ["never logged"]}, + }, + LLAMA_INVOKE_REPLY, + [{"role": "user", "content": "actual titan input"}], + {"operation": "invoke", "textGenerationConfig": {"maxTokenCount": 2}}, + id="invoke inputText", + ), + pytest.param( + "invoke", + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 8, + "system": "be terse", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}, + ], + }, + LLAMA_INVOKE_REPLY, + [ + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ], + {"operation": "invoke", "max_tokens": 8}, + id="invoke messages", + ), + pytest.param( + "invoke", + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 8, + "messages": [ + {"role": "user", "content": "weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t1", "name": "weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "18C"}, + { + "type": "tool_result", + "tool_use_id": "t2", + "content": [{"type": "text", "text": "sunny"}], + }, + ], + }, + ], + }, + LLAMA_INVOKE_REPLY, + [ + {"role": "user", "content": "weather in Paris?"}, + {"role": "assistant", "content": '{"city": "Paris"}'}, + {"role": "user", "content": "18C\nsunny"}, + ], + {"operation": "invoke", "max_tokens": 8}, + id="invoke tool exchange", + ), + pytest.param( + "converse", + { + "messages": [ + { + "role": "user", + "content": [ + {"guardContent": {"image": {"format": "png", "source": {"bytes": "aGk="}}}}, + {"text": "describe the picture"}, + ], + } + ], + "inferenceConfig": {"maxTokens": 2}, + }, + CONVERSE_REPLY, + [{"role": "user", "content": "describe the picture"}], + {"operation": "converse", "inferenceConfig": {"maxTokens": 2}}, + id="converse guarded image", + ), + pytest.param( + "converse", + { + "system": [{"text": "system instruction"}, {"guardContent": {"text": {"text": "guarded system"}}}], + "messages": [{"role": "user", "content": [{"text": "actual conversation"}]}], + "inferenceConfig": {"maxTokens": 2, "temperature": 0, "stopSequences": ["never logged"]}, + "promptVariables": {"name": {"text": "never logged"}}, + }, + CONVERSE_REPLY, + [ + {"role": "system", "content": "system instruction\nguarded system"}, + {"role": "user", "content": "actual conversation"}, + ], + {"operation": "converse", "inferenceConfig": {"maxTokens": 2, "temperature": 0}}, + id="converse", + ), + ], +) +async def test_bedrock_passthrough_logs_the_prompt_and_settings_the_caller_sent( + action: str, + body: dict[str, object], + reply: Mapping[str, object], + expected_messages: list[dict[str, str]], + expected_parameters: dict[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + logged: Final = await _logged_passthrough_request(action, body, httpx.Response(200, json=reply), monkeypatch) + + assert logged["messages"] == expected_messages + assert logged["model_parameters"] == expected_parameters + assert logged["stream"] is False + + +@pytest.mark.asyncio +async def test_bedrock_stream_passthrough_logs_the_prompt_and_that_it_streamed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logged: Final = await _logged_passthrough_request( + "converse-stream", + {"messages": [{"role": "user", "content": [{"text": "stream prompt"}]}]}, + httpx.Response(200, content=_converse_text_stream(["Converse response"])), + monkeypatch, + ) + + assert logged["messages"] == [{"role": "user", "content": "stream prompt"}] + assert logged["model_parameters"] == {"operation": "converse-stream"} + assert logged["stream"] is True + + +@pytest.mark.asyncio +async def test_bedrock_passthrough_with_an_unreadable_prompt_is_still_sent_and_logged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logged: Final = await _logged_passthrough_request( + "converse", {"messages": ["not a turn"]}, httpx.Response(200, json=CONVERSE_REPLY), monkeypatch + ) + + assert logged["status"] == "success" + assert logged["model_parameters"] == {}