mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): log the prompt, settings and stream flag of passthrough requests (#52146)
* fix(bedrock): log the prompt, settings and stream flag of passthrough requests * refactor: keep passthrough request logging within the lint gates * fix: keep the logging object deep-copyable * fix(bedrock): log Anthropic tool exchanges and skip guarded blocks without text
This commit is contained in:
parent
a14d17822b
commit
4e9c728d38
5 changed files with 301 additions and 10 deletions
|
|
@ -1043,6 +1043,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.message_copy_duration_ms: float = (time.time() - _copy_start) * 1000
|
||||
self.callback_duration_ms: float = 0.0
|
||||
self.stream = stream
|
||||
self.passthrough_model_parameters: Mapping[str, object] | None = None
|
||||
self.passthrough_stream: bool | None = None
|
||||
self.applied_timeout: float | Timeout | None = None
|
||||
self.start_time = start_time # log the call start time
|
||||
self.call_type = call_type
|
||||
|
|
@ -8741,7 +8743,7 @@ def get_standard_logging_object_payload(
|
|||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
stream: bool | None = None
|
||||
stream: bool | None = logging_obj.passthrough_stream
|
||||
if (
|
||||
kwargs.get("complete_streaming_response") is not None
|
||||
or kwargs.get("async_complete_streaming_response") is not None
|
||||
|
|
@ -8886,9 +8888,10 @@ def get_standard_logging_object_payload(
|
|||
)
|
||||
),
|
||||
response=final_response_obj,
|
||||
model_parameters=ModelParamHelper.get_standard_logging_model_parameters(
|
||||
kwargs.get("optional_params", None) or {}
|
||||
),
|
||||
model_parameters={
|
||||
**ModelParamHelper.get_standard_logging_model_parameters(kwargs.get("optional_params", None) or {}),
|
||||
**(logging_obj.passthrough_model_parameters or {}),
|
||||
},
|
||||
hidden_params=clean_hidden_params,
|
||||
model_map_information=model_cost_information,
|
||||
error_str=error_str,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ if TYPE_CHECKING:
|
|||
from httpx import URL, Headers, Response
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesTerminalEvent
|
||||
from litellm.types.llms.openai import AllMessageValues, ResponsesAPIResponse, ResponsesTerminalEvent
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.utils import CostResponseTypes, StandardPassThroughResponseObject
|
||||
|
||||
|
|
@ -135,6 +135,12 @@ class RawBytesStreamCollector:
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggedPassthroughRequest:
|
||||
messages: tuple[AllMessageValues, ...]
|
||||
model_parameters: Mapping[str, object]
|
||||
|
||||
|
||||
class BasePassthroughConfig(BaseLLMModelInfo):
|
||||
@abstractmethod
|
||||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
|
|
@ -142,6 +148,10 @@ class BasePassthroughConfig(BaseLLMModelInfo):
|
|||
Check if the request is a streaming request
|
||||
"""
|
||||
|
||||
def logged_request(self, endpoint: str, request_data: Mapping[str, object]) -> LoggedPassthroughRequest | None:
|
||||
"""The prompt and settings of a provider-native request body, for providers whose body shape is known."""
|
||||
return None
|
||||
|
||||
def format_url(
|
||||
self,
|
||||
endpoint: str,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging
|
|||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
|
||||
from litellm.llms.base_llm.passthrough.transformation import (
|
||||
BasePassthroughConfig,
|
||||
LoggedPassthroughRequest,
|
||||
PassthroughStreamCollector,
|
||||
relayed_json_object,
|
||||
)
|
||||
|
|
@ -22,7 +23,12 @@ from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
bedrock_invoke_guardrail_headers,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionSystemMessage,
|
||||
ChatCompletionUserMessage,
|
||||
)
|
||||
from litellm.types.utils import Embedding, EmbeddingResponse, ModelResponseStream, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
|
|
@ -331,18 +337,27 @@ def _leaf_texts(block: Mapping[str, object]) -> Iterator[str]:
|
|||
yield text
|
||||
if "json" in block:
|
||||
yield json.dumps(block["json"], default=str)
|
||||
guarded: Final = object_mapping((object_mapping(block.get("guardContent")) or {}).get("text")) or {}
|
||||
if isinstance(guarded.get("text"), str):
|
||||
yield str(guarded["text"])
|
||||
if "toolUse" in block:
|
||||
yield json.dumps(_JSON_OBJECT.validate_python(block["toolUse"]).get("input"), default=str)
|
||||
if block.get("type") == "tool_use":
|
||||
yield json.dumps(block.get("input"), default=str)
|
||||
|
||||
|
||||
def _block_texts(block: Mapping[str, object]) -> Iterator[str]:
|
||||
"""The prompt text a request content block carries, including the blocks of a tool result's content, which
|
||||
Bedrock does not nest further."""
|
||||
yield from _leaf_texts(block)
|
||||
if "toolResult" not in block:
|
||||
tool_result: Final = block if block.get("type") == "tool_result" else object_mapping(block.get("toolResult"))
|
||||
if tool_result is None:
|
||||
return
|
||||
tool_result: Final = _JSON_OBJECT.validate_python(block["toolResult"])
|
||||
yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(tool_result.get("content", ()))))
|
||||
result_content: Final = tool_result.get("content", ())
|
||||
if isinstance(result_content, str):
|
||||
yield result_content
|
||||
return
|
||||
yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(result_content)))
|
||||
|
||||
|
||||
def _content_text(content: object) -> str:
|
||||
|
|
@ -371,6 +386,60 @@ def _count_prompt_tokens(model: str, request_body: object) -> int:
|
|||
return token_counter(model=model, messages=messages)
|
||||
|
||||
|
||||
def _logged_turn(turn: Mapping[str, object]) -> AllMessageValues:
|
||||
content: Final = _content_text(turn.get("content", ""))
|
||||
if turn.get("role") == "assistant":
|
||||
return ChatCompletionAssistantMessage(role="assistant", content=content)
|
||||
return ChatCompletionUserMessage(role="user", content=content)
|
||||
|
||||
|
||||
_INVOKE_PROMPT_FIELDS: Final = ("prompt", "inputText")
|
||||
|
||||
|
||||
def _logged_messages(body: Mapping[str, object]) -> Iterator[AllMessageValues]:
|
||||
if body.get("system"):
|
||||
yield ChatCompletionSystemMessage(role="system", content=_content_text(body["system"]))
|
||||
if "messages" in body:
|
||||
yield from map(_logged_turn, _JSON_OBJECTS.validate_python(body["messages"]))
|
||||
yield from (
|
||||
ChatCompletionUserMessage(role="user", content=text)
|
||||
for text in map(body.get, _INVOKE_PROMPT_FIELDS)
|
||||
if isinstance(text, str)
|
||||
)
|
||||
|
||||
|
||||
def _numeric_setting(value: object) -> object:
|
||||
if isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, Mapping):
|
||||
return _numeric_settings(_JSON_OBJECT.validate_python(value)) or None
|
||||
return None
|
||||
|
||||
|
||||
def _is_set(setting: tuple[str, object]) -> bool:
|
||||
return setting[1] is not None
|
||||
|
||||
|
||||
def _numeric_settings(body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""A native body's numbers and booleans, nested ones included. Model-specific bodies name their prompt fields
|
||||
freely, so everything that could hold prompt text is left out of the logged settings."""
|
||||
settings: Final = ((name, _numeric_setting(value)) for name, value in body.items())
|
||||
return dict(filter(_is_set, settings))
|
||||
|
||||
|
||||
def _logged_request(endpoint: str, body: Mapping[str, object]) -> LoggedPassthroughRequest | None:
|
||||
action: Final = _bedrock_endpoint_action(endpoint)
|
||||
if action not in _BEDROCK_CONVERSE_ACTIONS | _BEDROCK_INVOKE_ACTIONS:
|
||||
return None
|
||||
try:
|
||||
messages: Final = tuple(_logged_messages(body))
|
||||
except (ValidationError, KeyError):
|
||||
return None
|
||||
return LoggedPassthroughRequest(
|
||||
messages=messages, model_parameters={**_numeric_settings(body), "operation": action}
|
||||
)
|
||||
|
||||
|
||||
def _invoke_guardrail_trace(httpx_response: Response) -> Mapping[str, object]:
|
||||
reply: Final = relayed_json_object(httpx_response) or {}
|
||||
return {BEDROCK_INVOKE_TRACE_FIELD: reply.get(BEDROCK_INVOKE_TRACE_FIELD)}
|
||||
|
|
@ -495,6 +564,9 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
return "stream" in endpoint
|
||||
|
||||
def logged_request(self, endpoint: str, request_data: Mapping[str, object]) -> LoggedPassthroughRequest | None:
|
||||
return _logged_request(endpoint, request_data)
|
||||
|
||||
def _encode_model_id_for_endpoint(self, model_id: str) -> str:
|
||||
"""
|
||||
Encode model_id (especially ARNs) for use in Bedrock endpoints.
|
||||
|
|
|
|||
|
|
@ -587,8 +587,14 @@ def llm_passthrough_route(
|
|||
request_data=_streaming_request_data,
|
||||
)
|
||||
|
||||
# Update logging object with streaming status
|
||||
litellm_logging_obj.stream = is_streaming_request
|
||||
litellm_logging_obj.model_call_details["stream"] = is_streaming_request
|
||||
litellm_logging_obj.passthrough_stream = is_streaming_request
|
||||
logged_request: Final = provider_config.logged_request(endpoint=endpoint, request_data=_streaming_request_data)
|
||||
if logged_request is not None:
|
||||
litellm_logging_obj.passthrough_model_parameters = logged_request.model_parameters
|
||||
if logged_request is not None and logged_request.messages:
|
||||
litellm_logging_obj.update_messages(list(logged_request.messages))
|
||||
|
||||
## LOGGING PRE-CALL
|
||||
request_data: Final = data if data else json
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import gc
|
||||
import json
|
||||
|
|
@ -13,6 +14,7 @@ from unittest.mock import patch
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -27,6 +29,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
|||
from litellm.passthrough.main import AsyncPassthroughStreamingResponse, PassthroughStreamingResponse
|
||||
from litellm.types.utils import EmbeddingResponse, ModelResponse
|
||||
|
||||
_STANDARD_LOGGING_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
CONVERSE_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
CONVERSE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/converse-stream"
|
||||
INVOKE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/invoke-with-response-stream"
|
||||
|
|
@ -1894,3 +1897,200 @@ def test_invoke_titan_text_reply_is_still_logged_as_a_chat_completion() -> None:
|
|||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.choices[0].message.content == "hi"
|
||||
|
||||
|
||||
CONVERSE_REPLY: Final = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "OK"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 25, "outputTokens": 2, "totalTokens": 27},
|
||||
}
|
||||
LLAMA_INVOKE_REPLY: Final = {"generation": "OK", "prompt_token_count": 25, "generation_token_count": 2}
|
||||
|
||||
|
||||
class _AwaitableSuccessCallback(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.logged: Final[asyncio.Future[dict[str, object]]] = asyncio.get_running_loop().create_future()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
if not self.logged.done():
|
||||
self.logged.set_result(kwargs)
|
||||
|
||||
|
||||
async def _logged_passthrough_request(
|
||||
action: str, body: dict[str, object], reply: httpx.Response, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Mapping[str, object]:
|
||||
callback: Final = _AwaitableSuccessCallback()
|
||||
monkeypatch.setattr(litellm, "callbacks", [callback])
|
||||
response: Final = await litellm.allm_passthrough_route(
|
||||
model="bedrock/meta.llama3-1-8b-instruct-v1:0",
|
||||
method="POST",
|
||||
endpoint=f"/model/meta.llama3-1-8b-instruct-v1:0/{action}",
|
||||
json=body,
|
||||
client=AsyncHTTPHandler(transport=httpx.MockTransport(lambda request: reply)),
|
||||
aws_access_key_id="AKIAROUTERKEY",
|
||||
aws_secret_access_key="router-secret",
|
||||
aws_region_name="us-west-2",
|
||||
)
|
||||
if not isinstance(response, httpx.Response):
|
||||
_ = [chunk async for chunk in response]
|
||||
assert await response.wait_for_spend_flush()
|
||||
return _STANDARD_LOGGING_OBJECT.validate_python((await callback.logged)["standard_logging_object"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("action", "body", "reply", "expected_messages", "expected_parameters"),
|
||||
[
|
||||
pytest.param(
|
||||
"invoke",
|
||||
{"prompt": "actual llama prompt", "max_gen_len": 2, "temperature": 0.5},
|
||||
LLAMA_INVOKE_REPLY,
|
||||
[{"role": "user", "content": "actual llama prompt"}],
|
||||
{"operation": "invoke", "max_gen_len": 2, "temperature": 0.5},
|
||||
id="invoke prompt",
|
||||
),
|
||||
pytest.param(
|
||||
"invoke",
|
||||
{
|
||||
"inputText": "actual titan input",
|
||||
"textGenerationConfig": {"maxTokenCount": 2, "stopSequences": ["never logged"]},
|
||||
},
|
||||
LLAMA_INVOKE_REPLY,
|
||||
[{"role": "user", "content": "actual titan input"}],
|
||||
{"operation": "invoke", "textGenerationConfig": {"maxTokenCount": 2}},
|
||||
id="invoke inputText",
|
||||
),
|
||||
pytest.param(
|
||||
"invoke",
|
||||
{
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 8,
|
||||
"system": "be terse",
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "hello"}]},
|
||||
],
|
||||
},
|
||||
LLAMA_INVOKE_REPLY,
|
||||
[
|
||||
{"role": "system", "content": "be terse"},
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
],
|
||||
{"operation": "invoke", "max_tokens": 8},
|
||||
id="invoke messages",
|
||||
),
|
||||
pytest.param(
|
||||
"invoke",
|
||||
{
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
"max_tokens": 8,
|
||||
"messages": [
|
||||
{"role": "user", "content": "weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": "t1", "name": "weather", "input": {"city": "Paris"}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "t1", "content": "18C"},
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "t2",
|
||||
"content": [{"type": "text", "text": "sunny"}],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
LLAMA_INVOKE_REPLY,
|
||||
[
|
||||
{"role": "user", "content": "weather in Paris?"},
|
||||
{"role": "assistant", "content": '{"city": "Paris"}'},
|
||||
{"role": "user", "content": "18C\nsunny"},
|
||||
],
|
||||
{"operation": "invoke", "max_tokens": 8},
|
||||
id="invoke tool exchange",
|
||||
),
|
||||
pytest.param(
|
||||
"converse",
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"guardContent": {"image": {"format": "png", "source": {"bytes": "aGk="}}}},
|
||||
{"text": "describe the picture"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"inferenceConfig": {"maxTokens": 2},
|
||||
},
|
||||
CONVERSE_REPLY,
|
||||
[{"role": "user", "content": "describe the picture"}],
|
||||
{"operation": "converse", "inferenceConfig": {"maxTokens": 2}},
|
||||
id="converse guarded image",
|
||||
),
|
||||
pytest.param(
|
||||
"converse",
|
||||
{
|
||||
"system": [{"text": "system instruction"}, {"guardContent": {"text": {"text": "guarded system"}}}],
|
||||
"messages": [{"role": "user", "content": [{"text": "actual conversation"}]}],
|
||||
"inferenceConfig": {"maxTokens": 2, "temperature": 0, "stopSequences": ["never logged"]},
|
||||
"promptVariables": {"name": {"text": "never logged"}},
|
||||
},
|
||||
CONVERSE_REPLY,
|
||||
[
|
||||
{"role": "system", "content": "system instruction\nguarded system"},
|
||||
{"role": "user", "content": "actual conversation"},
|
||||
],
|
||||
{"operation": "converse", "inferenceConfig": {"maxTokens": 2, "temperature": 0}},
|
||||
id="converse",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_bedrock_passthrough_logs_the_prompt_and_settings_the_caller_sent(
|
||||
action: str,
|
||||
body: dict[str, object],
|
||||
reply: Mapping[str, object],
|
||||
expected_messages: list[dict[str, str]],
|
||||
expected_parameters: dict[str, object],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged: Final = await _logged_passthrough_request(action, body, httpx.Response(200, json=reply), monkeypatch)
|
||||
|
||||
assert logged["messages"] == expected_messages
|
||||
assert logged["model_parameters"] == expected_parameters
|
||||
assert logged["stream"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_stream_passthrough_logs_the_prompt_and_that_it_streamed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged: Final = await _logged_passthrough_request(
|
||||
"converse-stream",
|
||||
{"messages": [{"role": "user", "content": [{"text": "stream prompt"}]}]},
|
||||
httpx.Response(200, content=_converse_text_stream(["Converse response"])),
|
||||
monkeypatch,
|
||||
)
|
||||
|
||||
assert logged["messages"] == [{"role": "user", "content": "stream prompt"}]
|
||||
assert logged["model_parameters"] == {"operation": "converse-stream"}
|
||||
assert logged["stream"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_passthrough_with_an_unreadable_prompt_is_still_sent_and_logged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged: Final = await _logged_passthrough_request(
|
||||
"converse", {"messages": ["not a turn"]}, httpx.Response(200, json=CONVERSE_REPLY), monkeypatch
|
||||
)
|
||||
|
||||
assert logged["status"] == "success"
|
||||
assert logged["model_parameters"] == {}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue