scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

fix: resolve root causes of Pub/Sub message data quality issues

Browse source 
Three production bugs fixed:

1. messages={} for all passthrough endpoints:
   - Anthropic handler's _create_anthropic_response_logging_payload never
     set kwargs["messages"] from request_body. Added request_body parameter
     and propagate messages to kwargs for get_standard_logging_object_payload().
   - Vertex handler's _create_vertex_response_logging_payload_for_generate_content
     same issue. Thread request_body through all call sites (generateContent,
     rawPredict, streaming) and set kwargs["messages"] from either
     request_body["messages"] or request_body["contents"].

2. model="unknown" for Meta/Llama via Vertex passthrough:
   - _get_vertex_publisher_or_api_spec_from_url() didn't detect "meta" publisher.
     Added "/publishers/meta/" check.
   - get_vertex_ai_partner_model_config() didn't handle "meta" publisher.
     Added "meta" to the condition that returns VertexAILlama3Config.

Tests updated: removed xfail markers, added kwargs["messages"] assertions.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Harshit28j 2026-03-02 17:13:44 +05:30
parent fa3d788a29
commit 47fea2acdf
5 changed files with 41 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py
View file

@ -13,10 +13,7 @@ def get_vertex_ai_partner_model_config(
from .ai21.transformation import VertexAIAi21Config
return VertexAIAi21Config()
elif (
vertex_publisher_or_api_spec == "openapi"
or vertex_publisher_or_api_spec == "mistralai"
):
elif vertex_publisher_or_api_spec in ("openapi", "mistralai", "meta"):
from .llama3.transformation import VertexAILlama3Config
return VertexAILlama3Config()

8
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -83,6 +83,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
request_body=request_body,
)
return {
@ -107,6 +108,7 @@ class AnthropicPassthroughLoggingHandler:
start_time: datetime,
end_time: datetime,
logging_obj: LiteLLMLoggingObj,
request_body: Optional[dict] = None,
):
"""
Create the standard logging object for Anthropic passthrough
@ -114,6 +116,11 @@ class AnthropicPassthroughLoggingHandler:
handles streaming and non-streaming responses
"""
try:
# Propagate request body messages to kwargs so that
# get_standard_logging_object_payload() can populate the messages field
if request_body and "messages" in request_body:
kwargs["messages"] = request_body["messages"]
# Get custom_llm_provider from logging object if available (e.g., azure_ai for Azure Anthropic)
custom_llm_provider = logging_obj.model_call_details.get(
"custom_llm_provider"
@ -216,6 +223,7 @@ class AnthropicPassthroughLoggingHandler:
start_time=start_time,
end_time=end_time,
logging_obj=litellm_logging_obj,
request_body=request_body,
)
return {

13
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -122,6 +122,7 @@ class VertexPassthroughLoggingHandler:
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(
url_route
),
request_body=request_body,
)
return {
@ -183,6 +184,7 @@ class VertexPassthroughLoggingHandler:
end_time=end_time,
logging_obj=logging_obj,
custom_llm_provider="vertex_ai",
request_body=request_body,
)
return {
@ -373,6 +375,7 @@ class VertexPassthroughLoggingHandler:
custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url(
url_route
),
request_body=request_body,
)
return {
@ -477,6 +480,8 @@ class VertexPassthroughLoggingHandler:
return "anthropic"
elif "/publishers/ai21/" in url:
return "ai21"
elif "/publishers/meta/" in url:
return "meta"
elif "/endpoints/openapi/" in url:
return "openapi"
return None
@ -532,11 +537,19 @@ class VertexPassthroughLoggingHandler:
end_time: datetime,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: str,
request_body: Optional[dict] = None,
) -> dict:
"""
Create the standard logging object for Vertex passthrough generateContent (streaming and non-streaming)
"""
# Propagate request body messages to kwargs so that
# get_standard_logging_object_payload() can populate the messages field
if request_body:
if "messages" in request_body:
kwargs["messages"] = request_body["messages"]
elif "contents" in request_body:
kwargs["messages"] = request_body["contents"]
response_cost = litellm.completion_cost(
completion_response=litellm_model_response,

16
tests/logging_callback_tests/test_gcs_pub_sub.py
View file

@ -571,6 +571,12 @@ async def test_pubsub_anthropic_passthrough_tokens_and_response(_mock_premium):
# The kwargs should include model
assert kwargs.get("model") == "claude-3-5-sonnet-20241022"
# Messages should be propagated from request_body to kwargs
assert isinstance(kwargs.get("messages"), list), (
f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}"
)
assert kwargs["messages"][0]["content"] == "Hi there!"
# ---------------------------------------------------------------------------
# Test 3d: Meta/Llama via Vertex passthrough — model and provider
@ -677,6 +683,11 @@ async def test_pubsub_vertex_meta_passthrough_model_and_provider():
f"completion_tokens should be > 0, got {litellm_model_response.usage.completion_tokens}"
)
# Messages should be propagated from request_body to kwargs
assert isinstance(kwargs.get("messages"), list), (
f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}"
)
# ---------------------------------------------------------------------------
# Test 3e: Vertex AI generateContent — messages must be array
@ -767,6 +778,11 @@ async def test_pubsub_vertex_generate_content_messages_is_array():
assert "response_cost" in kwargs
assert isinstance(kwargs["response_cost"], (int, float))
# Messages should be propagated from request_body contents to kwargs
assert isinstance(kwargs.get("messages"), list), (
f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}"
)
# ---------------------------------------------------------------------------
# Test 3f: Streaming response — all fields populated

11
tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py
View file

@ -133,7 +133,6 @@ class TestGetVertexPublisherFromUrl:
url = "https://us-central1-aiplatform.googleapis.com/v1/projects/proj/locations/us-central1/publishers/mistralai/models/mistral-large:rawPredict"
assert self._get_publisher(url) == "mistralai"
@pytest.mark.xfail(reason="Bug: _get_vertex_publisher_or_api_spec_from_url does not detect meta publisher")
def test_meta_publisher(self):
url = "https://us-central1-aiplatform.googleapis.com/v1/projects/proj/locations/us-central1/publishers/meta/models/llama-3.1-70b-instruct-maas:rawPredict"
assert self._get_publisher(url) == "meta"
@ -149,13 +148,11 @@ class TestGetVertexPublisherFromUrl:
class TestAnthropicPassthroughMessages:
"""Tests that Anthropic passthrough handler preserves request messages for logging."""
@pytest.mark.xfail(reason="Bug: anthropic_passthrough_handler does not set kwargs['messages'] from request body")
@pytest.mark.asyncio
async def test_anthropic_handler_includes_request_messages_in_kwargs(self):
"""
Bug: anthropic_passthrough_handler calls transform_response with messages=[]
(hardcoded at line 69). The request_body messages are never propagated to
kwargs for downstream standard logging, causing messages={} in Pub/Sub.
Verify that anthropic_passthrough_handler propagates request_body messages
to kwargs so get_standard_logging_object_payload() can populate the messages field.
"""
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
@ -223,12 +220,10 @@ class TestAnthropicPassthroughMessages:
# The kwargs should contain messages from the request body so that
# get_standard_logging_object_payload() can populate the messages field.
# Currently this is NOT the case — kwargs["messages"] is never set.
messages_in_kwargs = kwargs.get("messages")
assert messages_in_kwargs is not None, (
"kwargs['messages'] should be set from request_body['messages'] "
"so that the StandardLoggingPayload messages field is populated. "
"Currently the Anthropic passthrough handler does not set this."
"so that the StandardLoggingPayload messages field is populated."
)
assert isinstance(messages_in_kwargs, list), (
f"kwargs['messages'] should be a list, got {type(messages_in_kwargs)}"