From 47fea2acdf214de317795135a6473ed4b9464f48 Mon Sep 17 00:00:00 2001 From: Harshit28j Date: Mon, 2 Mar 2026 17:13:44 +0530 Subject: [PATCH] fix: resolve root causes of Pub/Sub message data quality issues Three production bugs fixed: 1. messages={} for all passthrough endpoints: - Anthropic handler's _create_anthropic_response_logging_payload never set kwargs["messages"] from request_body. Added request_body parameter and propagate messages to kwargs for get_standard_logging_object_payload(). - Vertex handler's _create_vertex_response_logging_payload_for_generate_content same issue. Thread request_body through all call sites (generateContent, rawPredict, streaming) and set kwargs["messages"] from either request_body["messages"] or request_body["contents"]. 2. model="unknown" for Meta/Llama via Vertex passthrough: - _get_vertex_publisher_or_api_spec_from_url() didn't detect "meta" publisher. Added "/publishers/meta/" check. - get_vertex_ai_partner_model_config() didn't handle "meta" publisher. Added "meta" to the condition that returns VertexAILlama3Config. Tests updated: removed xfail markers, added kwargs["messages"] assertions. Co-Authored-By: Claude Opus 4.6 --- .../vertex_ai_partner_models/__init__.py | 5 +---- .../anthropic_passthrough_logging_handler.py | 8 ++++++++ .../vertex_passthrough_logging_handler.py | 13 +++++++++++++ tests/logging_callback_tests/test_gcs_pub_sub.py | 16 ++++++++++++++++ .../integrations/gcs_pubsub/test_pub_sub.py | 11 +++-------- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py index cc0ecc2e3c6..7c338817828 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py @@ -13,10 +13,7 @@ def get_vertex_ai_partner_model_config( from .ai21.transformation import VertexAIAi21Config return VertexAIAi21Config() - elif ( - vertex_publisher_or_api_spec == "openapi" - or vertex_publisher_or_api_spec == "mistralai" - ): + elif vertex_publisher_or_api_spec in ("openapi", "mistralai", "meta"): from .llama3.transformation import VertexAILlama3Config return VertexAILlama3Config() diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index e70d6cb7fca..780a4df726f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -83,6 +83,7 @@ class AnthropicPassthroughLoggingHandler: start_time=start_time, end_time=end_time, logging_obj=logging_obj, + request_body=request_body, ) return { @@ -107,6 +108,7 @@ class AnthropicPassthroughLoggingHandler: start_time: datetime, end_time: datetime, logging_obj: LiteLLMLoggingObj, + request_body: Optional[dict] = None, ): """ Create the standard logging object for Anthropic passthrough @@ -114,6 +116,11 @@ class AnthropicPassthroughLoggingHandler: handles streaming and non-streaming responses """ try: + # Propagate request body messages to kwargs so that + # get_standard_logging_object_payload() can populate the messages field + if request_body and "messages" in request_body: + kwargs["messages"] = request_body["messages"] + # Get custom_llm_provider from logging object if available (e.g., azure_ai for Azure Anthropic) custom_llm_provider = logging_obj.model_call_details.get( "custom_llm_provider" @@ -216,6 +223,7 @@ class AnthropicPassthroughLoggingHandler: start_time=start_time, end_time=end_time, logging_obj=litellm_logging_obj, + request_body=request_body, ) return { diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 3d5c529a3bb..4a2e45ca2e7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -122,6 +122,7 @@ class VertexPassthroughLoggingHandler: custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url( url_route ), + request_body=request_body, ) return { @@ -183,6 +184,7 @@ class VertexPassthroughLoggingHandler: end_time=end_time, logging_obj=logging_obj, custom_llm_provider="vertex_ai", + request_body=request_body, ) return { @@ -373,6 +375,7 @@ class VertexPassthroughLoggingHandler: custom_llm_provider=VertexPassthroughLoggingHandler._get_custom_llm_provider_from_url( url_route ), + request_body=request_body, ) return { @@ -477,6 +480,8 @@ class VertexPassthroughLoggingHandler: return "anthropic" elif "/publishers/ai21/" in url: return "ai21" + elif "/publishers/meta/" in url: + return "meta" elif "/endpoints/openapi/" in url: return "openapi" return None @@ -532,11 +537,19 @@ class VertexPassthroughLoggingHandler: end_time: datetime, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str, + request_body: Optional[dict] = None, ) -> dict: """ Create the standard logging object for Vertex passthrough generateContent (streaming and non-streaming) """ + # Propagate request body messages to kwargs so that + # get_standard_logging_object_payload() can populate the messages field + if request_body: + if "messages" in request_body: + kwargs["messages"] = request_body["messages"] + elif "contents" in request_body: + kwargs["messages"] = request_body["contents"] response_cost = litellm.completion_cost( completion_response=litellm_model_response, diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 554d8610c27..84211c449b7 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -571,6 +571,12 @@ async def test_pubsub_anthropic_passthrough_tokens_and_response(_mock_premium): # The kwargs should include model assert kwargs.get("model") == "claude-3-5-sonnet-20241022" + # Messages should be propagated from request_body to kwargs + assert isinstance(kwargs.get("messages"), list), ( + f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}" + ) + assert kwargs["messages"][0]["content"] == "Hi there!" + # --------------------------------------------------------------------------- # Test 3d: Meta/Llama via Vertex passthrough — model and provider @@ -677,6 +683,11 @@ async def test_pubsub_vertex_meta_passthrough_model_and_provider(): f"completion_tokens should be > 0, got {litellm_model_response.usage.completion_tokens}" ) + # Messages should be propagated from request_body to kwargs + assert isinstance(kwargs.get("messages"), list), ( + f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}" + ) + # --------------------------------------------------------------------------- # Test 3e: Vertex AI generateContent — messages must be array @@ -767,6 +778,11 @@ async def test_pubsub_vertex_generate_content_messages_is_array(): assert "response_cost" in kwargs assert isinstance(kwargs["response_cost"], (int, float)) + # Messages should be propagated from request_body contents to kwargs + assert isinstance(kwargs.get("messages"), list), ( + f"kwargs['messages'] should be a list from request_body, got {type(kwargs.get('messages'))}" + ) + # --------------------------------------------------------------------------- # Test 3f: Streaming response — all fields populated diff --git a/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py b/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py index 950c9b469be..acf1a45bd44 100644 --- a/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py +++ b/tests/test_litellm/integrations/gcs_pubsub/test_pub_sub.py @@ -133,7 +133,6 @@ class TestGetVertexPublisherFromUrl: url = "https://us-central1-aiplatform.googleapis.com/v1/projects/proj/locations/us-central1/publishers/mistralai/models/mistral-large:rawPredict" assert self._get_publisher(url) == "mistralai" - @pytest.mark.xfail(reason="Bug: _get_vertex_publisher_or_api_spec_from_url does not detect meta publisher") def test_meta_publisher(self): url = "https://us-central1-aiplatform.googleapis.com/v1/projects/proj/locations/us-central1/publishers/meta/models/llama-3.1-70b-instruct-maas:rawPredict" assert self._get_publisher(url) == "meta" @@ -149,13 +148,11 @@ class TestGetVertexPublisherFromUrl: class TestAnthropicPassthroughMessages: """Tests that Anthropic passthrough handler preserves request messages for logging.""" - @pytest.mark.xfail(reason="Bug: anthropic_passthrough_handler does not set kwargs['messages'] from request body") @pytest.mark.asyncio async def test_anthropic_handler_includes_request_messages_in_kwargs(self): """ - Bug: anthropic_passthrough_handler calls transform_response with messages=[] - (hardcoded at line 69). The request_body messages are never propagated to - kwargs for downstream standard logging, causing messages={} in Pub/Sub. + Verify that anthropic_passthrough_handler propagates request_body messages + to kwargs so get_standard_logging_object_payload() can populate the messages field. """ from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, @@ -223,12 +220,10 @@ class TestAnthropicPassthroughMessages: # The kwargs should contain messages from the request body so that # get_standard_logging_object_payload() can populate the messages field. - # Currently this is NOT the case — kwargs["messages"] is never set. messages_in_kwargs = kwargs.get("messages") assert messages_in_kwargs is not None, ( "kwargs['messages'] should be set from request_body['messages'] " - "so that the StandardLoggingPayload messages field is populated. " - "Currently the Anthropic passthrough handler does not set this." + "so that the StandardLoggingPayload messages field is populated." ) assert isinstance(messages_in_kwargs, list), ( f"kwargs['messages'] should be a list, got {type(messages_in_kwargs)}"