diff --git a/litellm/proxy/pass_through_endpoints/architecture.md b/litellm/proxy/pass_through_endpoints/architecture.md index f7dd8077ab5..4d403bf451c 100644 --- a/litellm/proxy/pass_through_endpoints/architecture.md +++ b/litellm/proxy/pass_through_endpoints/architecture.md @@ -59,6 +59,7 @@ sequenceDiagram |-----------|-------------| | `x-pass-*` headers | Strip prefix and forward (e.g., `x-pass-anthropic-beta` → `anthropic-beta`) | | `x-litellm-tags` header | Extract tags and add to request metadata for logging | +| `x-litellm-spend-logs-metadata` header | Parse as JSON and propagate into `StandardLoggingPayload.metadata.spend_logs_metadata` (matches OpenAI-compatible endpoints) | | Streaming chunk collection | Collect chunks async for logging after stream completes | | Multipart form handling | Reconstruct multipart/form-data requests for file uploads | | Guardrails (opt-in) | Run content filtering when explicitly configured | diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 216eb61a9d1..83e522dde21 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -19,6 +19,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( ) from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse + if TYPE_CHECKING: from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType @@ -146,6 +147,7 @@ class AnthropicPassthroughLoggingHandler: kwargs["model"] = model passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( # type: ignore kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload") ) if passthrough_logging_payload: user = AnthropicPassthroughLoggingHandler._get_user_from_metadata( @@ -157,6 +159,11 @@ class AnthropicPassthroughLoggingHandler: {"proxy_server_request": {"body": {"user": user}}} ) + from .base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, passthrough_logging_payload + ) + # pretty print standard logging object verbose_proxy_logger.debug( "kwargs= %s", @@ -221,10 +228,11 @@ class AnthropicPassthroughLoggingHandler: "result": None, "kwargs": {}, } + from .base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( litellm_model_response=complete_streaming_response, model=model, - kwargs={}, + kwargs=BasePassthroughLoggingHandler._seed_streaming_kwargs_from_logging_obj(litellm_logging_obj), start_time=start_time, end_time=end_time, logging_obj=litellm_logging_obj, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py index b9df8ecede3..7de024669e5 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py @@ -93,6 +93,49 @@ class BasePassthroughLoggingHandler(ABC): return get_end_user_id_from_request_body(request_body) return None + @staticmethod + def _apply_spend_logs_metadata( + kwargs: dict, + passthrough_logging_payload: Optional["PassthroughStandardLoggingPayload"], + ) -> None: + """ + Propagate `spend_logs_metadata` from a pass-through logging payload into + `kwargs["litellm_params"]["metadata"]`. + + OpenAI-compatible endpoints write the value at + `data["metadata"]["spend_logs_metadata"]` before the call; this helper + matches that key path at logging time so downstream + `get_standard_logging_object_payload` surfaces the same field for + pass-through traffic. + """ + if not passthrough_logging_payload: + return + metadata = passthrough_logging_payload.get("spend_logs_metadata") + if not metadata: + return + kwargs.setdefault("litellm_params", {}) + kwargs["litellm_params"].setdefault("metadata", {}) + kwargs["litellm_params"]["metadata"]["spend_logs_metadata"] = metadata + + @staticmethod + def _seed_streaming_kwargs_from_logging_obj( + litellm_logging_obj: LiteLLMLoggingObj, + ) -> dict: + """ + Build the initial `kwargs` dict for streaming logging paths so the + pass-through payload (populated by `success_handler.pass_through_async_success_handler` + before any provider handler runs) survives into the response logging + builder. Without this seed, streaming paths would lose the + `spend_logs_metadata` that non-streaming traffic carries via kwargs. + """ + initial: dict = {} + passthrough_payload = litellm_logging_obj.model_call_details.get( + "passthrough_logging_payload" + ) + if passthrough_payload is not None: + initial["passthrough_logging_payload"] = passthrough_payload + return initial + def _create_response_logging_payload( self, litellm_model_response: Union[ModelResponse, TextCompletionResponse], @@ -118,6 +161,7 @@ class BasePassthroughLoggingHandler(ABC): kwargs["model"] = model passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( # type: ignore kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload") ) if passthrough_logging_payload: user = self._get_user_from_metadata( @@ -129,6 +173,8 @@ class BasePassthroughLoggingHandler(ABC): {"proxy_server_request": {"body": {"user": user}}} ) + self._apply_spend_logs_metadata(kwargs, passthrough_logging_payload) + # Make standard logging object for Anthropic standard_logging_object = get_standard_logging_object_payload( kwargs=kwargs, @@ -209,7 +255,7 @@ class BasePassthroughLoggingHandler(ABC): kwargs = self._create_response_logging_payload( litellm_model_response=complete_streaming_response, model=model, - kwargs={}, + kwargs=self._seed_streaming_kwargs_from_logging_obj(litellm_logging_obj), start_time=start_time, end_time=end_time, logging_obj=litellm_logging_obj, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index adb1278fee5..832b8764f93 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -131,7 +131,10 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): # Extract user information for tracking passthrough_logging_payload: Optional[ PassthroughStandardLoggingPayload - ] = kwargs.get("passthrough_logging_payload") + ] = ( + kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload") + ) if passthrough_logging_payload: user = handler_instance._get_user_from_metadata( passthrough_logging_payload=passthrough_logging_payload, @@ -142,6 +145,10 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): {"proxy_server_request": {"body": {"user": user}}} ) + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, passthrough_logging_payload + ) + # Create standard logging object if litellm_model_response is not None: get_standard_logging_object_payload( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index b05cb70f756..ca4cd13e44f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -17,6 +17,7 @@ from litellm.types.utils import ( TextCompletionResponse, ) + if TYPE_CHECKING: from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType @@ -139,7 +140,12 @@ class GeminiPassthroughLoggingHandler: - Creates standard logging object - Logs in litellm callbacks """ - kwargs: Dict[str, Any] = {} + from .base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 + kwargs: Dict[str, Any] = ( + BasePassthroughLoggingHandler._seed_streaming_kwargs_from_logging_obj( + litellm_logging_obj + ) + ) model = model or GeminiPassthroughLoggingHandler.extract_model_from_url( url_route ) @@ -242,6 +248,13 @@ class GeminiPassthroughLoggingHandler: kwargs["model"] = model kwargs["custom_llm_provider"] = custom_llm_provider + from .base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, + kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload"), + ) + # pretty print standard logging object verbose_proxy_logger.debug("kwargs= %s", kwargs) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index 29bbb37501f..7402681b67a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -367,8 +367,11 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): kwargs["custom_llm_provider"] = custom_llm_provider # Extract user information for tracking - passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = ( + passthrough_logging_payload: Optional[ + PassthroughStandardLoggingPayload + ] = ( kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload") ) if passthrough_logging_payload: user = handler_instance._get_user_from_metadata( @@ -379,6 +382,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): "proxy_server_request", {} ).setdefault("body", {})["user"] = user + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, passthrough_logging_payload + ) + # Create standard logging object if litellm_model_response is not None: get_standard_logging_object_payload( @@ -570,6 +577,10 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): "proxy_server_request", {} ).setdefault("body", {})["user"] = user + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, passthrough_logging_payload + ) + # Create standard logging object get_standard_logging_object_payload( kwargs=kwargs, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index d709956df5c..ce1e928dad7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -341,7 +341,12 @@ class VertexPassthroughLoggingHandler: - Creates standard logging object - Logs in litellm callbacks """ - kwargs: Dict[str, Any] = {} + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 + kwargs: Dict[str, Any] = ( + BasePassthroughLoggingHandler._seed_streaming_kwargs_from_logging_obj( + litellm_logging_obj + ) + ) model = model or VertexPassthroughLoggingHandler.extract_model_from_url( url_route ) @@ -547,6 +552,13 @@ class VertexPassthroughLoggingHandler: kwargs["response_cost"] = response_cost kwargs["model"] = model + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import BasePassthroughLoggingHandler # noqa: PLC0415 + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, + kwargs.get("passthrough_logging_payload") + or logging_obj.model_call_details.get("passthrough_logging_payload"), + ) + # pretty print standard logging object verbose_proxy_logger.debug("kwargs= %s", kwargs) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index d582240395f..fad398b60cf 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -752,11 +752,17 @@ async def pass_through_request( # noqa: PLR0915 params={"timeout": 600}, ) async_client = async_client_obj.client + spend_logs_metadata = ( + LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers( + _safe_get_request_headers(request) + ) + ) passthrough_logging_payload = PassthroughStandardLoggingPayload( url=str(url), request_body=_parsed_body, request_method=getattr(request, "method", None), cost_per_request=cost_per_request, + spend_logs_metadata=spend_logs_metadata, ) kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( user_api_key_dict=user_api_key_dict, @@ -1386,11 +1392,18 @@ async def websocket_passthrough_request( # noqa: PLR0915 ) # Create passthrough logging payload + websocket_headers = dict(websocket.headers) if hasattr(websocket, "headers") else {} + spend_logs_metadata = ( + LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers( + websocket_headers + ) + ) passthrough_logging_payload = PassthroughStandardLoggingPayload( url=target, request_body={}, # WebSocket doesn't have a traditional request body request_method="WEBSOCKET", cost_per_request=cost_per_request, + spend_logs_metadata=spend_logs_metadata, ) # Create a dummy request object for WebSocket connections to maintain compatibility diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 4a07fa5e849..cd6368840e2 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -46,3 +46,12 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False): Optional field, we use this for cost tracking only if it's set. """ + + spend_logs_metadata: Optional[dict] + """ + Parsed `x-litellm-spend-logs-metadata` request header, if present. + + Propagated into `StandardLoggingPayload.metadata.spend_logs_metadata` so + pass-through traffic surfaces the same per-call metadata (e.g. `task`, + `issue`) that OpenAI-compatible endpoints already emit. + """ diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_spend_logs_metadata_propagation.py b/tests/test_litellm/proxy/pass_through_endpoints/test_spend_logs_metadata_propagation.py new file mode 100644 index 00000000000..9aba45f0054 --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_spend_logs_metadata_propagation.py @@ -0,0 +1,135 @@ +"""Tests for `x-litellm-spend-logs-metadata` propagation through pass-through endpoints. + +OpenAI-compatible endpoints parse this header in +`LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers` and +stuff it into `data["metadata"]["spend_logs_metadata"]`. Pass-through endpoints +used to skip this extraction, so `/spend/logs` entries for +`call_type=pass_through_endpoint` always had +`metadata.spend_logs_metadata = None`. + +These tests pin the fix: + +1. Header extraction handles present/missing/malformed input. +2. `BasePassthroughLoggingHandler._apply_spend_logs_metadata` writes the + metadata to the same key path the OpenAI-compatible flow uses. +3. `_seed_streaming_kwargs_from_logging_obj` threads the pass-through payload + from `logging_obj.model_call_details` into fresh streaming kwargs, so + streaming responses retain the metadata. +""" + +import json +from unittest.mock import MagicMock + +import pytest + + +@pytest.mark.parametrize( + "headers,expected", + [ + ({"x-litellm-spend-logs-metadata": json.dumps({"task": "t1"})}, {"task": "t1"}), + ({}, None), + ({"x-litellm-spend-logs-metadata": "not-json"}, None), + ], +) +def test_header_extraction(headers, expected): + """Helper handles valid JSON, missing header, and malformed JSON.""" + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + assert ( + LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) + == expected + ) + + +def test_apply_spend_logs_metadata_writes_matching_key_path(): + """The helper must write to `kwargs["litellm_params"]["metadata"]["spend_logs_metadata"]` + — the same key path OpenAI-compatible flows use, so + `get_standard_logging_object_payload` picks it up identically. + """ + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import ( + BasePassthroughLoggingHandler, + ) + + kwargs: dict = {} + metadata = {"task": "cost-report", "issue": "#42"} + + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, + {"url": "x", "spend_logs_metadata": metadata}, + ) + + assert kwargs["litellm_params"]["metadata"]["spend_logs_metadata"] == metadata + + +def test_apply_spend_logs_metadata_preserves_existing_metadata(): + """Existing metadata keys must not be clobbered when attaching + `spend_logs_metadata`. + """ + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import ( + BasePassthroughLoggingHandler, + ) + + kwargs: dict = {"litellm_params": {"metadata": {"tags": ["prod"]}}} + + BasePassthroughLoggingHandler._apply_spend_logs_metadata( + kwargs, + {"url": "x", "spend_logs_metadata": {"task": "t1"}}, + ) + + assert kwargs["litellm_params"]["metadata"] == { + "tags": ["prod"], + "spend_logs_metadata": {"task": "t1"}, + } + + +@pytest.mark.parametrize( + "payload", + [None, {}, {"url": "x"}, {"url": "x", "spend_logs_metadata": None}], +) +def test_apply_spend_logs_metadata_is_noop_when_missing(payload): + """When the payload is missing or carries no metadata, the helper must + leave kwargs untouched — this is the path for existing callers that don't + send the header, and it must remain backward-compatible. + """ + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import ( + BasePassthroughLoggingHandler, + ) + + kwargs: dict = {} + BasePassthroughLoggingHandler._apply_spend_logs_metadata(kwargs, payload) + assert kwargs == {} + + +def test_seed_streaming_kwargs_copies_payload_from_logging_obj(): + """Streaming logging paths build fresh kwargs; the seed helper must lift + the pass-through payload stashed by `success_handler` onto that fresh dict + so `_create_*_response_logging_payload` can still see it. + """ + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import ( + BasePassthroughLoggingHandler, + ) + + payload = {"url": "x", "spend_logs_metadata": {"task": "t1"}} + logging_obj = MagicMock(model_call_details={"passthrough_logging_payload": payload}) + + seeded = BasePassthroughLoggingHandler._seed_streaming_kwargs_from_logging_obj( + logging_obj + ) + + assert seeded == {"passthrough_logging_payload": payload} + + +def test_seed_streaming_kwargs_empty_when_no_payload(): + """No pass-through payload on `logging_obj` → empty dict (prior behavior).""" + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import ( + BasePassthroughLoggingHandler, + ) + + logging_obj = MagicMock(model_call_details={}) + + assert ( + BasePassthroughLoggingHandler._seed_streaming_kwargs_from_logging_obj( + logging_obj + ) + == {} + )