From 8ff495be9624adbb45fd0e4c829d22f401ddab75 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 31 Oct 2026 07:34:29 -0700 Subject: [PATCH] fix(proxy): price OpenAI pass-through /v1/decisions calls like native Decisions (#49392) The OpenAI pass-through dispatcher kept its own list of endpoints it could price, separate from the handler, and /v1/decisions was on neither. Calls reached OpenAI but logged $0 and no tokens. The handler now owns the list the dispatcher reads, and prices a POST to /v1/decisions through the same Decisions response type and cost path native /v1/decisions uses, so the _decisions cost-map rates apply. Reads and sub-routes under /v1/decisions stay unbilled. Fixes #48543 --- litellm/proxy/_types.py | 2 + .../openai_passthrough_logging_handler.py | 60 ++++++++++++--- .../pass_through_endpoints/success_handler.py | 19 +---- ...test_openai_passthrough_logging_handler.py | 75 +++++++++++++++++++ 4 files changed, 126 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 173805543e8..7b7bfc55ade 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -45,6 +45,7 @@ from litellm.types.agents import ( from litellm.types.agents import ( AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export ) +from litellm.types.decisions import OpenAIDecisionResponse from litellm.types.integrations.compression_interception import ( CompressionSavingsMetadata, ) @@ -4601,6 +4602,7 @@ PassThroughEndpointLoggingResultValues = ( | StandardPassThroughResponseObject | ResponsesAPIResponse | TranscriptionResponse + | OpenAIDecisionResponse ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index b98dd42e3de..aabb83b3a5e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) +from litellm.types.decisions import OpenAIDecisionResponse from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, @@ -277,6 +278,22 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): def is_openai_agents_route(url_route: str) -> bool: return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path) + @staticmethod + def is_openai_decisions_route(url_route: str) -> bool: + return is_openai_compatible_url(url_route) and urlparse(url_route).path.rstrip("/").endswith("/v1/decisions") + + @staticmethod + def handles_route(url_route: str) -> bool: + return ( + OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) + or OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route) + ) + @staticmethod def _unbilled_call_model( url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object] @@ -479,8 +496,19 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): httpx_response.request.method, url_route, response_body ) is_responses: Final = responses_billing != "unbilled" + is_decisions: Final = ( + httpx_response.request.method == "POST" + and OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route) + ) - if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses): + if not ( + is_chat_completions + or is_embeddings + or is_image_generation + or is_image_editing + or is_responses + or is_decisions + ): OpenAIPassthroughLoggingHandler._label_unbilled_call( logging_obj, url_route, @@ -512,7 +540,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): try: response_cost = 0.0 litellm_model_response: ( - ModelResponse | TextCompletionResponse | EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None + ModelResponse + | TextCompletionResponse + | EmbeddingResponse + | ImageResponse + | ResponsesAPIResponse + | OpenAIDecisionResponse + | None ) = None handler_instance: Final = OpenAIPassthroughLoggingHandler() @@ -564,11 +598,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): response_body=response_body, request_body=request_body, ) - # Mark call type for downstream image-aware logic/metrics - try: - logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value - except Exception: - pass + logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value # Create a simple response object for logging litellm_model_response = ImageResponse( data=response_body.get("data", []), @@ -583,11 +613,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): response_body=response_body, request_body=request_body, ) - # Mark call type for downstream image-aware logic/metrics - try: - logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value - except Exception: - pass + logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value # Create a simple response object for logging litellm_model_response = ImageResponse( data=response_body.get("data", []), @@ -609,6 +635,14 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, ) + elif is_decisions: + litellm_model_response = OpenAIDecisionResponse.model_validate(response_body) # rebind-ok: elif-chain + response_cost = litellm.completion_cost( # rebind-ok: elif-chain + completion_response=litellm_model_response, + model=model, + custom_llm_provider=custom_llm_provider, + call_type="decisions", + ) # Update kwargs with cost information kwargs["response_cost"] = response_cost @@ -656,6 +690,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): if is_image_generation else "image_editing" if is_image_editing + else "decisions" + if is_decisions else "responses" ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 1f4e7b6df95..8b09b1cf7cd 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -666,28 +666,11 @@ class PassThroughEndpointLogging: return False def _is_supported_openai_endpoint(self, url_route: str) -> bool: - """Check if the OpenAI endpoint is supported by the passthrough logging handler. - - The Responses API route is included because - `openai_passthrough_handler` has a dedicated `elif is_responses:` - branch that knows how to extract usage + cost from the - Responses-API on-the-wire shape. Without including it here, the - outer dispatch filters Responses calls out before reaching the - handler — the inner branch is then unreachable and Responses - calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend. - """ from .llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, ) - return ( - OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route) - or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route) - or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route) - or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route) - or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route) - or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route) - ) + return OpenAIPassthroughLoggingHandler.handles_route(url_route) def _set_cost_per_request( self, diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py index a4f3fd0a86f..bb4b41fd09d 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py @@ -2585,6 +2585,81 @@ class TestOpenAIPassthroughResponsesSubRouteBilling: assert compact["kwargs"]["response_cost"] > 0 +DECISIONS_URL: Final = "https://api.openai.com/v1/decisions" +DECISIONS_MODEL: Final = "gpt-6-luna" +DECISIONS_INPUT_TOKENS: Final = 1000 +DECISIONS_OUTPUT_TOKENS: Final = 7 +DECISIONS_REQUEST: Final = { + "input": "The package arrived crushed and the screen is cracked.", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is the product defective?"}], +} +DECISION: Final = { + "model": DECISIONS_MODEL, + "answers": [{"type": "predicate", "name": "is_defect", "probability": 0.97}], + "usage": { + "input_tokens": DECISIONS_INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": DECISIONS_OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": DECISIONS_INPUT_TOKENS + DECISIONS_OUTPUT_TOKENS, + }, +} + + +class TestOpenAIPassthroughDecisionsSpendLog: + @staticmethod + async def _spend_row(method: str, url_route: str, request_body: dict[str, object]) -> SpendLogsPayload: + logged: Final = await TestOpenAIPassthroughResponsesSubRouteBilling()._logged_call( + method, "", request_body, DECISION, "4e9b2c71-8a3d-4f5e-b6c0-1d2e3f4a5b6c", url_route + ) + return logged.spend_row + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("url_route", "requested_model"), + [ + pytest.param(DECISIONS_URL, DECISIONS_MODEL, id="openai"), + pytest.param(f"{DECISIONS_URL}/", DECISIONS_MODEL, id="trailing_slash"), + pytest.param(DECISIONS_URL, "luna-prod-alias", id="unpriced_alias_billed_as_served_model"), + ], + ) + async def test_decisions_passthrough_spend_log_is_priced_like_native_decisions( + self, local_model_cost_map, url_route: str, requested_model: str + ): + decisions_rates: Final = litellm.model_cost[DECISIONS_MODEL] + + spend_log_row: Final = await self._spend_row( + "POST", url_route, {**DECISIONS_REQUEST, "model": requested_model} + ) + + assert spend_log_row["prompt_tokens"] == DECISIONS_INPUT_TOKENS + assert spend_log_row["completion_tokens"] == DECISIONS_OUTPUT_TOKENS + assert spend_log_row["spend"] == pytest.approx( + DECISIONS_INPUT_TOKENS * decisions_rates["input_cost_per_token_decisions"] + + DECISIONS_OUTPUT_TOKENS * decisions_rates["output_cost_per_token_decisions"] + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("method", "url_route"), + [ + pytest.param("GET", DECISIONS_URL, id="list"), + pytest.param("GET", f"{DECISIONS_URL}/dec_6a0b0f8e", id="retrieve"), + pytest.param("POST", f"{DECISIONS_URL}/dec_6a0b0f8e/cancel", id="sub_route"), + ], + ) + async def test_decisions_reads_and_sub_routes_are_not_billed( + self, local_model_cost_map, caplog: pytest.LogCaptureFixture, method: str, url_route: str + ): + with caplog.at_level(logging.WARNING): + spend_log_row: Final = await self._spend_row( + method, url_route, {**DECISIONS_REQUEST, "model": DECISIONS_MODEL} + ) + + assert not spend_log_row["spend"] + assert [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] == [] + + if __name__ == "__main__": pytest.main([__file__])