scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix(proxy): price OpenAI pass-through /v1/decisions calls like native Decisions (#49392)

Browse source 
The OpenAI pass-through dispatcher kept its own list of endpoints it could price,
separate from the handler, and /v1/decisions was on neither. Calls reached OpenAI
but logged $0 and no tokens.

The handler now owns the list the dispatcher reads, and prices a POST to
/v1/decisions through the same Decisions response type and cost path native
/v1/decisions uses, so the _decisions cost-map rates apply. Reads and sub-routes
under /v1/decisions stay unbilled.

Fixes #48543
This commit is contained in:
ryan-crabbe-berri 2026-10-31 07:34:29 -07:00 • committed by GitHub
parent be4f3d36ce
commit 8ff495be96
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 126 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_types.py
View file

@ -45,6 +45,7 @@ from litellm.types.agents import (
from litellm.types.agents import (
AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export
)
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.integrations.compression_interception import (
CompressionSavingsMetadata,
)
@ -4601,6 +4602,7 @@ PassThroughEndpointLoggingResultValues = (
| StandardPassThroughResponseObject
| ResponsesAPIResponse
| TranscriptionResponse
| OpenAIDecisionResponse
)

60
litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
View file

@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
@ -277,6 +278,22 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
def is_openai_agents_route(url_route: str) -> bool:
return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path)
@staticmethod
def is_openai_decisions_route(url_route: str) -> bool:
return is_openai_compatible_url(url_route) and urlparse(url_route).path.rstrip("/").endswith("/v1/decisions")
@staticmethod
def handles_route(url_route: str) -> bool:
return (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
)
@staticmethod
def _unbilled_call_model(
url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object]
@ -479,8 +496,19 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
httpx_response.request.method, url_route, response_body
)
is_responses: Final = responses_billing != "unbilled"
is_decisions: Final = (
httpx_response.request.method == "POST"
and OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
)
if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses):
if not (
is_chat_completions
or is_embeddings
or is_image_generation
or is_image_editing
or is_responses
or is_decisions
):
OpenAIPassthroughLoggingHandler._label_unbilled_call(
logging_obj,
url_route,
@ -512,7 +540,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
try:
response_cost = 0.0
litellm_model_response: (
ModelResponse | TextCompletionResponse | EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None
ModelResponse
| TextCompletionResponse
| EmbeddingResponse
| ImageResponse
| ResponsesAPIResponse
| OpenAIDecisionResponse
| None
) = None
handler_instance: Final = OpenAIPassthroughLoggingHandler()
@ -564,11 +598,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
response_body=response_body,
request_body=request_body,
)
# Mark call type for downstream image-aware logic/metrics
try:
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
except Exception:
pass
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
# Create a simple response object for logging
litellm_model_response = ImageResponse(
data=response_body.get("data", []),
@ -583,11 +613,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
response_body=response_body,
request_body=request_body,
)
# Mark call type for downstream image-aware logic/metrics
try:
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
except Exception:
pass
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
# Create a simple response object for logging
litellm_model_response = ImageResponse(
data=response_body.get("data", []),
@ -609,6 +635,14 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
)
elif is_decisions:
litellm_model_response = OpenAIDecisionResponse.model_validate(response_body) # rebind-ok: elif-chain
response_cost = litellm.completion_cost( # rebind-ok: elif-chain
completion_response=litellm_model_response,
model=model,
custom_llm_provider=custom_llm_provider,
call_type="decisions",
)
# Update kwargs with cost information
kwargs["response_cost"] = response_cost
@ -656,6 +690,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if is_image_generation
else "image_editing"
if is_image_editing
else "decisions"
if is_decisions
else "responses"
)
verbose_proxy_logger.debug(

19
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -666,28 +666,11 @@ class PassThroughEndpointLogging:
return False
def _is_supported_openai_endpoint(self, url_route: str) -> bool:
"""Check if the OpenAI endpoint is supported by the passthrough logging handler.
The Responses API route is included because
`openai_passthrough_handler` has a dedicated `elif is_responses:`
branch that knows how to extract usage + cost from the
Responses-API on-the-wire shape. Without including it here, the
outer dispatch filters Responses calls out before reaching the
handler — the inner branch is then unreachable and Responses
calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
"""
from .llm_provider_handlers.openai_passthrough_logging_handler import (
OpenAIPassthroughLoggingHandler,
)
return (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
)
return OpenAIPassthroughLoggingHandler.handles_route(url_route)
def _set_cost_per_request(
self,

75
tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py
View file

@ -2585,6 +2585,81 @@ class TestOpenAIPassthroughResponsesSubRouteBilling:
assert compact["kwargs"]["response_cost"] > 0
DECISIONS_URL: Final = "https://api.openai.com/v1/decisions"
DECISIONS_MODEL: Final = "gpt-6-luna"
DECISIONS_INPUT_TOKENS: Final = 1000
DECISIONS_OUTPUT_TOKENS: Final = 7
DECISIONS_REQUEST: Final = {
"input": "The package arrived crushed and the screen is cracked.",
"questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is the product defective?"}],
}
DECISION: Final = {
"model": DECISIONS_MODEL,
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.97}],
"usage": {
"input_tokens": DECISIONS_INPUT_TOKENS,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens": DECISIONS_OUTPUT_TOKENS,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": DECISIONS_INPUT_TOKENS + DECISIONS_OUTPUT_TOKENS,
},
}
class TestOpenAIPassthroughDecisionsSpendLog:
@staticmethod
async def _spend_row(method: str, url_route: str, request_body: dict[str, object]) -> SpendLogsPayload:
logged: Final = await TestOpenAIPassthroughResponsesSubRouteBilling()._logged_call(
method, "", request_body, DECISION, "4e9b2c71-8a3d-4f5e-b6c0-1d2e3f4a5b6c", url_route
)
return logged.spend_row
@pytest.mark.asyncio
@pytest.mark.parametrize(
("url_route", "requested_model"),
[
pytest.param(DECISIONS_URL, DECISIONS_MODEL, id="openai"),
pytest.param(f"{DECISIONS_URL}/", DECISIONS_MODEL, id="trailing_slash"),
pytest.param(DECISIONS_URL, "luna-prod-alias", id="unpriced_alias_billed_as_served_model"),
],
)
async def test_decisions_passthrough_spend_log_is_priced_like_native_decisions(
self, local_model_cost_map, url_route: str, requested_model: str
):
decisions_rates: Final = litellm.model_cost[DECISIONS_MODEL]
spend_log_row: Final = await self._spend_row(
"POST", url_route, {**DECISIONS_REQUEST, "model": requested_model}
)
assert spend_log_row["prompt_tokens"] == DECISIONS_INPUT_TOKENS
assert spend_log_row["completion_tokens"] == DECISIONS_OUTPUT_TOKENS
assert spend_log_row["spend"] == pytest.approx(
DECISIONS_INPUT_TOKENS * decisions_rates["input_cost_per_token_decisions"]
+ DECISIONS_OUTPUT_TOKENS * decisions_rates["output_cost_per_token_decisions"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("method", "url_route"),
[
pytest.param("GET", DECISIONS_URL, id="list"),
pytest.param("GET", f"{DECISIONS_URL}/dec_6a0b0f8e", id="retrieve"),
pytest.param("POST", f"{DECISIONS_URL}/dec_6a0b0f8e/cancel", id="sub_route"),
],
)
async def test_decisions_reads_and_sub_routes_are_not_billed(
self, local_model_cost_map, caplog: pytest.LogCaptureFixture, method: str, url_route: str
):
with caplog.at_level(logging.WARNING):
spend_log_row: Final = await self._spend_row(
method, url_route, {**DECISIONS_REQUEST, "model": DECISIONS_MODEL}
)
assert not spend_log_row["spend"]
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] == []
if __name__ == "__main__":
pytest.main([__file__])