scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

fix(openrouter): bill responses, decisions and pass-through from usage.cost and always allow reasoning params (#45640)

Browse source 
Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-09 13:28:31 -07:00 • committed by GitHub
parent 05166e2460
commit 465f20f274
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 240 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/decisions/main.py
View file

@ -7,6 +7,7 @@ from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
@ -205,11 +206,13 @@ def _prepare_call(
def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
formatted: Final = _formatted_response(response, call)
provider_cost: Final = call.provider_config.provider_reported_cost(response)
formatted.set_hidden_params(
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)
return formatted

3
litellm/llms/base_llm/decisions/transformation.py
View file

@ -308,6 +308,9 @@ class BaseDecisionsConfig(ABC):
def get_default_api_base(self) -> str | None:
return None
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
return None
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return f"api_base is required for Decisions provider '{custom_llm_provider}'"

13
litellm/llms/openrouter/chat/transformation.py
View file

@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
import litellm
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -41,18 +40,10 @@ class CacheControlSupportedModels(str, Enum):
class OpenrouterConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
"""
Allow reasoning parameters for models flagged as reasoning-capable.
Include reasoning parameters regardless of model-map capabilities.
"""
supported_params: Final = super().get_supported_openai_params(model=model)
try:
if litellm.supports_reasoning(model=model, custom_llm_provider="openrouter") or litellm.supports_reasoning(
model=model
):
supported_params.append("reasoning_effort")
supported_params.append("thinking")
except Exception:
pass
return list(dict.fromkeys(supported_params))
return list(dict.fromkeys([*supported_params, "reasoning_effort", "thinking"]))
def map_openai_params(
self,

9
litellm/llms/openrouter/decisions/transformation.py
View file

@ -1,4 +1,7 @@
from typing import Final
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.types.decisions import DecisionsIRResponse
class OpenRouterDecisionsConfig(BaseDecisionsConfig):
@ -8,3 +11,9 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig):
def get_default_api_base(self) -> str | None:
return "https://openrouter.ai/api"
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
cost: Final = response.usage.extra.get("cost")
if not isinstance(cost, (int, float)):
return None
return float(cost)

70
litellm/llms/openrouter/responses/transformation.py
View file

@ -8,11 +8,25 @@ encrypted_content for multi-turn stateless workflows.
Docs: https://openrouter.ai/docs/api/reference/responses/overview
"""
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponseFailedEvent,
ResponseIncompleteEvent,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
@ -33,6 +47,62 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.OPENROUTER
def transform_response_api_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
response: Final = super().transform_response_api_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
)
self._set_provider_reported_cost(response)
return response
def transform_streaming_response(
self,
model: str,
parsed_chunk: Mapping[str, object],
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIStreamingResponse:
response_event: Final = super().transform_streaming_response(
model=model,
parsed_chunk=dict(parsed_chunk),
logging_obj=logging_obj,
)
terminal_response: Final = (
response_event.response
if isinstance(
response_event,
(ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent),
)
else None
)
if terminal_response is not None:
self._set_provider_reported_cost(terminal_response)
return response_event
@staticmethod
def _set_provider_reported_cost(response: ResponsesAPIResponse) -> None:
cost: Final = response.usage.cost if response.usage is not None else None
if cost is None:
return
additional_headers: Final = TypeAdapter(dict[str, object]).validate_python(
response.hidden_params.get("additional_headers", {})
)
set_hidden_params(
response,
{
**response.hidden_params,
"additional_headers": {
**additional_headers,
RESPONSE_COST_HEADER: float(cost),
},
},
)
def validate_environment(
self,
headers: dict,

5
litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py
View file

@ -21,6 +21,7 @@ if TYPE_CHECKING:
class _TypeSafeUsage(LiteLLMBaseModel):
input_tokens: int = 0
output_tokens: int = 0
cost: float | None = None
class _TypeSafeResponse(LiteLLMBaseModel):
@ -88,7 +89,9 @@ class TypeSafePassthroughLoggingHandler:
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (
input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
usage.cost
if custom_llm_provider == "openrouter" and usage.cost is not None
else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
)
usage_object: Final = Usage(
prompt_tokens=input_tokens,

35
tests/integration/providers/test_decisions_wire.py
View file

@ -69,6 +69,7 @@ class _Provider:
api_key: str | None
wraps_result: bool
cost_map_key: str | None
provider_reported_cost: float | None = None
_PROVIDERS: Final = (
@ -89,7 +90,8 @@ _PROVIDERS: Final = (
"typesafe/jev-1.13",
_API_KEY,
False,
"openrouter/typesafe/jev-1.13",
None,
1.5834e-5,
),
_Provider(
"strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None
@ -133,17 +135,32 @@ def _number(value: JsonValue) -> float:
return float(value)
def _expected_spend(cost_map_key: str | None) -> float:
if cost_map_key is None:
def _expected_spend(provider: _Provider) -> float:
if provider.provider_reported_cost is not None:
return provider.provider_reported_cost
if provider.cost_map_key is None:
return 0.0
prices: Final = object_value(json.loads(Path("model_prices_and_context_window.json").read_text())[cost_map_key])
prices: Final = object_value(
json.loads(Path("model_prices_and_context_window.json").read_text())[provider.cost_map_key]
)
return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number(
_USAGE["output_tokens"]
) * _number(prices["output_cost_per_token"])
def _usage(provider: _Provider) -> dict[str, JsonValue]:
return {
**_USAGE,
**({"cost": provider.provider_reported_cost} if provider.provider_reported_cost is not None else {}),
}
def _answer_body(provider: _Provider) -> dict[str, JsonValue]:
answer: Final[dict[str, JsonValue]] = {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE}
answer: Final[dict[str, JsonValue]] = {
"model": provider.body_model,
"answers": _ANSWERS,
"usage": _usage(provider),
}
return {"result": answer, "success": True} if provider.wraps_result else answer
@ -226,16 +243,14 @@ def _pass_through_config(directory: Path, pass_through_target: str, native_api_b
@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name)
def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map(
gateway: Gateway, provider: _Provider
) -> None:
expected_spend: Final = _expected_spend(provider.cost_map_key)
def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None:
expected_spend: Final = _expected_spend(provider)
with gateway.scenario() as scenario:
handle: Final = _register(scenario, _answer_body(provider))
model: Final = _deployment(scenario, handle, provider)
response: Final = _decide(gateway, model)
assert response.status_code == 200, response.text
assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _USAGE}
assert response.json() == {"model": provider.body_model, "answers": _ANSWERS, "usage": _usage(provider)}
assert response.headers["x-litellm-model-group"] == model
assert math.isclose(float(response.headers.get("x-litellm-response-cost", "0")), expected_spend, rel_tol=1e-9)
(call,) = _upstream_calls(gateway, handle)

31
tests/unit/decisions/test_main.py
View file

@ -11,6 +11,7 @@ import pytest
import respx
import litellm
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.decisions import (
@ -1055,3 +1056,33 @@ async def test_requests_a_provider_cannot_serve_are_rejected_before_http(
await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs)
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
async def test_openrouter_decisions_uses_provider_reported_cost_without_cost_map(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False)
cost: Final = 1.1802e-5
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(
json={
"model": "typesafe/jev-1.13",
"answers": _RESPONSE["answers"],
"usage": {
"input_tokens": _INPUT_TOKENS,
"output_tokens": _OUTPUT_TOKENS,
"cost": cost,
},
}
)
response: Final = await litellm.adecisions(
model="openrouter/typesafe/jev-1.13",
state={"source": "unit-test"},
questions=_QUESTIONS,
api_key="caller-key",
)
assert route.called
assert get_response_cost_from_hidden_params(response.hidden_params) == cost

17
tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py
View file

@ -96,6 +96,15 @@ def test_openrouter_extra_body_transformation():
]
def test_openrouter_supported_params_include_reasoning_for_unmapped_model():
supported_params = OpenrouterConfig().get_supported_openai_params(
"openrouter/unmapped-model"
)
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
def test_openrouter_cache_control_flag_removal():
transformed_request = OpenrouterConfig().transform_request(
model="openrouter/deepseek/deepseek-chat",
@ -537,17 +546,15 @@ def test_openrouter_reasoning_models_allow_reasoning_effort_param():
assert supported_params.count("reasoning_effort") == 1
def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort():
"""
Models without reasoning support should not gain reasoning-specific params.
"""
def test_openrouter_non_reasoning_models_support_reasoning_params():
config = OpenrouterConfig()
supported_params = config.get_supported_openai_params(
model="openrouter/anthropic/claude-3-5-haiku"
)
assert "reasoning_effort" not in supported_params
assert "reasoning_effort" in supported_params
assert "thinking" in supported_params
def test_openrouter_reasoning_effort_max_maps_to_xhigh():

73
tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py
View file

@ -9,12 +9,22 @@ reasoning.encrypted_content for multi-turn stateless workflows.
Related issue: https://github.com/BerriAI/litellm/issues/22189
"""
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.cost_calculator import get_response_cost_from_hidden_params
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.llms.openrouter.responses.transformation import (
OpenRouterResponsesAPIConfig,
)
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponseFailedEvent,
ResponseIncompleteEvent,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -116,3 +126,66 @@ class TestOpenRouterResponsesAPIRegistration:
# The URL should point to OpenRouter's responses endpoint
url = config.get_complete_url(api_base=None, litellm_params={})
assert "/responses" in url
@pytest.mark.parametrize("cost", [1.23e-5, None])
def test_openrouter_responses_returns_provider_reported_cost(cost):
usage: dict[str, object] = {
"input_tokens": 2,
"output_tokens": 3,
"total_tokens": 5,
}
if cost is not None:
usage["cost"] = cost
response = OpenRouterResponsesAPIConfig().transform_response_api_response(
model="openrouter/unmapped-model",
raw_response=httpx.Response(
200,
json={
"id": "resp_test",
"created_at": 1,
"model": "unmapped-model",
"object": "response",
"output": [],
"status": "completed",
"usage": usage,
},
),
logging_obj=MagicMock(),
)
assert get_response_cost_from_hidden_params(response.hidden_params) == cost
@pytest.mark.parametrize(
"event_type",
["response.completed", "response.incomplete", "response.failed"],
)
def test_openrouter_terminal_stream_events_return_provider_reported_cost(event_type):
event = OpenRouterResponsesAPIConfig().transform_streaming_response(
model="openrouter/unmapped-model",
parsed_chunk={
"type": event_type,
"response": {
"id": "resp_test",
"created_at": 1,
"model": "unmapped-model",
"object": "response",
"output": [],
"status": "completed",
"usage": {
"input_tokens": 2,
"output_tokens": 3,
"total_tokens": 5,
"cost": 1.23e-5,
},
},
},
logging_obj=MagicMock(),
)
assert isinstance(
event,
(ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent),
)
assert event.response.hidden_params["additional_headers"][RESPONSE_COST_HEADER] == 1.23e-5

14
tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py
View file

@ -14,7 +14,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndp
@pytest.fixture(autouse=True)
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
@ -222,14 +222,17 @@ async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost(
await budget_limiter.is_key_within_model_budget(auth, f"{provider}/{requested}")
def test_openrouter_decisions_response_is_priced_from_request_model_registry_row():
def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row(
monkeypatch: pytest.MonkeyPatch,
):
logging_obj = _logging_obj()
model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"]
cost: Final = 1.5834e-5
monkeypatch.delitem(litellm.model_cost, "openrouter/typesafe/jev-1.13", raising=False)
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={
"model": "typesafe/jev-1.13-20260917",
"usage": {"input_tokens": 282, "output_tokens": 20},
"usage": {"input_tokens": 282, "output_tokens": 20, "cost": cost},
},
logging_obj=logging_obj,
url_route="https://openrouter.ai/api/alpha/decisions",
@ -241,10 +244,9 @@ def test_openrouter_decisions_response_is_priced_from_request_model_registry_row
custom_llm_provider="openrouter",
)
expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"]
assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917"
assert response["kwargs"]["custom_llm_provider"] == "openrouter"
assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost)
assert response["kwargs"]["response_cost"] == cost
assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282
assert response["kwargs"]["combined_usage_object"].completion_tokens == 20
assert response["kwargs"]["combined_usage_object"].total_tokens == 302