mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): bill Vercel AI Gateway decisions at the charge Vercel reports
Backport of #50642 to lts/1.108.x.
Cherry-picked from 7c40b75f56 (mishdotdev:vercel-ai-gateway-decisions).
Adapted to the line: its decisions usage is never None, so the None guards are dropped, and the resolve_credential override stays out as in the first pick.
This commit is contained in:
parent
5703b6295e
commit
3606c8ec07
6 changed files with 126 additions and 9 deletions
|
|
@ -461,6 +461,13 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
|
|||
return is_decisions_model_mode(model_mode)
|
||||
|
||||
|
||||
def usage_reported_cost(response: DecisionsIRResponse) -> float | None:
|
||||
cost: Final = response.usage.extra.get("cost")
|
||||
if not isinstance(cost, (int, float)):
|
||||
return None
|
||||
return float(cost)
|
||||
|
||||
|
||||
class BaseDecisionsConfig(ABC):
|
||||
path: str = "/v1/systemone"
|
||||
api_key_env: tuple[str, ...] = ()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, usage_reported_cost
|
||||
from litellm.types.decisions import DecisionsIRResponse
|
||||
|
||||
|
||||
|
|
@ -13,7 +11,4 @@ class OpenRouterDecisionsConfig(BaseDecisionsConfig):
|
|||
return "https://openrouter.ai/api"
|
||||
|
||||
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
|
||||
cost: Final = response.usage.extra.get("cost")
|
||||
if not isinstance(cost, (int, float)):
|
||||
return None
|
||||
return float(cost)
|
||||
return usage_reported_cost(response)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,36 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import usage_reported_cost
|
||||
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
|
||||
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
|
||||
|
||||
|
||||
class _GatewayCharge(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
cost: float | None = None
|
||||
|
||||
|
||||
class _GatewayProviderMetadata(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
gateway: _GatewayCharge = _GatewayCharge()
|
||||
|
||||
|
||||
class _GatewayDecisionResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
provider_metadata: _GatewayProviderMetadata = _GatewayProviderMetadata()
|
||||
|
||||
|
||||
def _gateway_charge(payload: object) -> float | None:
|
||||
try:
|
||||
return _GatewayDecisionResponse.model_validate(payload).provider_metadata.gateway.cost
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
|
||||
|
|
@ -6,3 +38,13 @@ class VercelAIGatewayDecisionsConfig(OpenAIDecisionsConfig):
|
|||
|
||||
def resolve_api_key(self, api_key: str | None) -> str | None:
|
||||
return api_key
|
||||
|
||||
def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
|
||||
response: Final = super().parse_response(payload, request)
|
||||
charge: Final = _gateway_charge(payload)
|
||||
if charge is None:
|
||||
return response
|
||||
return replace(response, usage=replace(response.usage, extra={**response.usage.extra, "cost": charge}))
|
||||
|
||||
def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None:
|
||||
return usage_reported_cost(response)
|
||||
|
|
|
|||
|
|
@ -51273,7 +51273,8 @@
|
|||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models"
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
|
|
@ -51305,6 +51306,7 @@
|
|||
"text",
|
||||
"image"
|
||||
],
|
||||
"supports_box_questions": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/vercel/v0-1.0-md": {
|
||||
|
|
|
|||
|
|
@ -51273,7 +51273,8 @@
|
|||
},
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models"
|
||||
"source": "https://ai-gateway.vercel.sh/v1/models",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"vercel_ai_gateway/typesafe-ai/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
|
|
@ -51305,6 +51306,7 @@
|
|||
"text",
|
||||
"image"
|
||||
],
|
||||
"supports_box_questions": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/vercel/v0-1.0-md": {
|
||||
|
|
|
|||
|
|
@ -3397,3 +3397,72 @@ async def test_vercel_ai_gateway_decisions_never_fall_back_to_the_openai_key(
|
|||
)
|
||||
|
||||
assert len(respx_mock.calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "gateway_cost", "reported_cost"),
|
||||
(
|
||||
("example/unpriced-decider", "0.000001386", 1.386e-06),
|
||||
("openai/gpt-6-luna-decisions", "0.00005722", 5.722e-05),
|
||||
("openai/gpt-6-luna-decisions", "not-a-number", None),
|
||||
),
|
||||
ids=("unpriced_model", "cache_hit_on_priced_model", "unreadable_charge"),
|
||||
)
|
||||
async def test_vercel_ai_gateway_decisions_bill_the_charge_the_gateway_reports(
|
||||
model: str,
|
||||
gateway_cost: str,
|
||||
reported_cost: float | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
|
||||
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
|
||||
respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
|
||||
json={
|
||||
"model": model,
|
||||
"answers": [{"type": "predicate", "name": "is_defect", "probability": 0.9}],
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
|
||||
"provider_metadata": {"gateway": {"cost": gateway_cost}},
|
||||
}
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=f"vercel_ai_gateway/{model}",
|
||||
input="The package arrived broken.",
|
||||
questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}],
|
||||
)
|
||||
|
||||
assert response.answers[0].probability == 0.9
|
||||
assert get_response_cost_from_hidden_params(response.hidden_params) == reported_cost
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ("typesafe-ai/jev", "typesafe-ai/jev-1.14.0"))
|
||||
async def test_vercel_ai_gateway_jev_sends_box_questions(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.delenv("VERCEL_AI_GATEWAY_API_BASE", raising=False)
|
||||
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "gateway-key")
|
||||
box_question: Final = {"type": "box", "name": "crack", "instructions": "the crack on the screen"}
|
||||
route: Final = respx_mock.post("https://ai-gateway.vercel.sh/v1/decisions").respond(
|
||||
json={
|
||||
"model": model,
|
||||
"answers": [
|
||||
{
|
||||
"type": "box",
|
||||
"name": "crack",
|
||||
"box": {"top": 0.1, "bottom": 0.9, "left": 0.1, "right": 0.9},
|
||||
"confidence": 0.8,
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0, "total_tokens": _INPUT_TOKENS},
|
||||
}
|
||||
)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model=f"vercel_ai_gateway/{model}", input="The screen arrived cracked.", questions=[box_question]
|
||||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["questions"] == [box_question]
|
||||
assert response.answers[0].confidence == 0.8
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue