This commit is contained in:
ishaan-j-berri 2026-10-24 17:13:45 +00:00 • committed by GitHub
commit 3322bfe995
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 2573 additions and 10 deletions

View file

@ -3,9 +3,9 @@
import logging
import math
import time
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from functools import lru_cache, partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -124,9 +124,10 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
)
from litellm.types.llms.xai import RealtimeMeteredUsage
from litellm.types.rerank import RerankBilledUnits, RerankResponse
from litellm.types.rerank import DecisionRerankResponse, RerankBilledUnits, RerankResponse
from litellm.types.utils import (
CallTypesLiteral,
CostBreakdown,
LiteLLMRealtimeStreamLoggingObject,
LlmProviders,
LlmProvidersSet,
@ -1500,6 +1501,77 @@ def _response_time_ms_for_cost(
return 0.0
def _decision_rerank_breakdown(
breakdowns: Sequence[CostBreakdown | None], repeated_fixed: float
) -> CostBreakdown | None:
complete: Final = tuple(item for item in breakdowns if item is not None)
if not complete or len(complete) != len(breakdowns):
return None
monetary_fields: Final = (
"input_cost",
"cache_read_cost",
"cache_creation_cost",
"output_cost",
"reasoning_cost",
"total_cost",
"tool_usage_cost",
"guardrail_cost",
"original_cost",
"discount_amount",
"margin_total_amount",
)
totals: Final = {
key: sum(value for item in complete if isinstance(value := item.get(key), (int, float)))
for key in monetary_fields
if any(key in item for item in complete)
}
additional_keys: Final = frozenset[str]().union(*(item.get("additional_costs", {}) for item in complete))
return TypeAdapter(CostBreakdown).validate_python(
{
**complete[-1],
**totals,
"additional_costs": {
key: sum(item.get("additional_costs", {}).get(key, 0.0) for item in complete) for key in additional_keys
},
**{key: totals[key] - repeated_fixed for key in ("total_cost", "margin_total_amount") if key in totals},
}
)
def _decision_rerank_cost(
responses: Sequence[OpenAIDecisionResponse],
price: Callable[[OpenAIDecisionResponse], float],
logging_obj: LitellmLoggingObject | None,
provider: str | None,
custom_pricing: bool,
) -> float:
def price_child(
response: OpenAIDecisionResponse,
) -> tuple[float, CostBreakdown | None, float]:
if logging_obj is not None:
logging_obj.cost_breakdown = None
provider_cost: Final = get_response_cost_from_hidden_params(response.hidden_params)
cost: Final = price(response) if custom_pricing or provider_cost is None else provider_cost
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
fixed: Final = (
_apply_cost_margin(
0.0,
provider or str(response.hidden_params.get("custom_llm_provider", "")),
)[2]
if custom_pricing or provider_cost is None
else 0.0
)
return cost, breakdown, fixed
prices: Final = tuple(price_child(response) for response in responses)
repeated_fixed: Final = sum(item[2] for item in prices) - max((item[2] for item in prices), default=0.0)
if logging_obj is not None and responses:
combined: Final = _decision_rerank_breakdown(tuple(item[1] for item in prices), repeated_fixed)
logging_obj.cost_breakdown = combined # rebind-ok: log the aggregate rather than the last child
return sum(item[0] for item in prices) - repeated_fixed
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1544,6 +1616,50 @@ def completion_cost(
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
decision_rerank_child: Final = (
completion_response.decision_responses[0]
if isinstance(completion_response, DecisionRerankResponse) and completion_response.decision_responses
else None
)
decision_rerank_model: Final = (
_get_hidden_str_for_cost_calc(decision_rerank_child.hidden_params, "model") or decision_rerank_child.model
if decision_rerank_child is not None
else None
)
decision_rerank_provider: Final = (
_get_hidden_str_for_cost_calc(decision_rerank_child.hidden_params, "custom_llm_provider")
if decision_rerank_child is not None
else None
)
if isinstance(completion_response, DecisionRerankResponse) and (
custom_cost_per_second is None
or custom_cost_per_token is not None
or not completion_response.decision_responses
):
return _decision_rerank_cost(
completion_response.decision_responses,
partial(
completion_cost,
model=model or (decision_rerank_child.model if decision_rerank_child is not None else ""),
call_type="decisions",
custom_llm_provider=custom_llm_provider,
optional_params=optional_params or {},
custom_pricing=custom_pricing,
base_model=base_model,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
pricing_provider=pricing_provider,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
region_name=region_name,
custom_cost_per_token=custom_cost_per_token,
),
litellm_logging_obj,
pricing_provider or custom_llm_provider,
bool(custom_pricing or custom_cost_per_token is not None),
)
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if pricing_provider is not None:
@ -1637,7 +1753,7 @@ def completion_cost(
litellm_logging_obj,
)
selected_model: Final = select_model_name_for_cost_calc(
model=model,
model=model or decision_rerank_model,
completion_response=completion_response,
custom_llm_provider=base_model_provider or custom_llm_provider,
custom_pricing=custom_pricing,
@ -1739,7 +1855,7 @@ def completion_cost(
hidden_params = getattr(completion_response, "_hidden_params", None)
if hidden_params is not None:
custom_llm_provider = pricing_provider or hidden_params.get(
"custom_llm_provider", custom_llm_provider or None
"custom_llm_provider", custom_llm_provider or decision_rerank_provider
)
region_name = hidden_params.get("region_name", region_name)

View file

@ -46328,6 +46328,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
},
@ -46336,6 +46337,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
},
@ -46344,6 +46346,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46352,6 +46355,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46360,6 +46364,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46368,6 +46373,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -72552,6 +72558,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
@ -72560,6 +72567,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
@ -72569,6 +72577,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
@ -72577,6 +72586,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true

View file

@ -0,0 +1,366 @@
"""Execute rerank requests through LiteLLM's existing Decisions transport."""
import asyncio
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor, wait
from dataclasses import dataclass, replace
from functools import partial
from queue import Empty, SimpleQueue
from threading import Event
from typing import Final, Literal
import httpx
import litellm
from litellm.decisions.call import (
DecisionsCall,
OpenAIDecisionRequestFields,
asend,
prepare_call,
send,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
DecisionRerankRequest,
DecisionRerankResult,
LiteLLMDecisionsRerankConfig,
invalid_response,
result_error,
)
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.rerank import DecisionRerankResponse
from litellm.types.utils import Usage
_CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"})
_MAX_CONCURRENCY: Final = 8
@dataclass(frozen=True, slots=True)
class _DocumentCall:
index: int
text: str
call: DecisionsCall
def _timeout(model: str, provider: str) -> litellm.Timeout:
return litellm.Timeout(
message="Decision reranking exceeded the request timeout",
model=model,
llm_provider=provider,
)
def _call_with_deadline(document: _DocumentCall, deadline: float) -> DecisionsCall | litellm.Timeout:
remaining: Final = deadline - time.monotonic()
if remaining <= 0:
return _timeout(document.call.requested_model, document.call.custom_llm_provider)
timeout: Final = document.call.timeout
if isinstance(timeout, httpx.Timeout):
return replace(
document.call,
timeout=httpx.Timeout(
connect=remaining if timeout.connect is None else min(timeout.connect, remaining),
read=remaining if timeout.read is None else min(timeout.read, remaining),
write=remaining if timeout.write is None else min(timeout.write, remaining),
pool=remaining if timeout.pool is None else min(timeout.pool, remaining),
),
)
return replace(document.call, timeout=remaining)
def _send(
document: _DocumentCall,
deadline: float,
) -> DecisionRerankResult:
call: Final = _call_with_deadline(document, deadline)
if isinstance(call, litellm.Timeout):
return DecisionRerankResult(document.index, document.text, error=call)
try:
response: Final = send(call)
if isinstance(response, OpenAIDecisionResponse):
return DecisionRerankResult(document.index, document.text, response=response)
return DecisionRerankResult(
document.index,
document.text,
error=invalid_response(
document.index, "expected a Decisions response", call.requested_model, call.custom_llm_provider
),
)
except Exception as error: # noqa: BLE001 # preserve mapped provider failures alongside completed billable calls
return DecisionRerankResult(document.index, document.text, error=error)
async def _asend(
document: _DocumentCall,
deadline: float,
cancellation_requested: asyncio.Event,
) -> DecisionRerankResult:
call: Final = _call_with_deadline(document, deadline)
if isinstance(call, litellm.Timeout):
return DecisionRerankResult(document.index, document.text, error=call)
try:
response: Final = await asend(call)
if isinstance(response, OpenAIDecisionResponse):
return DecisionRerankResult(document.index, document.text, response=response)
return DecisionRerankResult(
document.index,
document.text,
error=invalid_response(
document.index, "expected a Decisions response", call.requested_model, call.custom_llm_provider
),
)
except asyncio.CancelledError:
if cancellation_requested.is_set():
raise
return DecisionRerankResult(
document.index,
document.text,
error=litellm.APIConnectionError(
message="Decision rerank provider request was cancelled",
model=document.call.requested_model,
llm_provider=document.call.custom_llm_provider,
),
)
except Exception as error: # noqa: BLE001 # preserve mapped provider failures alongside completed billable calls
return DecisionRerankResult(document.index, document.text, error=error)
def _aggregate_usage(response: DecisionRerankResponse) -> Usage:
prompt_tokens: Final = sum(item.usage.input_tokens for item in response.decision_responses)
completion_tokens: Final = sum(item.usage.output_tokens for item in response.decision_responses)
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
def _record_partial(response: DecisionRerankResponse, logging_obj: Logging) -> None:
if not response.decision_responses:
return
logging_obj.record_partial_usage_for_failure(
_aggregate_usage(response),
logging_obj.response_cost_calculator(result=response) or 0.0,
)
def _finish(
outcomes: Sequence[DecisionRerankResult],
request: DecisionRerankRequest,
logging_obj: Logging,
timed_out: bool,
model: str,
provider: str,
) -> DecisionRerankResponse:
response, validation_failure = LiteLLMDecisionsRerankConfig.transform_decisions_response(
outcomes, request, model, provider
)
failure: Final = validation_failure or (_timeout(model, provider) if timed_out else None)
if failure is not None:
_record_partial(response, logging_obj)
raise failure
usage: Final = _aggregate_usage(response)
logging_obj.model_call_details["combined_usage_object"] = usage # rebind-ok: log aggregate usage
return response
def _rerank(
prepare: Callable[[int], _DocumentCall],
request: DecisionRerankRequest,
logging_obj: Logging,
model: str,
provider: str,
deadline: float,
) -> DecisionRerankResponse:
count: Final = request.request_count
workers: Final = min(count, _MAX_CONCURRENCY)
completed: Final[SimpleQueue[DecisionRerankResult]] = SimpleQueue()
stopped: Final = Event()
def record(outcome: DecisionRerankResult) -> None:
if result_error(outcome, request, model, provider) is not None:
stopped.set()
completed.put(outcome)
def run_one(index: int) -> None:
try:
document: Final = prepare(index)
if stopped.is_set():
return
record(_send(document, deadline))
except Exception as error: # noqa: BLE001 # preparation failures retain completed billable calls
record(DecisionRerankResult(index, request.document_text(index), error=error))
def run(indices: range) -> None:
for index in indices:
if stopped.is_set() or time.monotonic() >= deadline:
return
run_one(index)
executor: Final = ThreadPoolExecutor(max_workers=workers)
futures: Final = tuple(executor.submit(run, range(worker, count, workers)) for worker in range(workers))
try:
waited: Final = wait(futures, timeout=max(0.0, deadline - time.monotonic()))
timed_out: Final = bool(waited.not_done) or time.monotonic() >= deadline
finally:
stopped.set()
executor.shutdown(wait=True, cancel_futures=True)
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
while True:
try:
outcomes.append(completed.get_nowait())
except Empty:
break
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
async def _arerank(
prepare: Callable[[int], _DocumentCall],
request: DecisionRerankRequest,
logging_obj: Logging,
model: str,
provider: str,
deadline: float,
) -> DecisionRerankResponse:
count: Final = request.request_count
workers: Final = min(count, _MAX_CONCURRENCY)
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
cancellation_requested: Final = asyncio.Event()
stopped: Final = asyncio.Event()
def record(outcome: DecisionRerankResult) -> None:
if result_error(outcome, request, model, provider) is not None:
stopped.set()
outcomes.append(outcome)
async def run(indices: range) -> None:
for index in indices:
if stopped.is_set() or time.monotonic() >= deadline:
return
try:
record(
await _asend(
prepare(index),
deadline,
cancellation_requested,
)
)
except Exception as error: # noqa: BLE001 # preparation failures retain completed billable calls
record(DecisionRerankResult(index, request.document_text(index), error=error))
tasks: Final = tuple(asyncio.create_task(run(range(worker, count, workers))) for worker in range(workers))
try:
try:
_, pending = await asyncio.wait(tasks, timeout=max(0.0, deadline - time.monotonic()))
timed_out: Final = bool(pending) or time.monotonic() >= deadline
finally:
stopped.set()
cancellation_requested.set()
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
except asyncio.CancelledError:
response_on_cancel, _ = LiteLLMDecisionsRerankConfig.transform_decisions_response(
outcomes, request, model, provider
)
_record_partial(response_on_cancel, logging_obj)
raise
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
def _prepare_document(
index: int,
request: DecisionRerankRequest,
model: str,
provider: str,
api_key: str | None,
api_base: str | None,
transport_timeout: float | httpx.Timeout,
kwargs: Mapping[str, object],
) -> _DocumentCall:
text: Final = request.document_text(index)
transformed: Final = LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index)
return _DocumentCall(
index=index,
text=text,
call=prepare_call(
model=model,
fields=OpenAIDecisionRequestFields(input=transformed.input, questions=transformed.questions),
safety_identifier=request.safety_identifier,
api_key=api_key,
api_base=api_base,
timeout=transport_timeout,
custom_llm_provider=provider,
extra_headers=request.extra_headers,
kwargs={key: value for key, value in kwargs.items() if key not in _CHILD_OMIT},
),
)
def rerank_with_decisions(
*,
model: str,
provider: str,
api_key: str | None,
api_base: str | None,
logging_obj: Logging,
is_async: bool,
strategy: Literal["pointwise", "multiple_choice"],
transport_timeout: float | httpx.Timeout,
params: Mapping[str, object],
kwargs: Mapping[str, object],
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy)
deadline: Final = time.monotonic() + request.timeout
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs
kwargs=dict(kwargs),
model=model,
custom_llm_provider=provider,
optional_params=request.model_dump(
exclude={"extra_headers", "safety_identifier", "timeout"}, exclude_none=True
),
litellm_params={key: value for key, value in kwargs.items() if key != "litellm_logging_obj"},
)
prepare: Final = partial(
_prepare_document,
request=request,
model=model,
provider=provider,
api_key=api_key,
api_base=api_base,
transport_timeout=transport_timeout,
kwargs=kwargs,
)
first: Final = prepare(0)
def prepare_next(index: int) -> _DocumentCall:
return first if index == 0 else prepare(index)
logging_obj.pre_call( # pyright: ignore[reportUnknownMemberType] # Logging accepts the logical rerank request
input=request.query,
api_key=first.call.api_key,
additional_args={
"complete_input_dict": {
"model": model,
**request.model_dump(
mode="json",
exclude={"extra_headers", "safety_identifier", "timeout"},
exclude_none=True,
),
},
"api_base": first.call.api_base,
"headers": dict(first.call.headers),
},
)
if is_async:
return _arerank(
prepare_next,
request,
logging_obj,
model,
provider,
deadline,
)
return _rerank(prepare_next, request, logging_obj, model, provider, deadline)

View file

@ -0,0 +1,301 @@
"""Convert rerank requests and Decisions responses independently of provider transport."""
import json
import math
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Annotated, Final, Literal
from pydantic import BaseModel, ConfigDict, Field, StrictInt, ValidationError, model_validator
import litellm
from litellm.types.decisions import (
OpenAIChoiceAnswer,
OpenAIChoiceOption,
OpenAIChoiceQuestion,
OpenAIDecisionRequestBody,
OpenAIDecisionResponse,
OpenAIPredicateAnswer,
OpenAIPredicateQuestion,
OpenAIRefusalAnswer,
)
from litellm.types.rerank import DecisionRerankResponse, RerankResponseResult
_RELEVANCE: Final = (
"Is the passage relevant to the query, meaning it contains information that answers, supports or contradicts it?"
)
_UNSUPPORTED: Final = frozenset(
{
"rerank_mode",
"decision_rerank",
"strategy",
"rank_fields",
"max_chunks_per_doc",
"max_tokens_per_doc",
"instruction",
"truncate_prompt_tokens",
"max_tokens_per_query",
"truncation_side",
}
)
class _TextDocument(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
text: str
class DecisionRerankRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
strategy: Literal["pointwise", "multiple_choice"] = Field(exclude=True)
query: str
documents: Annotated[tuple[str | _TextDocument, ...], Field(min_length=1, max_length=1000)]
top_n: Annotated[StrictInt | None, Field(ge=1)] = None
return_documents: bool | None = None
timeout: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 600.0
extra_headers: Mapping[str, str] | None = None
safety_identifier: str | None = None
@model_validator(mode="after")
def validate_choice_request(self) -> "DecisionRerankRequest":
if self.strategy == "multiple_choice" and not 2 <= len(self.documents) <= 255:
raise ValueError("multiple_choice requires between 2 and 255 documents")
return self
def document_text(self, index: int) -> str:
document: Final = self.documents[index]
return document if isinstance(document, str) else document.text
@property
def request_count(self) -> int:
return 1 if self.strategy == "multiple_choice" else len(self.documents)
@dataclass(frozen=True, slots=True)
class DecisionRerankResult:
index: int
text: str
response: OpenAIDecisionResponse | None = None
error: Exception | None = None
def invalid_response(index: int, reason: str, model: str, provider: str) -> litellm.InternalServerError:
return litellm.InternalServerError(
message=f"Invalid decision rerank response for document {index}: {reason}",
model=model,
llm_provider=provider,
)
def _score(outcome: DecisionRerankResult, model: str, provider: str) -> float | Exception:
if outcome.error is not None:
return outcome.error
response: Final = outcome.response
if response is None or len(response.answers) != 1:
return invalid_response(outcome.index, "expected one relevance answer", model, provider)
answer: Final = response.answers[0]
if isinstance(answer, OpenAIRefusalAnswer):
return litellm.ContentPolicyViolationError(
message=f"Decision model refused to score document {outcome.index}",
model=model,
llm_provider=provider,
)
if not isinstance(answer, OpenAIPredicateAnswer) or answer.name != "relevance":
return invalid_response(outcome.index, "expected the relevance predicate", model, provider)
if not math.isfinite(answer.probability) or not 0 <= answer.probability <= 1:
return invalid_response(
outcome.index,
"probability must be finite and between zero and one",
model,
provider,
)
return answer.probability
def _result(index: int, text: str, score: float, return_documents: bool) -> RerankResponseResult:
if return_documents:
return RerankResponseResult(
index=index,
relevance_score=score,
document={"text": text},
)
return RerankResponseResult(index=index, relevance_score=score)
def _choice_scores(
outcome: DecisionRerankResult, request: DecisionRerankRequest, model: str, provider: str
) -> tuple[float, ...] | Exception:
if outcome.error is not None:
return outcome.error
response: Final = outcome.response
if response is None or len(response.answers) != 1:
return invalid_response(outcome.index, "expected one choice answer", model, provider)
answer: Final = response.answers[0]
if isinstance(answer, OpenAIRefusalAnswer):
return litellm.ContentPolicyViolationError(
message="Decision model refused to rank the candidate set",
model=model,
llm_provider=provider,
)
if not isinstance(answer, OpenAIChoiceAnswer) or answer.name != "relevance":
return invalid_response(outcome.index, "expected the relevance choice", model, provider)
labels: Final = tuple(f"p{index}" for index in range(len(request.documents)))
probabilities: Final = {item.value: item.probability for item in answer.probabilities}
if len(answer.probabilities) != len(labels) or set(probabilities) != set(labels):
return invalid_response(outcome.index, "expected each document label exactly once", model, provider)
scores: Final = tuple(probabilities[label] for label in labels)
if any(not math.isfinite(score) or not 0 <= score <= 1 for score in scores):
return invalid_response(outcome.index, "probabilities must be finite and between zero and one", model, provider)
# Two-decimal provider rounding can move the total by half a cent per option.
if (
sum(scores) <= 0
or sum(max(0.0, score - 0.005) for score in scores) > 1.0 + 1e-9
or sum(min(1.0, score + 0.005) for score in scores) < 1.0 - 1e-9
):
return invalid_response(
outcome.index,
"choice probabilities must sum to one within rounding precision",
model,
provider,
)
return scores
def result_error(
outcome: DecisionRerankResult, request: DecisionRerankRequest, model: str, provider: str
) -> Exception | None:
score: Final = (
_choice_scores(outcome, request, model, provider)
if request.strategy == "multiple_choice"
else _score(outcome, model, provider)
)
return score if isinstance(score, Exception) else None
def _scores(
outcomes: Sequence[DecisionRerankResult],
request: DecisionRerankRequest,
model: str,
provider: str,
) -> tuple[float | Exception, ...]:
if request.strategy == "multiple_choice":
scores: Final = _choice_scores(outcomes[0], request, model, provider) if outcomes else ()
return (scores,) if isinstance(scores, Exception) else scores
return tuple(_score(outcome, model, provider) for outcome in outcomes)
def _ranked_results(
outcomes: Sequence[DecisionRerankResult],
request: DecisionRerankRequest,
scores: Sequence[float | Exception],
) -> tuple[RerankResponseResult, ...]:
if request.strategy == "multiple_choice":
return tuple(
_result(
index,
request.document_text(index),
score,
bool(request.return_documents),
)
for index, score in enumerate(scores)
if isinstance(score, float)
)
return tuple(
_result(outcome.index, outcome.text, score, bool(request.return_documents))
for outcome, score in zip(outcomes, scores, strict=True)
if isinstance(score, float)
)
def _response(
outcomes: Sequence[DecisionRerankResult], request: DecisionRerankRequest, scores: Sequence[float | Exception]
) -> DecisionRerankResponse:
responses: Final = tuple(outcome.response for outcome in outcomes if outcome.response is not None)
results: Final = _ranked_results(outcomes, request, scores)
ordered: Final = sorted(results, key=lambda result: (-result["relevance_score"], result["index"]))
return DecisionRerankResponse(
results=ordered[: request.top_n],
meta={
"tokens": {
"input_tokens": sum(response.usage.input_tokens for response in responses),
"output_tokens": sum(response.usage.output_tokens for response in responses),
},
},
decision_responses=responses,
)
class LiteLLMDecisionsRerankConfig:
@staticmethod
def validate_request(
params: Mapping[str, object],
model: str,
provider: str,
strategy: Literal["pointwise", "multiple_choice"],
) -> DecisionRerankRequest:
if params.get("stream") is True:
raise litellm.UnsupportedParamsError(
message="Decision reranking does not support stream=True",
model=model,
llm_provider=provider,
)
unsupported: Final = tuple(key for key in _UNSUPPORTED if params.get(key) is not None)
if unsupported and not (params.get("drop_params") or litellm.drop_params):
raise litellm.UnsupportedParamsError(
message=f"Decision reranking does not support: {', '.join(sorted(unsupported))}",
model=model,
llm_provider=provider,
)
try:
supported_params: Final = {key: value for key, value in params.items() if key not in _UNSUPPORTED}
return DecisionRerankRequest.model_validate({**supported_params, "strategy": strategy})
except ValidationError as error:
raise litellm.BadRequestError(
message="Invalid decision rerank request: "
+ "; ".join(detail["msg"] for detail in error.errors(include_input=False, include_url=False)),
model=model,
llm_provider=provider,
) from error
@staticmethod
def transform_rerank_request(request: DecisionRerankRequest, index: int) -> OpenAIDecisionRequestBody:
if request.strategy == "multiple_choice":
return OpenAIDecisionRequestBody(
input=json.dumps(
{
"query": request.query,
"passages": {
f"p{index}": request.document_text(index) for index in range(len(request.documents))
},
},
ensure_ascii=False,
),
questions=(
OpenAIChoiceQuestion(
type="choice",
name="relevance",
instructions="Which passage is the most relevant to the query?",
choices=tuple(OpenAIChoiceOption(value=f"p{index}") for index in range(len(request.documents))),
),
),
)
return OpenAIDecisionRequestBody(
input=json.dumps(
{"query": request.query, "passage": request.document_text(index)},
ensure_ascii=False,
),
questions=(OpenAIPredicateQuestion(type="predicate", name="relevance", instructions=_RELEVANCE),),
)
@staticmethod
def transform_decisions_response(
outcomes: Sequence[DecisionRerankResult],
request: DecisionRerankRequest,
model: str,
provider: str,
) -> tuple[DecisionRerankResponse, Exception | None]:
"""Retain completed responses for billing even when another result fails validation."""
scores: Final = _scores(outcomes, request, model, provider)
return _response(outcomes, request, scores), next(
(score for score in scores if isinstance(score, Exception)), None
)

View file

@ -1,11 +1,15 @@
import asyncio
import contextvars
from collections.abc import Awaitable
from collections.abc import Awaitable, Mapping
from functools import partial
from typing import Final, Literal
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
@ -13,12 +17,17 @@ from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.together_ai.rerank.handler import TogetherAIRerank
from litellm.llms.watsonx.common_utils import IBMWatsonXMixin
from litellm.rerank_api.rerank_utils import get_optional_rerank_params
from litellm.rerank_api.rerank_utils import (
get_decision_rerank_strategy,
get_optional_rerank_params,
)
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.rerank import RerankResponse
from litellm.types.router import *
from litellm.utils import ProviderConfigManager, client, exception_type
_DECISION_KWARGS: Final = TypeAdapter(Mapping[str, object])
####### ENVIRONMENT VARIABLES ###################
# Initialize any necessary instances or variables here
together_rerank: Final = TogetherAIRerank()
@ -27,6 +36,12 @@ base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
def _decision_rerank_timeout(timeout: float | httpx.Timeout) -> float:
if isinstance(timeout, httpx.Timeout):
return float(litellm.request_timeout) if timeout.read is None else timeout.read
return timeout
@client
async def arerank(
model: str,
@ -154,6 +169,42 @@ def rerank(
api_key=optional_params.api_key,
)
decision_strategy: Final = get_decision_rerank_strategy(model, litellm.LlmProviders(_custom_llm_provider))
if decision_strategy is not None:
decision_kwargs: Final = _DECISION_KWARGS.validate_python(kwargs)
decision_timeout: Final = CompletionTimeout.as_limit(
litellm.request_timeout if optional_params.timeout is None else optional_params.timeout
)
from litellm.rerank_api.litellm_decisions_transformation.handler import (
rerank_with_decisions,
)
return rerank_with_decisions(
model=model,
provider=_custom_llm_provider,
api_key=dynamic_api_key or optional_params.api_key,
api_base=dynamic_api_base or optional_params.api_base,
logging_obj=litellm_logging_obj,
is_async=_is_async,
strategy=decision_strategy,
transport_timeout=decision_timeout,
params=_DECISION_KWARGS.validate_python(
{
**decision_kwargs,
"query": query,
"documents": documents,
"top_n": top_n,
"return_documents": return_documents,
"rank_fields": rank_fields,
"max_chunks_per_doc": max_chunks_per_doc,
"max_tokens_per_doc": max_tokens_per_doc,
"timeout": _decision_rerank_timeout(decision_timeout),
"extra_headers": decision_kwargs.get("extra_headers") or headers,
}
),
kwargs=decision_kwargs,
)
rerank_provider_config: Final[BaseRerankConfig] = ProviderConfigManager.get_provider_rerank_config(
model=model,
provider=litellm.LlmProviders(_custom_llm_provider),

View file

@ -1,6 +1,31 @@
from typing import Any, Final
from typing import Any, Final, Literal
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
def _catalog_rerank_strategy(model: str) -> Literal["pointwise", "multiple_choice"] | None:
info: Final = as_str_mapping(litellm.model_cost.get(model)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Narrow the legacy untyped catalog entry at the boundary.
strategy: Final = None if info is None else info.get("rerank_strategy")
match strategy:
case "pointwise":
return "pointwise"
case "multiple_choice":
return "multiple_choice"
case _:
return None
def get_decision_rerank_strategy(
model: str, custom_llm_provider: LlmProviders
) -> Literal["pointwise", "multiple_choice"] | None:
"""Read the catalog method for an already resolved model and provider."""
if ProviderConfigManager.get_provider_decisions_config(model, custom_llm_provider) is None:
return None
return _catalog_rerank_strategy(f"{custom_llm_provider.value}/{model}") or _catalog_rerank_strategy(model)
def get_optional_rerank_params(

View file

@ -6,9 +6,10 @@ https://docs.cohere.com/reference/rerank
from typing import Literal
from pydantic import ConfigDict, PrivateAttr
from pydantic import ConfigDict, Field, PrivateAttr
from typing_extensions import ReadOnly, Required, TypedDict
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.llms.base import LiteLLMBaseModel
@ -103,3 +104,9 @@ class RerankResponse(LiteLLMBaseModel):
def __contains__(self, key) -> bool:
return key in self.__dict__
class DecisionRerankResponse(RerankResponse):
model_config = ConfigDict(frozen=True)
decision_responses: tuple[OpenAIDecisionResponse, ...] = Field(exclude=True)

View file

@ -203,6 +203,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_audio_input: bool | None
supports_video_input: ReadOnly[bool | None]
supports_box_questions: ReadOnly[bool | None]
rerank_strategy: ReadOnly[Literal["pointwise", "multiple_choice"] | None]
supports_embedding_image_input: bool | None
supports_audio_output: bool | None
supports_pdf_input: bool | None

View file

@ -6967,6 +6967,7 @@ def get_model_info_helper(
supports_audio_input=_model_info.get("supports_audio_input", None),
supports_video_input=_model_info.get("supports_video_input", None),
supports_box_questions=_model_info.get("supports_box_questions", None),
rerank_strategy=_model_info.get("rerank_strategy", None),
supports_audio_output=_model_info.get("supports_audio_output", None),
supports_pdf_input=_model_info.get("supports_pdf_input", None),
supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None),

View file

@ -46328,6 +46328,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
},
@ -46336,6 +46337,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
},
@ -46344,6 +46346,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46352,6 +46355,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46360,6 +46364,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -46368,6 +46373,7 @@
"litellm_provider": "perplexity",
"max_input_tokens": 262144,
"mode": "evaluation",
"rerank_strategy": "multiple_choice",
"output_cost_per_token": 0.0,
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
},
@ -72552,6 +72558,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
@ -72560,6 +72567,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
@ -72569,6 +72577,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
},
@ -72577,6 +72586,7 @@
"litellm_provider": "typesafe",
"max_input_tokens": 65536,
"mode": "evaluation",
"rerank_strategy": "pointwise",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,118 @@
from typing import Final
import pytest
import litellm
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
DecisionRerankRequest,
DecisionRerankResult,
LiteLLMDecisionsRerankConfig,
)
from litellm.types.decisions import (
OpenAIDecisionResponse,
OpenAIDecisionUsage,
OpenAIPredicateAnswer,
OpenAIRefusalAnswer,
)
def test_invalid_score_keeps_completed_responses_and_usage_for_failure_billing():
request: Final = DecisionRerankRequest(strategy="pointwise", query="warranty", documents=("relevant", "invalid"))
responses: Final = tuple(
OpenAIDecisionResponse(
model="decision-unit-test",
answers=(OpenAIPredicateAnswer(name="relevance", probability=score),),
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
)
for score in (0.9, 1.4)
)
outcomes: Final = tuple(
DecisionRerankResult(index, request.document_text(index), response=response)
for index, response in enumerate(responses)
)
result, error = LiteLLMDecisionsRerankConfig.transform_decisions_response(
outcomes, request, "decision-unit-test", "typesafe"
)
assert isinstance(error, litellm.InternalServerError)
assert result.results == [{"index": 0, "relevance_score": 0.9}]
assert result.meta["tokens"] == {"input_tokens": 20, "output_tokens": 4}
assert result.decision_responses[0] is responses[0]
assert result.decision_responses[1] is responses[1]
@pytest.mark.parametrize("strategy,limit", [("pointwise", 1000), ("multiple_choice", 255)])
def test_document_limit_is_enforced_at_request_boundary(strategy, limit):
params: Final = {"query": "warranty", "documents": ("passage",) * limit}
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, "decision-unit-test", "typesafe", strategy)
assert len(request.documents) == limit
with pytest.raises(litellm.BadRequestError, match=str(limit)):
LiteLLMDecisionsRerankConfig.validate_request(
{**params, "documents": ("passage",) * (limit + 1)}, "decision-unit-test", "typesafe", strategy
)
@pytest.mark.parametrize("per_request,global_default", [(True, False), (None, True), (False, True)])
@pytest.mark.parametrize(
"unsupported",
[
{"rank_fields": ["text"]},
{"max_chunks_per_doc": 1},
{"max_tokens_per_doc": 10},
{"instruction": "Rank only identical text"},
],
)
def test_drop_params_preserves_supported_request_fields(per_request, global_default, unsupported, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", global_default)
request: Final = LiteLLMDecisionsRerankConfig.validate_request(
{
"query": "warranty",
"documents": ("passage",),
"top_n": 1,
"return_documents": True,
"drop_params": per_request,
"extra_headers": {"x-request-id": "supported-header"},
"safety_identifier": "supported-identifier",
**unsupported,
},
"decision-unit-test",
"typesafe",
"pointwise",
)
assert request.query == "warranty"
assert request.documents == ("passage",)
assert request.top_n == 1
assert request.return_documents is True
assert request.extra_headers == {"x-request-id": "supported-header"}
assert request.safety_identifier == "supported-identifier"
assert request.timeout == 600.0
def test_unsupported_parameters_still_fail_when_dropping_is_disabled(monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(litellm.UnsupportedParamsError, match="rank_fields"):
LiteLLMDecisionsRerankConfig.validate_request(
{"query": "warranty", "documents": ("passage",), "rank_fields": ["text"], "drop_params": False},
"decision-unit-test",
"typesafe",
"pointwise",
)
@pytest.mark.parametrize("strategy", ["pointwise", "multiple_choice"])
def test_refusal_retains_billable_usage_and_returns_content_policy_error(strategy):
request: Final = DecisionRerankRequest(strategy=strategy, query="warranty", documents=("first", "second"))
response: Final = OpenAIDecisionResponse(
model="decision-unit-test",
answers=(OpenAIRefusalAnswer(name="relevance"),),
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
)
result, error = LiteLLMDecisionsRerankConfig.transform_decisions_response(
(DecisionRerankResult(0, request.document_text(0), response=response),),
request,
"decision-unit-test",
"typesafe",
)
assert isinstance(error, litellm.ContentPolicyViolationError)
assert result.results == []
assert result.meta["tokens"] == {"input_tokens": 10, "output_tokens": 2}
assert result.decision_responses == (response,)

View file

@ -34,7 +34,8 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.llms.xai import RealtimeMeteredUsage
from litellm.types.rerank import RerankResponse
from litellm.types.decisions import OpenAIDecisionResponse, OpenAIDecisionUsage
from litellm.types.rerank import DecisionRerankResponse, RerankResponse
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
from litellm.types.videos.main import VideoObject
from litellm.utils import supports_prompt_caching
@ -4033,6 +4034,130 @@ def test_completion_cost_per_second_deployment_bills_the_call_duration(
assert cost == pytest.approx(0.02 * expected_seconds)
@pytest.mark.parametrize(
("document_count", "rate", "stamped_ms", "total_time", "logged_ms", "expected_seconds"),
[
(1, 0.5, None, 2000.0, 4000.0, 2.0),
(3, 0.5, None, 2000.0, 4000.0, 2.0),
(3, 0.5, 3000.0, 2000.0, 4000.0, 3.0),
(3, 0.5, None, 0.0, 4000.0, 4.0),
(3, 0.0, None, 2000.0, 4000.0, 2.0),
],
)
def test_decision_rerank_custom_per_second_price_bills_outer_duration_once(
monkeypatch: pytest.MonkeyPatch,
document_count: int,
rate: float,
stamped_ms: float | None,
total_time: float,
logged_ms: float,
expected_seconds: float,
) -> None:
monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1})
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"percentage": 0.2, "fixed_amount": 0.05}})
child: Final = OpenAIDecisionResponse(
model="openai/rerank-time-test",
answers=(),
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=0, total_tokens=10),
)
child.set_hidden_params({"additional_headers": {"llm_provider-x-litellm-response-cost": "50.0"}})
response: Final = DecisionRerankResponse(decision_responses=(child,) * document_count).model_copy(
update={"_response_ms": stamped_ms}
)
logging_obj: Final = _logging_obj_with_call_window(logged_ms)
cost: Final = completion_cost(
completion_response=response,
custom_cost_per_second=rate,
total_time=total_time,
litellm_logging_obj=logging_obj,
)
expected_base: Final = rate * expected_seconds
assert cost == pytest.approx(expected_base * 0.9 * 1.2 + 0.05)
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["original_cost"] == pytest.approx(expected_base)
assert logging_obj.cost_breakdown["total_cost"] == pytest.approx(cost)
@pytest.mark.parametrize(
("model", "provider", "pricing_provider", "expected_cost"),
[
(None, None, None, 1.25),
("unprefixed-deployment", None, None, 1.25),
(None, "openai", None, 1.6),
(None, "typesafe", "openai", 1.6),
],
)
def test_decision_rerank_per_second_price_preserves_child_provider_with_explicit_overrides(
monkeypatch: pytest.MonkeyPatch,
model: str | None,
provider: str | None,
pricing_provider: str | None,
expected_cost: float,
) -> None:
monkeypatch.setattr(litellm, "cost_discount_config", {})
monkeypatch.setattr(
litellm,
"cost_margin_config",
{"typesafe": {"percentage": 0.2, "fixed_amount": 0.05}, "openai": {"percentage": 0.5, "fixed_amount": 0.1}},
)
child: Final = OpenAIDecisionResponse(
model="rerank-time-test",
answers=(),
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=0, total_tokens=10),
)
child.set_hidden_params({"model": "typesafe/rerank-time-test", "custom_llm_provider": "typesafe"})
cost: Final = completion_cost(
completion_response=DecisionRerankResponse(decision_responses=(child, child)),
model=model,
custom_cost_per_second=0.5,
total_time=2000.0,
custom_llm_provider=provider,
pricing_provider=pricing_provider,
)
assert cost == pytest.approx(expected_cost)
def test_decision_rerank_without_billable_responses_has_no_time_cost() -> None:
assert (
completion_cost(
completion_response=DecisionRerankResponse(decision_responses=()),
custom_cost_per_second=0.5,
total_time=2000.0,
)
== 0.0
)
def test_decision_rerank_custom_token_price_takes_precedence_over_per_second_price(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cost_discount_config", {})
monkeypatch.setattr(litellm, "cost_margin_config", {})
response: Final = DecisionRerankResponse(
decision_responses=(
OpenAIDecisionResponse(
model="openai/rerank-time-test",
answers=(),
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
),
)
* 3
)
cost: Final = completion_cost(
completion_response=response,
custom_cost_per_token={"input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
custom_cost_per_second=0.5,
total_time=2000.0,
)
assert cost == pytest.approx(3 * (10 * 0.01 + 2 * 0.02))
@pytest.mark.parametrize("mode", ["audio_speech", "video_generation", "realtime"])
def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str):
"""

View file

@ -1167,6 +1167,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_tool_search": {"type": "boolean"},
"supports_video_input": {"type": "boolean"},
"supports_box_questions": {"type": "boolean"},
"rerank_strategy": {"type": "string", "enum": ["pointwise", "multiple_choice"]},
"supports_vision": {"type": "boolean"},
"supports_web_search": {"type": "boolean"},
"token_count_framing": {"type": "integer"},