mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge dbfb175303 into dc0237ccd7
This commit is contained in:
commit
3322bfe995
16 changed files with 2573 additions and 10 deletions
|
|
@ -3,9 +3,9 @@
|
|||
import logging
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from functools import lru_cache, partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
|
||||
|
||||
|
|
@ -124,9 +124,10 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.rerank import RerankBilledUnits, RerankResponse
|
||||
from litellm.types.rerank import DecisionRerankResponse, RerankBilledUnits, RerankResponse
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
CostBreakdown,
|
||||
LiteLLMRealtimeStreamLoggingObject,
|
||||
LlmProviders,
|
||||
LlmProvidersSet,
|
||||
|
|
@ -1500,6 +1501,77 @@ def _response_time_ms_for_cost(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _decision_rerank_breakdown(
|
||||
breakdowns: Sequence[CostBreakdown | None], repeated_fixed: float
|
||||
) -> CostBreakdown | None:
|
||||
complete: Final = tuple(item for item in breakdowns if item is not None)
|
||||
if not complete or len(complete) != len(breakdowns):
|
||||
return None
|
||||
monetary_fields: Final = (
|
||||
"input_cost",
|
||||
"cache_read_cost",
|
||||
"cache_creation_cost",
|
||||
"output_cost",
|
||||
"reasoning_cost",
|
||||
"total_cost",
|
||||
"tool_usage_cost",
|
||||
"guardrail_cost",
|
||||
"original_cost",
|
||||
"discount_amount",
|
||||
"margin_total_amount",
|
||||
)
|
||||
totals: Final = {
|
||||
key: sum(value for item in complete if isinstance(value := item.get(key), (int, float)))
|
||||
for key in monetary_fields
|
||||
if any(key in item for item in complete)
|
||||
}
|
||||
additional_keys: Final = frozenset[str]().union(*(item.get("additional_costs", {}) for item in complete))
|
||||
return TypeAdapter(CostBreakdown).validate_python(
|
||||
{
|
||||
**complete[-1],
|
||||
**totals,
|
||||
"additional_costs": {
|
||||
key: sum(item.get("additional_costs", {}).get(key, 0.0) for item in complete) for key in additional_keys
|
||||
},
|
||||
**{key: totals[key] - repeated_fixed for key in ("total_cost", "margin_total_amount") if key in totals},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _decision_rerank_cost(
|
||||
responses: Sequence[OpenAIDecisionResponse],
|
||||
price: Callable[[OpenAIDecisionResponse], float],
|
||||
logging_obj: LitellmLoggingObject | None,
|
||||
provider: str | None,
|
||||
custom_pricing: bool,
|
||||
) -> float:
|
||||
def price_child(
|
||||
response: OpenAIDecisionResponse,
|
||||
) -> tuple[float, CostBreakdown | None, float]:
|
||||
if logging_obj is not None:
|
||||
logging_obj.cost_breakdown = None
|
||||
provider_cost: Final = get_response_cost_from_hidden_params(response.hidden_params)
|
||||
cost: Final = price(response) if custom_pricing or provider_cost is None else provider_cost
|
||||
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
|
||||
fixed: Final = (
|
||||
_apply_cost_margin(
|
||||
0.0,
|
||||
provider or str(response.hidden_params.get("custom_llm_provider", "")),
|
||||
)[2]
|
||||
if custom_pricing or provider_cost is None
|
||||
else 0.0
|
||||
)
|
||||
return cost, breakdown, fixed
|
||||
|
||||
prices: Final = tuple(price_child(response) for response in responses)
|
||||
repeated_fixed: Final = sum(item[2] for item in prices) - max((item[2] for item in prices), default=0.0)
|
||||
|
||||
if logging_obj is not None and responses:
|
||||
combined: Final = _decision_rerank_breakdown(tuple(item[1] for item in prices), repeated_fixed)
|
||||
logging_obj.cost_breakdown = combined # rebind-ok: log the aggregate rather than the last child
|
||||
return sum(item[0] for item in prices) - repeated_fixed
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -1544,6 +1616,50 @@ def completion_cost(
|
|||
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
|
||||
or add it to model_prices_and_context_window.json.
|
||||
"""
|
||||
decision_rerank_child: Final = (
|
||||
completion_response.decision_responses[0]
|
||||
if isinstance(completion_response, DecisionRerankResponse) and completion_response.decision_responses
|
||||
else None
|
||||
)
|
||||
decision_rerank_model: Final = (
|
||||
_get_hidden_str_for_cost_calc(decision_rerank_child.hidden_params, "model") or decision_rerank_child.model
|
||||
if decision_rerank_child is not None
|
||||
else None
|
||||
)
|
||||
decision_rerank_provider: Final = (
|
||||
_get_hidden_str_for_cost_calc(decision_rerank_child.hidden_params, "custom_llm_provider")
|
||||
if decision_rerank_child is not None
|
||||
else None
|
||||
)
|
||||
if isinstance(completion_response, DecisionRerankResponse) and (
|
||||
custom_cost_per_second is None
|
||||
or custom_cost_per_token is not None
|
||||
or not completion_response.decision_responses
|
||||
):
|
||||
return _decision_rerank_cost(
|
||||
completion_response.decision_responses,
|
||||
partial(
|
||||
completion_cost,
|
||||
model=model or (decision_rerank_child.model if decision_rerank_child is not None else ""),
|
||||
call_type="decisions",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=optional_params or {},
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
pricing_provider=pricing_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
),
|
||||
litellm_logging_obj,
|
||||
pricing_provider or custom_llm_provider,
|
||||
bool(custom_pricing or custom_cost_per_token is not None),
|
||||
)
|
||||
try:
|
||||
call_type = infer_call_type(call_type, completion_response) or "completion"
|
||||
if pricing_provider is not None:
|
||||
|
|
@ -1637,7 +1753,7 @@ def completion_cost(
|
|||
litellm_logging_obj,
|
||||
)
|
||||
selected_model: Final = select_model_name_for_cost_calc(
|
||||
model=model,
|
||||
model=model or decision_rerank_model,
|
||||
completion_response=completion_response,
|
||||
custom_llm_provider=base_model_provider or custom_llm_provider,
|
||||
custom_pricing=custom_pricing,
|
||||
|
|
@ -1739,7 +1855,7 @@ def completion_cost(
|
|||
hidden_params = getattr(completion_response, "_hidden_params", None)
|
||||
if hidden_params is not None:
|
||||
custom_llm_provider = pricing_provider or hidden_params.get(
|
||||
"custom_llm_provider", custom_llm_provider or None
|
||||
"custom_llm_provider", custom_llm_provider or decision_rerank_provider
|
||||
)
|
||||
region_name = hidden_params.get("region_name", region_name)
|
||||
|
||||
|
|
|
|||
|
|
@ -46328,6 +46328,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
|
|
@ -46336,6 +46337,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
|
||||
},
|
||||
|
|
@ -46344,6 +46346,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46352,6 +46355,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46360,6 +46364,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46368,6 +46373,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -72552,6 +72558,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
|
|
@ -72560,6 +72567,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
|
|
@ -72569,6 +72577,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
|
|
@ -72577,6 +72586,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
|
|
|
|||
366
litellm/rerank_api/litellm_decisions_transformation/handler.py
Normal file
366
litellm/rerank_api/litellm_decisions_transformation/handler.py
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
"""Execute rerank requests through LiteLLM's existing Decisions transport."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import partial
|
||||
from queue import Empty, SimpleQueue
|
||||
from threading import Event
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.decisions.call import (
|
||||
DecisionsCall,
|
||||
OpenAIDecisionRequestFields,
|
||||
asend,
|
||||
prepare_call,
|
||||
send,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
|
||||
DecisionRerankRequest,
|
||||
DecisionRerankResult,
|
||||
LiteLLMDecisionsRerankConfig,
|
||||
invalid_response,
|
||||
result_error,
|
||||
)
|
||||
from litellm.types.decisions import OpenAIDecisionResponse
|
||||
from litellm.types.rerank import DecisionRerankResponse
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
_CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"})
|
||||
_MAX_CONCURRENCY: Final = 8
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DocumentCall:
|
||||
index: int
|
||||
text: str
|
||||
call: DecisionsCall
|
||||
|
||||
|
||||
def _timeout(model: str, provider: str) -> litellm.Timeout:
|
||||
return litellm.Timeout(
|
||||
message="Decision reranking exceeded the request timeout",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
|
||||
def _call_with_deadline(document: _DocumentCall, deadline: float) -> DecisionsCall | litellm.Timeout:
|
||||
remaining: Final = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return _timeout(document.call.requested_model, document.call.custom_llm_provider)
|
||||
timeout: Final = document.call.timeout
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
return replace(
|
||||
document.call,
|
||||
timeout=httpx.Timeout(
|
||||
connect=remaining if timeout.connect is None else min(timeout.connect, remaining),
|
||||
read=remaining if timeout.read is None else min(timeout.read, remaining),
|
||||
write=remaining if timeout.write is None else min(timeout.write, remaining),
|
||||
pool=remaining if timeout.pool is None else min(timeout.pool, remaining),
|
||||
),
|
||||
)
|
||||
return replace(document.call, timeout=remaining)
|
||||
|
||||
|
||||
def _send(
|
||||
document: _DocumentCall,
|
||||
deadline: float,
|
||||
) -> DecisionRerankResult:
|
||||
call: Final = _call_with_deadline(document, deadline)
|
||||
if isinstance(call, litellm.Timeout):
|
||||
return DecisionRerankResult(document.index, document.text, error=call)
|
||||
try:
|
||||
response: Final = send(call)
|
||||
if isinstance(response, OpenAIDecisionResponse):
|
||||
return DecisionRerankResult(document.index, document.text, response=response)
|
||||
return DecisionRerankResult(
|
||||
document.index,
|
||||
document.text,
|
||||
error=invalid_response(
|
||||
document.index, "expected a Decisions response", call.requested_model, call.custom_llm_provider
|
||||
),
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # preserve mapped provider failures alongside completed billable calls
|
||||
return DecisionRerankResult(document.index, document.text, error=error)
|
||||
|
||||
|
||||
async def _asend(
|
||||
document: _DocumentCall,
|
||||
deadline: float,
|
||||
cancellation_requested: asyncio.Event,
|
||||
) -> DecisionRerankResult:
|
||||
call: Final = _call_with_deadline(document, deadline)
|
||||
if isinstance(call, litellm.Timeout):
|
||||
return DecisionRerankResult(document.index, document.text, error=call)
|
||||
try:
|
||||
response: Final = await asend(call)
|
||||
if isinstance(response, OpenAIDecisionResponse):
|
||||
return DecisionRerankResult(document.index, document.text, response=response)
|
||||
return DecisionRerankResult(
|
||||
document.index,
|
||||
document.text,
|
||||
error=invalid_response(
|
||||
document.index, "expected a Decisions response", call.requested_model, call.custom_llm_provider
|
||||
),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
if cancellation_requested.is_set():
|
||||
raise
|
||||
return DecisionRerankResult(
|
||||
document.index,
|
||||
document.text,
|
||||
error=litellm.APIConnectionError(
|
||||
message="Decision rerank provider request was cancelled",
|
||||
model=document.call.requested_model,
|
||||
llm_provider=document.call.custom_llm_provider,
|
||||
),
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # preserve mapped provider failures alongside completed billable calls
|
||||
return DecisionRerankResult(document.index, document.text, error=error)
|
||||
|
||||
|
||||
def _aggregate_usage(response: DecisionRerankResponse) -> Usage:
|
||||
prompt_tokens: Final = sum(item.usage.input_tokens for item in response.decision_responses)
|
||||
completion_tokens: Final = sum(item.usage.output_tokens for item in response.decision_responses)
|
||||
return Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _record_partial(response: DecisionRerankResponse, logging_obj: Logging) -> None:
|
||||
if not response.decision_responses:
|
||||
return
|
||||
logging_obj.record_partial_usage_for_failure(
|
||||
_aggregate_usage(response),
|
||||
logging_obj.response_cost_calculator(result=response) or 0.0,
|
||||
)
|
||||
|
||||
|
||||
def _finish(
|
||||
outcomes: Sequence[DecisionRerankResult],
|
||||
request: DecisionRerankRequest,
|
||||
logging_obj: Logging,
|
||||
timed_out: bool,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> DecisionRerankResponse:
|
||||
response, validation_failure = LiteLLMDecisionsRerankConfig.transform_decisions_response(
|
||||
outcomes, request, model, provider
|
||||
)
|
||||
failure: Final = validation_failure or (_timeout(model, provider) if timed_out else None)
|
||||
if failure is not None:
|
||||
_record_partial(response, logging_obj)
|
||||
raise failure
|
||||
usage: Final = _aggregate_usage(response)
|
||||
logging_obj.model_call_details["combined_usage_object"] = usage # rebind-ok: log aggregate usage
|
||||
return response
|
||||
|
||||
|
||||
def _rerank(
|
||||
prepare: Callable[[int], _DocumentCall],
|
||||
request: DecisionRerankRequest,
|
||||
logging_obj: Logging,
|
||||
model: str,
|
||||
provider: str,
|
||||
deadline: float,
|
||||
) -> DecisionRerankResponse:
|
||||
count: Final = request.request_count
|
||||
workers: Final = min(count, _MAX_CONCURRENCY)
|
||||
completed: Final[SimpleQueue[DecisionRerankResult]] = SimpleQueue()
|
||||
stopped: Final = Event()
|
||||
|
||||
def record(outcome: DecisionRerankResult) -> None:
|
||||
if result_error(outcome, request, model, provider) is not None:
|
||||
stopped.set()
|
||||
completed.put(outcome)
|
||||
|
||||
def run_one(index: int) -> None:
|
||||
try:
|
||||
document: Final = prepare(index)
|
||||
if stopped.is_set():
|
||||
return
|
||||
record(_send(document, deadline))
|
||||
except Exception as error: # noqa: BLE001 # preparation failures retain completed billable calls
|
||||
record(DecisionRerankResult(index, request.document_text(index), error=error))
|
||||
|
||||
def run(indices: range) -> None:
|
||||
for index in indices:
|
||||
if stopped.is_set() or time.monotonic() >= deadline:
|
||||
return
|
||||
run_one(index)
|
||||
|
||||
executor: Final = ThreadPoolExecutor(max_workers=workers)
|
||||
futures: Final = tuple(executor.submit(run, range(worker, count, workers)) for worker in range(workers))
|
||||
try:
|
||||
waited: Final = wait(futures, timeout=max(0.0, deadline - time.monotonic()))
|
||||
timed_out: Final = bool(waited.not_done) or time.monotonic() >= deadline
|
||||
finally:
|
||||
stopped.set()
|
||||
executor.shutdown(wait=True, cancel_futures=True)
|
||||
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
|
||||
while True:
|
||||
try:
|
||||
outcomes.append(completed.get_nowait())
|
||||
except Empty:
|
||||
break
|
||||
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
|
||||
|
||||
|
||||
async def _arerank(
|
||||
prepare: Callable[[int], _DocumentCall],
|
||||
request: DecisionRerankRequest,
|
||||
logging_obj: Logging,
|
||||
model: str,
|
||||
provider: str,
|
||||
deadline: float,
|
||||
) -> DecisionRerankResponse:
|
||||
count: Final = request.request_count
|
||||
workers: Final = min(count, _MAX_CONCURRENCY)
|
||||
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
|
||||
cancellation_requested: Final = asyncio.Event()
|
||||
stopped: Final = asyncio.Event()
|
||||
|
||||
def record(outcome: DecisionRerankResult) -> None:
|
||||
if result_error(outcome, request, model, provider) is not None:
|
||||
stopped.set()
|
||||
outcomes.append(outcome)
|
||||
|
||||
async def run(indices: range) -> None:
|
||||
for index in indices:
|
||||
if stopped.is_set() or time.monotonic() >= deadline:
|
||||
return
|
||||
try:
|
||||
record(
|
||||
await _asend(
|
||||
prepare(index),
|
||||
deadline,
|
||||
cancellation_requested,
|
||||
)
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # preparation failures retain completed billable calls
|
||||
record(DecisionRerankResult(index, request.document_text(index), error=error))
|
||||
|
||||
tasks: Final = tuple(asyncio.create_task(run(range(worker, count, workers))) for worker in range(workers))
|
||||
try:
|
||||
try:
|
||||
_, pending = await asyncio.wait(tasks, timeout=max(0.0, deadline - time.monotonic()))
|
||||
timed_out: Final = bool(pending) or time.monotonic() >= deadline
|
||||
finally:
|
||||
stopped.set()
|
||||
cancellation_requested.set()
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
except asyncio.CancelledError:
|
||||
response_on_cancel, _ = LiteLLMDecisionsRerankConfig.transform_decisions_response(
|
||||
outcomes, request, model, provider
|
||||
)
|
||||
_record_partial(response_on_cancel, logging_obj)
|
||||
raise
|
||||
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
|
||||
|
||||
|
||||
def _prepare_document(
|
||||
index: int,
|
||||
request: DecisionRerankRequest,
|
||||
model: str,
|
||||
provider: str,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
transport_timeout: float | httpx.Timeout,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _DocumentCall:
|
||||
text: Final = request.document_text(index)
|
||||
transformed: Final = LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index)
|
||||
return _DocumentCall(
|
||||
index=index,
|
||||
text=text,
|
||||
call=prepare_call(
|
||||
model=model,
|
||||
fields=OpenAIDecisionRequestFields(input=transformed.input, questions=transformed.questions),
|
||||
safety_identifier=request.safety_identifier,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=transport_timeout,
|
||||
custom_llm_provider=provider,
|
||||
extra_headers=request.extra_headers,
|
||||
kwargs={key: value for key, value in kwargs.items() if key not in _CHILD_OMIT},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def rerank_with_decisions(
|
||||
*,
|
||||
model: str,
|
||||
provider: str,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
logging_obj: Logging,
|
||||
is_async: bool,
|
||||
strategy: Literal["pointwise", "multiple_choice"],
|
||||
transport_timeout: float | httpx.Timeout,
|
||||
params: Mapping[str, object],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy)
|
||||
deadline: Final = time.monotonic() + request.timeout
|
||||
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs
|
||||
kwargs=dict(kwargs),
|
||||
model=model,
|
||||
custom_llm_provider=provider,
|
||||
optional_params=request.model_dump(
|
||||
exclude={"extra_headers", "safety_identifier", "timeout"}, exclude_none=True
|
||||
),
|
||||
litellm_params={key: value for key, value in kwargs.items() if key != "litellm_logging_obj"},
|
||||
)
|
||||
prepare: Final = partial(
|
||||
_prepare_document,
|
||||
request=request,
|
||||
model=model,
|
||||
provider=provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
transport_timeout=transport_timeout,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
first: Final = prepare(0)
|
||||
|
||||
def prepare_next(index: int) -> _DocumentCall:
|
||||
return first if index == 0 else prepare(index)
|
||||
|
||||
logging_obj.pre_call( # pyright: ignore[reportUnknownMemberType] # Logging accepts the logical rerank request
|
||||
input=request.query,
|
||||
api_key=first.call.api_key,
|
||||
additional_args={
|
||||
"complete_input_dict": {
|
||||
"model": model,
|
||||
**request.model_dump(
|
||||
mode="json",
|
||||
exclude={"extra_headers", "safety_identifier", "timeout"},
|
||||
exclude_none=True,
|
||||
),
|
||||
},
|
||||
"api_base": first.call.api_base,
|
||||
"headers": dict(first.call.headers),
|
||||
},
|
||||
)
|
||||
if is_async:
|
||||
return _arerank(
|
||||
prepare_next,
|
||||
request,
|
||||
logging_obj,
|
||||
model,
|
||||
provider,
|
||||
deadline,
|
||||
)
|
||||
return _rerank(prepare_next, request, logging_obj, model, provider, deadline)
|
||||
|
|
@ -0,0 +1,301 @@
|
|||
"""Convert rerank requests and Decisions responses independently of provider transport."""
|
||||
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt, ValidationError, model_validator
|
||||
|
||||
import litellm
|
||||
from litellm.types.decisions import (
|
||||
OpenAIChoiceAnswer,
|
||||
OpenAIChoiceOption,
|
||||
OpenAIChoiceQuestion,
|
||||
OpenAIDecisionRequestBody,
|
||||
OpenAIDecisionResponse,
|
||||
OpenAIPredicateAnswer,
|
||||
OpenAIPredicateQuestion,
|
||||
OpenAIRefusalAnswer,
|
||||
)
|
||||
from litellm.types.rerank import DecisionRerankResponse, RerankResponseResult
|
||||
|
||||
_RELEVANCE: Final = (
|
||||
"Is the passage relevant to the query, meaning it contains information that answers, supports or contradicts it?"
|
||||
)
|
||||
_UNSUPPORTED: Final = frozenset(
|
||||
{
|
||||
"rerank_mode",
|
||||
"decision_rerank",
|
||||
"strategy",
|
||||
"rank_fields",
|
||||
"max_chunks_per_doc",
|
||||
"max_tokens_per_doc",
|
||||
"instruction",
|
||||
"truncate_prompt_tokens",
|
||||
"max_tokens_per_query",
|
||||
"truncation_side",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _TextDocument(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
text: str
|
||||
|
||||
|
||||
class DecisionRerankRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
strategy: Literal["pointwise", "multiple_choice"] = Field(exclude=True)
|
||||
query: str
|
||||
documents: Annotated[tuple[str | _TextDocument, ...], Field(min_length=1, max_length=1000)]
|
||||
top_n: Annotated[StrictInt | None, Field(ge=1)] = None
|
||||
return_documents: bool | None = None
|
||||
timeout: Annotated[float, Field(gt=0, allow_inf_nan=False)] = 600.0
|
||||
extra_headers: Mapping[str, str] | None = None
|
||||
safety_identifier: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_choice_request(self) -> "DecisionRerankRequest":
|
||||
if self.strategy == "multiple_choice" and not 2 <= len(self.documents) <= 255:
|
||||
raise ValueError("multiple_choice requires between 2 and 255 documents")
|
||||
return self
|
||||
|
||||
def document_text(self, index: int) -> str:
|
||||
document: Final = self.documents[index]
|
||||
return document if isinstance(document, str) else document.text
|
||||
|
||||
@property
|
||||
def request_count(self) -> int:
|
||||
return 1 if self.strategy == "multiple_choice" else len(self.documents)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DecisionRerankResult:
|
||||
index: int
|
||||
text: str
|
||||
response: OpenAIDecisionResponse | None = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
def invalid_response(index: int, reason: str, model: str, provider: str) -> litellm.InternalServerError:
|
||||
return litellm.InternalServerError(
|
||||
message=f"Invalid decision rerank response for document {index}: {reason}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
|
||||
|
||||
def _score(outcome: DecisionRerankResult, model: str, provider: str) -> float | Exception:
|
||||
if outcome.error is not None:
|
||||
return outcome.error
|
||||
response: Final = outcome.response
|
||||
if response is None or len(response.answers) != 1:
|
||||
return invalid_response(outcome.index, "expected one relevance answer", model, provider)
|
||||
answer: Final = response.answers[0]
|
||||
if isinstance(answer, OpenAIRefusalAnswer):
|
||||
return litellm.ContentPolicyViolationError(
|
||||
message=f"Decision model refused to score document {outcome.index}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
if not isinstance(answer, OpenAIPredicateAnswer) or answer.name != "relevance":
|
||||
return invalid_response(outcome.index, "expected the relevance predicate", model, provider)
|
||||
if not math.isfinite(answer.probability) or not 0 <= answer.probability <= 1:
|
||||
return invalid_response(
|
||||
outcome.index,
|
||||
"probability must be finite and between zero and one",
|
||||
model,
|
||||
provider,
|
||||
)
|
||||
return answer.probability
|
||||
|
||||
|
||||
def _result(index: int, text: str, score: float, return_documents: bool) -> RerankResponseResult:
|
||||
if return_documents:
|
||||
return RerankResponseResult(
|
||||
index=index,
|
||||
relevance_score=score,
|
||||
document={"text": text},
|
||||
)
|
||||
return RerankResponseResult(index=index, relevance_score=score)
|
||||
|
||||
|
||||
def _choice_scores(
|
||||
outcome: DecisionRerankResult, request: DecisionRerankRequest, model: str, provider: str
|
||||
) -> tuple[float, ...] | Exception:
|
||||
if outcome.error is not None:
|
||||
return outcome.error
|
||||
response: Final = outcome.response
|
||||
if response is None or len(response.answers) != 1:
|
||||
return invalid_response(outcome.index, "expected one choice answer", model, provider)
|
||||
answer: Final = response.answers[0]
|
||||
if isinstance(answer, OpenAIRefusalAnswer):
|
||||
return litellm.ContentPolicyViolationError(
|
||||
message="Decision model refused to rank the candidate set",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
if not isinstance(answer, OpenAIChoiceAnswer) or answer.name != "relevance":
|
||||
return invalid_response(outcome.index, "expected the relevance choice", model, provider)
|
||||
labels: Final = tuple(f"p{index}" for index in range(len(request.documents)))
|
||||
probabilities: Final = {item.value: item.probability for item in answer.probabilities}
|
||||
if len(answer.probabilities) != len(labels) or set(probabilities) != set(labels):
|
||||
return invalid_response(outcome.index, "expected each document label exactly once", model, provider)
|
||||
scores: Final = tuple(probabilities[label] for label in labels)
|
||||
if any(not math.isfinite(score) or not 0 <= score <= 1 for score in scores):
|
||||
return invalid_response(outcome.index, "probabilities must be finite and between zero and one", model, provider)
|
||||
# Two-decimal provider rounding can move the total by half a cent per option.
|
||||
if (
|
||||
sum(scores) <= 0
|
||||
or sum(max(0.0, score - 0.005) for score in scores) > 1.0 + 1e-9
|
||||
or sum(min(1.0, score + 0.005) for score in scores) < 1.0 - 1e-9
|
||||
):
|
||||
return invalid_response(
|
||||
outcome.index,
|
||||
"choice probabilities must sum to one within rounding precision",
|
||||
model,
|
||||
provider,
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
def result_error(
|
||||
outcome: DecisionRerankResult, request: DecisionRerankRequest, model: str, provider: str
|
||||
) -> Exception | None:
|
||||
score: Final = (
|
||||
_choice_scores(outcome, request, model, provider)
|
||||
if request.strategy == "multiple_choice"
|
||||
else _score(outcome, model, provider)
|
||||
)
|
||||
return score if isinstance(score, Exception) else None
|
||||
|
||||
|
||||
def _scores(
|
||||
outcomes: Sequence[DecisionRerankResult],
|
||||
request: DecisionRerankRequest,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> tuple[float | Exception, ...]:
|
||||
if request.strategy == "multiple_choice":
|
||||
scores: Final = _choice_scores(outcomes[0], request, model, provider) if outcomes else ()
|
||||
return (scores,) if isinstance(scores, Exception) else scores
|
||||
return tuple(_score(outcome, model, provider) for outcome in outcomes)
|
||||
|
||||
|
||||
def _ranked_results(
|
||||
outcomes: Sequence[DecisionRerankResult],
|
||||
request: DecisionRerankRequest,
|
||||
scores: Sequence[float | Exception],
|
||||
) -> tuple[RerankResponseResult, ...]:
|
||||
if request.strategy == "multiple_choice":
|
||||
return tuple(
|
||||
_result(
|
||||
index,
|
||||
request.document_text(index),
|
||||
score,
|
||||
bool(request.return_documents),
|
||||
)
|
||||
for index, score in enumerate(scores)
|
||||
if isinstance(score, float)
|
||||
)
|
||||
return tuple(
|
||||
_result(outcome.index, outcome.text, score, bool(request.return_documents))
|
||||
for outcome, score in zip(outcomes, scores, strict=True)
|
||||
if isinstance(score, float)
|
||||
)
|
||||
|
||||
|
||||
def _response(
|
||||
outcomes: Sequence[DecisionRerankResult], request: DecisionRerankRequest, scores: Sequence[float | Exception]
|
||||
) -> DecisionRerankResponse:
|
||||
responses: Final = tuple(outcome.response for outcome in outcomes if outcome.response is not None)
|
||||
results: Final = _ranked_results(outcomes, request, scores)
|
||||
ordered: Final = sorted(results, key=lambda result: (-result["relevance_score"], result["index"]))
|
||||
return DecisionRerankResponse(
|
||||
results=ordered[: request.top_n],
|
||||
meta={
|
||||
"tokens": {
|
||||
"input_tokens": sum(response.usage.input_tokens for response in responses),
|
||||
"output_tokens": sum(response.usage.output_tokens for response in responses),
|
||||
},
|
||||
},
|
||||
decision_responses=responses,
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMDecisionsRerankConfig:
|
||||
@staticmethod
|
||||
def validate_request(
|
||||
params: Mapping[str, object],
|
||||
model: str,
|
||||
provider: str,
|
||||
strategy: Literal["pointwise", "multiple_choice"],
|
||||
) -> DecisionRerankRequest:
|
||||
if params.get("stream") is True:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message="Decision reranking does not support stream=True",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
unsupported: Final = tuple(key for key in _UNSUPPORTED if params.get(key) is not None)
|
||||
if unsupported and not (params.get("drop_params") or litellm.drop_params):
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"Decision reranking does not support: {', '.join(sorted(unsupported))}",
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
)
|
||||
try:
|
||||
supported_params: Final = {key: value for key, value in params.items() if key not in _UNSUPPORTED}
|
||||
return DecisionRerankRequest.model_validate({**supported_params, "strategy": strategy})
|
||||
except ValidationError as error:
|
||||
raise litellm.BadRequestError(
|
||||
message="Invalid decision rerank request: "
|
||||
+ "; ".join(detail["msg"] for detail in error.errors(include_input=False, include_url=False)),
|
||||
model=model,
|
||||
llm_provider=provider,
|
||||
) from error
|
||||
|
||||
@staticmethod
|
||||
def transform_rerank_request(request: DecisionRerankRequest, index: int) -> OpenAIDecisionRequestBody:
|
||||
if request.strategy == "multiple_choice":
|
||||
return OpenAIDecisionRequestBody(
|
||||
input=json.dumps(
|
||||
{
|
||||
"query": request.query,
|
||||
"passages": {
|
||||
f"p{index}": request.document_text(index) for index in range(len(request.documents))
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
questions=(
|
||||
OpenAIChoiceQuestion(
|
||||
type="choice",
|
||||
name="relevance",
|
||||
instructions="Which passage is the most relevant to the query?",
|
||||
choices=tuple(OpenAIChoiceOption(value=f"p{index}") for index in range(len(request.documents))),
|
||||
),
|
||||
),
|
||||
)
|
||||
return OpenAIDecisionRequestBody(
|
||||
input=json.dumps(
|
||||
{"query": request.query, "passage": request.document_text(index)},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
questions=(OpenAIPredicateQuestion(type="predicate", name="relevance", instructions=_RELEVANCE),),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def transform_decisions_response(
|
||||
outcomes: Sequence[DecisionRerankResult],
|
||||
request: DecisionRerankRequest,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> tuple[DecisionRerankResponse, Exception | None]:
|
||||
"""Retain completed responses for billing even when another result fails validation."""
|
||||
scores: Final = _scores(outcomes, request, model, provider)
|
||||
return _response(outcomes, request, scores), next(
|
||||
(score for score in scores if isinstance(score, Exception)), None
|
||||
)
|
||||
|
|
@ -1,11 +1,15 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from functools import partial
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
|
||||
|
|
@ -13,12 +17,17 @@ from litellm.llms.bedrock.rerank.handler import BedrockRerankHandler
|
|||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.together_ai.rerank.handler import TogetherAIRerank
|
||||
from litellm.llms.watsonx.common_utils import IBMWatsonXMixin
|
||||
from litellm.rerank_api.rerank_utils import get_optional_rerank_params
|
||||
from litellm.rerank_api.rerank_utils import (
|
||||
get_decision_rerank_strategy,
|
||||
get_optional_rerank_params,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret, get_secret_str
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.router import *
|
||||
from litellm.utils import ProviderConfigManager, client, exception_type
|
||||
|
||||
_DECISION_KWARGS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
####### ENVIRONMENT VARIABLES ###################
|
||||
# Initialize any necessary instances or variables here
|
||||
together_rerank: Final = TogetherAIRerank()
|
||||
|
|
@ -27,6 +36,12 @@ base_llm_http_handler = BaseLLMHTTPHandler()
|
|||
#################################################
|
||||
|
||||
|
||||
def _decision_rerank_timeout(timeout: float | httpx.Timeout) -> float:
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
return float(litellm.request_timeout) if timeout.read is None else timeout.read
|
||||
return timeout
|
||||
|
||||
|
||||
@client
|
||||
async def arerank(
|
||||
model: str,
|
||||
|
|
@ -154,6 +169,42 @@ def rerank(
|
|||
api_key=optional_params.api_key,
|
||||
)
|
||||
|
||||
decision_strategy: Final = get_decision_rerank_strategy(model, litellm.LlmProviders(_custom_llm_provider))
|
||||
if decision_strategy is not None:
|
||||
decision_kwargs: Final = _DECISION_KWARGS.validate_python(kwargs)
|
||||
decision_timeout: Final = CompletionTimeout.as_limit(
|
||||
litellm.request_timeout if optional_params.timeout is None else optional_params.timeout
|
||||
)
|
||||
from litellm.rerank_api.litellm_decisions_transformation.handler import (
|
||||
rerank_with_decisions,
|
||||
)
|
||||
|
||||
return rerank_with_decisions(
|
||||
model=model,
|
||||
provider=_custom_llm_provider,
|
||||
api_key=dynamic_api_key or optional_params.api_key,
|
||||
api_base=dynamic_api_base or optional_params.api_base,
|
||||
logging_obj=litellm_logging_obj,
|
||||
is_async=_is_async,
|
||||
strategy=decision_strategy,
|
||||
transport_timeout=decision_timeout,
|
||||
params=_DECISION_KWARGS.validate_python(
|
||||
{
|
||||
**decision_kwargs,
|
||||
"query": query,
|
||||
"documents": documents,
|
||||
"top_n": top_n,
|
||||
"return_documents": return_documents,
|
||||
"rank_fields": rank_fields,
|
||||
"max_chunks_per_doc": max_chunks_per_doc,
|
||||
"max_tokens_per_doc": max_tokens_per_doc,
|
||||
"timeout": _decision_rerank_timeout(decision_timeout),
|
||||
"extra_headers": decision_kwargs.get("extra_headers") or headers,
|
||||
}
|
||||
),
|
||||
kwargs=decision_kwargs,
|
||||
)
|
||||
|
||||
rerank_provider_config: Final[BaseRerankConfig] = ProviderConfigManager.get_provider_rerank_config(
|
||||
model=model,
|
||||
provider=litellm.LlmProviders(_custom_llm_provider),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,31 @@
|
|||
from typing import Any, Final
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
||||
def _catalog_rerank_strategy(model: str) -> Literal["pointwise", "multiple_choice"] | None:
|
||||
info: Final = as_str_mapping(litellm.model_cost.get(model)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Narrow the legacy untyped catalog entry at the boundary.
|
||||
strategy: Final = None if info is None else info.get("rerank_strategy")
|
||||
match strategy:
|
||||
case "pointwise":
|
||||
return "pointwise"
|
||||
case "multiple_choice":
|
||||
return "multiple_choice"
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def get_decision_rerank_strategy(
|
||||
model: str, custom_llm_provider: LlmProviders
|
||||
) -> Literal["pointwise", "multiple_choice"] | None:
|
||||
"""Read the catalog method for an already resolved model and provider."""
|
||||
if ProviderConfigManager.get_provider_decisions_config(model, custom_llm_provider) is None:
|
||||
return None
|
||||
return _catalog_rerank_strategy(f"{custom_llm_provider.value}/{model}") or _catalog_rerank_strategy(model)
|
||||
|
||||
|
||||
def get_optional_rerank_params(
|
||||
|
|
|
|||
|
|
@ -6,9 +6,10 @@ https://docs.cohere.com/reference/rerank
|
|||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import ConfigDict, PrivateAttr
|
||||
from pydantic import ConfigDict, Field, PrivateAttr
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
|
||||
from litellm.types.decisions import OpenAIDecisionResponse
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
||||
|
|
@ -103,3 +104,9 @@ class RerankResponse(LiteLLMBaseModel):
|
|||
|
||||
def __contains__(self, key) -> bool:
|
||||
return key in self.__dict__
|
||||
|
||||
|
||||
class DecisionRerankResponse(RerankResponse):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
decision_responses: tuple[OpenAIDecisionResponse, ...] = Field(exclude=True)
|
||||
|
|
|
|||
|
|
@ -203,6 +203,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
|
|||
supports_audio_input: bool | None
|
||||
supports_video_input: ReadOnly[bool | None]
|
||||
supports_box_questions: ReadOnly[bool | None]
|
||||
rerank_strategy: ReadOnly[Literal["pointwise", "multiple_choice"] | None]
|
||||
supports_embedding_image_input: bool | None
|
||||
supports_audio_output: bool | None
|
||||
supports_pdf_input: bool | None
|
||||
|
|
|
|||
|
|
@ -6967,6 +6967,7 @@ def get_model_info_helper(
|
|||
supports_audio_input=_model_info.get("supports_audio_input", None),
|
||||
supports_video_input=_model_info.get("supports_video_input", None),
|
||||
supports_box_questions=_model_info.get("supports_box_questions", None),
|
||||
rerank_strategy=_model_info.get("rerank_strategy", None),
|
||||
supports_audio_output=_model_info.get("supports_audio_output", None),
|
||||
supports_pdf_input=_model_info.get("supports_pdf_input", None),
|
||||
supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None),
|
||||
|
|
|
|||
|
|
@ -46328,6 +46328,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/api-reference/decisions-post"
|
||||
},
|
||||
|
|
@ -46336,6 +46337,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/getting-started/pricing"
|
||||
},
|
||||
|
|
@ -46344,6 +46346,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46352,6 +46355,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46360,6 +46364,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -46368,6 +46373,7 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "multiple_choice",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.perplexity.ai/docs/decisions/quickstart"
|
||||
},
|
||||
|
|
@ -72552,6 +72558,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
|
|
@ -72560,6 +72567,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
|
|
@ -72569,6 +72577,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models"
|
||||
},
|
||||
|
|
@ -72577,6 +72586,7 @@
|
|||
"litellm_provider": "typesafe",
|
||||
"max_input_tokens": 65536,
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,118 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
|
||||
DecisionRerankRequest,
|
||||
DecisionRerankResult,
|
||||
LiteLLMDecisionsRerankConfig,
|
||||
)
|
||||
from litellm.types.decisions import (
|
||||
OpenAIDecisionResponse,
|
||||
OpenAIDecisionUsage,
|
||||
OpenAIPredicateAnswer,
|
||||
OpenAIRefusalAnswer,
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_score_keeps_completed_responses_and_usage_for_failure_billing():
|
||||
request: Final = DecisionRerankRequest(strategy="pointwise", query="warranty", documents=("relevant", "invalid"))
|
||||
responses: Final = tuple(
|
||||
OpenAIDecisionResponse(
|
||||
model="decision-unit-test",
|
||||
answers=(OpenAIPredicateAnswer(name="relevance", probability=score),),
|
||||
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
|
||||
)
|
||||
for score in (0.9, 1.4)
|
||||
)
|
||||
outcomes: Final = tuple(
|
||||
DecisionRerankResult(index, request.document_text(index), response=response)
|
||||
for index, response in enumerate(responses)
|
||||
)
|
||||
result, error = LiteLLMDecisionsRerankConfig.transform_decisions_response(
|
||||
outcomes, request, "decision-unit-test", "typesafe"
|
||||
)
|
||||
assert isinstance(error, litellm.InternalServerError)
|
||||
assert result.results == [{"index": 0, "relevance_score": 0.9}]
|
||||
assert result.meta["tokens"] == {"input_tokens": 20, "output_tokens": 4}
|
||||
assert result.decision_responses[0] is responses[0]
|
||||
assert result.decision_responses[1] is responses[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strategy,limit", [("pointwise", 1000), ("multiple_choice", 255)])
|
||||
def test_document_limit_is_enforced_at_request_boundary(strategy, limit):
|
||||
params: Final = {"query": "warranty", "documents": ("passage",) * limit}
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, "decision-unit-test", "typesafe", strategy)
|
||||
assert len(request.documents) == limit
|
||||
with pytest.raises(litellm.BadRequestError, match=str(limit)):
|
||||
LiteLLMDecisionsRerankConfig.validate_request(
|
||||
{**params, "documents": ("passage",) * (limit + 1)}, "decision-unit-test", "typesafe", strategy
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("per_request,global_default", [(True, False), (None, True), (False, True)])
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
{"rank_fields": ["text"]},
|
||||
{"max_chunks_per_doc": 1},
|
||||
{"max_tokens_per_doc": 10},
|
||||
{"instruction": "Rank only identical text"},
|
||||
],
|
||||
)
|
||||
def test_drop_params_preserves_supported_request_fields(per_request, global_default, unsupported, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", global_default)
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(
|
||||
{
|
||||
"query": "warranty",
|
||||
"documents": ("passage",),
|
||||
"top_n": 1,
|
||||
"return_documents": True,
|
||||
"drop_params": per_request,
|
||||
"extra_headers": {"x-request-id": "supported-header"},
|
||||
"safety_identifier": "supported-identifier",
|
||||
**unsupported,
|
||||
},
|
||||
"decision-unit-test",
|
||||
"typesafe",
|
||||
"pointwise",
|
||||
)
|
||||
assert request.query == "warranty"
|
||||
assert request.documents == ("passage",)
|
||||
assert request.top_n == 1
|
||||
assert request.return_documents is True
|
||||
assert request.extra_headers == {"x-request-id": "supported-header"}
|
||||
assert request.safety_identifier == "supported-identifier"
|
||||
assert request.timeout == 600.0
|
||||
|
||||
|
||||
def test_unsupported_parameters_still_fail_when_dropping_is_disabled(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="rank_fields"):
|
||||
LiteLLMDecisionsRerankConfig.validate_request(
|
||||
{"query": "warranty", "documents": ("passage",), "rank_fields": ["text"], "drop_params": False},
|
||||
"decision-unit-test",
|
||||
"typesafe",
|
||||
"pointwise",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strategy", ["pointwise", "multiple_choice"])
|
||||
def test_refusal_retains_billable_usage_and_returns_content_policy_error(strategy):
|
||||
request: Final = DecisionRerankRequest(strategy=strategy, query="warranty", documents=("first", "second"))
|
||||
response: Final = OpenAIDecisionResponse(
|
||||
model="decision-unit-test",
|
||||
answers=(OpenAIRefusalAnswer(name="relevance"),),
|
||||
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
|
||||
)
|
||||
result, error = LiteLLMDecisionsRerankConfig.transform_decisions_response(
|
||||
(DecisionRerankResult(0, request.document_text(0), response=response),),
|
||||
request,
|
||||
"decision-unit-test",
|
||||
"typesafe",
|
||||
)
|
||||
assert isinstance(error, litellm.ContentPolicyViolationError)
|
||||
assert result.results == []
|
||||
assert result.meta["tokens"] == {"input_tokens": 10, "output_tokens": 2}
|
||||
assert result.decision_responses == (response,)
|
||||
|
|
@ -34,7 +34,8 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego
|
|||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.decisions import OpenAIDecisionResponse, OpenAIDecisionUsage
|
||||
from litellm.types.rerank import DecisionRerankResponse, RerankResponse
|
||||
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
@ -4033,6 +4034,130 @@ def test_completion_cost_per_second_deployment_bills_the_call_duration(
|
|||
assert cost == pytest.approx(0.02 * expected_seconds)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("document_count", "rate", "stamped_ms", "total_time", "logged_ms", "expected_seconds"),
|
||||
[
|
||||
(1, 0.5, None, 2000.0, 4000.0, 2.0),
|
||||
(3, 0.5, None, 2000.0, 4000.0, 2.0),
|
||||
(3, 0.5, 3000.0, 2000.0, 4000.0, 3.0),
|
||||
(3, 0.5, None, 0.0, 4000.0, 4.0),
|
||||
(3, 0.0, None, 2000.0, 4000.0, 2.0),
|
||||
],
|
||||
)
|
||||
def test_decision_rerank_custom_per_second_price_bills_outer_duration_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
document_count: int,
|
||||
rate: float,
|
||||
stamped_ms: float | None,
|
||||
total_time: float,
|
||||
logged_ms: float,
|
||||
expected_seconds: float,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"percentage": 0.2, "fixed_amount": 0.05}})
|
||||
child: Final = OpenAIDecisionResponse(
|
||||
model="openai/rerank-time-test",
|
||||
answers=(),
|
||||
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=0, total_tokens=10),
|
||||
)
|
||||
child.set_hidden_params({"additional_headers": {"llm_provider-x-litellm-response-cost": "50.0"}})
|
||||
response: Final = DecisionRerankResponse(decision_responses=(child,) * document_count).model_copy(
|
||||
update={"_response_ms": stamped_ms}
|
||||
)
|
||||
logging_obj: Final = _logging_obj_with_call_window(logged_ms)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response,
|
||||
custom_cost_per_second=rate,
|
||||
total_time=total_time,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
expected_base: Final = rate * expected_seconds
|
||||
assert cost == pytest.approx(expected_base * 0.9 * 1.2 + 0.05)
|
||||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["original_cost"] == pytest.approx(expected_base)
|
||||
assert logging_obj.cost_breakdown["total_cost"] == pytest.approx(cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider", "pricing_provider", "expected_cost"),
|
||||
[
|
||||
(None, None, None, 1.25),
|
||||
("unprefixed-deployment", None, None, 1.25),
|
||||
(None, "openai", None, 1.6),
|
||||
(None, "typesafe", "openai", 1.6),
|
||||
],
|
||||
)
|
||||
def test_decision_rerank_per_second_price_preserves_child_provider_with_explicit_overrides(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str | None,
|
||||
provider: str | None,
|
||||
pricing_provider: str | None,
|
||||
expected_cost: float,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {})
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"cost_margin_config",
|
||||
{"typesafe": {"percentage": 0.2, "fixed_amount": 0.05}, "openai": {"percentage": 0.5, "fixed_amount": 0.1}},
|
||||
)
|
||||
child: Final = OpenAIDecisionResponse(
|
||||
model="rerank-time-test",
|
||||
answers=(),
|
||||
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=0, total_tokens=10),
|
||||
)
|
||||
child.set_hidden_params({"model": "typesafe/rerank-time-test", "custom_llm_provider": "typesafe"})
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=DecisionRerankResponse(decision_responses=(child, child)),
|
||||
model=model,
|
||||
custom_cost_per_second=0.5,
|
||||
total_time=2000.0,
|
||||
custom_llm_provider=provider,
|
||||
pricing_provider=pricing_provider,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_decision_rerank_without_billable_responses_has_no_time_cost() -> None:
|
||||
assert (
|
||||
completion_cost(
|
||||
completion_response=DecisionRerankResponse(decision_responses=()),
|
||||
custom_cost_per_second=0.5,
|
||||
total_time=2000.0,
|
||||
)
|
||||
== 0.0
|
||||
)
|
||||
|
||||
|
||||
def test_decision_rerank_custom_token_price_takes_precedence_over_per_second_price(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {})
|
||||
response: Final = DecisionRerankResponse(
|
||||
decision_responses=(
|
||||
OpenAIDecisionResponse(
|
||||
model="openai/rerank-time-test",
|
||||
answers=(),
|
||||
usage=OpenAIDecisionUsage(input_tokens=10, output_tokens=2, total_tokens=12),
|
||||
),
|
||||
)
|
||||
* 3
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response,
|
||||
custom_cost_per_token={"input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
|
||||
custom_cost_per_second=0.5,
|
||||
total_time=2000.0,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(3 * (10 * 0.01 + 2 * 0.02))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["audio_speech", "video_generation", "realtime"])
|
||||
def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1167,6 +1167,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_tool_search": {"type": "boolean"},
|
||||
"supports_video_input": {"type": "boolean"},
|
||||
"supports_box_questions": {"type": "boolean"},
|
||||
"rerank_strategy": {"type": "string", "enum": ["pointwise", "multiple_choice"]},
|
||||
"supports_vision": {"type": "boolean"},
|
||||
"supports_web_search": {"type": "boolean"},
|
||||
"token_count_framing": {"type": "integer"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue