fix(rerank): scope decision adapter to SDK integration

Restore proxy endpoints, spend tracking, and logs UI to the base branch. Remove endpoint-specific logging capture and preserve decision cost handling through the existing calculator API
This commit is contained in:
ishaan-j-berri 2026-10-23 18:01:12 -07:00
parent 72bcd5d945
commit 314c52bcf4
12 changed files with 56 additions and 765 deletions

View file

@ -1550,9 +1550,9 @@ def _decision_rerank_cost(
) -> tuple[float, CostBreakdown | None, float]:
if logging_obj is not None:
logging_obj.cost_breakdown = None
cost: Final = price(response)
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
provider_cost: Final = get_response_cost_from_hidden_params(response.hidden_params)
cost: Final = price(response) if custom_pricing or provider_cost is None else provider_cost
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
fixed: Final = (
_apply_cost_margin(
0.0,
@ -1639,7 +1639,7 @@ def completion_cost(
return _decision_rerank_cost(
completion_response.decision_responses,
partial(
response_cost_calculator,
completion_cost,
model=model or (decision_rerank_child.model if decision_rerank_child is not None else ""),
call_type="decisions",
custom_llm_provider=custom_llm_provider,
@ -2380,7 +2380,6 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
) -> float:
"""
Returns
@ -2398,7 +2397,7 @@ def response_cost_calculator(
getattr(response_object, HIDDEN_PARAMS_ATTR),
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing and custom_cost_per_token is None:
if provider_response_cost is not None and not custom_pricing:
return provider_response_cost
response_cost = completion_cost(
@ -2419,7 +2418,6 @@ def response_cost_calculator(
data_residency=data_residency,
vertex_location=vertex_location,
region_name=region_name,
custom_cost_per_token=custom_cost_per_token,
)
return response_cost
except Exception as e:

View file

@ -7,9 +7,7 @@ import orjson
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import ORJSONResponse
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.proxy._types import *
from litellm.proxy.auth.auth_utils import get_request_route
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
@ -22,9 +20,6 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_param,
openai_error_type,
)
from litellm.rerank_api.litellm_decisions_transformation.handler import (
DecisionRerankLoggingCapture,
)
router: Final = APIRouter()
@ -64,8 +59,7 @@ async def rerank(
)
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
logging_capture: Final = DecisionRerankLoggingCapture(litellm_call_id)
data: dict[str, object] = {"litellm_call_id": litellm_call_id} # mutable-ok: proxy hooks enrich request data
data = {"litellm_call_id": litellm_call_id}
try:
body: Final = await request.body()
data = orjson.loads(body) | data
@ -79,28 +73,18 @@ async def rerank(
version=version,
proxy_config=proxy_config,
)
metadata: Final = as_str_mapping(data.get("metadata")) or {}
spend_metadata: Final = as_str_mapping(metadata.get("spend_logs_metadata")) or {}
data["metadata"] = {
**metadata,
"spend_logs_metadata": {
**spend_metadata,
"gateway_endpoint": {"method": request.method, "path": get_request_route(request)},
},
}
### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook(user_api_key_dict=user_api_key_dict, data=data, call_type="rerank")
## ROUTE TO CORRECT ENDPOINT ##
with logging_capture.capture():
llm_call: Final = await route_request(
data=data,
route_type="arerank",
llm_router=llm_router,
user_model=user_model,
)
response: Final = await llm_call
llm_call: Final = await route_request(
data=data,
route_type="arerank",
llm_router=llm_router,
user_model=user_model,
)
response: Final = await llm_call
### ALERTING ###
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
@ -129,9 +113,6 @@ async def rerank(
return response
except Exception as e:
partial_logging: Final = logging_capture.logging_obj
if partial_logging is not None:
data["litellm_logging_obj"] = partial_logging
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)

View file

@ -16,7 +16,6 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.proxy._types import (
@ -40,22 +39,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
tag_cache_key,
team_membership_reservation_cache_key,
)
from litellm.proxy.spend_tracking.input_tokens import (
count_input_tokens,
count_input_tokens_for_model,
count_text_tokens,
)
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
DecisionRerankRequest,
LiteLLMDecisionsRerankConfig,
)
from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy
from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
from litellm.types.utils import LlmProviders
@dataclass
@ -84,7 +73,7 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
}
_CACHED_VALUE: Final = TypeAdapter(object)
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
class _CounterReservationUnavailable(Exception):
@ -895,7 +884,7 @@ def _coerce_window(window: object) -> Mapping[str, object]:
return window
if isinstance(window, str):
try:
return _OBJECT_MAPPING.validate_python(json.loads(window))
return _WINDOW_OBJECT.validate_python(json.loads(window))
except Exception:
return {}
model_dump: Final = getattr(window, "model_dump", None)
@ -1352,19 +1341,13 @@ def _estimate_request_input_cost_for_model(
llm_router: Router | None,
input_tokens: int | None = None,
) -> float | None:
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router)
counted: Final = (
_decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens
)
estimates: Final = [
_input_cost_for_cost_info(
request_body=request_body,
route=route,
model=model,
model_info=model_info,
input_tokens=counted,
is_decision_rerank=bool(plans),
input_tokens=input_tokens,
)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
]
@ -1378,7 +1361,6 @@ def _input_cost_for_cost_info(
model: str,
model_info: Mapping[str, object],
input_tokens: int | None = None,
is_decision_rerank: bool = False,
) -> float | None:
estimated_input_tokens: Final = _estimate_input_tokens(
request_body=request_body,
@ -1390,8 +1372,6 @@ def _input_cost_for_cost_info(
if estimated_input_tokens is None:
return None
tiered_pricing: Final = model_info.get("tiered_pricing")
if is_decision_rerank and tiered_pricing:
return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token")
if isinstance(tiered_pricing, list) and tiered_pricing:
tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens)
if tier is not None:
@ -1409,20 +1389,13 @@ def _estimate_request_max_cost_for_model(
llm_router: Router | None,
input_tokens: int | None = None,
) -> float | None:
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router)
counted: Final = (
_decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens
)
request_count: Final = max((request.request_count for _, request in plans), default=0)
estimates: Final = [
_max_cost_for_cost_info(
request_body=request_body,
route=route,
model=model,
model_info=model_info,
input_tokens=counted,
decision_request_count=request_count,
input_tokens=input_tokens,
)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
]
@ -1436,7 +1409,6 @@ def _max_cost_for_cost_info(
model: str,
model_info: Mapping[str, object],
input_tokens: int | None = None,
decision_request_count: int = 0,
) -> float | None:
image_cost: Final = _estimate_image_generation_cost(
request_body=request_body,
@ -1456,18 +1428,12 @@ def _max_cost_for_cost_info(
request_body=request_body,
route=route,
model_info=model_info,
decision_request_count=decision_request_count,
)
if estimated_input_tokens is None or output_tokens is None:
return None
output_multiplier: Final = _get_output_multiplier(request_body=request_body)
tiered_pricing: Final = model_info.get("tiered_pricing")
if decision_request_count and tiered_pricing:
return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token") + output_tokens * max(
_max_tier_rate(tiered_pricing, "output_cost_per_token"),
_max_tier_rate(tiered_pricing, "output_cost_per_reasoning_token"),
)
if isinstance(tiered_pricing, list) and tiered_pricing:
tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens)
if tier is not None:
@ -1500,14 +1466,6 @@ def _max_cost_for_cost_info(
return cost
_TIER_COSTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def _max_tier_rate(tiered_pricing: object, cost_key: str) -> float:
tiers: Final = _TIER_COSTS.validate_python(tiered_pricing)
return max(tier_rate(dict(tier), cost_key) for tier in tiers)
def _estimate_image_generation_cost(
request_body: dict,
model_info: Mapping[str, object],
@ -1639,84 +1597,7 @@ async def count_request_input_tokens(
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
if not models:
return MappingProxyType({})
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
plans: Final = {model: _decision_rerank_plans(typed_body, route, model, llm_router) for model in models}
regular_models: Final = tuple(model for model in models if not plans[model])
regular_counts: Final = await count_input_tokens(
request_body=request_body, raw_body=raw_body, models=regular_models
)
decision_counts: Final = {
model: await asyncio.to_thread(_decision_rerank_input_tokens, typed_body, model, plan)
for model, plan in plans.items()
if plan
}
return MappingProxyType({**regular_counts, **decision_counts})
def _decision_rerank_plan(
request_body: Mapping[str, object], model: str, custom_llm_provider: str | None
) -> tuple[str, DecisionRerankRequest] | None:
try:
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
strategy: Final = get_decision_rerank_strategy(resolved_model, LlmProviders(provider))
if strategy is None:
return None
pricing_params: Final = {key: value for key, value in request_body.items() if key != "timeout"}
request: Final = LiteLLMDecisionsRerankConfig.validate_request(
pricing_params, resolved_model, provider, strategy
)
except (litellm.BadRequestError, litellm.UnsupportedParamsError, ValueError):
return None
return f"{provider}/{resolved_model}", request
def _decision_rerank_plans(
request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None
) -> tuple[tuple[str, DecisionRerankRequest], ...]:
if "rerank" not in route:
return ()
deployments: Final = () if llm_router is None else (llm_router.get_model_list(model_name=model) or ())
if not deployments:
provider: Final = request_body.get("custom_llm_provider")
direct_plan: Final = _decision_rerank_plan(request_body, model, provider if isinstance(provider, str) else None)
return () if direct_plan is None else (direct_plan,)
return tuple(
plan
for deployment in deployments
if (plan := _decision_rerank_deployment_plan(request_body, deployment)) is not None
)
def _decision_rerank_deployment_plan(
request_body: Mapping[str, object], deployment: DeploymentTypedDict
) -> tuple[str, DecisionRerankRequest] | None:
params: Final = as_str_mapping(deployment["litellm_params"]) if "litellm_params" in deployment else None
if params is None:
return None
model: Final = params.get("model")
provider: Final = params.get("custom_llm_provider")
if not isinstance(model, str):
return None
return _decision_rerank_plan({**params, **request_body}, model, provider if isinstance(provider, str) else None)
def _decision_rerank_input_tokens(
request_body: Mapping[str, object], requested_model: str, plans: Sequence[tuple[str, DecisionRerankRequest]]
) -> int:
native_count: Final = count_input_tokens_for_model(dict(request_body), requested_model) or 0
decision_counts: Final = (
sum(
count_text_tokens(
model=model,
text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json(
exclude_none=True
),
)
for index in range(request.request_count)
)
for model, request in plans
)
return max(native_count, *decision_counts)
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
def _estimate_input_tokens(
@ -1748,11 +1629,7 @@ def _estimate_output_tokens(
request_body: dict,
route: str,
model_info: Mapping[str, object],
decision_request_count: int = 0,
) -> int | None:
if decision_request_count:
ceiling: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK
return decision_request_count * ceiling
if _is_input_only_route(route=route):
return 0

View file

@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
tool_choice=request_body.get("tool_choice"),
)
except ValueError:
return count_text_tokens(model=model, text=request_body.get("messages"))
return _count_text_tokens(model=model, text=request_body.get("messages"))
if "prompt" in request_body:
return count_text_tokens(model=model, text=request_body.get("prompt"))
return _count_text_tokens(model=model, text=request_body.get("prompt"))
if "input" in request_body:
return count_text_tokens(
return _count_text_tokens(
model=model, text=billable_input_since_newest_compaction(request_body.get("input"))
)
if "query" in request_body or "documents" in request_body:
query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query"))
document_tokens: Final = count_text_tokens(
query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query"))
document_tokens: Final = _count_text_tokens(
model=model,
text=request_body.get("documents"),
)
@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
return None
def count_text_tokens(model: str, text: object) -> int:
def _count_text_tokens(model: str, text: object) -> int:
if text is None:
return 0

View file

@ -2,10 +2,8 @@
import asyncio
import time
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor, wait
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, replace
from functools import partial
from queue import Empty, SimpleQueue
@ -36,29 +34,6 @@ _CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"})
_MAX_CONCURRENCY: Final = 8
class DecisionRerankLoggingCapture:
def __init__(self, call_id: str) -> None:
self.call_id: Final = call_id
self.logging_obj: Logging | None = None
@contextmanager
def capture(self) -> Generator[None]:
token: Final = _LOGGING_CAPTURE.set(self)
try:
yield
finally:
_LOGGING_CAPTURE.reset(token)
def record(self, logging_obj: Logging) -> None:
if logging_obj.litellm_call_id == self.call_id:
self.logging_obj = logging_obj
_LOGGING_CAPTURE: Final[ContextVar[DecisionRerankLoggingCapture | None]] = ContextVar(
"decision_rerank_logging_capture", default=None
)
@dataclass(frozen=True, slots=True)
class _DocumentCall:
index: int
@ -322,9 +297,6 @@ def rerank_with_decisions(
params: Mapping[str, object],
kwargs: Mapping[str, object],
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
logging_capture: Final = _LOGGING_CAPTURE.get()
if logging_capture is not None:
logging_capture.record(logging_obj)
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy)
deadline: Final = time.monotonic() + request.timeout
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs

View file

@ -2,32 +2,19 @@
Tests for rerank_endpoints/endpoints.py response headers.
"""
import asyncio
import json
import logging
from collections.abc import Iterator, Mapping
from datetime import datetime, timezone
from queue import SimpleQueue
from typing import Final
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
from collections.abc import Iterator
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import HTTPException, Request, Response
import litellm
import litellm.proxy.common_request_processing as common_request_processing_mod
import litellm.proxy.proxy_server as proxy_server_mod
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.rerank_endpoints.endpoints import rerank
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import ProxyLogging
from litellm.types.utils import RerankResponse
HIDDEN_PARAMS = {
@ -44,21 +31,8 @@ HIDDEN_PARAMS = {
}
def _build_request(
headers: tuple[tuple[bytes, bytes], ...] = (),
documents: tuple[str, ...] = ("a", "b"),
path: str = "/rerank",
root_path: str = "",
metadata: Mapping[str, object] | None = None,
) -> Request:
body = json.dumps(
{
"model": "rerank-model",
"query": "q",
"documents": documents,
**({"metadata": metadata} if metadata is not None else {}),
}
).encode()
def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request:
body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode()
async def receive():
return {"type": "http.request", "body": body, "more_body": False}
@ -67,8 +41,7 @@ def _build_request(
scope={
"type": "http",
"method": "POST",
"path": path,
"root_path": root_path,
"path": "/rerank",
"headers": [(b"content-type", b"application/json"), *headers],
"query_string": b"",
},
@ -236,205 +209,3 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc
error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch)
assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "root_path", "expected_path"),
[
("/rerank", "", "/rerank"),
("/v1/rerank", "", "/v1/rerank"),
("/v2/rerank", "", "/v2/rerank"),
("/gateway/v2/rerank", "/gateway", "/v2/rerank"),
],
)
@pytest.mark.parametrize("partial_failure", [False, True])
async def test_decision_rerank_writes_one_log_with_gateway_endpoint_and_aggregate_usage(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
path: str,
root_path: str,
expected_path: str,
partial_failure: bool,
) -> None:
model: Final = "typesafe/jevrank-spend-unit-test"
rate: Final = 0.004
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "typesafe",
"mode": "evaluation",
"rerank_strategy": "pointwise",
"input_cost_per_token": rate,
"output_cost_per_token": 0,
},
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "cache", None)
llm_router: Final = litellm.Router(
model_list=[
{
"model_name": "rerank-model",
"litellm_params": {"model": model, "api_key": "test-key"},
}
],
num_retries=0,
)
writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
db_logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
proxy_logging.db_spend_update_writer = writer
monkeypatch.setattr(litellm, "callbacks", [db_logger])
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", proxy_logging)
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
def score(request: httpx.Request) -> httpx.Response:
passage: Final = json.loads(json.loads(request.content)["state"])["passage"]
if passage == "failed":
return httpx.Response(503, json={"error": "unavailable"})
return httpx.Response(
200,
json={
"model": "jevrank-spend-unit-test",
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
"usage": {
"input_tokens": 100 if passage == "first" else 80,
"output_tokens": 0,
},
},
)
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").mock(
side_effect=score
)
request: Final = _build_request(
headers=((b"x-litellm-call-id", b"rerank-spend"),),
documents=("first", "failed" if partial_failure else "middle", "last"),
path=path,
root_path=root_path,
metadata={
"tags": ["search"],
"spend_logs_metadata": {
"customer": "retained",
"gateway_endpoint": {"method": "DELETE", "path": "/client-forged"},
},
},
)
call: Final = rerank(
request=request,
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=expected_path),
)
if partial_failure:
with pytest.raises(ProxyException) as failure:
await call
assert failure.value.code == "503"
else:
await call
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await GLOBAL_LOGGING_WORKER.flush()
assert upstream.call_count == (2 if partial_failure else 3)
writer.update_database.assert_awaited_once()
write: Final = writer.update_database.call_args.kwargs
recorded_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
row: Final = get_logging_payload(
kwargs={**write["kwargs"], "response_cost": write["response_cost"]},
response_obj=write["completion_response"],
start_time=recorded_at,
end_time=recorded_at,
)
tokens: Final = 100 if partial_failure else 260
assert row["request_id"] == "rerank-spend"
assert row["call_type"] == "arerank"
assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (tokens, 0, tokens)
assert row["spend"] == pytest.approx(tokens * rate)
assert json.loads(row["metadata"])["spend_logs_metadata"] == {
"customer": "retained",
"gateway_endpoint": {"method": "POST", "path": expected_path},
}
assert json.loads(row["request_tags"]) == ["search"]
@pytest.mark.asyncio
@pytest.mark.parametrize("decision_model", [True, False])
async def test_rerank_keeps_the_selected_deployment_success_callback(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, decision_model: bool
) -> None:
provider: Final = "typesafe" if decision_model else "cohere"
model: Final = f"{provider}/rerank-callback-unit-test"
events: Final[SimpleQueue[object]] = SimpleQueue()
async def record_success(
kwargs: Mapping[str, object],
response: object,
start_time: datetime,
end_time: datetime,
) -> None:
events.put(response)
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": provider,
"mode": "evaluation" if decision_model else "rerank",
**({"rerank_strategy": "pointwise"} if decision_model else {}),
"input_cost_per_token": 0.004,
"output_cost_per_token": 0,
},
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "cache", None)
llm_router: Final = litellm.Router(
model_list=[
{
"model_name": "rerank-model",
"litellm_params": {
"model": model,
"api_key": "test-key",
"success_callback": [record_success],
},
}
],
num_retries=0,
)
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
monkeypatch.setattr(
proxy_server_mod,
"proxy_logging_obj",
ProxyLogging(user_api_key_cache=UserApiKeyCache()),
)
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
if decision_model:
respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(
json={
"model": "rerank-callback-unit-test",
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
"usage": {"input_tokens": 10, "output_tokens": 0},
}
)
else:
respx_mock.post("https://api.cohere.com/v2/rerank").respond(
json={
"id": "native-rerank",
"results": [{"index": 0, "relevance_score": 0.8}],
}
)
response: Final = await rerank(
request=_build_request(),
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/rerank"),
)
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await GLOBAL_LOGGING_WORKER.flush()
assert events.qsize() == 1
assert events.get_nowait() is response

View file

@ -1,11 +1,9 @@
from __future__ import annotations
import json
import math
from datetime import datetime, timedelta, timezone
from typing import Final
import httpx
import pytest
import litellm
@ -27,17 +25,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
)
from litellm.proxy.spend_tracking.budget_reservation import (
_get_team_member_budget_counter,
count_request_input_tokens,
estimate_request_input_cost,
estimate_request_max_cost,
get_budget_window_start,
release_unbound_budget_reservation,
reserve_budget_for_request,
)
from litellm.proxy.utils import ProxyLogging
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
LiteLLMDecisionsRerankConfig,
)
from litellm.router import Router
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -433,231 +426,3 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
)
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
assert get_budget_window_start(window) == expected
@pytest.fixture
def decision_rerank_models(monkeypatch: pytest.MonkeyPatch) -> tuple[str, str]:
models: Final = ("typesafe/budget-pointwise", "typesafe/budget-choice")
for model, strategy in zip(models, ("pointwise", "multiple_choice"), strict=True):
monkeypatch.setitem(
litellm.model_cost,
model,
{
"mode": "evaluation",
"litellm_provider": "typesafe",
"rerank_strategy": strategy,
"max_input_tokens": 65536,
"max_output_tokens": 32,
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
},
)
return models
def _decision_rerank_body(model: str, count: int) -> dict[str, object]:
return {
"model": model,
"query": "Which passage explains the solar cycle?",
"documents": ["A solar passage."] * count,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("route", ["/rerank", "/v1/rerank", "/v2/rerank"])
async def test_pointwise_reserves_every_transformed_input_and_output(
decision_rerank_models: tuple[str, str], route: str
) -> None:
model: Final = decision_rerank_models[0]
single: Final = _decision_rerank_body(model, 1)
multiple: Final = {**_decision_rerank_body(model, 3), "top_n": 1, "max_tokens": 1}
single_counts: Final = await count_request_input_tokens(single, route, None, json.dumps(single).encode())
counts: Final = await count_request_input_tokens(multiple, route, None, json.dumps(multiple).encode())
raw_prompt_tokens: Final = litellm.token_counter(model=model, text=str(single["query"])) + litellm.token_counter(
model=model, text="A solar passage."
)
expected_input_cost: Final = counts[model] * 0.001
expected_output_cost: Final = 3 * 32 * 0.002
request: Final = LiteLLMDecisionsRerankConfig.validate_request(multiple, model, "typesafe", "pointwise")
sent_inputs: Final = tuple(
LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index) for index in range(request.request_count)
)
assert single_counts[model] > raw_prompt_tokens
assert counts[model] == sum(
litellm.token_counter(model=model, text=sent.model_dump_json(exclude_none=True)) for sent in sent_inputs
)
assert counts[model] == 3 * single_counts[model]
assert estimate_request_max_cost(multiple, route, None, counts) == pytest.approx(
expected_input_cost + expected_output_cost
)
assert estimate_request_max_cost(multiple, route, None) == pytest.approx(expected_input_cost + expected_output_cost)
assert estimate_request_input_cost(multiple, route, None, counts) == pytest.approx(expected_input_cost)
@pytest.mark.asyncio
async def test_multiple_choice_reserves_one_decision_output(decision_rerank_models: tuple[str, str]) -> None:
model: Final = decision_rerank_models[1]
body: Final = _decision_rerank_body(model, 3)
counts: Final = await count_request_input_tokens(body, "/rerank", None)
input_cost: Final = estimate_request_input_cost(body, "/rerank", None, counts)
total_cost: Final = estimate_request_max_cost(body, "/rerank", None, counts)
assert input_cost is not None
assert total_cost == pytest.approx(input_cost + 32 * 0.002)
@pytest.mark.asyncio
async def test_alias_reserves_decision_fanout_at_deployment_prices(decision_rerank_models: tuple[str, str]) -> None:
backend: Final = decision_rerank_models[0]
alias: Final = "rerank-budget-group"
router: Final = Router(
model_list=[
{
"model_name": alias,
"litellm_params": {
"model": backend,
"api_key": "test-key",
"input_cost_per_token": 0.003,
"output_cost_per_token": 0.004,
},
}
]
)
body: Final = _decision_rerank_body(alias, 3)
counts: Final = await count_request_input_tokens(body, "/rerank", router, json.dumps(body).encode())
direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None)
assert counts[alias] == direct_counts[backend]
assert estimate_request_max_cost(body, "/rerank", router, counts) == pytest.approx(
counts[alias] * 0.003 + 3 * 32 * 0.004
)
@pytest.mark.asyncio
async def test_budget_rejects_fanout_that_only_fits_one_decision(
decision_rerank_models: tuple[str, str], spend_counter_cache: DualCache
) -> None:
model: Final = decision_rerank_models[0]
single_cost: Final = estimate_request_max_cost(_decision_rerank_body(model, 1), "/rerank", None)
assert single_cost is not None
with pytest.raises(litellm.BudgetExceededError):
await reserve_budget_for_request(
request_body=_decision_rerank_body(model, 3),
route="/rerank",
llm_router=None,
valid_token=UserAPIKeyAuth(token="decision-fanout-budget", max_budget=single_cost * 2, spend=0.0),
team_object=None,
user_object=None,
prisma_client=None,
user_api_key_cache=UserApiKeyCache(),
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
fail_closed_budget_enforcement=True,
)
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-fanout-budget") == pytest.approx(0.0)
def test_native_rerank_still_reserves_only_query_and_document_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "cohere/budget-native-rerank"
monkeypatch.setitem(
litellm.model_cost,
model,
{"mode": "rerank", "litellm_provider": "cohere", "input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
)
body: Final = _decision_rerank_body(model, 3)
input_cost: Final = estimate_request_input_cost(body, "/rerank", None)
assert input_cost is not None
assert estimate_request_max_cost(body, "/rerank", None) == input_cost
@pytest.mark.asyncio
async def test_decision_fanout_does_not_apply_aggregate_tier_discount(
decision_rerank_models: tuple[str, str], monkeypatch: pytest.MonkeyPatch
) -> None:
model: Final = decision_rerank_models[0]
single: Final = _decision_rerank_body(model, 1)
single_counts: Final = await count_request_input_tokens(single, "/rerank", None)
monkeypatch.setitem(
litellm.model_cost,
model,
{
**litellm.model_cost[model],
"tiered_pricing": [
{"range": [0, single_counts[model]], "input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
{
"range": [single_counts[model], 100000],
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
},
],
},
)
single_cost: Final = estimate_request_max_cost(single, "/rerank", None)
single_input: Final = estimate_request_input_cost(single, "/rerank", None)
multiple: Final = _decision_rerank_body(model, 3)
assert single_cost is not None
assert single_input is not None
assert estimate_request_max_cost(multiple, "/rerank", None) == pytest.approx(single_cost * 3)
assert estimate_request_input_cost(multiple, "/rerank", None) == pytest.approx(single_input * 3)
@pytest.mark.asyncio
async def test_budget_honors_deployment_drop_params_and_timeout(decision_rerank_models: tuple[str, str]) -> None:
backend: Final = decision_rerank_models[0]
alias: Final = "rerank-drop-params-group"
router: Final = Router(
model_list=[
{
"model_name": alias,
"litellm_params": {
"model": backend,
"api_key": "test-key",
"drop_params": True,
"timeout": httpx.Timeout(30.0),
},
}
]
)
plain: Final = _decision_rerank_body(alias, 3)
with_dropped: Final = {**plain, "rank_fields": ["text"]}
plain_counts: Final = await count_request_input_tokens(plain, "/rerank", router)
dropped_counts: Final = await count_request_input_tokens(with_dropped, "/rerank", router)
direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None)
assert plain_counts[alias] == direct_counts[backend]
assert plain_counts == dropped_counts
assert estimate_request_max_cost(plain, "/rerank", router) == estimate_request_max_cost(
with_dropped, "/rerank", router
)
def test_invalid_decision_candidate_does_not_reject_native_rerank_group(
decision_rerank_models: tuple[str, str],
) -> None:
alias: Final = "rerank-mixed-group"
router: Final = Router(
model_list=[
{"model_name": alias, "litellm_params": {"model": decision_rerank_models[0], "api_key": "test-key"}},
{"model_name": alias, "litellm_params": {"model": "cohere/rerank-v3.5", "api_key": "test-key"}},
]
)
body: Final = {**_decision_rerank_body(alias, 3), "rank_fields": ["text"]}
assert estimate_request_max_cost(body, "/rerank", router) is not None
def test_pointwise_candidate_reserves_when_choice_candidate_is_ineligible(
decision_rerank_models: tuple[str, str],
) -> None:
alias: Final = "rerank-mixed-strategy-group"
router: Final = Router(
model_list=[
{"model_name": alias, "litellm_params": {"model": model, "api_key": "test-key"}}
for model in decision_rerank_models
]
)
body: Final = _decision_rerank_body(alias, 1)
direct: Final = _decision_rerank_body(decision_rerank_models[0], 1)
assert estimate_request_max_cost(body, "/rerank", router) == estimate_request_max_cost(direct, "/rerank", None)

View file

@ -1,5 +1,6 @@
import asyncio
import json
from datetime import datetime, timezone
from threading import Event
from types import SimpleNamespace
from typing import Final
@ -13,6 +14,7 @@ import litellm
from litellm.caching.caching import Cache
from litellm.cost_calculator import _decision_rerank_breakdown
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import CallTypes, CostBreakdown
from litellm.utils import return_raw_request
@ -825,12 +827,16 @@ async def test_async_deadline_bills_responses_returned_during_cancellation_once(
async def test_caller_cancellation_retains_usage_returned_during_transport_cleanup(
respx_mock,
):
from litellm.rerank_api.litellm_decisions_transformation.handler import (
DecisionRerankLoggingCapture,
)
started = asyncio.Event()
capture = DecisionRerankLoggingCapture("cancelled-rerank")
logging_obj = Logging(
model=MODEL,
messages=[],
stream=False,
call_type="arerank",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="cancelled-rerank",
function_id="cancelled-rerank",
)
async def score(request):
started.set()
@ -840,28 +846,26 @@ async def test_caller_cancellation_retains_usage_returned_during_transport_clean
return response(output_tokens=3)
route = respx_mock.post(URL).mock(side_effect=score)
with capture.capture():
task = asyncio.create_task(
rerank(
False,
documents=["one", "two"],
concurrency=1,
litellm_call_id=capture.call_id,
)
task = asyncio.create_task(
rerank(
False,
documents=["one", "two"],
concurrency=1,
litellm_logging_obj=logging_obj,
)
await asyncio.wait_for(started.wait(), 5)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
)
await asyncio.wait_for(started.wait(), 5)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert route.call_count == 1
assert capture.logging_obj is not None
usage = capture.logging_obj.model_call_details["combined_usage_object"]
usage = logging_obj.model_call_details["combined_usage_object"]
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
10,
3,
13,
)
assert capture.logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE)
assert logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE)
async def test_direct_token_price_override_is_respected(respx_mock):

View file

@ -56,39 +56,6 @@ describe("LogDetailContent", () => {
expect(screen.getByText("completion")).toBeInTheDocument();
});
it("should distinguish the recorded gateway endpoint from the provider API base", () => {
render(
<LogDetailContent
logEntry={createLogEntry({
call_type: "arerank",
api_base: "https://api.typesafe.ai/v1",
metadata: { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/v1/rerank" } } },
})}
/>,
);
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
expect(screen.getByText("POST /v1/rerank")).toBeInTheDocument();
expect(screen.getByText("Provider API base")).toBeInTheDocument();
expect(screen.getByText("https://api.typesafe.ai/v1")).toBeInTheDocument();
expect(screen.getByText("arerank")).toBeInTheDocument();
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
});
it.each([
{ status: "success" },
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
])(
"should show that the gateway endpoint was not recorded when its metadata is unavailable or invalid",
(metadata) => {
render(<LogDetailContent logEntry={createLogEntry({ call_type: "arerank", metadata })} />);
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
expect(screen.getByText("Not recorded")).toBeInTheDocument();
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
},
);
it("shows the requesting user's email and id in Request Details when the email is resolved", () => {
render(
<LogDetailContent logEntry={createLogEntry({ user: "106514937785257944828" })} userEmail="alice@example.com" />,

View file

@ -30,7 +30,6 @@ import { JsonViewer } from "./JsonViewer";
import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard";
import {
formatData,
formatGatewayEndpoint,
checkHasMessages,
checkHasResponse,
normalizeGuardrailEntries,
@ -151,9 +150,6 @@ export function LogDetailContent({
<DescriptionItem label="Model">{logEntry.model}</DescriptionItem>
<DescriptionItem label="Provider">{logEntry.custom_llm_provider || "-"}</DescriptionItem>
<DescriptionItem label="Call Type">{logEntry.call_type}</DescriptionItem>
<DescriptionItem label="Gateway endpoint">
{formatGatewayEndpoint(metadata) ?? "Not recorded"}
</DescriptionItem>
{logEntry.user && (
<DescriptionItem label="User">
<UserIdentity userId={logEntry.user} email={userEmail} />
@ -162,7 +158,7 @@ export function LogDetailContent({
<DescriptionItem label="Model ID">
<TruncatedValue value={logEntry.model_id} />
</DescriptionItem>
<DescriptionItem label="Provider API base">
<DescriptionItem label="API Base">
<TruncatedValue value={logEntry.api_base} maxWidth={API_BASE_MAX_WIDTH} />
</DescriptionItem>
{podRouting && typeof podRouting === "object" && (

View file

@ -1,27 +1,5 @@
import { describe, expect, it } from "vitest";
import { formatGatewayEndpoint, sortSessionLogs } from "./utils";
describe("formatGatewayEndpoint", () => {
it.each(["/rerank", "/v1/rerank", "/v2/rerank"])("should preserve the recorded route %s", (path) => {
expect(formatGatewayEndpoint({ spend_logs_metadata: { gateway_endpoint: { method: "POST", path } } })).toBe(
`POST ${path}`,
);
});
it.each([
undefined,
null,
{},
{ spend_logs_metadata: null },
{ spend_logs_metadata: { gateway_endpoint: "POST /rerank" } },
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "", path: "/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "https://api.example.com/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/rerank?api_key=hidden" } } },
])("should return no endpoint for missing or malformed metadata %j", (metadata) => {
expect(formatGatewayEndpoint(metadata)).toBeNull();
});
});
import { sortSessionLogs } from "./utils";
const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({
request_id: id,

View file

@ -1,5 +1,3 @@
import { z } from "zod";
/**
* Utility functions for LogDetailsDrawer component.
* These functions handle data formatting, validation, and guardrail calculations.
@ -7,22 +5,6 @@ import { z } from "zod";
export type SessionLogSortMode = "duration" | "start_time";
const gatewayEndpointMetadataSchema = z.object({
spend_logs_metadata: z.object({
gateway_endpoint: z.object({
method: z.string().regex(/^[A-Z]+$/),
path: z.string().regex(/^\/(?!\/)[^?#\s]*$/),
}),
}),
});
export function formatGatewayEndpoint(metadata: unknown): string | null {
const parsed = gatewayEndpointMetadataSchema.safeParse(metadata);
if (!parsed.success) return null;
const { method, path } = parsed.data.spend_logs_metadata.gateway_endpoint;
return `${method} ${path}`;
}
type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number };
const durationMs = (row: SortableSessionLog): number =>