mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
fix(rerank): scope decision adapter to SDK integration
Restore proxy endpoints, spend tracking, and logs UI to the base branch. Remove endpoint-specific logging capture and preserve decision cost handling through the existing calculator API
This commit is contained in:
parent
72bcd5d945
commit
314c52bcf4
12 changed files with 56 additions and 765 deletions
|
|
@ -1550,9 +1550,9 @@ def _decision_rerank_cost(
|
|||
) -> tuple[float, CostBreakdown | None, float]:
|
||||
if logging_obj is not None:
|
||||
logging_obj.cost_breakdown = None
|
||||
cost: Final = price(response)
|
||||
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
|
||||
provider_cost: Final = get_response_cost_from_hidden_params(response.hidden_params)
|
||||
cost: Final = price(response) if custom_pricing or provider_cost is None else provider_cost
|
||||
breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None
|
||||
fixed: Final = (
|
||||
_apply_cost_margin(
|
||||
0.0,
|
||||
|
|
@ -1639,7 +1639,7 @@ def completion_cost(
|
|||
return _decision_rerank_cost(
|
||||
completion_response.decision_responses,
|
||||
partial(
|
||||
response_cost_calculator,
|
||||
completion_cost,
|
||||
model=model or (decision_rerank_child.model if decision_rerank_child is not None else ""),
|
||||
call_type="decisions",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -2380,7 +2380,6 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2398,7 +2397,7 @@ def response_cost_calculator(
|
|||
getattr(response_object, HIDDEN_PARAMS_ATTR),
|
||||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing and custom_cost_per_token is None:
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
@ -2419,7 +2418,6 @@ def response_cost_calculator(
|
|||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
)
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ import orjson
|
|||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.auth_utils import get_request_route
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
|
|
@ -22,9 +20,6 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_param,
|
||||
openai_error_type,
|
||||
)
|
||||
from litellm.rerank_api.litellm_decisions_transformation.handler import (
|
||||
DecisionRerankLoggingCapture,
|
||||
)
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -64,8 +59,7 @@ async def rerank(
|
|||
)
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
|
||||
logging_capture: Final = DecisionRerankLoggingCapture(litellm_call_id)
|
||||
data: dict[str, object] = {"litellm_call_id": litellm_call_id} # mutable-ok: proxy hooks enrich request data
|
||||
data = {"litellm_call_id": litellm_call_id}
|
||||
try:
|
||||
body: Final = await request.body()
|
||||
data = orjson.loads(body) | data
|
||||
|
|
@ -79,28 +73,18 @@ async def rerank(
|
|||
version=version,
|
||||
proxy_config=proxy_config,
|
||||
)
|
||||
metadata: Final = as_str_mapping(data.get("metadata")) or {}
|
||||
spend_metadata: Final = as_str_mapping(metadata.get("spend_logs_metadata")) or {}
|
||||
data["metadata"] = {
|
||||
**metadata,
|
||||
"spend_logs_metadata": {
|
||||
**spend_metadata,
|
||||
"gateway_endpoint": {"method": request.method, "path": get_request_route(request)},
|
||||
},
|
||||
}
|
||||
|
||||
### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(user_api_key_dict=user_api_key_dict, data=data, call_type="rerank")
|
||||
|
||||
## ROUTE TO CORRECT ENDPOINT ##
|
||||
with logging_capture.capture():
|
||||
llm_call: Final = await route_request(
|
||||
data=data,
|
||||
route_type="arerank",
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
response: Final = await llm_call
|
||||
llm_call: Final = await route_request(
|
||||
data=data,
|
||||
route_type="arerank",
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
response: Final = await llm_call
|
||||
|
||||
### ALERTING ###
|
||||
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
|
||||
|
|
@ -129,9 +113,6 @@ async def rerank(
|
|||
|
||||
return response
|
||||
except Exception as e:
|
||||
partial_logging: Final = logging_capture.logging_obj
|
||||
if partial_logging is not None:
|
||||
data["litellm_logging_obj"] = partial_logging
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -40,22 +39,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
tag_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
count_input_tokens,
|
||||
count_input_tokens_for_model,
|
||||
count_text_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
|
||||
DecisionRerankRequest,
|
||||
LiteLLMDecisionsRerankConfig,
|
||||
)
|
||||
from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -84,7 +73,7 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
|
|||
}
|
||||
|
||||
_CACHED_VALUE: Final = TypeAdapter(object)
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _CounterReservationUnavailable(Exception):
|
||||
|
|
@ -895,7 +884,7 @@ def _coerce_window(window: object) -> Mapping[str, object]:
|
|||
return window
|
||||
if isinstance(window, str):
|
||||
try:
|
||||
return _OBJECT_MAPPING.validate_python(json.loads(window))
|
||||
return _WINDOW_OBJECT.validate_python(json.loads(window))
|
||||
except Exception:
|
||||
return {}
|
||||
model_dump: Final = getattr(window, "model_dump", None)
|
||||
|
|
@ -1352,19 +1341,13 @@ def _estimate_request_input_cost_for_model(
|
|||
llm_router: Router | None,
|
||||
input_tokens: int | None = None,
|
||||
) -> float | None:
|
||||
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
|
||||
plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router)
|
||||
counted: Final = (
|
||||
_decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens
|
||||
)
|
||||
estimates: Final = [
|
||||
_input_cost_for_cost_info(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=counted,
|
||||
is_decision_rerank=bool(plans),
|
||||
input_tokens=input_tokens,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
]
|
||||
|
|
@ -1378,7 +1361,6 @@ def _input_cost_for_cost_info(
|
|||
model: str,
|
||||
model_info: Mapping[str, object],
|
||||
input_tokens: int | None = None,
|
||||
is_decision_rerank: bool = False,
|
||||
) -> float | None:
|
||||
estimated_input_tokens: Final = _estimate_input_tokens(
|
||||
request_body=request_body,
|
||||
|
|
@ -1390,8 +1372,6 @@ def _input_cost_for_cost_info(
|
|||
if estimated_input_tokens is None:
|
||||
return None
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if is_decision_rerank and tiered_pricing:
|
||||
return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token")
|
||||
if isinstance(tiered_pricing, list) and tiered_pricing:
|
||||
tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens)
|
||||
if tier is not None:
|
||||
|
|
@ -1409,20 +1389,13 @@ def _estimate_request_max_cost_for_model(
|
|||
llm_router: Router | None,
|
||||
input_tokens: int | None = None,
|
||||
) -> float | None:
|
||||
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
|
||||
plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router)
|
||||
counted: Final = (
|
||||
_decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens
|
||||
)
|
||||
request_count: Final = max((request.request_count for _, request in plans), default=0)
|
||||
estimates: Final = [
|
||||
_max_cost_for_cost_info(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
model=model,
|
||||
model_info=model_info,
|
||||
input_tokens=counted,
|
||||
decision_request_count=request_count,
|
||||
input_tokens=input_tokens,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
]
|
||||
|
|
@ -1436,7 +1409,6 @@ def _max_cost_for_cost_info(
|
|||
model: str,
|
||||
model_info: Mapping[str, object],
|
||||
input_tokens: int | None = None,
|
||||
decision_request_count: int = 0,
|
||||
) -> float | None:
|
||||
image_cost: Final = _estimate_image_generation_cost(
|
||||
request_body=request_body,
|
||||
|
|
@ -1456,18 +1428,12 @@ def _max_cost_for_cost_info(
|
|||
request_body=request_body,
|
||||
route=route,
|
||||
model_info=model_info,
|
||||
decision_request_count=decision_request_count,
|
||||
)
|
||||
if estimated_input_tokens is None or output_tokens is None:
|
||||
return None
|
||||
|
||||
output_multiplier: Final = _get_output_multiplier(request_body=request_body)
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if decision_request_count and tiered_pricing:
|
||||
return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token") + output_tokens * max(
|
||||
_max_tier_rate(tiered_pricing, "output_cost_per_token"),
|
||||
_max_tier_rate(tiered_pricing, "output_cost_per_reasoning_token"),
|
||||
)
|
||||
if isinstance(tiered_pricing, list) and tiered_pricing:
|
||||
tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens)
|
||||
if tier is not None:
|
||||
|
|
@ -1500,14 +1466,6 @@ def _max_cost_for_cost_info(
|
|||
return cost
|
||||
|
||||
|
||||
_TIER_COSTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def _max_tier_rate(tiered_pricing: object, cost_key: str) -> float:
|
||||
tiers: Final = _TIER_COSTS.validate_python(tiered_pricing)
|
||||
return max(tier_rate(dict(tier), cost_key) for tier in tiers)
|
||||
|
||||
|
||||
def _estimate_image_generation_cost(
|
||||
request_body: dict,
|
||||
model_info: Mapping[str, object],
|
||||
|
|
@ -1639,84 +1597,7 @@ async def count_request_input_tokens(
|
|||
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
|
||||
if not models:
|
||||
return MappingProxyType({})
|
||||
typed_body: Final = _OBJECT_MAPPING.validate_python(request_body)
|
||||
plans: Final = {model: _decision_rerank_plans(typed_body, route, model, llm_router) for model in models}
|
||||
regular_models: Final = tuple(model for model in models if not plans[model])
|
||||
regular_counts: Final = await count_input_tokens(
|
||||
request_body=request_body, raw_body=raw_body, models=regular_models
|
||||
)
|
||||
decision_counts: Final = {
|
||||
model: await asyncio.to_thread(_decision_rerank_input_tokens, typed_body, model, plan)
|
||||
for model, plan in plans.items()
|
||||
if plan
|
||||
}
|
||||
return MappingProxyType({**regular_counts, **decision_counts})
|
||||
|
||||
|
||||
def _decision_rerank_plan(
|
||||
request_body: Mapping[str, object], model: str, custom_llm_provider: str | None
|
||||
) -> tuple[str, DecisionRerankRequest] | None:
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
strategy: Final = get_decision_rerank_strategy(resolved_model, LlmProviders(provider))
|
||||
if strategy is None:
|
||||
return None
|
||||
pricing_params: Final = {key: value for key, value in request_body.items() if key != "timeout"}
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(
|
||||
pricing_params, resolved_model, provider, strategy
|
||||
)
|
||||
except (litellm.BadRequestError, litellm.UnsupportedParamsError, ValueError):
|
||||
return None
|
||||
return f"{provider}/{resolved_model}", request
|
||||
|
||||
|
||||
def _decision_rerank_plans(
|
||||
request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None
|
||||
) -> tuple[tuple[str, DecisionRerankRequest], ...]:
|
||||
if "rerank" not in route:
|
||||
return ()
|
||||
deployments: Final = () if llm_router is None else (llm_router.get_model_list(model_name=model) or ())
|
||||
if not deployments:
|
||||
provider: Final = request_body.get("custom_llm_provider")
|
||||
direct_plan: Final = _decision_rerank_plan(request_body, model, provider if isinstance(provider, str) else None)
|
||||
return () if direct_plan is None else (direct_plan,)
|
||||
return tuple(
|
||||
plan
|
||||
for deployment in deployments
|
||||
if (plan := _decision_rerank_deployment_plan(request_body, deployment)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _decision_rerank_deployment_plan(
|
||||
request_body: Mapping[str, object], deployment: DeploymentTypedDict
|
||||
) -> tuple[str, DecisionRerankRequest] | None:
|
||||
params: Final = as_str_mapping(deployment["litellm_params"]) if "litellm_params" in deployment else None
|
||||
if params is None:
|
||||
return None
|
||||
model: Final = params.get("model")
|
||||
provider: Final = params.get("custom_llm_provider")
|
||||
if not isinstance(model, str):
|
||||
return None
|
||||
return _decision_rerank_plan({**params, **request_body}, model, provider if isinstance(provider, str) else None)
|
||||
|
||||
|
||||
def _decision_rerank_input_tokens(
|
||||
request_body: Mapping[str, object], requested_model: str, plans: Sequence[tuple[str, DecisionRerankRequest]]
|
||||
) -> int:
|
||||
native_count: Final = count_input_tokens_for_model(dict(request_body), requested_model) or 0
|
||||
decision_counts: Final = (
|
||||
sum(
|
||||
count_text_tokens(
|
||||
model=model,
|
||||
text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json(
|
||||
exclude_none=True
|
||||
),
|
||||
)
|
||||
for index in range(request.request_count)
|
||||
)
|
||||
for model, request in plans
|
||||
)
|
||||
return max(native_count, *decision_counts)
|
||||
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
|
||||
|
||||
|
||||
def _estimate_input_tokens(
|
||||
|
|
@ -1748,11 +1629,7 @@ def _estimate_output_tokens(
|
|||
request_body: dict,
|
||||
route: str,
|
||||
model_info: Mapping[str, object],
|
||||
decision_request_count: int = 0,
|
||||
) -> int | None:
|
||||
if decision_request_count:
|
||||
ceiling: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK
|
||||
return decision_request_count * ceiling
|
||||
if _is_input_only_route(route=route):
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
|||
tool_choice=request_body.get("tool_choice"),
|
||||
)
|
||||
except ValueError:
|
||||
return count_text_tokens(model=model, text=request_body.get("messages"))
|
||||
return _count_text_tokens(model=model, text=request_body.get("messages"))
|
||||
if "prompt" in request_body:
|
||||
return count_text_tokens(model=model, text=request_body.get("prompt"))
|
||||
return _count_text_tokens(model=model, text=request_body.get("prompt"))
|
||||
if "input" in request_body:
|
||||
return count_text_tokens(
|
||||
return _count_text_tokens(
|
||||
model=model, text=billable_input_since_newest_compaction(request_body.get("input"))
|
||||
)
|
||||
if "query" in request_body or "documents" in request_body:
|
||||
query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query"))
|
||||
document_tokens: Final = count_text_tokens(
|
||||
query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query"))
|
||||
document_tokens: Final = _count_text_tokens(
|
||||
model=model,
|
||||
text=request_body.get("documents"),
|
||||
)
|
||||
|
|
@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
def count_text_tokens(model: str, text: object) -> int:
|
||||
def _count_text_tokens(model: str, text: object) -> int:
|
||||
if text is None:
|
||||
return 0
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,8 @@
|
|||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import partial
|
||||
from queue import Empty, SimpleQueue
|
||||
|
|
@ -36,29 +34,6 @@ _CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"})
|
|||
_MAX_CONCURRENCY: Final = 8
|
||||
|
||||
|
||||
class DecisionRerankLoggingCapture:
|
||||
def __init__(self, call_id: str) -> None:
|
||||
self.call_id: Final = call_id
|
||||
self.logging_obj: Logging | None = None
|
||||
|
||||
@contextmanager
|
||||
def capture(self) -> Generator[None]:
|
||||
token: Final = _LOGGING_CAPTURE.set(self)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_LOGGING_CAPTURE.reset(token)
|
||||
|
||||
def record(self, logging_obj: Logging) -> None:
|
||||
if logging_obj.litellm_call_id == self.call_id:
|
||||
self.logging_obj = logging_obj
|
||||
|
||||
|
||||
_LOGGING_CAPTURE: Final[ContextVar[DecisionRerankLoggingCapture | None]] = ContextVar(
|
||||
"decision_rerank_logging_capture", default=None
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DocumentCall:
|
||||
index: int
|
||||
|
|
@ -322,9 +297,6 @@ def rerank_with_decisions(
|
|||
params: Mapping[str, object],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
|
||||
logging_capture: Final = _LOGGING_CAPTURE.get()
|
||||
if logging_capture is not None:
|
||||
logging_capture.record(logging_obj)
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy)
|
||||
deadline: Final = time.monotonic() + request.timeout
|
||||
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs
|
||||
|
|
|
|||
|
|
@ -2,32 +2,19 @@
|
|||
Tests for rerank_endpoints/endpoints.py response headers.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from queue import SimpleQueue
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
|
||||
from collections.abc import Iterator
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException, Request, Response
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.common_request_processing as common_request_processing_mod
|
||||
import litellm.proxy.proxy_server as proxy_server_mod
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
|
||||
from litellm.proxy.rerank_endpoints.endpoints import rerank
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.types.utils import RerankResponse
|
||||
|
||||
HIDDEN_PARAMS = {
|
||||
|
|
@ -44,21 +31,8 @@ HIDDEN_PARAMS = {
|
|||
}
|
||||
|
||||
|
||||
def _build_request(
|
||||
headers: tuple[tuple[bytes, bytes], ...] = (),
|
||||
documents: tuple[str, ...] = ("a", "b"),
|
||||
path: str = "/rerank",
|
||||
root_path: str = "",
|
||||
metadata: Mapping[str, object] | None = None,
|
||||
) -> Request:
|
||||
body = json.dumps(
|
||||
{
|
||||
"model": "rerank-model",
|
||||
"query": "q",
|
||||
"documents": documents,
|
||||
**({"metadata": metadata} if metadata is not None else {}),
|
||||
}
|
||||
).encode()
|
||||
def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request:
|
||||
body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode()
|
||||
|
||||
async def receive():
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
|
@ -67,8 +41,7 @@ def _build_request(
|
|||
scope={
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"root_path": root_path,
|
||||
"path": "/rerank",
|
||||
"headers": [(b"content-type", b"application/json"), *headers],
|
||||
"query_string": b"",
|
||||
},
|
||||
|
|
@ -236,205 +209,3 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc
|
|||
error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch)
|
||||
|
||||
assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("path", "root_path", "expected_path"),
|
||||
[
|
||||
("/rerank", "", "/rerank"),
|
||||
("/v1/rerank", "", "/v1/rerank"),
|
||||
("/v2/rerank", "", "/v2/rerank"),
|
||||
("/gateway/v2/rerank", "/gateway", "/v2/rerank"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("partial_failure", [False, True])
|
||||
async def test_decision_rerank_writes_one_log_with_gateway_endpoint_and_aggregate_usage(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
path: str,
|
||||
root_path: str,
|
||||
expected_path: str,
|
||||
partial_failure: bool,
|
||||
) -> None:
|
||||
model: Final = "typesafe/jevrank-spend-unit-test"
|
||||
rate: Final = 0.004
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "typesafe",
|
||||
"mode": "evaluation",
|
||||
"rerank_strategy": "pointwise",
|
||||
"input_cost_per_token": rate,
|
||||
"output_cost_per_token": 0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "cache", None)
|
||||
llm_router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "rerank-model",
|
||||
"litellm_params": {"model": model, "api_key": "test-key"},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
|
||||
db_logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
|
||||
proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
|
||||
proxy_logging.db_spend_update_writer = writer
|
||||
monkeypatch.setattr(litellm, "callbacks", [db_logger])
|
||||
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
|
||||
monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", proxy_logging)
|
||||
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
|
||||
|
||||
def score(request: httpx.Request) -> httpx.Response:
|
||||
passage: Final = json.loads(json.loads(request.content)["state"])["passage"]
|
||||
if passage == "failed":
|
||||
return httpx.Response(503, json={"error": "unavailable"})
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jevrank-spend-unit-test",
|
||||
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
|
||||
"usage": {
|
||||
"input_tokens": 100 if passage == "first" else 80,
|
||||
"output_tokens": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").mock(
|
||||
side_effect=score
|
||||
)
|
||||
request: Final = _build_request(
|
||||
headers=((b"x-litellm-call-id", b"rerank-spend"),),
|
||||
documents=("first", "failed" if partial_failure else "middle", "last"),
|
||||
path=path,
|
||||
root_path=root_path,
|
||||
metadata={
|
||||
"tags": ["search"],
|
||||
"spend_logs_metadata": {
|
||||
"customer": "retained",
|
||||
"gateway_endpoint": {"method": "DELETE", "path": "/client-forged"},
|
||||
},
|
||||
},
|
||||
)
|
||||
call: Final = rerank(
|
||||
request=request,
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=expected_path),
|
||||
)
|
||||
if partial_failure:
|
||||
with pytest.raises(ProxyException) as failure:
|
||||
await call
|
||||
assert failure.value.code == "503"
|
||||
else:
|
||||
await call
|
||||
await asyncio.sleep(0)
|
||||
GLOBAL_LOGGING_WORKER.start()
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
assert upstream.call_count == (2 if partial_failure else 3)
|
||||
writer.update_database.assert_awaited_once()
|
||||
write: Final = writer.update_database.call_args.kwargs
|
||||
recorded_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
row: Final = get_logging_payload(
|
||||
kwargs={**write["kwargs"], "response_cost": write["response_cost"]},
|
||||
response_obj=write["completion_response"],
|
||||
start_time=recorded_at,
|
||||
end_time=recorded_at,
|
||||
)
|
||||
tokens: Final = 100 if partial_failure else 260
|
||||
assert row["request_id"] == "rerank-spend"
|
||||
assert row["call_type"] == "arerank"
|
||||
assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (tokens, 0, tokens)
|
||||
assert row["spend"] == pytest.approx(tokens * rate)
|
||||
assert json.loads(row["metadata"])["spend_logs_metadata"] == {
|
||||
"customer": "retained",
|
||||
"gateway_endpoint": {"method": "POST", "path": expected_path},
|
||||
}
|
||||
assert json.loads(row["request_tags"]) == ["search"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("decision_model", [True, False])
|
||||
async def test_rerank_keeps_the_selected_deployment_success_callback(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, decision_model: bool
|
||||
) -> None:
|
||||
provider: Final = "typesafe" if decision_model else "cohere"
|
||||
model: Final = f"{provider}/rerank-callback-unit-test"
|
||||
events: Final[SimpleQueue[object]] = SimpleQueue()
|
||||
|
||||
async def record_success(
|
||||
kwargs: Mapping[str, object],
|
||||
response: object,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> None:
|
||||
events.put(response)
|
||||
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": provider,
|
||||
"mode": "evaluation" if decision_model else "rerank",
|
||||
**({"rerank_strategy": "pointwise"} if decision_model else {}),
|
||||
"input_cost_per_token": 0.004,
|
||||
"output_cost_per_token": 0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "cache", None)
|
||||
llm_router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "rerank-model",
|
||||
"litellm_params": {
|
||||
"model": model,
|
||||
"api_key": "test-key",
|
||||
"success_callback": [record_success],
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
|
||||
monkeypatch.setattr(
|
||||
proxy_server_mod,
|
||||
"proxy_logging_obj",
|
||||
ProxyLogging(user_api_key_cache=UserApiKeyCache()),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
|
||||
if decision_model:
|
||||
respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(
|
||||
json={
|
||||
"model": "rerank-callback-unit-test",
|
||||
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
|
||||
"usage": {"input_tokens": 10, "output_tokens": 0},
|
||||
}
|
||||
)
|
||||
else:
|
||||
respx_mock.post("https://api.cohere.com/v2/rerank").respond(
|
||||
json={
|
||||
"id": "native-rerank",
|
||||
"results": [{"index": 0, "relevance_score": 0.8}],
|
||||
}
|
||||
)
|
||||
response: Final = await rerank(
|
||||
request=_build_request(),
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/rerank"),
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
GLOBAL_LOGGING_WORKER.start()
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
assert events.qsize() == 1
|
||||
assert events.get_nowait() is response
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
|
@ -27,17 +25,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
_get_team_member_budget_counter,
|
||||
count_request_input_tokens,
|
||||
estimate_request_input_cost,
|
||||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
release_unbound_budget_reservation,
|
||||
reserve_budget_for_request,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.rerank_api.litellm_decisions_transformation.transformation import (
|
||||
LiteLLMDecisionsRerankConfig,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -433,231 +426,3 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
|
|||
)
|
||||
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
|
||||
assert get_budget_window_start(window) == expected
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def decision_rerank_models(monkeypatch: pytest.MonkeyPatch) -> tuple[str, str]:
|
||||
models: Final = ("typesafe/budget-pointwise", "typesafe/budget-choice")
|
||||
for model, strategy in zip(models, ("pointwise", "multiple_choice"), strict=True):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"mode": "evaluation",
|
||||
"litellm_provider": "typesafe",
|
||||
"rerank_strategy": strategy,
|
||||
"max_input_tokens": 65536,
|
||||
"max_output_tokens": 32,
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
},
|
||||
)
|
||||
return models
|
||||
|
||||
|
||||
def _decision_rerank_body(model: str, count: int) -> dict[str, object]:
|
||||
return {
|
||||
"model": model,
|
||||
"query": "Which passage explains the solar cycle?",
|
||||
"documents": ["A solar passage."] * count,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", ["/rerank", "/v1/rerank", "/v2/rerank"])
|
||||
async def test_pointwise_reserves_every_transformed_input_and_output(
|
||||
decision_rerank_models: tuple[str, str], route: str
|
||||
) -> None:
|
||||
model: Final = decision_rerank_models[0]
|
||||
single: Final = _decision_rerank_body(model, 1)
|
||||
multiple: Final = {**_decision_rerank_body(model, 3), "top_n": 1, "max_tokens": 1}
|
||||
single_counts: Final = await count_request_input_tokens(single, route, None, json.dumps(single).encode())
|
||||
counts: Final = await count_request_input_tokens(multiple, route, None, json.dumps(multiple).encode())
|
||||
raw_prompt_tokens: Final = litellm.token_counter(model=model, text=str(single["query"])) + litellm.token_counter(
|
||||
model=model, text="A solar passage."
|
||||
)
|
||||
expected_input_cost: Final = counts[model] * 0.001
|
||||
expected_output_cost: Final = 3 * 32 * 0.002
|
||||
|
||||
request: Final = LiteLLMDecisionsRerankConfig.validate_request(multiple, model, "typesafe", "pointwise")
|
||||
sent_inputs: Final = tuple(
|
||||
LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index) for index in range(request.request_count)
|
||||
)
|
||||
assert single_counts[model] > raw_prompt_tokens
|
||||
assert counts[model] == sum(
|
||||
litellm.token_counter(model=model, text=sent.model_dump_json(exclude_none=True)) for sent in sent_inputs
|
||||
)
|
||||
assert counts[model] == 3 * single_counts[model]
|
||||
assert estimate_request_max_cost(multiple, route, None, counts) == pytest.approx(
|
||||
expected_input_cost + expected_output_cost
|
||||
)
|
||||
assert estimate_request_max_cost(multiple, route, None) == pytest.approx(expected_input_cost + expected_output_cost)
|
||||
assert estimate_request_input_cost(multiple, route, None, counts) == pytest.approx(expected_input_cost)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_choice_reserves_one_decision_output(decision_rerank_models: tuple[str, str]) -> None:
|
||||
model: Final = decision_rerank_models[1]
|
||||
body: Final = _decision_rerank_body(model, 3)
|
||||
counts: Final = await count_request_input_tokens(body, "/rerank", None)
|
||||
input_cost: Final = estimate_request_input_cost(body, "/rerank", None, counts)
|
||||
total_cost: Final = estimate_request_max_cost(body, "/rerank", None, counts)
|
||||
|
||||
assert input_cost is not None
|
||||
assert total_cost == pytest.approx(input_cost + 32 * 0.002)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_reserves_decision_fanout_at_deployment_prices(decision_rerank_models: tuple[str, str]) -> None:
|
||||
backend: Final = decision_rerank_models[0]
|
||||
alias: Final = "rerank-budget-group"
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": alias,
|
||||
"litellm_params": {
|
||||
"model": backend,
|
||||
"api_key": "test-key",
|
||||
"input_cost_per_token": 0.003,
|
||||
"output_cost_per_token": 0.004,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
body: Final = _decision_rerank_body(alias, 3)
|
||||
counts: Final = await count_request_input_tokens(body, "/rerank", router, json.dumps(body).encode())
|
||||
direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None)
|
||||
|
||||
assert counts[alias] == direct_counts[backend]
|
||||
assert estimate_request_max_cost(body, "/rerank", router, counts) == pytest.approx(
|
||||
counts[alias] * 0.003 + 3 * 32 * 0.004
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_rejects_fanout_that_only_fits_one_decision(
|
||||
decision_rerank_models: tuple[str, str], spend_counter_cache: DualCache
|
||||
) -> None:
|
||||
model: Final = decision_rerank_models[0]
|
||||
single_cost: Final = estimate_request_max_cost(_decision_rerank_body(model, 1), "/rerank", None)
|
||||
assert single_cost is not None
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await reserve_budget_for_request(
|
||||
request_body=_decision_rerank_body(model, 3),
|
||||
route="/rerank",
|
||||
llm_router=None,
|
||||
valid_token=UserAPIKeyAuth(token="decision-fanout-budget", max_budget=single_cost * 2, spend=0.0),
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=UserApiKeyCache(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
fail_closed_budget_enforcement=True,
|
||||
)
|
||||
assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-fanout-budget") == pytest.approx(0.0)
|
||||
|
||||
|
||||
def test_native_rerank_still_reserves_only_query_and_document_tokens(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "cohere/budget-native-rerank"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"mode": "rerank", "litellm_provider": "cohere", "input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
|
||||
)
|
||||
body: Final = _decision_rerank_body(model, 3)
|
||||
input_cost: Final = estimate_request_input_cost(body, "/rerank", None)
|
||||
assert input_cost is not None
|
||||
assert estimate_request_max_cost(body, "/rerank", None) == input_cost
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decision_fanout_does_not_apply_aggregate_tier_discount(
|
||||
decision_rerank_models: tuple[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
model: Final = decision_rerank_models[0]
|
||||
single: Final = _decision_rerank_body(model, 1)
|
||||
single_counts: Final = await count_request_input_tokens(single, "/rerank", None)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
**litellm.model_cost[model],
|
||||
"tiered_pricing": [
|
||||
{"range": [0, single_counts[model]], "input_cost_per_token": 0.01, "output_cost_per_token": 0.02},
|
||||
{
|
||||
"range": [single_counts[model], 100000],
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
single_cost: Final = estimate_request_max_cost(single, "/rerank", None)
|
||||
single_input: Final = estimate_request_input_cost(single, "/rerank", None)
|
||||
multiple: Final = _decision_rerank_body(model, 3)
|
||||
|
||||
assert single_cost is not None
|
||||
assert single_input is not None
|
||||
assert estimate_request_max_cost(multiple, "/rerank", None) == pytest.approx(single_cost * 3)
|
||||
assert estimate_request_input_cost(multiple, "/rerank", None) == pytest.approx(single_input * 3)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_honors_deployment_drop_params_and_timeout(decision_rerank_models: tuple[str, str]) -> None:
|
||||
backend: Final = decision_rerank_models[0]
|
||||
alias: Final = "rerank-drop-params-group"
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": alias,
|
||||
"litellm_params": {
|
||||
"model": backend,
|
||||
"api_key": "test-key",
|
||||
"drop_params": True,
|
||||
"timeout": httpx.Timeout(30.0),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
plain: Final = _decision_rerank_body(alias, 3)
|
||||
with_dropped: Final = {**plain, "rank_fields": ["text"]}
|
||||
plain_counts: Final = await count_request_input_tokens(plain, "/rerank", router)
|
||||
dropped_counts: Final = await count_request_input_tokens(with_dropped, "/rerank", router)
|
||||
|
||||
direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None)
|
||||
assert plain_counts[alias] == direct_counts[backend]
|
||||
assert plain_counts == dropped_counts
|
||||
assert estimate_request_max_cost(plain, "/rerank", router) == estimate_request_max_cost(
|
||||
with_dropped, "/rerank", router
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_decision_candidate_does_not_reject_native_rerank_group(
|
||||
decision_rerank_models: tuple[str, str],
|
||||
) -> None:
|
||||
alias: Final = "rerank-mixed-group"
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": alias, "litellm_params": {"model": decision_rerank_models[0], "api_key": "test-key"}},
|
||||
{"model_name": alias, "litellm_params": {"model": "cohere/rerank-v3.5", "api_key": "test-key"}},
|
||||
]
|
||||
)
|
||||
body: Final = {**_decision_rerank_body(alias, 3), "rank_fields": ["text"]}
|
||||
|
||||
assert estimate_request_max_cost(body, "/rerank", router) is not None
|
||||
|
||||
|
||||
def test_pointwise_candidate_reserves_when_choice_candidate_is_ineligible(
|
||||
decision_rerank_models: tuple[str, str],
|
||||
) -> None:
|
||||
alias: Final = "rerank-mixed-strategy-group"
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": alias, "litellm_params": {"model": model, "api_key": "test-key"}}
|
||||
for model in decision_rerank_models
|
||||
]
|
||||
)
|
||||
body: Final = _decision_rerank_body(alias, 1)
|
||||
direct: Final = _decision_rerank_body(decision_rerank_models[0], 1)
|
||||
|
||||
assert estimate_request_max_cost(body, "/rerank", router) == estimate_request_max_cost(direct, "/rerank", None)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from threading import Event
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -13,6 +14,7 @@ import litellm
|
|||
from litellm.caching.caching import Cache
|
||||
from litellm.cost_calculator import _decision_rerank_breakdown
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.utils import CallTypes, CostBreakdown
|
||||
from litellm.utils import return_raw_request
|
||||
|
|
@ -825,12 +827,16 @@ async def test_async_deadline_bills_responses_returned_during_cancellation_once(
|
|||
async def test_caller_cancellation_retains_usage_returned_during_transport_cleanup(
|
||||
respx_mock,
|
||||
):
|
||||
from litellm.rerank_api.litellm_decisions_transformation.handler import (
|
||||
DecisionRerankLoggingCapture,
|
||||
)
|
||||
|
||||
started = asyncio.Event()
|
||||
capture = DecisionRerankLoggingCapture("cancelled-rerank")
|
||||
logging_obj = Logging(
|
||||
model=MODEL,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="arerank",
|
||||
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
litellm_call_id="cancelled-rerank",
|
||||
function_id="cancelled-rerank",
|
||||
)
|
||||
|
||||
async def score(request):
|
||||
started.set()
|
||||
|
|
@ -840,28 +846,26 @@ async def test_caller_cancellation_retains_usage_returned_during_transport_clean
|
|||
return response(output_tokens=3)
|
||||
|
||||
route = respx_mock.post(URL).mock(side_effect=score)
|
||||
with capture.capture():
|
||||
task = asyncio.create_task(
|
||||
rerank(
|
||||
False,
|
||||
documents=["one", "two"],
|
||||
concurrency=1,
|
||||
litellm_call_id=capture.call_id,
|
||||
)
|
||||
task = asyncio.create_task(
|
||||
rerank(
|
||||
False,
|
||||
documents=["one", "two"],
|
||||
concurrency=1,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 5)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), 5)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert route.call_count == 1
|
||||
assert capture.logging_obj is not None
|
||||
usage = capture.logging_obj.model_call_details["combined_usage_object"]
|
||||
usage = logging_obj.model_call_details["combined_usage_object"]
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
10,
|
||||
3,
|
||||
13,
|
||||
)
|
||||
assert capture.logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE)
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE)
|
||||
|
||||
|
||||
async def test_direct_token_price_override_is_respected(respx_mock):
|
||||
|
|
|
|||
|
|
@ -56,39 +56,6 @@ describe("LogDetailContent", () => {
|
|||
expect(screen.getByText("completion")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should distinguish the recorded gateway endpoint from the provider API base", () => {
|
||||
render(
|
||||
<LogDetailContent
|
||||
logEntry={createLogEntry({
|
||||
call_type: "arerank",
|
||||
api_base: "https://api.typesafe.ai/v1",
|
||||
metadata: { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/v1/rerank" } } },
|
||||
})}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
|
||||
expect(screen.getByText("POST /v1/rerank")).toBeInTheDocument();
|
||||
expect(screen.getByText("Provider API base")).toBeInTheDocument();
|
||||
expect(screen.getByText("https://api.typesafe.ai/v1")).toBeInTheDocument();
|
||||
expect(screen.getByText("arerank")).toBeInTheDocument();
|
||||
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ status: "success" },
|
||||
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
|
||||
])(
|
||||
"should show that the gateway endpoint was not recorded when its metadata is unavailable or invalid",
|
||||
(metadata) => {
|
||||
render(<LogDetailContent logEntry={createLogEntry({ call_type: "arerank", metadata })} />);
|
||||
|
||||
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
|
||||
expect(screen.getByText("Not recorded")).toBeInTheDocument();
|
||||
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
|
||||
},
|
||||
);
|
||||
|
||||
it("shows the requesting user's email and id in Request Details when the email is resolved", () => {
|
||||
render(
|
||||
<LogDetailContent logEntry={createLogEntry({ user: "106514937785257944828" })} userEmail="alice@example.com" />,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import { JsonViewer } from "./JsonViewer";
|
|||
import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard";
|
||||
import {
|
||||
formatData,
|
||||
formatGatewayEndpoint,
|
||||
checkHasMessages,
|
||||
checkHasResponse,
|
||||
normalizeGuardrailEntries,
|
||||
|
|
@ -151,9 +150,6 @@ export function LogDetailContent({
|
|||
<DescriptionItem label="Model">{logEntry.model}</DescriptionItem>
|
||||
<DescriptionItem label="Provider">{logEntry.custom_llm_provider || "-"}</DescriptionItem>
|
||||
<DescriptionItem label="Call Type">{logEntry.call_type}</DescriptionItem>
|
||||
<DescriptionItem label="Gateway endpoint">
|
||||
{formatGatewayEndpoint(metadata) ?? "Not recorded"}
|
||||
</DescriptionItem>
|
||||
{logEntry.user && (
|
||||
<DescriptionItem label="User">
|
||||
<UserIdentity userId={logEntry.user} email={userEmail} />
|
||||
|
|
@ -162,7 +158,7 @@ export function LogDetailContent({
|
|||
<DescriptionItem label="Model ID">
|
||||
<TruncatedValue value={logEntry.model_id} />
|
||||
</DescriptionItem>
|
||||
<DescriptionItem label="Provider API base">
|
||||
<DescriptionItem label="API Base">
|
||||
<TruncatedValue value={logEntry.api_base} maxWidth={API_BASE_MAX_WIDTH} />
|
||||
</DescriptionItem>
|
||||
{podRouting && typeof podRouting === "object" && (
|
||||
|
|
|
|||
|
|
@ -1,27 +1,5 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { formatGatewayEndpoint, sortSessionLogs } from "./utils";
|
||||
|
||||
describe("formatGatewayEndpoint", () => {
|
||||
it.each(["/rerank", "/v1/rerank", "/v2/rerank"])("should preserve the recorded route %s", (path) => {
|
||||
expect(formatGatewayEndpoint({ spend_logs_metadata: { gateway_endpoint: { method: "POST", path } } })).toBe(
|
||||
`POST ${path}`,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
undefined,
|
||||
null,
|
||||
{},
|
||||
{ spend_logs_metadata: null },
|
||||
{ spend_logs_metadata: { gateway_endpoint: "POST /rerank" } },
|
||||
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
|
||||
{ spend_logs_metadata: { gateway_endpoint: { method: "", path: "/rerank" } } },
|
||||
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "https://api.example.com/rerank" } } },
|
||||
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/rerank?api_key=hidden" } } },
|
||||
])("should return no endpoint for missing or malformed metadata %j", (metadata) => {
|
||||
expect(formatGatewayEndpoint(metadata)).toBeNull();
|
||||
});
|
||||
});
|
||||
import { sortSessionLogs } from "./utils";
|
||||
|
||||
const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({
|
||||
request_id: id,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* Utility functions for LogDetailsDrawer component.
|
||||
* These functions handle data formatting, validation, and guardrail calculations.
|
||||
|
|
@ -7,22 +5,6 @@ import { z } from "zod";
|
|||
|
||||
export type SessionLogSortMode = "duration" | "start_time";
|
||||
|
||||
const gatewayEndpointMetadataSchema = z.object({
|
||||
spend_logs_metadata: z.object({
|
||||
gateway_endpoint: z.object({
|
||||
method: z.string().regex(/^[A-Z]+$/),
|
||||
path: z.string().regex(/^\/(?!\/)[^?#\s]*$/),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export function formatGatewayEndpoint(metadata: unknown): string | null {
|
||||
const parsed = gatewayEndpointMetadataSchema.safeParse(metadata);
|
||||
if (!parsed.success) return null;
|
||||
const { method, path } = parsed.data.spend_logs_metadata.gateway_endpoint;
|
||||
return `${method} ${path}`;
|
||||
}
|
||||
|
||||
type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number };
|
||||
|
||||
const durationMs = (row: SortableSessionLog): number =>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue