fix(rerank): preserve consolidated usage and show gateway endpoint

This commit is contained in:
Ishaan 2026-10-23 12:54:05 -07:00
parent 55692c25dc
commit ef332d3ab2
8 changed files with 390 additions and 29 deletions

View file

@ -7,7 +7,9 @@ import orjson
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import ORJSONResponse
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.proxy._types import *
from litellm.proxy.auth.auth_utils import get_request_route
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
@ -20,6 +22,9 @@ from litellm.proxy.common_utils.openai_error_payload import (
openai_error_param,
openai_error_type,
)
from litellm.rerank_api.litellm_decisions_transformation.handler import (
DecisionRerankLoggingCapture,
)
router: Final = APIRouter()
@ -59,7 +64,8 @@ async def rerank(
)
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data = {"litellm_call_id": litellm_call_id}
logging_capture: Final = DecisionRerankLoggingCapture(litellm_call_id)
data: dict[str, object] = {"litellm_call_id": litellm_call_id} # mutable-ok: proxy hooks enrich request data
try:
body: Final = await request.body()
data = orjson.loads(body) | data
@ -73,18 +79,28 @@ async def rerank(
version=version,
proxy_config=proxy_config,
)
metadata: Final = as_str_mapping(data.get("metadata")) or {}
spend_metadata: Final = as_str_mapping(metadata.get("spend_logs_metadata")) or {}
data["metadata"] = {
**metadata,
"spend_logs_metadata": {
**spend_metadata,
"gateway_endpoint": {"method": request.method, "path": get_request_route(request)},
},
}
### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook(user_api_key_dict=user_api_key_dict, data=data, call_type="rerank")
## ROUTE TO CORRECT ENDPOINT ##
llm_call: Final = await route_request(
data=data,
route_type="arerank",
llm_router=llm_router,
user_model=user_model,
)
response: Final = await llm_call
with logging_capture.capture():
llm_call: Final = await route_request(
data=data,
route_type="arerank",
llm_router=llm_router,
user_model=user_model,
)
response: Final = await llm_call
### ALERTING ###
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
@ -113,6 +129,9 @@ async def rerank(
return response
except Exception as e:
partial_logging: Final = logging_capture.logging_obj
if partial_logging is not None:
data["litellm_logging_obj"] = partial_logging
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)

View file

@ -2,8 +2,10 @@
import asyncio
import time
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor, wait
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, replace
from functools import partial
from queue import Empty, SimpleQueue
@ -31,6 +33,29 @@ _CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"})
_MAX_CONCURRENCY: Final = 8
class DecisionRerankLoggingCapture:
def __init__(self, call_id: str) -> None:
self.call_id: Final = call_id
self.logging_obj: Logging | None = None
@contextmanager
def capture(self) -> Generator[None]:
token: Final = _LOGGING_CAPTURE.set(self)
try:
yield
finally:
_LOGGING_CAPTURE.reset(token)
def record(self, logging_obj: Logging) -> None:
if logging_obj.litellm_call_id == self.call_id:
self.logging_obj = logging_obj
_LOGGING_CAPTURE: Final[ContextVar[DecisionRerankLoggingCapture | None]] = ContextVar(
"decision_rerank_logging_capture", default=None
)
@dataclass(frozen=True, slots=True)
class _DocumentCall:
index: int
@ -114,15 +139,21 @@ def _invalid_response(document: _DocumentCall, reason: str) -> litellm.InternalS
)
def _aggregate_usage(response: DecisionRerankResponse) -> Usage:
prompt_tokens: Final = sum(item.usage.input_tokens for item in response.decision_responses)
completion_tokens: Final = sum(item.usage.output_tokens for item in response.decision_responses)
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
def _record_partial(response: DecisionRerankResponse, logging_obj: Logging) -> None:
responses: Final = response.decision_responses
if not responses:
if not response.decision_responses:
return
logging_obj.record_partial_usage_for_failure(
Usage(
prompt_tokens=sum(item.usage.input_tokens for item in responses),
completion_tokens=sum(item.usage.output_tokens for item in responses),
),
_aggregate_usage(response),
logging_obj.response_cost_calculator(result=response) or 0.0,
)
@ -142,10 +173,8 @@ def _finish(
if failure is not None:
_record_partial(response, logging_obj)
raise failure
logging_obj.model_call_details["combined_usage_object"] = Usage( # rebind-ok: log aggregate usage
prompt_tokens=sum(item.usage.input_tokens for item in response.decision_responses),
completion_tokens=sum(item.usage.output_tokens for item in response.decision_responses),
)
usage: Final = _aggregate_usage(response)
logging_obj.model_call_details["combined_usage_object"] = usage # rebind-ok: log aggregate usage
return response
@ -275,6 +304,9 @@ def rerank_with_decisions(
params: Mapping[str, object],
kwargs: Mapping[str, object],
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
logging_capture: Final = _LOGGING_CAPTURE.get()
if logging_capture is not None:
logging_capture.record(logging_obj)
request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy)
deadline: Final = time.monotonic() + request.timeout
logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs

View file

@ -2,19 +2,32 @@
Tests for rerank_endpoints/endpoints.py response headers.
"""
import asyncio
import json
import logging
from collections.abc import Iterator
from unittest.mock import AsyncMock, MagicMock, patch
from collections.abc import Iterator, Mapping
from datetime import datetime, timezone
from queue import SimpleQueue
from typing import Final
from unittest.mock import AsyncMock, MagicMock, create_autospec, patch
import httpx
import pytest
import respx
from fastapi import HTTPException, Request, Response
import litellm
import litellm.proxy.common_request_processing as common_request_processing_mod
import litellm.proxy.proxy_server as proxy_server_mod
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.rerank_endpoints.endpoints import rerank
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import ProxyLogging
from litellm.types.utils import RerankResponse
HIDDEN_PARAMS = {
@ -31,8 +44,21 @@ HIDDEN_PARAMS = {
}
def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request:
body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode()
def _build_request(
headers: tuple[tuple[bytes, bytes], ...] = (),
documents: tuple[str, ...] = ("a", "b"),
path: str = "/rerank",
root_path: str = "",
metadata: Mapping[str, object] | None = None,
) -> Request:
body = json.dumps(
{
"model": "rerank-model",
"query": "q",
"documents": documents,
**({"metadata": metadata} if metadata is not None else {}),
}
).encode()
async def receive():
return {"type": "http.request", "body": body, "more_body": False}
@ -41,7 +67,8 @@ def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request:
scope={
"type": "http",
"method": "POST",
"path": "/rerank",
"path": path,
"root_path": root_path,
"headers": [(b"content-type", b"application/json"), *headers],
"query_string": b"",
},
@ -209,3 +236,205 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc
error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch)
assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "root_path", "expected_path"),
[
("/rerank", "", "/rerank"),
("/v1/rerank", "", "/v1/rerank"),
("/v2/rerank", "", "/v2/rerank"),
("/gateway/v2/rerank", "/gateway", "/v2/rerank"),
],
)
@pytest.mark.parametrize("partial_failure", [False, True])
async def test_decision_rerank_writes_one_log_with_gateway_endpoint_and_aggregate_usage(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
path: str,
root_path: str,
expected_path: str,
partial_failure: bool,
) -> None:
model: Final = "typesafe/jevrank-spend-unit-test"
rate: Final = 0.004
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "typesafe",
"mode": "evaluation",
"rerank_strategy": "pointwise",
"input_cost_per_token": rate,
"output_cost_per_token": 0,
},
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "cache", None)
llm_router: Final = litellm.Router(
model_list=[
{
"model_name": "rerank-model",
"litellm_params": {"model": model, "api_key": "test-key"},
}
],
num_retries=0,
)
writer: Final = create_autospec(DBSpendUpdateWriter, instance=True)
db_logger: Final = ProxyDBLogger(spend_writer=lambda: writer)
proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
proxy_logging.db_spend_update_writer = writer
monkeypatch.setattr(litellm, "callbacks", [db_logger])
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", proxy_logging)
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
def score(request: httpx.Request) -> httpx.Response:
passage: Final = json.loads(json.loads(request.content)["state"])["passage"]
if passage == "failed":
return httpx.Response(503, json={"error": "unavailable"})
return httpx.Response(
200,
json={
"model": "jevrank-spend-unit-test",
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
"usage": {
"input_tokens": 100 if passage == "first" else 80,
"output_tokens": 0,
},
},
)
upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").mock(
side_effect=score
)
request: Final = _build_request(
headers=((b"x-litellm-call-id", b"rerank-spend"),),
documents=("first", "failed" if partial_failure else "middle", "last"),
path=path,
root_path=root_path,
metadata={
"tags": ["search"],
"spend_logs_metadata": {
"customer": "retained",
"gateway_endpoint": {"method": "DELETE", "path": "/client-forged"},
},
},
)
call: Final = rerank(
request=request,
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=expected_path),
)
if partial_failure:
with pytest.raises(ProxyException) as failure:
await call
assert failure.value.code == "503"
else:
await call
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await GLOBAL_LOGGING_WORKER.flush()
assert upstream.call_count == 3
writer.update_database.assert_awaited_once()
write: Final = writer.update_database.call_args.kwargs
recorded_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
row: Final = get_logging_payload(
kwargs={**write["kwargs"], "response_cost": write["response_cost"]},
response_obj=write["completion_response"],
start_time=recorded_at,
end_time=recorded_at,
)
tokens: Final = 180 if partial_failure else 260
assert row["request_id"] == "rerank-spend"
assert row["call_type"] == "arerank"
assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (tokens, 0, tokens)
assert row["spend"] == pytest.approx(tokens * rate)
assert json.loads(row["metadata"])["spend_logs_metadata"] == {
"customer": "retained",
"gateway_endpoint": {"method": "POST", "path": expected_path},
}
assert json.loads(row["request_tags"]) == ["search"]
@pytest.mark.asyncio
@pytest.mark.parametrize("decision_model", [True, False])
async def test_rerank_keeps_the_selected_deployment_success_callback(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, decision_model: bool
) -> None:
provider: Final = "typesafe" if decision_model else "cohere"
model: Final = f"{provider}/rerank-callback-unit-test"
events: Final[SimpleQueue[object]] = SimpleQueue()
async def record_success(
kwargs: Mapping[str, object],
response: object,
start_time: datetime,
end_time: datetime,
) -> None:
events.put(response)
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": provider,
"mode": "evaluation" if decision_model else "rerank",
**({"rerank_strategy": "pointwise"} if decision_model else {}),
"input_cost_per_token": 0.004,
"output_cost_per_token": 0,
},
)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "cache", None)
llm_router: Final = litellm.Router(
model_list=[
{
"model_name": "rerank-model",
"litellm_params": {
"model": model,
"api_key": "test-key",
"success_callback": [record_success],
},
}
],
num_retries=0,
)
monkeypatch.setattr(litellm, "callbacks", [])
monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router)
monkeypatch.setattr(
proxy_server_mod,
"proxy_logging_obj",
ProxyLogging(user_api_key_cache=UserApiKeyCache()),
)
monkeypatch.setattr(proxy_server_mod, "general_settings", {})
monkeypatch.setattr(proxy_server_mod, "prisma_client", None)
if decision_model:
respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(
json={
"model": "rerank-callback-unit-test",
"answers": {"relevance": {"type": "noul", "noul": 0.8}},
"usage": {"input_tokens": 10, "output_tokens": 0},
}
)
else:
respx_mock.post("https://api.cohere.com/v2/rerank").respond(
json={
"id": "native-rerank",
"results": [{"index": 0, "relevance_score": 0.8}],
}
)
response: Final = await rerank(
request=_build_request(),
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/rerank"),
)
await asyncio.sleep(0)
GLOBAL_LOGGING_WORKER.start()
await GLOBAL_LOGGING_WORKER.flush()
assert events.qsize() == 1
assert events.get_nowait() is response

View file

@ -68,13 +68,13 @@ def setup_model(monkeypatch: pytest.MonkeyPatch):
)
def response(probability=0.8, *, tokens=10):
def response(probability=0.8, *, tokens=10, output_tokens=0):
return httpx.Response(
200,
json={
"model": "jevrank-unit-test",
"answers": {"relevance": {"type": "noul", "noul": probability}},
"usage": {"input_tokens": tokens, "output_tokens": 0},
"usage": {"input_tokens": tokens, "output_tokens": output_tokens},
},
)
@ -305,7 +305,7 @@ async def test_success_is_billed_once_and_failed_request_retains_completed_usage
def score(request):
passage = json.loads(json.loads(request.content)["state"])["passage"]
return httpx.Response(503, json={"error": "unavailable"}) if passage == "failed" else response()
return httpx.Response(503, json={"error": "unavailable"}) if passage == "failed" else response(output_tokens=3)
respx_mock.post(URL).mock(side_effect=score)
await rerank(False, documents=["one", "two"])
@ -317,6 +317,10 @@ async def test_success_is_billed_once_and_failed_request_retains_completed_usage
await flush_logging()
assert len(logger.events) == 2
assert logger.events[1]["response_cost"] == pytest.approx(10 * RATE)
assert [(event["prompt_tokens"], event["completion_tokens"], event["total_tokens"]) for event in logger.events] == [
(20, 6, 26),
(10, 3, 13),
]
async def test_custom_token_prices_apply_to_decision_usage(respx_mock):

View file

@ -56,6 +56,39 @@ describe("LogDetailContent", () => {
expect(screen.getByText("completion")).toBeInTheDocument();
});
it("should distinguish the recorded gateway endpoint from the provider API base", () => {
render(
<LogDetailContent
logEntry={createLogEntry({
call_type: "arerank",
api_base: "https://api.typesafe.ai/v1",
metadata: { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/v1/rerank" } } },
})}
/>,
);
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
expect(screen.getByText("POST /v1/rerank")).toBeInTheDocument();
expect(screen.getByText("Provider API base")).toBeInTheDocument();
expect(screen.getByText("https://api.typesafe.ai/v1")).toBeInTheDocument();
expect(screen.getByText("arerank")).toBeInTheDocument();
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
});
it.each([
{ status: "success" },
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
])(
"should show that the gateway endpoint was not recorded when its metadata is unavailable or invalid",
(metadata) => {
render(<LogDetailContent logEntry={createLogEntry({ call_type: "arerank", metadata })} />);
expect(screen.getByText("Gateway endpoint")).toBeInTheDocument();
expect(screen.getByText("Not recorded")).toBeInTheDocument();
expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument();
},
);
it("shows the requesting user's email and id in Request Details when the email is resolved", () => {
render(
<LogDetailContent logEntry={createLogEntry({ user: "106514937785257944828" })} userEmail="alice@example.com" />,

View file

@ -30,6 +30,7 @@ import { JsonViewer } from "./JsonViewer";
import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard";
import {
formatData,
formatGatewayEndpoint,
checkHasMessages,
checkHasResponse,
normalizeGuardrailEntries,
@ -150,6 +151,9 @@ export function LogDetailContent({
<DescriptionItem label="Model">{logEntry.model}</DescriptionItem>
<DescriptionItem label="Provider">{logEntry.custom_llm_provider || "-"}</DescriptionItem>
<DescriptionItem label="Call Type">{logEntry.call_type}</DescriptionItem>
<DescriptionItem label="Gateway endpoint">
{formatGatewayEndpoint(metadata) ?? "Not recorded"}
</DescriptionItem>
{logEntry.user && (
<DescriptionItem label="User">
<UserIdentity userId={logEntry.user} email={userEmail} />
@ -158,7 +162,7 @@ export function LogDetailContent({
<DescriptionItem label="Model ID">
<TruncatedValue value={logEntry.model_id} />
</DescriptionItem>
<DescriptionItem label="API Base">
<DescriptionItem label="Provider API base">
<TruncatedValue value={logEntry.api_base} maxWidth={API_BASE_MAX_WIDTH} />
</DescriptionItem>
{podRouting && typeof podRouting === "object" && (

View file

@ -1,5 +1,27 @@
import { describe, expect, it } from "vitest";
import { sortSessionLogs } from "./utils";
import { formatGatewayEndpoint, sortSessionLogs } from "./utils";
describe("formatGatewayEndpoint", () => {
it.each(["/rerank", "/v1/rerank", "/v2/rerank"])("should preserve the recorded route %s", (path) => {
expect(formatGatewayEndpoint({ spend_logs_metadata: { gateway_endpoint: { method: "POST", path } } })).toBe(
`POST ${path}`,
);
});
it.each([
undefined,
null,
{},
{ spend_logs_metadata: null },
{ spend_logs_metadata: { gateway_endpoint: "POST /rerank" } },
{ spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "", path: "/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "https://api.example.com/rerank" } } },
{ spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/rerank?api_key=hidden" } } },
])("should return no endpoint for missing or malformed metadata %j", (metadata) => {
expect(formatGatewayEndpoint(metadata)).toBeNull();
});
});
const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({
request_id: id,

View file

@ -1,3 +1,5 @@
import { z } from "zod";
/**
* Utility functions for LogDetailsDrawer component.
* These functions handle data formatting, validation, and guardrail calculations.
@ -5,6 +7,22 @@
export type SessionLogSortMode = "duration" | "start_time";
const gatewayEndpointMetadataSchema = z.object({
spend_logs_metadata: z.object({
gateway_endpoint: z.object({
method: z.string().regex(/^[A-Z]+$/),
path: z.string().regex(/^\/(?!\/)[^?#\s]*$/),
}),
}),
});
export function formatGatewayEndpoint(metadata: unknown): string | null {
const parsed = gatewayEndpointMetadataSchema.safeParse(metadata);
if (!parsed.success) return null;
const { method, path } = parsed.data.spend_logs_metadata.gateway_endpoint;
return `${method} ${path}`;
}
type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number };
const durationMs = (row: SortableSessionLog): number =>