diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index 4f2daed15ed..73dded8e7fe 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -7,7 +7,9 @@ import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import ORJSONResponse +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import * +from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -20,6 +22,9 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) +from litellm.rerank_api.litellm_decisions_transformation.handler import ( + DecisionRerankLoggingCapture, +) router: Final = APIRouter() @@ -59,7 +64,8 @@ async def rerank( ) litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) - data = {"litellm_call_id": litellm_call_id} + logging_capture: Final = DecisionRerankLoggingCapture(litellm_call_id) + data: dict[str, object] = {"litellm_call_id": litellm_call_id} # mutable-ok: proxy hooks enrich request data try: body: Final = await request.body() data = orjson.loads(body) | data @@ -73,18 +79,28 @@ async def rerank( version=version, proxy_config=proxy_config, ) + metadata: Final = as_str_mapping(data.get("metadata")) or {} + spend_metadata: Final = as_str_mapping(metadata.get("spend_logs_metadata")) or {} + data["metadata"] = { + **metadata, + "spend_logs_metadata": { + **spend_metadata, + "gateway_endpoint": {"method": request.method, "path": get_request_route(request)}, + }, + } ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook(user_api_key_dict=user_api_key_dict, data=data, call_type="rerank") ## ROUTE TO CORRECT ENDPOINT ## - llm_call: Final = await route_request( - data=data, - route_type="arerank", - llm_router=llm_router, - user_model=user_model, - ) - response: Final = await llm_call + with logging_capture.capture(): + llm_call: Final = await route_request( + data=data, + route_type="arerank", + llm_router=llm_router, + user_model=user_model, + ) + response: Final = await llm_call ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -113,6 +129,9 @@ async def rerank( return response except Exception as e: + partial_logging: Final = logging_capture.logging_obj + if partial_logging is not None: + data["litellm_logging_obj"] = partial_logging await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) diff --git a/litellm/rerank_api/litellm_decisions_transformation/handler.py b/litellm/rerank_api/litellm_decisions_transformation/handler.py index 366935f4977..e188bdaa164 100644 --- a/litellm/rerank_api/litellm_decisions_transformation/handler.py +++ b/litellm/rerank_api/litellm_decisions_transformation/handler.py @@ -2,8 +2,10 @@ import asyncio import time -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor, wait +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass, replace from functools import partial from queue import Empty, SimpleQueue @@ -31,6 +33,29 @@ _CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"}) _MAX_CONCURRENCY: Final = 8 +class DecisionRerankLoggingCapture: + def __init__(self, call_id: str) -> None: + self.call_id: Final = call_id + self.logging_obj: Logging | None = None + + @contextmanager + def capture(self) -> Generator[None]: + token: Final = _LOGGING_CAPTURE.set(self) + try: + yield + finally: + _LOGGING_CAPTURE.reset(token) + + def record(self, logging_obj: Logging) -> None: + if logging_obj.litellm_call_id == self.call_id: + self.logging_obj = logging_obj + + +_LOGGING_CAPTURE: Final[ContextVar[DecisionRerankLoggingCapture | None]] = ContextVar( + "decision_rerank_logging_capture", default=None +) + + @dataclass(frozen=True, slots=True) class _DocumentCall: index: int @@ -114,15 +139,21 @@ def _invalid_response(document: _DocumentCall, reason: str) -> litellm.InternalS ) +def _aggregate_usage(response: DecisionRerankResponse) -> Usage: + prompt_tokens: Final = sum(item.usage.input_tokens for item in response.decision_responses) + completion_tokens: Final = sum(item.usage.output_tokens for item in response.decision_responses) + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + def _record_partial(response: DecisionRerankResponse, logging_obj: Logging) -> None: - responses: Final = response.decision_responses - if not responses: + if not response.decision_responses: return logging_obj.record_partial_usage_for_failure( - Usage( - prompt_tokens=sum(item.usage.input_tokens for item in responses), - completion_tokens=sum(item.usage.output_tokens for item in responses), - ), + _aggregate_usage(response), logging_obj.response_cost_calculator(result=response) or 0.0, ) @@ -142,10 +173,8 @@ def _finish( if failure is not None: _record_partial(response, logging_obj) raise failure - logging_obj.model_call_details["combined_usage_object"] = Usage( # rebind-ok: log aggregate usage - prompt_tokens=sum(item.usage.input_tokens for item in response.decision_responses), - completion_tokens=sum(item.usage.output_tokens for item in response.decision_responses), - ) + usage: Final = _aggregate_usage(response) + logging_obj.model_call_details["combined_usage_object"] = usage # rebind-ok: log aggregate usage return response @@ -275,6 +304,9 @@ def rerank_with_decisions( params: Mapping[str, object], kwargs: Mapping[str, object], ) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]: + logging_capture: Final = _LOGGING_CAPTURE.get() + if logging_capture is not None: + logging_capture.record(logging_obj) request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy) deadline: Final = time.monotonic() + request.timeout logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs diff --git a/tests/unit/proxy/rerank_endpoints/test_endpoints.py b/tests/unit/proxy/rerank_endpoints/test_endpoints.py index 52d12dd1813..e3041e87096 100644 --- a/tests/unit/proxy/rerank_endpoints/test_endpoints.py +++ b/tests/unit/proxy/rerank_endpoints/test_endpoints.py @@ -2,19 +2,32 @@ Tests for rerank_endpoints/endpoints.py response headers. """ +import asyncio import json import logging -from collections.abc import Iterator -from unittest.mock import AsyncMock, MagicMock, patch +from collections.abc import Iterator, Mapping +from datetime import datetime, timezone +from queue import SimpleQueue +from typing import Final +from unittest.mock import AsyncMock, MagicMock, create_autospec, patch +import httpx import pytest +import respx from fastapi import HTTPException, Request, Response +import litellm import litellm.proxy.common_request_processing as common_request_processing_mod import litellm.proxy.proxy_server as proxy_server_mod from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.rerank_endpoints.endpoints import rerank +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.proxy.utils import ProxyLogging from litellm.types.utils import RerankResponse HIDDEN_PARAMS = { @@ -31,8 +44,21 @@ HIDDEN_PARAMS = { } -def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request: - body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode() +def _build_request( + headers: tuple[tuple[bytes, bytes], ...] = (), + documents: tuple[str, ...] = ("a", "b"), + path: str = "/rerank", + root_path: str = "", + metadata: Mapping[str, object] | None = None, +) -> Request: + body = json.dumps( + { + "model": "rerank-model", + "query": "q", + "documents": documents, + **({"metadata": metadata} if metadata is not None else {}), + } + ).encode() async def receive(): return {"type": "http.request", "body": body, "more_body": False} @@ -41,7 +67,8 @@ def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request: scope={ "type": "http", "method": "POST", - "path": "/rerank", + "path": path, + "root_path": root_path, "headers": [(b"content-type", b"application/json"), *headers], "query_string": b"", }, @@ -209,3 +236,205 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch) assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "root_path", "expected_path"), + [ + ("/rerank", "", "/rerank"), + ("/v1/rerank", "", "/v1/rerank"), + ("/v2/rerank", "", "/v2/rerank"), + ("/gateway/v2/rerank", "/gateway", "/v2/rerank"), + ], +) +@pytest.mark.parametrize("partial_failure", [False, True]) +async def test_decision_rerank_writes_one_log_with_gateway_endpoint_and_aggregate_usage( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + path: str, + root_path: str, + expected_path: str, + partial_failure: bool, +) -> None: + model: Final = "typesafe/jevrank-spend-unit-test" + rate: Final = 0.004 + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "typesafe", + "mode": "evaluation", + "rerank_strategy": "pointwise", + "input_cost_per_token": rate, + "output_cost_per_token": 0, + }, + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "cache", None) + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "rerank-model", + "litellm_params": {"model": model, "api_key": "test-key"}, + } + ], + num_retries=0, + ) + writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) + db_logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + proxy_logging.db_spend_update_writer = writer + monkeypatch.setattr(litellm, "callbacks", [db_logger]) + monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router) + monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", proxy_logging) + monkeypatch.setattr(proxy_server_mod, "general_settings", {}) + monkeypatch.setattr(proxy_server_mod, "prisma_client", None) + + def score(request: httpx.Request) -> httpx.Response: + passage: Final = json.loads(json.loads(request.content)["state"])["passage"] + if passage == "failed": + return httpx.Response(503, json={"error": "unavailable"}) + return httpx.Response( + 200, + json={ + "model": "jevrank-spend-unit-test", + "answers": {"relevance": {"type": "noul", "noul": 0.8}}, + "usage": { + "input_tokens": 100 if passage == "first" else 80, + "output_tokens": 0, + }, + }, + ) + + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").mock( + side_effect=score + ) + request: Final = _build_request( + headers=((b"x-litellm-call-id", b"rerank-spend"),), + documents=("first", "failed" if partial_failure else "middle", "last"), + path=path, + root_path=root_path, + metadata={ + "tags": ["search"], + "spend_logs_metadata": { + "customer": "retained", + "gateway_endpoint": {"method": "DELETE", "path": "/client-forged"}, + }, + }, + ) + call: Final = rerank( + request=request, + fastapi_response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=expected_path), + ) + if partial_failure: + with pytest.raises(ProxyException) as failure: + await call + assert failure.value.code == "503" + else: + await call + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await GLOBAL_LOGGING_WORKER.flush() + + assert upstream.call_count == 3 + writer.update_database.assert_awaited_once() + write: Final = writer.update_database.call_args.kwargs + recorded_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + row: Final = get_logging_payload( + kwargs={**write["kwargs"], "response_cost": write["response_cost"]}, + response_obj=write["completion_response"], + start_time=recorded_at, + end_time=recorded_at, + ) + tokens: Final = 180 if partial_failure else 260 + assert row["request_id"] == "rerank-spend" + assert row["call_type"] == "arerank" + assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (tokens, 0, tokens) + assert row["spend"] == pytest.approx(tokens * rate) + assert json.loads(row["metadata"])["spend_logs_metadata"] == { + "customer": "retained", + "gateway_endpoint": {"method": "POST", "path": expected_path}, + } + assert json.loads(row["request_tags"]) == ["search"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("decision_model", [True, False]) +async def test_rerank_keeps_the_selected_deployment_success_callback( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, decision_model: bool +) -> None: + provider: Final = "typesafe" if decision_model else "cohere" + model: Final = f"{provider}/rerank-callback-unit-test" + events: Final[SimpleQueue[object]] = SimpleQueue() + + async def record_success( + kwargs: Mapping[str, object], + response: object, + start_time: datetime, + end_time: datetime, + ) -> None: + events.put(response) + + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": provider, + "mode": "evaluation" if decision_model else "rerank", + **({"rerank_strategy": "pointwise"} if decision_model else {}), + "input_cost_per_token": 0.004, + "output_cost_per_token": 0, + }, + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "cache", None) + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "rerank-model", + "litellm_params": { + "model": model, + "api_key": "test-key", + "success_callback": [record_success], + }, + } + ], + num_retries=0, + ) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router) + monkeypatch.setattr( + proxy_server_mod, + "proxy_logging_obj", + ProxyLogging(user_api_key_cache=UserApiKeyCache()), + ) + monkeypatch.setattr(proxy_server_mod, "general_settings", {}) + monkeypatch.setattr(proxy_server_mod, "prisma_client", None) + if decision_model: + respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( + json={ + "model": "rerank-callback-unit-test", + "answers": {"relevance": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 0}, + } + ) + else: + respx_mock.post("https://api.cohere.com/v2/rerank").respond( + json={ + "id": "native-rerank", + "results": [{"index": 0, "relevance_score": 0.8}], + } + ) + response: Final = await rerank( + request=_build_request(), + fastapi_response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/rerank"), + ) + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await GLOBAL_LOGGING_WORKER.flush() + + assert events.qsize() == 1 + assert events.get_nowait() is response diff --git a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py index d6f37269dab..bda54ba080e 100644 --- a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py +++ b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py @@ -68,13 +68,13 @@ def setup_model(monkeypatch: pytest.MonkeyPatch): ) -def response(probability=0.8, *, tokens=10): +def response(probability=0.8, *, tokens=10, output_tokens=0): return httpx.Response( 200, json={ "model": "jevrank-unit-test", "answers": {"relevance": {"type": "noul", "noul": probability}}, - "usage": {"input_tokens": tokens, "output_tokens": 0}, + "usage": {"input_tokens": tokens, "output_tokens": output_tokens}, }, ) @@ -305,7 +305,7 @@ async def test_success_is_billed_once_and_failed_request_retains_completed_usage def score(request): passage = json.loads(json.loads(request.content)["state"])["passage"] - return httpx.Response(503, json={"error": "unavailable"}) if passage == "failed" else response() + return httpx.Response(503, json={"error": "unavailable"}) if passage == "failed" else response(output_tokens=3) respx_mock.post(URL).mock(side_effect=score) await rerank(False, documents=["one", "two"]) @@ -317,6 +317,10 @@ async def test_success_is_billed_once_and_failed_request_retains_completed_usage await flush_logging() assert len(logger.events) == 2 assert logger.events[1]["response_cost"] == pytest.approx(10 * RATE) + assert [(event["prompt_tokens"], event["completion_tokens"], event["total_tokens"]) for event in logger.events] == [ + (20, 6, 26), + (10, 3, 13), + ] async def test_custom_token_prices_apply_to_decision_usage(respx_mock): diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx index 7f39a0d8fe5..79cff240ab9 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx @@ -56,6 +56,39 @@ describe("LogDetailContent", () => { expect(screen.getByText("completion")).toBeInTheDocument(); }); + it("should distinguish the recorded gateway endpoint from the provider API base", () => { + render( + , + ); + + expect(screen.getByText("Gateway endpoint")).toBeInTheDocument(); + expect(screen.getByText("POST /v1/rerank")).toBeInTheDocument(); + expect(screen.getByText("Provider API base")).toBeInTheDocument(); + expect(screen.getByText("https://api.typesafe.ai/v1")).toBeInTheDocument(); + expect(screen.getByText("arerank")).toBeInTheDocument(); + expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument(); + }); + + it.each([ + { status: "success" }, + { spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } }, + ])( + "should show that the gateway endpoint was not recorded when its metadata is unavailable or invalid", + (metadata) => { + render(); + + expect(screen.getByText("Gateway endpoint")).toBeInTheDocument(); + expect(screen.getByText("Not recorded")).toBeInTheDocument(); + expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument(); + }, + ); + it("shows the requesting user's email and id in Request Details when the email is resolved", () => { render( , diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx index f6c63582938..9ef3c640d5c 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx @@ -30,6 +30,7 @@ import { JsonViewer } from "./JsonViewer"; import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard"; import { formatData, + formatGatewayEndpoint, checkHasMessages, checkHasResponse, normalizeGuardrailEntries, @@ -150,6 +151,9 @@ export function LogDetailContent({ {logEntry.model} {logEntry.custom_llm_provider || "-"} {logEntry.call_type} + + {formatGatewayEndpoint(metadata) ?? "Not recorded"} + {logEntry.user && ( @@ -158,7 +162,7 @@ export function LogDetailContent({ - + {podRouting && typeof podRouting === "object" && ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts index f59a20529d0..c91e028f9c4 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts @@ -1,5 +1,27 @@ import { describe, expect, it } from "vitest"; -import { sortSessionLogs } from "./utils"; +import { formatGatewayEndpoint, sortSessionLogs } from "./utils"; + +describe("formatGatewayEndpoint", () => { + it.each(["/rerank", "/v1/rerank", "/v2/rerank"])("should preserve the recorded route %s", (path) => { + expect(formatGatewayEndpoint({ spend_logs_metadata: { gateway_endpoint: { method: "POST", path } } })).toBe( + `POST ${path}`, + ); + }); + + it.each([ + undefined, + null, + {}, + { spend_logs_metadata: null }, + { spend_logs_metadata: { gateway_endpoint: "POST /rerank" } }, + { spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } }, + { spend_logs_metadata: { gateway_endpoint: { method: "", path: "/rerank" } } }, + { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "https://api.example.com/rerank" } } }, + { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/rerank?api_key=hidden" } } }, + ])("should return no endpoint for missing or malformed metadata %j", (metadata) => { + expect(formatGatewayEndpoint(metadata)).toBeNull(); + }); +}); const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({ request_id: id, diff --git a/ui/litellm-dashboard/src/components/logs/detail/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/utils.ts index d313d07361c..e9ffb1f429e 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/utils.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + /** * Utility functions for LogDetailsDrawer component. * These functions handle data formatting, validation, and guardrail calculations. @@ -5,6 +7,22 @@ export type SessionLogSortMode = "duration" | "start_time"; +const gatewayEndpointMetadataSchema = z.object({ + spend_logs_metadata: z.object({ + gateway_endpoint: z.object({ + method: z.string().regex(/^[A-Z]+$/), + path: z.string().regex(/^\/(?!\/)[^?#\s]*$/), + }), + }), +}); + +export function formatGatewayEndpoint(metadata: unknown): string | null { + const parsed = gatewayEndpointMetadataSchema.safeParse(metadata); + if (!parsed.success) return null; + const { method, path } = parsed.data.spend_logs_metadata.gateway_endpoint; + return `${method} ${path}`; +} + type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number }; const durationMs = (row: SortableSessionLog): number =>