From 314c52bcf48627195b736d9fd4807d199fafe264 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 23 Oct 2026 18:01:12 -0700 Subject: [PATCH] fix(rerank): scope decision adapter to SDK integration Restore proxy endpoints, spend tracking, and logs UI to the base branch. Remove endpoint-specific logging capture and preserve decision cost handling through the existing calculator API --- litellm/cost_calculator.py | 10 +- litellm/proxy/rerank_endpoints/endpoints.py | 35 +-- .../spend_tracking/budget_reservation.py | 135 +--------- litellm/proxy/spend_tracking/input_tokens.py | 12 +- .../handler.py | 30 +-- .../proxy/rerank_endpoints/test_endpoints.py | 239 +----------------- .../spend_tracking/test_budget_reservation.py | 235 ----------------- .../test_handler.py | 44 ++-- .../LogDetailContent.integration.test.tsx | 33 --- .../logs/detail/LogDetailContent.tsx | 6 +- .../src/components/logs/detail/utils.test.ts | 24 +- .../src/components/logs/detail/utils.ts | 18 -- 12 files changed, 56 insertions(+), 765 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index c9594b7e716..b8d304ff6a1 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1550,9 +1550,9 @@ def _decision_rerank_cost( ) -> tuple[float, CostBreakdown | None, float]: if logging_obj is not None: logging_obj.cost_breakdown = None - cost: Final = price(response) - breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None provider_cost: Final = get_response_cost_from_hidden_params(response.hidden_params) + cost: Final = price(response) if custom_pricing or provider_cost is None else provider_cost + breakdown: Final = logging_obj.cost_breakdown if logging_obj is not None else None fixed: Final = ( _apply_cost_margin( 0.0, @@ -1639,7 +1639,7 @@ def completion_cost( return _decision_rerank_cost( completion_response.decision_responses, partial( - response_cost_calculator, + completion_cost, model=model or (decision_rerank_child.model if decision_rerank_child is not None else ""), call_type="decisions", custom_llm_provider=custom_llm_provider, @@ -2380,7 +2380,6 @@ def response_cost_calculator( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") region_name: str | None = None, - custom_cost_per_token: CostPerToken | None = None, ) -> float: """ Returns @@ -2398,7 +2397,7 @@ def response_cost_calculator( getattr(response_object, HIDDEN_PARAMS_ATTR), ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) - if provider_response_cost is not None and not custom_pricing and custom_cost_per_token is None: + if provider_response_cost is not None and not custom_pricing: return provider_response_cost response_cost = completion_cost( @@ -2419,7 +2418,6 @@ def response_cost_calculator( data_residency=data_residency, vertex_location=vertex_location, region_name=region_name, - custom_cost_per_token=custom_cost_per_token, ) return response_cost except Exception as e: diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py index 73dded8e7fe..4f2daed15ed 100644 --- a/litellm/proxy/rerank_endpoints/endpoints.py +++ b/litellm/proxy/rerank_endpoints/endpoints.py @@ -7,9 +7,7 @@ import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import ORJSONResponse -from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import * -from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -22,9 +20,6 @@ from litellm.proxy.common_utils.openai_error_payload import ( openai_error_param, openai_error_type, ) -from litellm.rerank_api.litellm_decisions_transformation.handler import ( - DecisionRerankLoggingCapture, -) router: Final = APIRouter() @@ -64,8 +59,7 @@ async def rerank( ) litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) - logging_capture: Final = DecisionRerankLoggingCapture(litellm_call_id) - data: dict[str, object] = {"litellm_call_id": litellm_call_id} # mutable-ok: proxy hooks enrich request data + data = {"litellm_call_id": litellm_call_id} try: body: Final = await request.body() data = orjson.loads(body) | data @@ -79,28 +73,18 @@ async def rerank( version=version, proxy_config=proxy_config, ) - metadata: Final = as_str_mapping(data.get("metadata")) or {} - spend_metadata: Final = as_str_mapping(metadata.get("spend_logs_metadata")) or {} - data["metadata"] = { - **metadata, - "spend_logs_metadata": { - **spend_metadata, - "gateway_endpoint": {"method": request.method, "path": get_request_route(request)}, - }, - } ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook(user_api_key_dict=user_api_key_dict, data=data, call_type="rerank") ## ROUTE TO CORRECT ENDPOINT ## - with logging_capture.capture(): - llm_call: Final = await route_request( - data=data, - route_type="arerank", - llm_router=llm_router, - user_model=user_model, - ) - response: Final = await llm_call + llm_call: Final = await route_request( + data=data, + route_type="arerank", + llm_router=llm_router, + user_model=user_model, + ) + response: Final = await llm_call ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -129,9 +113,6 @@ async def rerank( return response except Exception as e: - partial_logging: Final = logging_capture.logging_obj - if partial_logging is not None: - data["litellm_logging_obj"] = partial_logging await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index a628d919766..803df9f83ca 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -16,7 +16,6 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.proxy._types import ( @@ -40,22 +39,12 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import ( - count_input_tokens, - count_input_tokens_for_model, - count_text_tokens, -) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging -from litellm.rerank_api.litellm_decisions_transformation.transformation import ( - DecisionRerankRequest, - LiteLLMDecisionsRerankConfig, -) -from litellm.rerank_api.rerank_utils import get_decision_rerank_strategy from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict -from litellm.types.utils import LlmProviders @dataclass @@ -84,7 +73,7 @@ _COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = { } _CACHED_VALUE: Final = TypeAdapter(object) -_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) class _CounterReservationUnavailable(Exception): @@ -895,7 +884,7 @@ def _coerce_window(window: object) -> Mapping[str, object]: return window if isinstance(window, str): try: - return _OBJECT_MAPPING.validate_python(json.loads(window)) + return _WINDOW_OBJECT.validate_python(json.loads(window)) except Exception: return {} model_dump: Final = getattr(window, "model_dump", None) @@ -1352,19 +1341,13 @@ def _estimate_request_input_cost_for_model( llm_router: Router | None, input_tokens: int | None = None, ) -> float | None: - typed_body: Final = _OBJECT_MAPPING.validate_python(request_body) - plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router) - counted: Final = ( - _decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens - ) estimates: Final = [ _input_cost_for_cost_info( request_body=request_body, route=route, model=model, model_info=model_info, - input_tokens=counted, - is_decision_rerank=bool(plans), + input_tokens=input_tokens, ) for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) ] @@ -1378,7 +1361,6 @@ def _input_cost_for_cost_info( model: str, model_info: Mapping[str, object], input_tokens: int | None = None, - is_decision_rerank: bool = False, ) -> float | None: estimated_input_tokens: Final = _estimate_input_tokens( request_body=request_body, @@ -1390,8 +1372,6 @@ def _input_cost_for_cost_info( if estimated_input_tokens is None: return None tiered_pricing: Final = model_info.get("tiered_pricing") - if is_decision_rerank and tiered_pricing: - return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token") if isinstance(tiered_pricing, list) and tiered_pricing: tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens) if tier is not None: @@ -1409,20 +1389,13 @@ def _estimate_request_max_cost_for_model( llm_router: Router | None, input_tokens: int | None = None, ) -> float | None: - typed_body: Final = _OBJECT_MAPPING.validate_python(request_body) - plans: Final = _decision_rerank_plans(typed_body, route, model, llm_router) - counted: Final = ( - _decision_rerank_input_tokens(typed_body, model, plans) if input_tokens is None and plans else input_tokens - ) - request_count: Final = max((request.request_count for _, request in plans), default=0) estimates: Final = [ _max_cost_for_cost_info( request_body=request_body, route=route, model=model, model_info=model_info, - input_tokens=counted, - decision_request_count=request_count, + input_tokens=input_tokens, ) for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) ] @@ -1436,7 +1409,6 @@ def _max_cost_for_cost_info( model: str, model_info: Mapping[str, object], input_tokens: int | None = None, - decision_request_count: int = 0, ) -> float | None: image_cost: Final = _estimate_image_generation_cost( request_body=request_body, @@ -1456,18 +1428,12 @@ def _max_cost_for_cost_info( request_body=request_body, route=route, model_info=model_info, - decision_request_count=decision_request_count, ) if estimated_input_tokens is None or output_tokens is None: return None output_multiplier: Final = _get_output_multiplier(request_body=request_body) tiered_pricing: Final = model_info.get("tiered_pricing") - if decision_request_count and tiered_pricing: - return estimated_input_tokens * _max_tier_rate(tiered_pricing, "input_cost_per_token") + output_tokens * max( - _max_tier_rate(tiered_pricing, "output_cost_per_token"), - _max_tier_rate(tiered_pricing, "output_cost_per_reasoning_token"), - ) if isinstance(tiered_pricing, list) and tiered_pricing: tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=estimated_input_tokens) if tier is not None: @@ -1500,14 +1466,6 @@ def _max_cost_for_cost_info( return cost -_TIER_COSTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) - - -def _max_tier_rate(tiered_pricing: object, cost_key: str) -> float: - tiers: Final = _TIER_COSTS.validate_python(tiered_pricing) - return max(tier_rate(dict(tier), cost_key) for tier in tiers) - - def _estimate_image_generation_cost( request_body: dict, model_info: Mapping[str, object], @@ -1639,84 +1597,7 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - typed_body: Final = _OBJECT_MAPPING.validate_python(request_body) - plans: Final = {model: _decision_rerank_plans(typed_body, route, model, llm_router) for model in models} - regular_models: Final = tuple(model for model in models if not plans[model]) - regular_counts: Final = await count_input_tokens( - request_body=request_body, raw_body=raw_body, models=regular_models - ) - decision_counts: Final = { - model: await asyncio.to_thread(_decision_rerank_input_tokens, typed_body, model, plan) - for model, plan in plans.items() - if plan - } - return MappingProxyType({**regular_counts, **decision_counts}) - - -def _decision_rerank_plan( - request_body: Mapping[str, object], model: str, custom_llm_provider: str | None -) -> tuple[str, DecisionRerankRequest] | None: - try: - resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) - strategy: Final = get_decision_rerank_strategy(resolved_model, LlmProviders(provider)) - if strategy is None: - return None - pricing_params: Final = {key: value for key, value in request_body.items() if key != "timeout"} - request: Final = LiteLLMDecisionsRerankConfig.validate_request( - pricing_params, resolved_model, provider, strategy - ) - except (litellm.BadRequestError, litellm.UnsupportedParamsError, ValueError): - return None - return f"{provider}/{resolved_model}", request - - -def _decision_rerank_plans( - request_body: Mapping[str, object], route: str, model: str, llm_router: Router | None -) -> tuple[tuple[str, DecisionRerankRequest], ...]: - if "rerank" not in route: - return () - deployments: Final = () if llm_router is None else (llm_router.get_model_list(model_name=model) or ()) - if not deployments: - provider: Final = request_body.get("custom_llm_provider") - direct_plan: Final = _decision_rerank_plan(request_body, model, provider if isinstance(provider, str) else None) - return () if direct_plan is None else (direct_plan,) - return tuple( - plan - for deployment in deployments - if (plan := _decision_rerank_deployment_plan(request_body, deployment)) is not None - ) - - -def _decision_rerank_deployment_plan( - request_body: Mapping[str, object], deployment: DeploymentTypedDict -) -> tuple[str, DecisionRerankRequest] | None: - params: Final = as_str_mapping(deployment["litellm_params"]) if "litellm_params" in deployment else None - if params is None: - return None - model: Final = params.get("model") - provider: Final = params.get("custom_llm_provider") - if not isinstance(model, str): - return None - return _decision_rerank_plan({**params, **request_body}, model, provider if isinstance(provider, str) else None) - - -def _decision_rerank_input_tokens( - request_body: Mapping[str, object], requested_model: str, plans: Sequence[tuple[str, DecisionRerankRequest]] -) -> int: - native_count: Final = count_input_tokens_for_model(dict(request_body), requested_model) or 0 - decision_counts: Final = ( - sum( - count_text_tokens( - model=model, - text=LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index).model_dump_json( - exclude_none=True - ), - ) - for index in range(request.request_count) - ) - for model, request in plans - ) - return max(native_count, *decision_counts) + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( @@ -1748,11 +1629,7 @@ def _estimate_output_tokens( request_body: dict, route: str, model_info: Mapping[str, object], - decision_request_count: int = 0, ) -> int | None: - if decision_request_count: - ceiling: Final = _to_int(model_info.get("max_output_tokens")) or DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK - return decision_request_count * ceiling if _is_input_only_route(route=route): return 0 diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 763807dd52a..927152296e9 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -141,16 +141,16 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: tool_choice=request_body.get("tool_choice"), ) except ValueError: - return count_text_tokens(model=model, text=request_body.get("messages")) + return _count_text_tokens(model=model, text=request_body.get("messages")) if "prompt" in request_body: - return count_text_tokens(model=model, text=request_body.get("prompt")) + return _count_text_tokens(model=model, text=request_body.get("prompt")) if "input" in request_body: - return count_text_tokens( + return _count_text_tokens( model=model, text=billable_input_since_newest_compaction(request_body.get("input")) ) if "query" in request_body or "documents" in request_body: - query_tokens: Final = count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = count_text_tokens( + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( model=model, text=request_body.get("documents"), ) @@ -160,7 +160,7 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: return None -def count_text_tokens(model: str, text: object) -> int: +def _count_text_tokens(model: str, text: object) -> int: if text is None: return 0 diff --git a/litellm/rerank_api/litellm_decisions_transformation/handler.py b/litellm/rerank_api/litellm_decisions_transformation/handler.py index cab2444808c..c8d64d70290 100644 --- a/litellm/rerank_api/litellm_decisions_transformation/handler.py +++ b/litellm/rerank_api/litellm_decisions_transformation/handler.py @@ -2,10 +2,8 @@ import asyncio import time -from collections.abc import Awaitable, Callable, Generator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor, wait -from contextlib import contextmanager -from contextvars import ContextVar from dataclasses import dataclass, replace from functools import partial from queue import Empty, SimpleQueue @@ -36,29 +34,6 @@ _CHILD_OMIT: Final = frozenset({"litellm_logging_obj", "litellm_call_id"}) _MAX_CONCURRENCY: Final = 8 -class DecisionRerankLoggingCapture: - def __init__(self, call_id: str) -> None: - self.call_id: Final = call_id - self.logging_obj: Logging | None = None - - @contextmanager - def capture(self) -> Generator[None]: - token: Final = _LOGGING_CAPTURE.set(self) - try: - yield - finally: - _LOGGING_CAPTURE.reset(token) - - def record(self, logging_obj: Logging) -> None: - if logging_obj.litellm_call_id == self.call_id: - self.logging_obj = logging_obj - - -_LOGGING_CAPTURE: Final[ContextVar[DecisionRerankLoggingCapture | None]] = ContextVar( - "decision_rerank_logging_capture", default=None -) - - @dataclass(frozen=True, slots=True) class _DocumentCall: index: int @@ -322,9 +297,6 @@ def rerank_with_decisions( params: Mapping[str, object], kwargs: Mapping[str, object], ) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]: - logging_capture: Final = _LOGGING_CAPTURE.get() - if logging_capture is not None: - logging_capture.record(logging_obj) request: Final = LiteLLMDecisionsRerankConfig.validate_request(params, model, provider, strategy) deadline: Final = time.monotonic() + request.timeout logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging accepts legacy provider kwargs diff --git a/tests/unit/proxy/rerank_endpoints/test_endpoints.py b/tests/unit/proxy/rerank_endpoints/test_endpoints.py index ebf4a80470b..52d12dd1813 100644 --- a/tests/unit/proxy/rerank_endpoints/test_endpoints.py +++ b/tests/unit/proxy/rerank_endpoints/test_endpoints.py @@ -2,32 +2,19 @@ Tests for rerank_endpoints/endpoints.py response headers. """ -import asyncio import json import logging -from collections.abc import Iterator, Mapping -from datetime import datetime, timezone -from queue import SimpleQueue -from typing import Final -from unittest.mock import AsyncMock, MagicMock, create_autospec, patch +from collections.abc import Iterator +from unittest.mock import AsyncMock, MagicMock, patch -import httpx import pytest -import respx from fastapi import HTTPException, Request, Response -import litellm import litellm.proxy.common_request_processing as common_request_processing_mod import litellm.proxy.proxy_server as proxy_server_mod from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy._types import ProxyException, UserAPIKeyAuth -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache -from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter -from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.rerank_endpoints.endpoints import rerank -from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.proxy.utils import ProxyLogging from litellm.types.utils import RerankResponse HIDDEN_PARAMS = { @@ -44,21 +31,8 @@ HIDDEN_PARAMS = { } -def _build_request( - headers: tuple[tuple[bytes, bytes], ...] = (), - documents: tuple[str, ...] = ("a", "b"), - path: str = "/rerank", - root_path: str = "", - metadata: Mapping[str, object] | None = None, -) -> Request: - body = json.dumps( - { - "model": "rerank-model", - "query": "q", - "documents": documents, - **({"metadata": metadata} if metadata is not None else {}), - } - ).encode() +def _build_request(headers: tuple[tuple[bytes, bytes], ...] = ()) -> Request: + body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode() async def receive(): return {"type": "http.request", "body": body, "more_body": False} @@ -67,8 +41,7 @@ def _build_request( scope={ "type": "http", "method": "POST", - "path": path, - "root_path": root_path, + "path": "/rerank", "headers": [(b"content-type", b"application/json"), *headers], "query_string": b"", }, @@ -236,205 +209,3 @@ async def test_a_rejection_raised_before_routing_keeps_its_own_status(monkeypatc error = await _rerank_failure(rejection, raised_before_routing=True, monkeypatch=monkeypatch) assert (error.type, error.param, error.code) == ("bad_request_error", "session_id", "400") - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("path", "root_path", "expected_path"), - [ - ("/rerank", "", "/rerank"), - ("/v1/rerank", "", "/v1/rerank"), - ("/v2/rerank", "", "/v2/rerank"), - ("/gateway/v2/rerank", "/gateway", "/v2/rerank"), - ], -) -@pytest.mark.parametrize("partial_failure", [False, True]) -async def test_decision_rerank_writes_one_log_with_gateway_endpoint_and_aggregate_usage( - monkeypatch: pytest.MonkeyPatch, - respx_mock: respx.MockRouter, - path: str, - root_path: str, - expected_path: str, - partial_failure: bool, -) -> None: - model: Final = "typesafe/jevrank-spend-unit-test" - rate: Final = 0.004 - monkeypatch.setitem( - litellm.model_cost, - model, - { - "litellm_provider": "typesafe", - "mode": "evaluation", - "rerank_strategy": "pointwise", - "input_cost_per_token": rate, - "output_cost_per_token": 0, - }, - ) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "cache", None) - llm_router: Final = litellm.Router( - model_list=[ - { - "model_name": "rerank-model", - "litellm_params": {"model": model, "api_key": "test-key"}, - } - ], - num_retries=0, - ) - writer: Final = create_autospec(DBSpendUpdateWriter, instance=True) - db_logger: Final = ProxyDBLogger(spend_writer=lambda: writer) - proxy_logging: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) - proxy_logging.db_spend_update_writer = writer - monkeypatch.setattr(litellm, "callbacks", [db_logger]) - monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router) - monkeypatch.setattr(proxy_server_mod, "proxy_logging_obj", proxy_logging) - monkeypatch.setattr(proxy_server_mod, "general_settings", {}) - monkeypatch.setattr(proxy_server_mod, "prisma_client", None) - - def score(request: httpx.Request) -> httpx.Response: - passage: Final = json.loads(json.loads(request.content)["state"])["passage"] - if passage == "failed": - return httpx.Response(503, json={"error": "unavailable"}) - return httpx.Response( - 200, - json={ - "model": "jevrank-spend-unit-test", - "answers": {"relevance": {"type": "noul", "noul": 0.8}}, - "usage": { - "input_tokens": 100 if passage == "first" else 80, - "output_tokens": 0, - }, - }, - ) - - upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").mock( - side_effect=score - ) - request: Final = _build_request( - headers=((b"x-litellm-call-id", b"rerank-spend"),), - documents=("first", "failed" if partial_failure else "middle", "last"), - path=path, - root_path=root_path, - metadata={ - "tags": ["search"], - "spend_logs_metadata": { - "customer": "retained", - "gateway_endpoint": {"method": "DELETE", "path": "/client-forged"}, - }, - }, - ) - call: Final = rerank( - request=request, - fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=expected_path), - ) - if partial_failure: - with pytest.raises(ProxyException) as failure: - await call - assert failure.value.code == "503" - else: - await call - await asyncio.sleep(0) - GLOBAL_LOGGING_WORKER.start() - await GLOBAL_LOGGING_WORKER.flush() - - assert upstream.call_count == (2 if partial_failure else 3) - writer.update_database.assert_awaited_once() - write: Final = writer.update_database.call_args.kwargs - recorded_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) - row: Final = get_logging_payload( - kwargs={**write["kwargs"], "response_cost": write["response_cost"]}, - response_obj=write["completion_response"], - start_time=recorded_at, - end_time=recorded_at, - ) - tokens: Final = 100 if partial_failure else 260 - assert row["request_id"] == "rerank-spend" - assert row["call_type"] == "arerank" - assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (tokens, 0, tokens) - assert row["spend"] == pytest.approx(tokens * rate) - assert json.loads(row["metadata"])["spend_logs_metadata"] == { - "customer": "retained", - "gateway_endpoint": {"method": "POST", "path": expected_path}, - } - assert json.loads(row["request_tags"]) == ["search"] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("decision_model", [True, False]) -async def test_rerank_keeps_the_selected_deployment_success_callback( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, decision_model: bool -) -> None: - provider: Final = "typesafe" if decision_model else "cohere" - model: Final = f"{provider}/rerank-callback-unit-test" - events: Final[SimpleQueue[object]] = SimpleQueue() - - async def record_success( - kwargs: Mapping[str, object], - response: object, - start_time: datetime, - end_time: datetime, - ) -> None: - events.put(response) - - monkeypatch.setitem( - litellm.model_cost, - model, - { - "litellm_provider": provider, - "mode": "evaluation" if decision_model else "rerank", - **({"rerank_strategy": "pointwise"} if decision_model else {}), - "input_cost_per_token": 0.004, - "output_cost_per_token": 0, - }, - ) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "cache", None) - llm_router: Final = litellm.Router( - model_list=[ - { - "model_name": "rerank-model", - "litellm_params": { - "model": model, - "api_key": "test-key", - "success_callback": [record_success], - }, - } - ], - num_retries=0, - ) - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(proxy_server_mod, "llm_router", llm_router) - monkeypatch.setattr( - proxy_server_mod, - "proxy_logging_obj", - ProxyLogging(user_api_key_cache=UserApiKeyCache()), - ) - monkeypatch.setattr(proxy_server_mod, "general_settings", {}) - monkeypatch.setattr(proxy_server_mod, "prisma_client", None) - if decision_model: - respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( - json={ - "model": "rerank-callback-unit-test", - "answers": {"relevance": {"type": "noul", "noul": 0.8}}, - "usage": {"input_tokens": 10, "output_tokens": 0}, - } - ) - else: - respx_mock.post("https://api.cohere.com/v2/rerank").respond( - json={ - "id": "native-rerank", - "results": [{"index": 0, "relevance_score": 0.8}], - } - ) - response: Final = await rerank( - request=_build_request(), - fastapi_response=Response(), - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route="/rerank"), - ) - await asyncio.sleep(0) - GLOBAL_LOGGING_WORKER.start() - await GLOBAL_LOGGING_WORKER.flush() - - assert events.qsize() == 1 - assert events.get_nowait() is response diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 92a197e81b0..024fe5229be 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -1,11 +1,9 @@ from __future__ import annotations -import json import math from datetime import datetime, timedelta, timezone from typing import Final -import httpx import pytest import litellm @@ -27,17 +25,12 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, - count_request_input_tokens, - estimate_request_input_cost, estimate_request_max_cost, get_budget_window_start, release_unbound_budget_reservation, reserve_budget_for_request, ) from litellm.proxy.utils import ProxyLogging -from litellm.rerank_api.litellm_decisions_transformation.transformation import ( - LiteLLMDecisionsRerankConfig, -) from litellm.router import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo @@ -433,231 +426,3 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe ) def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None: assert get_budget_window_start(window) == expected - - -@pytest.fixture -def decision_rerank_models(monkeypatch: pytest.MonkeyPatch) -> tuple[str, str]: - models: Final = ("typesafe/budget-pointwise", "typesafe/budget-choice") - for model, strategy in zip(models, ("pointwise", "multiple_choice"), strict=True): - monkeypatch.setitem( - litellm.model_cost, - model, - { - "mode": "evaluation", - "litellm_provider": "typesafe", - "rerank_strategy": strategy, - "max_input_tokens": 65536, - "max_output_tokens": 32, - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - }, - ) - return models - - -def _decision_rerank_body(model: str, count: int) -> dict[str, object]: - return { - "model": model, - "query": "Which passage explains the solar cycle?", - "documents": ["A solar passage."] * count, - } - - -@pytest.mark.asyncio -@pytest.mark.parametrize("route", ["/rerank", "/v1/rerank", "/v2/rerank"]) -async def test_pointwise_reserves_every_transformed_input_and_output( - decision_rerank_models: tuple[str, str], route: str -) -> None: - model: Final = decision_rerank_models[0] - single: Final = _decision_rerank_body(model, 1) - multiple: Final = {**_decision_rerank_body(model, 3), "top_n": 1, "max_tokens": 1} - single_counts: Final = await count_request_input_tokens(single, route, None, json.dumps(single).encode()) - counts: Final = await count_request_input_tokens(multiple, route, None, json.dumps(multiple).encode()) - raw_prompt_tokens: Final = litellm.token_counter(model=model, text=str(single["query"])) + litellm.token_counter( - model=model, text="A solar passage." - ) - expected_input_cost: Final = counts[model] * 0.001 - expected_output_cost: Final = 3 * 32 * 0.002 - - request: Final = LiteLLMDecisionsRerankConfig.validate_request(multiple, model, "typesafe", "pointwise") - sent_inputs: Final = tuple( - LiteLLMDecisionsRerankConfig.transform_rerank_request(request, index) for index in range(request.request_count) - ) - assert single_counts[model] > raw_prompt_tokens - assert counts[model] == sum( - litellm.token_counter(model=model, text=sent.model_dump_json(exclude_none=True)) for sent in sent_inputs - ) - assert counts[model] == 3 * single_counts[model] - assert estimate_request_max_cost(multiple, route, None, counts) == pytest.approx( - expected_input_cost + expected_output_cost - ) - assert estimate_request_max_cost(multiple, route, None) == pytest.approx(expected_input_cost + expected_output_cost) - assert estimate_request_input_cost(multiple, route, None, counts) == pytest.approx(expected_input_cost) - - -@pytest.mark.asyncio -async def test_multiple_choice_reserves_one_decision_output(decision_rerank_models: tuple[str, str]) -> None: - model: Final = decision_rerank_models[1] - body: Final = _decision_rerank_body(model, 3) - counts: Final = await count_request_input_tokens(body, "/rerank", None) - input_cost: Final = estimate_request_input_cost(body, "/rerank", None, counts) - total_cost: Final = estimate_request_max_cost(body, "/rerank", None, counts) - - assert input_cost is not None - assert total_cost == pytest.approx(input_cost + 32 * 0.002) - - -@pytest.mark.asyncio -async def test_alias_reserves_decision_fanout_at_deployment_prices(decision_rerank_models: tuple[str, str]) -> None: - backend: Final = decision_rerank_models[0] - alias: Final = "rerank-budget-group" - router: Final = Router( - model_list=[ - { - "model_name": alias, - "litellm_params": { - "model": backend, - "api_key": "test-key", - "input_cost_per_token": 0.003, - "output_cost_per_token": 0.004, - }, - } - ] - ) - body: Final = _decision_rerank_body(alias, 3) - counts: Final = await count_request_input_tokens(body, "/rerank", router, json.dumps(body).encode()) - direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None) - - assert counts[alias] == direct_counts[backend] - assert estimate_request_max_cost(body, "/rerank", router, counts) == pytest.approx( - counts[alias] * 0.003 + 3 * 32 * 0.004 - ) - - -@pytest.mark.asyncio -async def test_budget_rejects_fanout_that_only_fits_one_decision( - decision_rerank_models: tuple[str, str], spend_counter_cache: DualCache -) -> None: - model: Final = decision_rerank_models[0] - single_cost: Final = estimate_request_max_cost(_decision_rerank_body(model, 1), "/rerank", None) - assert single_cost is not None - with pytest.raises(litellm.BudgetExceededError): - await reserve_budget_for_request( - request_body=_decision_rerank_body(model, 3), - route="/rerank", - llm_router=None, - valid_token=UserAPIKeyAuth(token="decision-fanout-budget", max_budget=single_cost * 2, spend=0.0), - team_object=None, - user_object=None, - prisma_client=None, - user_api_key_cache=UserApiKeyCache(), - proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), - fail_closed_budget_enforcement=True, - ) - assert spend_counter_cache.in_memory_cache.get_cache(key="spend:key:decision-fanout-budget") == pytest.approx(0.0) - - -def test_native_rerank_still_reserves_only_query_and_document_tokens(monkeypatch: pytest.MonkeyPatch) -> None: - model: Final = "cohere/budget-native-rerank" - monkeypatch.setitem( - litellm.model_cost, - model, - {"mode": "rerank", "litellm_provider": "cohere", "input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, - ) - body: Final = _decision_rerank_body(model, 3) - input_cost: Final = estimate_request_input_cost(body, "/rerank", None) - assert input_cost is not None - assert estimate_request_max_cost(body, "/rerank", None) == input_cost - - -@pytest.mark.asyncio -async def test_decision_fanout_does_not_apply_aggregate_tier_discount( - decision_rerank_models: tuple[str, str], monkeypatch: pytest.MonkeyPatch -) -> None: - model: Final = decision_rerank_models[0] - single: Final = _decision_rerank_body(model, 1) - single_counts: Final = await count_request_input_tokens(single, "/rerank", None) - monkeypatch.setitem( - litellm.model_cost, - model, - { - **litellm.model_cost[model], - "tiered_pricing": [ - {"range": [0, single_counts[model]], "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, - { - "range": [single_counts[model], 100000], - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - }, - ], - }, - ) - single_cost: Final = estimate_request_max_cost(single, "/rerank", None) - single_input: Final = estimate_request_input_cost(single, "/rerank", None) - multiple: Final = _decision_rerank_body(model, 3) - - assert single_cost is not None - assert single_input is not None - assert estimate_request_max_cost(multiple, "/rerank", None) == pytest.approx(single_cost * 3) - assert estimate_request_input_cost(multiple, "/rerank", None) == pytest.approx(single_input * 3) - - -@pytest.mark.asyncio -async def test_budget_honors_deployment_drop_params_and_timeout(decision_rerank_models: tuple[str, str]) -> None: - backend: Final = decision_rerank_models[0] - alias: Final = "rerank-drop-params-group" - router: Final = Router( - model_list=[ - { - "model_name": alias, - "litellm_params": { - "model": backend, - "api_key": "test-key", - "drop_params": True, - "timeout": httpx.Timeout(30.0), - }, - } - ] - ) - plain: Final = _decision_rerank_body(alias, 3) - with_dropped: Final = {**plain, "rank_fields": ["text"]} - plain_counts: Final = await count_request_input_tokens(plain, "/rerank", router) - dropped_counts: Final = await count_request_input_tokens(with_dropped, "/rerank", router) - - direct_counts: Final = await count_request_input_tokens(_decision_rerank_body(backend, 3), "/rerank", None) - assert plain_counts[alias] == direct_counts[backend] - assert plain_counts == dropped_counts - assert estimate_request_max_cost(plain, "/rerank", router) == estimate_request_max_cost( - with_dropped, "/rerank", router - ) - - -def test_invalid_decision_candidate_does_not_reject_native_rerank_group( - decision_rerank_models: tuple[str, str], -) -> None: - alias: Final = "rerank-mixed-group" - router: Final = Router( - model_list=[ - {"model_name": alias, "litellm_params": {"model": decision_rerank_models[0], "api_key": "test-key"}}, - {"model_name": alias, "litellm_params": {"model": "cohere/rerank-v3.5", "api_key": "test-key"}}, - ] - ) - body: Final = {**_decision_rerank_body(alias, 3), "rank_fields": ["text"]} - - assert estimate_request_max_cost(body, "/rerank", router) is not None - - -def test_pointwise_candidate_reserves_when_choice_candidate_is_ineligible( - decision_rerank_models: tuple[str, str], -) -> None: - alias: Final = "rerank-mixed-strategy-group" - router: Final = Router( - model_list=[ - {"model_name": alias, "litellm_params": {"model": model, "api_key": "test-key"}} - for model in decision_rerank_models - ] - ) - body: Final = _decision_rerank_body(alias, 1) - direct: Final = _decision_rerank_body(decision_rerank_models[0], 1) - - assert estimate_request_max_cost(body, "/rerank", router) == estimate_request_max_cost(direct, "/rerank", None) diff --git a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py index 9f8244a4932..232039defad 100644 --- a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py +++ b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py @@ -1,5 +1,6 @@ import asyncio import json +from datetime import datetime, timezone from threading import Event from types import SimpleNamespace from typing import Final @@ -13,6 +14,7 @@ import litellm from litellm.caching.caching import Cache from litellm.cost_calculator import _decision_rerank_breakdown from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import CallTypes, CostBreakdown from litellm.utils import return_raw_request @@ -825,12 +827,16 @@ async def test_async_deadline_bills_responses_returned_during_cancellation_once( async def test_caller_cancellation_retains_usage_returned_during_transport_cleanup( respx_mock, ): - from litellm.rerank_api.litellm_decisions_transformation.handler import ( - DecisionRerankLoggingCapture, - ) - started = asyncio.Event() - capture = DecisionRerankLoggingCapture("cancelled-rerank") + logging_obj = Logging( + model=MODEL, + messages=[], + stream=False, + call_type="arerank", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="cancelled-rerank", + function_id="cancelled-rerank", + ) async def score(request): started.set() @@ -840,28 +846,26 @@ async def test_caller_cancellation_retains_usage_returned_during_transport_clean return response(output_tokens=3) route = respx_mock.post(URL).mock(side_effect=score) - with capture.capture(): - task = asyncio.create_task( - rerank( - False, - documents=["one", "two"], - concurrency=1, - litellm_call_id=capture.call_id, - ) + task = asyncio.create_task( + rerank( + False, + documents=["one", "two"], + concurrency=1, + litellm_logging_obj=logging_obj, ) - await asyncio.wait_for(started.wait(), 5) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task + ) + await asyncio.wait_for(started.wait(), 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task assert route.call_count == 1 - assert capture.logging_obj is not None - usage = capture.logging_obj.model_call_details["combined_usage_object"] + usage = logging_obj.model_call_details["combined_usage_object"] assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( 10, 3, 13, ) - assert capture.logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE) async def test_direct_token_price_override_is_respected(respx_mock): diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx index 79cff240ab9..7f39a0d8fe5 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx @@ -56,39 +56,6 @@ describe("LogDetailContent", () => { expect(screen.getByText("completion")).toBeInTheDocument(); }); - it("should distinguish the recorded gateway endpoint from the provider API base", () => { - render( - , - ); - - expect(screen.getByText("Gateway endpoint")).toBeInTheDocument(); - expect(screen.getByText("POST /v1/rerank")).toBeInTheDocument(); - expect(screen.getByText("Provider API base")).toBeInTheDocument(); - expect(screen.getByText("https://api.typesafe.ai/v1")).toBeInTheDocument(); - expect(screen.getByText("arerank")).toBeInTheDocument(); - expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument(); - }); - - it.each([ - { status: "success" }, - { spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } }, - ])( - "should show that the gateway endpoint was not recorded when its metadata is unavailable or invalid", - (metadata) => { - render(); - - expect(screen.getByText("Gateway endpoint")).toBeInTheDocument(); - expect(screen.getByText("Not recorded")).toBeInTheDocument(); - expect(screen.queryByText("POST /rerank")).not.toBeInTheDocument(); - }, - ); - it("shows the requesting user's email and id in Request Details when the email is resolved", () => { render( , diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx index 9ef3c640d5c..f6c63582938 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.tsx @@ -30,7 +30,6 @@ import { JsonViewer } from "./JsonViewer"; import { RoutingDecisionCard, type RoutingDecision } from "./RoutingDecisionCard"; import { formatData, - formatGatewayEndpoint, checkHasMessages, checkHasResponse, normalizeGuardrailEntries, @@ -151,9 +150,6 @@ export function LogDetailContent({ {logEntry.model} {logEntry.custom_llm_provider || "-"} {logEntry.call_type} - - {formatGatewayEndpoint(metadata) ?? "Not recorded"} - {logEntry.user && ( @@ -162,7 +158,7 @@ export function LogDetailContent({ - + {podRouting && typeof podRouting === "object" && ( diff --git a/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts b/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts index c91e028f9c4..f59a20529d0 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/utils.test.ts @@ -1,27 +1,5 @@ import { describe, expect, it } from "vitest"; -import { formatGatewayEndpoint, sortSessionLogs } from "./utils"; - -describe("formatGatewayEndpoint", () => { - it.each(["/rerank", "/v1/rerank", "/v2/rerank"])("should preserve the recorded route %s", (path) => { - expect(formatGatewayEndpoint({ spend_logs_metadata: { gateway_endpoint: { method: "POST", path } } })).toBe( - `POST ${path}`, - ); - }); - - it.each([ - undefined, - null, - {}, - { spend_logs_metadata: null }, - { spend_logs_metadata: { gateway_endpoint: "POST /rerank" } }, - { spend_logs_metadata: { gateway_endpoint: { method: ["POST"], path: "/rerank" } } }, - { spend_logs_metadata: { gateway_endpoint: { method: "", path: "/rerank" } } }, - { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "https://api.example.com/rerank" } } }, - { spend_logs_metadata: { gateway_endpoint: { method: "POST", path: "/rerank?api_key=hidden" } } }, - ])("should return no endpoint for missing or malformed metadata %j", (metadata) => { - expect(formatGatewayEndpoint(metadata)).toBeNull(); - }); -}); +import { sortSessionLogs } from "./utils"; const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({ request_id: id, diff --git a/ui/litellm-dashboard/src/components/logs/detail/utils.ts b/ui/litellm-dashboard/src/components/logs/detail/utils.ts index e9ffb1f429e..d313d07361c 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/utils.ts +++ b/ui/litellm-dashboard/src/components/logs/detail/utils.ts @@ -1,5 +1,3 @@ -import { z } from "zod"; - /** * Utility functions for LogDetailsDrawer component. * These functions handle data formatting, validation, and guardrail calculations. @@ -7,22 +5,6 @@ import { z } from "zod"; export type SessionLogSortMode = "duration" | "start_time"; -const gatewayEndpointMetadataSchema = z.object({ - spend_logs_metadata: z.object({ - gateway_endpoint: z.object({ - method: z.string().regex(/^[A-Z]+$/), - path: z.string().regex(/^\/(?!\/)[^?#\s]*$/), - }), - }), -}); - -export function formatGatewayEndpoint(metadata: unknown): string | null { - const parsed = gatewayEndpointMetadataSchema.safeParse(metadata); - if (!parsed.success) return null; - const { method, path } = parsed.data.spend_logs_metadata.gateway_endpoint; - return `${method} ${path}`; -} - type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number }; const durationMs = (row: SortableSessionLog): number =>