scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_gemini_prompt_cache_min_tokens_4096

Browse source 
# Conflicts:
#	litellm/model_prices_and_context_window_backup.json
#	model_prices_and_context_window.json
This commit is contained in:
mateo-berri 2026-08-19 14:55:44 -07:00
commit 45884b9bd3
84 changed files with 4722 additions and 1065 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,5 +1,6 @@
.python-version
.venv
tests/e2e/.fixtures/
.venv-typecheck
.venv_policy_test
.env

7
litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql Normal file
View file

@ -0,0 +1,7 @@
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT;
UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL;
ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL;
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id");

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "stopped_by" TEXT;
UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown'
WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc');

30
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
// either direction. forward duplicates the requests the keys did not route through the
// router through it, answering whether they should adopt it; reverse duplicates the
// requests the router did serve against a fixed baseline model, answering whether a key
// already on it still benefits. Either way a sampled slice runs in a detached task and an
// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job:
// immutable config plus that key's own turn budget and stop state, so one key exhausting
// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id
// (the id the API reports), written together by one atomic create_many with identical
// config; single-key jobs predating group_id were backfilled group_id = id. "One active
// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE
// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state
// partial indexes; it is what makes a concurrent start on another pod race-safe rather
// than read-then-create. Every count, status, and spend figure is derived from the
// append-only attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
group_id String // legs of one job share this; the API's job id
api_key_id String // hashed virtual key whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
max_turns Int // this key's sample budget: judge at most this many turns
created_at DateTime @default(now())
created_by String?
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
@@index([group_id])
@@index([api_key_id])
@@index([created_at])
}

19
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1830,6 +1830,20 @@ class CustomStreamWrapper:
return
self.chunks.append(model_response.model_copy(update={"choices": []}))
@staticmethod
def _resolve_provider_reported_cost(usage_cost: object) -> float | None:
"""
Providers report usage.cost either as a number or, for Perplexity, as a
breakdown object whose total lives under ``total_cost``.
"""
if isinstance(usage_cost, bool):
return None
if isinstance(usage_cost, (int, float)):
return float(usage_cost)
if isinstance(usage_cost, dict):
return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost"))
return None
@staticmethod
def _propagate_usage_cost_to_hidden_params(
response: "ModelResponse",
@ -1840,10 +1854,11 @@ class CustomStreamWrapper:
calculator uses it instead of a token-based estimate.
"""
_usage: Final[Usage | None] = getattr(response, "usage", None)
if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None:
_cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None))
if _cost is not None:
if "additional_headers" not in response._hidden_params:
response._hidden_params["additional_headers"] = {}
response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost)
response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = _cost
def __next__(self) -> "ModelResponseStream":
cache_hit = False

205
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

18
litellm/proxy/_experimental/mcp_server/exceptions.py
View file

@ -75,6 +75,24 @@ class MCPUpstreamAuthError(Exception):
)
class MCPOpenApiUpstreamError(Exception):
"""An OpenAPI-backed MCP tool's upstream answered with a non-2xx that is not a 401.
Carries the status only. The upstream's response body is deliberately dropped rather than served
as tool content: it crosses a trust boundary and may hold prose, urls, or an error document that
reads as data, which is how these failures came to be reported as successful tool output. This
matches ``outcome_wire_value``'s contract for listing faults, category and status and nothing
else. A 401 is raised as ``MCPUpstreamAuthError`` instead, so the caller learns to
re-authenticate; every other status stays here, mirroring the regular MCP path where a 403
deliberately does not produce a challenge.
"""
def __init__(self, status_code: int, server_name: str) -> None:
self.status_code = status_code
self.server_name = server_name
super().__init__(f"upstream returned HTTP {status_code}")
class MCPToolResultError(Exception):
"""An MCP tool call completed with ``isError=True`` in its result.

16
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -2330,7 +2330,15 @@ class MCPServerManager:
input_schema = build_input_schema(resolved_operation)
# Create tool function with headers using imported function
tool_func = create_tool_function(path, method, resolved_operation, base_url, headers=headers)
tool_func = create_tool_function(
path,
method,
resolved_operation,
base_url,
headers=headers,
server_label=server.name or server.server_name or server.alias or server.server_id,
relays_upstream_auth=server.is_client_forwarded_token,
)
tool_func.__name__ = prefixed_tool_name
tool_func.__doc__ = description
@ -4979,6 +4987,12 @@ class MCPServerManager:
return result
except MCPUpstreamAuthError:
# The caller must re-authenticate upstream, so this keeps its type all the way to the
# renderers: the streamable path turns it into an isError result naming the status, and
# the REST path relays a real 401 with the upstream's WWW-Authenticate. Flattening it
# into the generic message below would lose both.
raise
except Exception as e:
error_msg = f"Error calling OpenAPI tool {tool_name}: {e}"
verbose_logger.error(error_msg)

64
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -15,6 +15,12 @@ from urllib.parse import quote
import httpx
from typing_extensions import ReadOnly, Required
from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPOpenApiUpstreamError,
MCPUpstreamAuthError,
)
# Tool names emitted from OpenAPI specs must work across all major LLM providers.
# OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to
# ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use
@ -392,12 +398,40 @@ def _merge_openapi_tool_request_headers(
return effective_headers
def _raise_for_upstream_failure(
response: httpx.Response,
upstream: str,
relays_upstream_auth: bool,
) -> None:
"""Turn a non-2xx upstream response into the right typed failure, or return for a 2xx.
Both call sites feed this: ``get`` hands back the response for a 4xx, while post/put/patch/delete
raise ``MaskedHTTPStatusError`` from inside the HTTP handler, so without one classifier the
non-GET tools would keep serving an error body as tool output.
Only the client-forwarded modes carry the caller's own upstream token, so only they can act on a
401 by re-authenticating; ``_call_regular_mcp_tool`` gates its re-auth signal the same way. Every
other status carries the code alone, never the upstream's body, which crosses a trust boundary.
"""
if response.status_code < 400:
return
if response.status_code == 401 and relays_upstream_auth:
raise MCPUpstreamAuthError(
status_code=response.status_code,
www_authenticate=response.headers.get("www-authenticate"),
server_name=upstream,
)
raise MCPOpenApiUpstreamError(response.status_code, upstream)
def create_tool_function(
path: str,
method: str,
operation: _OpenAPIOperation,
base_url: str,
headers: dict[str, str] | None = None,
server_label: str | None = None,
relays_upstream_auth: bool = False,
):
"""Create a tool function for an OpenAPI operation.
@ -477,20 +511,26 @@ def create_tool_function(
json_body = {"data": body_value}
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
upstream: Final = server_label or f"{original_method.upper()} {path}"
if original_method == "get":
response = await client.get(url, params=params, headers=effective_headers)
elif original_method == "post":
response = await client.post(url, params=params, json=json_body, headers=effective_headers)
elif original_method == "put":
response = await client.put(url, params=params, json=json_body, headers=effective_headers)
elif original_method == "delete":
response = await client.delete(url, params=params, headers=effective_headers)
elif original_method == "patch":
response = await client.patch(url, params=params, json=json_body, headers=effective_headers)
else:
return f"Unsupported HTTP method: {original_method}"
try:
if original_method == "get":
response = await client.get(url, params=params, headers=effective_headers)
elif original_method == "post":
response = await client.post(url, params=params, json=json_body, headers=effective_headers)
elif original_method == "put":
response = await client.put(url, params=params, json=json_body, headers=effective_headers)
elif original_method == "delete":
response = await client.delete(url, params=params, headers=effective_headers)
elif original_method == "patch":
response = await client.patch(url, params=params, json=json_body, headers=effective_headers)
else:
return f"Unsupported HTTP method: {original_method}"
except MaskedHTTPStatusError as e:
_raise_for_upstream_failure(e.response, upstream, relays_upstream_auth)
raise
_raise_for_upstream_failure(response, upstream, relays_upstream_auth)
return response.text
return tool_function

32
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -407,8 +407,6 @@ if MCP_AVAILABLE:
StreamableHTTPSessionManager = None
from mcp.types import (
CallToolResult,
EmbeddedResource,
ImageContent,
ListToolsResult,
Prompt,
TextContent,
@ -2861,12 +2859,11 @@ if MCP_AVAILABLE:
_extra_token: Final = _request_extra_headers.set(forwarded_headers)
_resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers)
try:
local_content = await _handle_local_mcp_tool(name, arguments)
response = await _handle_local_mcp_tool(name, arguments)
finally:
_request_auth_header.reset(_auth_token)
_request_extra_headers.reset(_extra_token)
_request_resolved_auth_headers.reset(_resolved_token)
response = CallToolResult(content=local_content, isError=False)
# Try managed MCP server tool (the name is bare; the prefix boundary was
# already resolved above against this server's registered prefixes)
@ -2940,8 +2937,7 @@ if MCP_AVAILABLE:
if "arguments" in hook_result:
arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args
local_content = await _handle_local_mcp_tool(original_tool_name, arguments)
response = CallToolResult(content=local_content, isError=False)
response = await _handle_local_mcp_tool(original_tool_name, arguments)
return await _run_post_mcp_call_guardrails(
result=response,
@ -3319,11 +3315,18 @@ if MCP_AVAILABLE:
verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
return call_tool_result
async def _handle_local_mcp_tool(
name: str, arguments: dict[str, object]
) -> list[TextContent | ImageContent | EmbeddedResource]:
"""
Handle tool execution for local registry tools
async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult:
"""Execute a local-registry tool and report whether it succeeded.
Returns the result rather than bare content because the verdict is part of it: the content
alone cannot say whether the handler failed, so callers used to stamp isError=False on every
outcome and an upstream rejection was served as tool output.
A failure is reported as ``isError=True`` here rather than raised, because the REST surface
turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash.
``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to
re-authenticate, which both renderers already know how to say.
Note: Local tools don't use prefixes, so we use the original name
"""
import inspect
@ -3333,15 +3336,16 @@ if MCP_AVAILABLE:
raise HTTPException(status_code=404, detail=f"Tool '{name}' not found")
try:
# Check if handler is async or sync
if inspect.iscoroutinefunction(tool.handler):
result = await tool.handler(**arguments)
else:
result = tool.handler(**arguments)
return [TextContent(text=str(result), type="text")]
except MCPUpstreamAuthError:
raise
except Exception as e:
verbose_logger.exception("Error executing local tool %s: %s", name, e)
return [TextContent(text=f"Error: {e}", type="text")]
return CallToolResult(content=[TextContent(text=f"Error: {e}", type="text")], isError=True)
return CallToolResult(content=[TextContent(text=str(result), type="text")], isError=False)
def _get_mcp_servers_in_path(path: str) -> list[str] | None:
"""

378
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -6,10 +6,13 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity-
from collections.abc import Mapping, Sequence
from datetime import datetime, timedelta, timezone
from itertools import groupby
from operator import attrgetter
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, Protocol
from uuid import uuid4
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import BudgetExceededError
@ -41,6 +44,8 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterRoutingTestRequest,
AutoRouterRoutingTestResponse,
RequestComplexityRouterConfig,
ShadowEvalDirection,
ShadowEvalJobKeyResponse,
ShadowEvalJobResponse,
ShadowEvalResult,
ShadowEvalSlice,
@ -89,17 +94,9 @@ class _ShadowEvalJobRow(Protocol):
class _ShadowEvalJobTable(Protocol):
async def find_unique(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ...
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ...
async def find_first(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ...
async def find_many(
self, *, where: Mapping[str, object], order: Mapping[str, str], take: int
) -> Sequence[_ShadowEvalJobRow]: ...
async def create(self, data: Mapping[str, object]) -> _ShadowEvalJobRow: ...
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> _ShadowEvalJobRow | None: ...
async def create_many(self, data: Sequence[Mapping[str, object]]) -> int: ...
class _ShadowEvalAttemptRow(Protocol):
@ -606,18 +603,19 @@ _ATTEMPT_AGG_SELECT: Final = """
COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties,
AVG(confidence)::float AS avg_confidence
FROM "LiteLLM_ShadowEvalAttempt"
WHERE job_id = $1 AND outcome != 'error'
WHERE job_id = ANY($1::text[]) AND outcome != 'error'
GROUP BY 1
"""
_ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT
_ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT
_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT
_SWEEP_FINISHED_JOBS_SQL: Final = """
UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = NOW()
WHERE j.api_key_id = $1 AND j.stopped_at IS NULL
UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc')
WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
AND (
j.ends_at <= NOW()
j.ends_at <= (NOW() AT TIME ZONE 'utc')
OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
)
"""
@ -628,7 +626,52 @@ SELECT
COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count,
COALESCE(SUM(judge_cost), 0)::float AS judge_spend
FROM "LiteLLM_ShadowEvalAttempt"
WHERE job_id = $1
WHERE job_id = ANY($1::text[])
"""
_ATTEMPT_COUNTS_SQL: Final = """
SELECT a.job_id, COUNT(*)::int AS attempt_count
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at)
GROUP BY a.job_id
"""
_STOP_JOB_SQL: Final = """
UPDATE "LiteLLM_ShadowEvalJob"
SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)
WHERE group_id = $1 AND stopped_by IS NULL
AND ends_at > (NOW() AT TIME ZONE 'utc')
AND EXISTS (
SELECT 1 FROM "LiteLLM_ShadowEvalJob" k
WHERE k.group_id = $1 AND k.stopped_at IS NULL
AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns
)
"""
class _AttemptCountRow(BaseModel):
job_id: str
attempt_count: int
_ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow])
_LIST_LEGS_SQL: Final = """
SELECT * FROM "LiteLLM_ShadowEvalJob"
WHERE group_id IN (
SELECT group_id FROM "LiteLLM_ShadowEvalJob"
GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int
)
"""
_LIST_LEGS_BY_KEY_SQL: Final = """
SELECT * FROM "LiteLLM_ShadowEvalJob"
WHERE group_id IN (
SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2
GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int
)
"""
@ -659,18 +702,98 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
)
class _LegRow(BaseModel):
"""One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is
one key's leg of a job; the legs of a job share group_id and identical config, written
together by one create_many. The API's job id is the group id, so leg ids never leave
the server (attempts reference them internally)."""
model_config = ConfigDict(from_attributes=True)
id: str
group_id: str
api_key_id: str
router_name: str
direction: ShadowEvalDirection
baseline_model: str | None = None
judge_model: str
shadow_percentage: float
max_turns: int
created_at: datetime
ends_at: datetime
stopped_at: datetime | None = None
stopped_by: str | None = None
@field_validator("created_at", "ends_at", "stopped_at")
@classmethod
def _as_aware_utc(cls, value: datetime | None) -> datetime | None:
"""The columns store naive UTC wall time (prisma's convention); prisma reads hand
back aware datetimes while raw SQL reads hand back naive ones, so this boundary
makes every read aware UTC before anything compares or serializes them."""
if value is None or value.tzinfo is not None:
return value
return value.replace(tzinfo=timezone.utc)
_LEG_ROWS: Final = TypeAdapter(list[_LegRow])
async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]:
"""Each leg's attempt count by leg id, judged and errored alike, in one grouped read.
It is the same count the sampler budgets against max_turns, so the derived status
flips to completed exactly when sampling actually ends. A stamped leg's count freezes
at its stopped_at: in-flight attempts that land after the stamp are excluded, so they
can never reclassify a leg that was stopped under budget as budget-spent."""
if not legs:
return MappingProxyType({})
rows: Final = _ATTEMPT_COUNT_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param
or ()
)
return MappingProxyType({row.job_id: row.attempt_count for row in rows})
def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse:
"""The one constructor of a job response: the caller names the group and passes that
group's legs. Config is read off the first leg because every leg carries the same copy,
written by one create_many. No caller may serialize a raw row (that would leak a leg id
as the job id)."""
first: Final = legs[0]
return ShadowEvalJobResponse(
job_id=group_id,
keys=tuple(
ShadowEvalJobKeyResponse(
api_key_id=leg.api_key_id,
max_turns=leg.max_turns,
stopped_at=leg.stopped_at,
attempt_count=attempt_counts.get(leg.id, 0),
)
for leg in sorted(legs, key=lambda leg: leg.api_key_id)
),
router_name=first.router_name,
direction=first.direction,
baseline_model=first.baseline_model,
judge_model=first.judge_model,
shadow_percentage=first.shadow_percentage,
created_at=first.created_at,
ends_at=first.ends_at,
stopped_by=next((leg.stopped_by for leg in legs if leg.stopped_by is not None), None),
)
_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None)
async def _with_key_labels(
prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse]
) -> tuple[ShadowEvalJobResponse, ...]:
"""Resolve each job's key hash to the key's alias and masked name in one batched read,
"""Resolve every scoped key's hash to its alias and masked name in one batched read,
so the UI can say whose traffic a job shadows. Deleted keys resolve to None."""
if not responses:
return ()
tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys))
key_rows: Final = await _verification_tokens(prisma_client).find_many(
where={"token": {"in": sorted({response.api_key_id for response in responses})}} # mutable-ok: Prisma filter
where={"token": {"in": tokens}} # mutable-ok: Prisma filter
)
labels: Final[Mapping[str, tuple[str | None, str | None]]] = {
row.token: (row.key_alias, row.key_name) for row in key_rows or ()
@ -678,32 +801,50 @@ async def _with_key_labels(
return tuple(
response.model_copy(
update={ # mutable-ok: pydantic update payload
"key_alias": labels.get(response.api_key_id, _NO_KEY_LABELS)[0],
"key_name": labels.get(response.api_key_id, _NO_KEY_LABELS)[1],
"keys": tuple(
key.model_copy(
update={ # mutable-ok: pydantic update payload
"key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0],
"key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1],
}
)
for key in response.keys
)
}
)
for response in responses
)
async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None:
"""Both stratifications of one job's verdicts. Tier answers "where does the router do
well"; the model stratification groups by whichever model served the real arm, so it
answers "which of the models this key uses today would the router beat" forward, and
"for the turns the router sent to X, did X beat the baseline" in reverse. Reads are
bounded by the job's own attempts (<= max_turns) via the job_id index."""
async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None:
"""All three stratifications of one job's verdicts. Tier answers "where does the router
do well"; the model stratification groups by whichever model served the real arm, so it
answers "which of the models these keys use today would the router beat" forward, and
"for the turns the router sent to X, did X beat the baseline" in reverse; key answers
"which key's traffic does the router suit". Reads are bounded by the job's own attempts
(<= the sum of its keys' max_turns) via the job_id index."""
leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param
by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, job_id) or ()
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or ()
)
if not by_tier:
return None
by_model: Final = _ATTEMPT_AGG_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, job_id) or ()
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or ()
)
key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs})
by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or ()
)
by_key: Final = tuple(
row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload
for row in by_leg
)
total_turns: Final = sum(r.turn_count for r in by_tier)
return ShadowEvalResult(
by_tier=_slices(by_tier),
by_current_model=_slices(by_model),
by_key=_slices(by_key),
overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
)
@ -721,20 +862,21 @@ async def start_shadow_eval(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ShadowEvalJobResponse:
"""
Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second
arm, judge the two responses blind, and stratify win rates by tier and by the model that
served the real arm.
Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against
a second arm, judge the two responses blind, and stratify win rates by tier, by the model
that served the real arm, and by key.
A forward job answers whether the key should adopt router_name: it samples the requests
A forward job answers whether the keys should adopt router_name: it samples the requests
the router did not serve and duplicates them through it. A reverse job answers whether a
key already on the router still gains from it: it samples the requests the router did
serve and duplicates them against baseline_model. A key can hold one active job per
direction, so both questions can run at once.
Shadow responses are never served to users. The job samples until it has judged
max_turns turns, reaches the end of its window, or is stopped; sampling changes
propagate to pods within about 10 seconds. Shadow and judge calls bill to the
shadowed key but are excluded from request counts and auto-router adoption metrics.
Shadow responses are never served to users. Each key samples until it has judged
max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one
key running out of budget does not end sampling for the others; sampling changes
propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed
key but are excluded from request counts and auto-router adoption metrics.
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
@ -746,48 +888,58 @@ async def start_shadow_eval(
_validate_plain_model(llm_router, data.judge_model, "judge_model")
if data.baseline_model is not None:
_validate_plain_model(llm_router, data.baseline_model, "baseline_model")
key_row: Final = await _verification_tokens(prisma_client).find_unique(
where={"token": data.api_key_id} # mutable-ok: Prisma filter
token_rows: Final = await _verification_tokens(prisma_client).find_many(
where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter
)
if key_row is None:
unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ())))
if unknown:
raise HTTPException(
status_code=400,
detail=(
f"api_key_id '{data.api_key_id}' is not a key on this proxy; pass the key's token hash, "
f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, "
"the value the key list and key info endpoints report"
),
)
# A job that expired or exhausted its turn budget stopped sampling on its own, but
# still holds its slot in the per-key, per-direction partial unique index until
# stamped; free it so a new eval can start. Sweeping both directions is deliberate.
await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id)
active: Final = await _shadow_eval_jobs(prisma_client).find_first(
# A job whose window passed or whose turn budget ran out stopped sampling on its own,
# but its legs still hold their slots in the per-key, per-direction partial unique index
# until stamped; free them so a new eval can start. Sweeping both directions is deliberate.
requested: Final = list(data.api_key_ids) # mutable-ok: query param
await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested)
claimed: Final = await _shadow_eval_jobs(prisma_client).find_many(
where={ # mutable-ok: Prisma filter
"api_key_id": data.api_key_id,
"api_key_id": {"in": requested}, # mutable-ok: Prisma filter
"direction": data.direction,
"stopped_at": None,
},
)
if active is not None:
if claimed:
raise HTTPException(
status_code=409,
detail=f"Key already has an active {data.direction} shadow eval job ({active.id}). Stop it first.",
detail=(
f"Already in an active {data.direction} shadow eval job: "
+ ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed))
+ ". Stop it first."
),
)
now: Final = datetime.now(timezone.utc)
group_id: Final = str(uuid4())
ends_at: Final = now + timedelta(days=data.duration_days)
shared_config: Final = { # mutable-ok: Prisma payload
"group_id": group_id,
"router_name": data.router_name,
"direction": data.direction,
"baseline_model": data.baseline_model,
"judge_model": data.judge_model,
"shadow_percentage": data.shadow_percentage,
"max_turns": data.max_turns,
"created_by": user_api_key_dict.user_id,
"created_at": now,
"ends_at": ends_at,
}
try:
job: Final = await _shadow_eval_jobs(prisma_client).create(
data={ # mutable-ok: Prisma payload
"api_key_id": data.api_key_id,
"router_name": data.router_name,
"direction": data.direction,
"baseline_model": data.baseline_model,
"judge_model": data.judge_model,
"shadow_percentage": data.shadow_percentage,
"max_turns": data.max_turns,
"created_by": user_api_key_dict.user_id,
"ends_at": now + timedelta(days=data.duration_days),
}
await _shadow_eval_jobs(prisma_client).create_many(
data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload
)
except Exception as e:
if not _is_unique_violation(e):
@ -795,11 +947,28 @@ async def start_shadow_eval(
raise HTTPException(
status_code=409,
detail=(
f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first."
f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first."
),
) from e
return ShadowEvalJobResponse.model_validate(job, from_attributes=True).model_copy(
update={"key_alias": key_row.key_alias, "key_name": key_row.key_name} # mutable-ok: pydantic update payload
labels: Final = MappingProxyType({row.token: row for row in token_rows})
return ShadowEvalJobResponse(
job_id=group_id,
keys=tuple(
ShadowEvalJobKeyResponse(
api_key_id=api_key_id,
max_turns=data.max_turns,
key_alias=labels[api_key_id].key_alias,
key_name=labels[api_key_id].key_name,
)
for api_key_id in sorted(data.api_key_ids)
),
router_name=data.router_name,
direction=data.direction,
baseline_model=data.baseline_model,
judge_model=data.judge_model,
shadow_percentage=data.shadow_percentage,
created_at=now,
ends_at=ends_at,
)
@ -811,23 +980,38 @@ async def start_shadow_eval(
)
async def list_shadow_eval_jobs(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None,
api_key_id: Annotated[
str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others")
] = None,
limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50,
) -> tuple[ShadowEvalJobResponse, ...]:
"""List shadow eval jobs, newest first. Counts and results ride the detail endpoint only."""
"""List shadow eval jobs, newest first, each key with its attempt count so status is
accurate. Judged counts, spend, and results ride the detail endpoint only."""
from litellm.proxy.proxy_server import prisma_client
_require_admin_viewer(user_api_key_dict, "view shadow evals")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
records: Final = await _shadow_eval_jobs(prisma_client).find_many(
where={"api_key_id": api_key_id} if api_key_id else {}, # mutable-ok: Prisma filter
order={"created_at": "desc"}, # mutable-ok: Prisma order
take=limit,
legs: Final = _LEG_ROWS.validate_python(
(
await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id)
if api_key_id
else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit)
)
or ()
)
by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType(
{
group_id: tuple(group)
for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id"))
}
)
newest_first: Final = sorted(
by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True
)
counts: Final = await _leg_attempt_counts(prisma_client, legs)
return await _with_key_labels(
prisma_client,
tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()),
prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first)
)
@ -847,20 +1031,24 @@ async def get_shadow_eval_job(
_require_admin_viewer(user_api_key_dict, "view shadow evals")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
record: Final = await _shadow_eval_jobs(prisma_client).find_unique(
where={"id": job_id} # mutable-ok: Prisma filter
legs: Final = _LEG_ROWS.validate_python(
await _shadow_eval_jobs(prisma_client).find_many(
where={"group_id": job_id} # mutable-ok: Prisma filter
)
or ()
)
if record is None:
if not legs:
raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}")
leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param
totals: Final = _ATTEMPT_TOTALS_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, job_id) or ()
await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, leg_ids) or ()
)
latest_error: Final = await _shadow_eval_attempts(prisma_client).find_first(
where={"job_id": job_id, "outcome": "error"}, # mutable-ok: Prisma filter
where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter
order={"created_at": "desc"}, # mutable-ok: Prisma order
)
labeled: Final = await _with_key_labels(
prisma_client, (ShadowEvalJobResponse.model_validate(record, from_attributes=True),)
prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),)
)
return labeled[0].model_copy(
update={ # mutable-ok: pydantic update payload
@ -868,7 +1056,7 @@ async def get_shadow_eval_job(
"error_count": totals[0].error_count if totals else 0,
"judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0,
"last_error": latest_error.error if latest_error else None,
"results": await _shadow_eval_results(prisma_client, job_id),
"results": await _shadow_eval_results(prisma_client, legs),
}
)
@ -883,25 +1071,33 @@ async def stop_shadow_eval_job(
job_id: str,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ShadowEvalJobResponse:
"""Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s."""
"""Stop an active shadow eval job, every key it scopes at once. Attempts are kept;
sampling halts within ~10s. Keys that already stopped on their own budget keep the
stopped_at they earned. The statement is the whole state machine: it claims the job
only while a leg still samples inside the window with no stop recorded, so a racing
operator, a same-instant budget spend, and a repeat stop all read the same 400 with
the status the job actually holds."""
from litellm.proxy.proxy_server import prisma_client
_require_admin_writer(user_api_key_dict, "stop a shadow eval")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
record: Final = await _shadow_eval_jobs(prisma_client).find_unique(
where={"id": job_id} # mutable-ok: Prisma filter
stamp: Final = datetime.now(timezone.utc)
operator: Final = user_api_key_dict.user_id or "operator"
claimed: Final = await prisma_client.db.execute_raw(
_STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat()
)
if record is None:
legs: Final = _LEG_ROWS.validate_python(
await _shadow_eval_jobs(prisma_client).find_many(
where={"group_id": job_id} # mutable-ok: Prisma filter
)
or ()
)
if not legs:
raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}")
current: Final = ShadowEvalJobResponse.model_validate(record, from_attributes=True)
if current.status != "running":
counts: Final = await _leg_attempt_counts(prisma_client, legs)
current: Final = _group_response(job_id, legs, counts)
if claimed == 0:
raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}")
updated: Final = await _shadow_eval_jobs(prisma_client).update(
where={"id": job_id}, # mutable-ok: Prisma filter
data={"stopped_at": datetime.now(timezone.utc)}, # mutable-ok: Prisma payload
)
labeled: Final = await _with_key_labels(
prisma_client, (ShadowEvalJobResponse.model_validate(updated, from_attributes=True),)
)
labeled: Final = await _with_key_labels(prisma_client, (current,))
return labeled[0]

10
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -1416,6 +1416,11 @@ async def _check_team_key_limits(
)
_INHERITED_MODEL_SENTINELS: Final = frozenset(
{SpecialModelNames.all_team_models.value, SpecialModelNames.all_proxy_models.value}
)
async def _check_project_key_limits(
project_id: str,
data: GenerateKeyRequest | UpdateKeyRequest,
@ -1425,7 +1430,8 @@ async def _check_project_key_limits(
"""
Validate that key's models and budget respect its project's limits.
- Key models must be a subset of project models
- Key models must be a subset of project models, except the all-team-models / all-proxy-models
sentinels, which inherit a parent scope and are narrowed by the project at request time
- Key max_budget must be <= project max_budget
"""
project_obj: Final = await get_project_object(
@ -1443,7 +1449,7 @@ async def _check_project_key_limits(
# Validate key models are a subset of project models
if data.models and len(project_obj.models) > 0:
for m in data.models:
if m not in project_obj.models:
if m not in project_obj.models and m not in _INHERITED_MODEL_SENTINELS:
raise HTTPException(
status_code=400,
detail={

30
litellm/proxy/schema.prisma
View file

@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
// either direction. forward duplicates the requests the keys did not route through the
// router through it, answering whether they should adopt it; reverse duplicates the
// requests the router did serve against a fixed baseline model, answering whether a key
// already on it still benefits. Either way a sampled slice runs in a detached task and an
// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job:
// immutable config plus that key's own turn budget and stop state, so one key exhausting
// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id
// (the id the API reports), written together by one atomic create_many with identical
// config; single-key jobs predating group_id were backfilled group_id = id. "One active
// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE
// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state
// partial indexes; it is what makes a concurrent start on another pod race-safe rather
// than read-then-create. Every count, status, and spend figure is derived from the
// append-only attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
group_id String // legs of one job share this; the API's job id
api_key_id String // hashed virtual key whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
max_turns Int // this key's sample budget: judge at most this many turns
created_at DateTime @default(now())
created_by String?
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
@@index([group_id])
@@index([api_key_id])
@@index([created_at])
}

46
litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py
View file

@ -17,6 +17,7 @@ import json
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
@ -110,17 +111,56 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str:
def _decode_model_info(raw: object) -> "Mapping[str, object] | None":
"""A deployment's model_info as a dict, decoding a JSON string, else None."""
"""A deployment's model_info as a mapping, decoding a JSON string, else None.
Valid JSON that is not an object decodes to a list or a scalar, which every caller
would then read fields off, so it is rejected here rather than raised past them.
"""
if isinstance(raw, str):
try:
return json.loads(raw)
decoded: Final = json.loads(raw)
except (TypeError, ValueError):
return None
if isinstance(raw, dict):
return decoded if isinstance(decoded, dict) else None
if isinstance(raw, Mapping):
return raw
return None
@dataclass(frozen=True, slots=True)
class _PTUDeployment:
"""A deployment in the shape ``_parse_ptu_model`` reads, whatever declared it.
A ``LiteLLM_ProxyModelTable`` row already has it. A router entry does not: its id
lives in ``model_info.id`` rather than on the entry itself.
"""
model_id: str
model_name: str
model_info: Mapping[str, object]
def _router_deployment(deployment: Mapping[str, object]) -> _PTUDeployment | None:
"""A router ``model_list`` entry in the shape the parser reads, else None.
An id is required rather than defaulted because it keys the sentinel row: every
deployment without one would collapse onto a single row per team and only the last
would be billed. The mapping is copied because the router rewrites entries in place
while the rollup runs.
"""
model_info: Final = _decode_model_info(deployment.get("model_info"))
if model_info is None:
return None
model_id: Final = model_info.get("id")
if not isinstance(model_id, str) or not model_id:
return None
return _PTUDeployment(
model_id=model_id,
model_name=str(deployment.get("model_name") or ""),
model_info=MappingProxyType(dict(model_info)),
)
def _parse_ptu_model(row: object) -> PTUModel | None:
"""Return a PTUModel when the deployment carries valid manual PTU config, else None.

7
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -1020,13 +1020,14 @@ class ComplexityRouter(CustomLogger):
weights: Final = self.config.dimension_weights
weighted_score: Final = sum(d.score * weights.get(d.name, 0) for d in dimensions)
# Check for reasoning override (2+ reasoning markers)
boundaries: Final = self._effective_tier_boundaries()
scored_above_simple: Final = weighted_score >= boundaries["simple_medium"]
# Reuse match count from _score_keyword_match to avoid scanning twice
if reasoning_match_count >= 2:
if reasoning_match_count >= 2 and scored_above_simple:
return ComplexityTier.REASONING, weighted_score, tuple(signals), "reasoning_override"
# Map score to tier
boundaries: Final = self._effective_tier_boundaries()
if weighted_score < boundaries["simple_medium"]:
tier = ComplexityTier.SIMPLE
elif weighted_score < boundaries["medium_complex"]:

103
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -6,7 +6,7 @@ from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final, Literal, TypeAlias
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator
from pydantic import BaseModel, Field, computed_field, field_validator, model_validator
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.types.utils import StandardLoggingRoutingDecision
@ -155,13 +155,17 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
class StartShadowEvalRequest(BaseModel):
"""Start duplicating a key's traffic for blind comparison against an auto-router."""
"""Start duplicating one or more keys' traffic for blind comparison against an auto-router."""
api_key_id: str = Field(
api_key_ids: tuple[str, ...] = Field(
min_length=1,
max_length=100,
description=(
"The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this "
"key's traffic; requests made with any other key are not sampled."
)
"The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these "
"keys' traffic; requests made with any other key are not sampled. Each key carries its own "
"max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 "
"keys per job, which also bounds every read the job's endpoints make."
),
)
router_name: str = Field(description="The auto-router under evaluation, in either direction")
direction: ShadowEvalDirection = Field(
@ -204,8 +208,9 @@ class StartShadowEvalRequest(BaseModel):
ge=1,
le=2000,
description=(
"Sample budget: the job judges at most this many turns, then completes. This is also the spend "
"bound; expected judge cost is roughly max_turns times one judge call"
"Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, "
"so a job over N keys judges at most N times max_turns turns. This is also the spend bound; "
"expected judge cost is roughly that turn ceiling times one judge call"
),
)
@ -214,6 +219,12 @@ class StartShadowEvalRequest(BaseModel):
def _round_percentage(cls, value: float) -> float:
return round(value, 2)
@field_validator("api_key_ids")
@classmethod
def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]:
"""A key named twice would collide with itself on the one-active-per-(key, direction) index."""
return tuple(dict.fromkeys(value))
@model_validator(mode="after")
def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest":
if self.direction == "reverse" and self.baseline_model is None:
@ -251,24 +262,46 @@ class ShadowEvalResult(BaseModel):
by_tier: tuple[ShadowEvalSlice, ...]
by_current_model: tuple[ShadowEvalSlice, ...] = Field(
description=(
"Sliced by the model that served the real arm: the key's incumbent models in forward mode, "
"Sliced by the model that served the real arm: the keys' incumbent models in forward mode, "
"and in reverse the models the router itself picked"
)
)
by_key: tuple[ShadowEvalSlice, ...] = Field(
description=(
"One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job "
"scopes but has not judged a turn for yet are absent rather than reported as zero"
),
)
overall_shadow_win_rate_pct: float
overall_tie_rate_pct: float
class ShadowEvalJobResponse(BaseModel):
"""A shadow-eval job. Validates directly from the prisma record (job_id reads the
row's id); status is derived from stopped_at and ends_at, never stored, so no writer
anywhere can produce an inconsistent one. Aggregate fields are populated by the
detail endpoint only and stay None on list responses."""
class ShadowEvalJobKeyResponse(BaseModel):
"""One key a job shadows, with its own budget and stop state."""
model_config = ConfigDict(from_attributes=True, populate_by_name=True)
api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes")
max_turns: int = Field(description="This key's own sample budget, independent of its siblings'")
stopped_at: datetime | None = Field(
default=None,
description=(
"When this key's slot was stamped free, whether its own budget ran out, the window closed, "
"or an operator stopped the job; status is derived, so a spent budget reads completed even "
"while this is still unset"
),
)
attempt_count: int | None = Field(
default=None,
description=(
"This key's sampled attempts so far, judged and errored alike, the same count the sampler "
"budgets against max_turns; populated on list and detail responses. Frozen at stopped_at "
"once the key is stamped, so in-flight attempts landing after a stop never reclassify it"
),
)
@property
def budget_spent(self) -> bool:
return self.attempt_count is not None and self.attempt_count >= self.max_turns
job_id: str = Field(validation_alias=AliasChoices("id", "job_id"))
api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's")
key_alias: str | None = Field(
default=None,
description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted",
@ -277,15 +310,34 @@ class ShadowEvalJobResponse(BaseModel):
default=None,
description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias",
)
class ShadowEvalJobResponse(BaseModel):
"""A shadow-eval job over one or more keys, each with its own budget and stop state;
status is derived from stopped_by, the keys' stop and budget state, and ends_at,
never stored, so no writer anywhere can produce an inconsistent one. Aggregate
fields are populated by the detail endpoint only and stay None on list responses."""
job_id: str
keys: tuple[ShadowEvalJobKeyResponse, ...] = Field(
min_length=1,
description="The keys whose traffic this job evaluates, and only those keys', each with its own budget",
)
router_name: str
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
judge_model: str
shadow_percentage: float
max_turns: int
created_at: datetime
ends_at: datetime
stopped_at: datetime | None = None
stopped_by: str | None = Field(
default=None,
description=(
"The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled "
"by migration for jobs that displayed stopped when the column arrived; None when the job "
"ended on its own. Its presence is what makes a job read stopped rather than completed"
),
)
judged_count: int | None = Field(default=None, description="Verdicts recorded; detail endpoint only")
error_count: int | None = Field(default=None, description="Sampled attempts that errored; detail endpoint only")
@ -296,12 +348,19 @@ class ShadowEvalJobResponse(BaseModel):
@computed_field
@property
def status(self) -> ShadowEvalStatus:
"""A job whose window has passed reads completed even if a later sweep stamped
stopped_at; stopped means sampling ended before the window did."""
"""Three recorded facts, no history-guessing: a stop is stopped_by (the migration
backfills it for every job that displayed stopped when the column arrived, so the
pre-column population is closed), completion is the window passing or every key
spending its budget, and anything else is running. The all-keys-stamped fallback
covers only stops written by pre-column pods during a rolling deploy."""
if self.stopped_by is not None:
return "stopped"
if datetime.now(timezone.utc) >= (
self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc)
):
return "completed"
if self.stopped_at is not None:
if all(key.budget_spent for key in self.keys):
return "completed"
if all(key.stopped_at is not None for key in self.keys):
return "stopped"
return "running"

205
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

30
schema.prisma
View file

@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession {
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
}
// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
// direction. forward duplicates the requests the key did not route through the router
// through it, answering whether the key should adopt it; reverse duplicates the requests
// the router did serve against a fixed baseline model, answering whether a key already on
// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
// compares real vs shadow responses blind. The job row is immutable config plus
// stopped_at; every count, status, and spend figure is derived from the append-only
// attempt rows, so nothing can disagree across pods or stop races.
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
// either direction. forward duplicates the requests the keys did not route through the
// router through it, answering whether they should adopt it; reverse duplicates the
// requests the router did serve against a fixed baseline model, answering whether a key
// already on it still benefits. Either way a sampled slice runs in a detached task and an
// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job:
// immutable config plus that key's own turn budget and stop state, so one key exhausting
// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id
// (the id the API reports), written together by one atomic create_many with identical
// config; single-key jobs predating group_id were backfilled group_id = id. "One active
// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE
// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state
// partial indexes; it is what makes a concurrent start on another pod race-safe rather
// than read-then-create. Every count, status, and spend figure is derived from the
// append-only attempt rows, so nothing can disagree across pods or stop races.
model LiteLLM_ShadowEvalJob {
id String @id @default(cuid())
api_key_id String // hashed virtual key whose traffic is shadowed
group_id String // legs of one job share this; the API's job id
api_key_id String // hashed virtual key whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // sample budget: judge at most this many turns
max_turns Int // this key's sample budget: judge at most this many turns
created_at DateTime @default(now())
created_by String?
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
@@index([group_id])
@@index([api_key_id])
@@index([created_at])
}

10
tests/e2e/CLAUDE.md
View file

@ -71,6 +71,16 @@ Request and response bodies are typed pydantic models in `models.py`; only the f
Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure coverage of the harness itself carries no marker and runs regardless. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache
## Record and replay fixtures
`E2E_FIXTURE_MODE` selects the transport every client is built on: `live` (the default, and what an unset variable means: nothing changes), `record` (run against the live proxy and write every interaction to a fixture bundle), or `replay` (serve every interaction back from the bundle with no HTTP at all, so a replay run needs no proxy and cannot bill a provider). The seam is `select_transport` in `fixture_transport.py`, applied inside `build_proxy_client`; both transports fulfil the same `Transport` protocol, so no test or client changes shape in any mode
A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per transport call in call order (`0000-post-chat-completions.json`). Auth header values are redacted on write, and file uploads store a sha256 digest instead of the bytes; response bodies are stored verbatim (a /key/generate response keeps the ephemeral virtual key it minted), which is part of why bundles are gitignored. `fixture_bundle.py` owns the format
Replay matches calls per test by transport verb and path in recorded order and raises `ReplayMiss` on any drift, naming the recorded and the actual call; a passed test must also consume its whole recording, or teardown fails it naming the first leftover interaction. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live proxy
Deliberately not here yet: canonical content-based match keys (LIT-5741), streaming chunk fidelity (LIT-5742), and scoping record/replay to provider-bound traffic (LIT-5745)
## Typing
The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test

11
tests/e2e/CONTRIBUTING.md
View file

@ -52,6 +52,17 @@ The suites run against a live proxy, so bring one up first by running the litell
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy
### Record and replay
`E2E_FIXTURE_MODE=record` runs a suite against the live proxy as usual while writing every request/response pair to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`); `E2E_FIXTURE_MODE=replay` then runs the same suite entirely from that bundle, with no proxy traffic and no provider spend; the proxy liveness gate is skipped, so replay runs with no proxy up at all. Unset (or `live`) behaves exactly as before the knob existed
```bash
E2E_FIXTURE_MODE=record uv run pytest tests/e2e/llm_translation/ -v
E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/llm_translation/ -v
```
Replay fails hard (`ReplayMiss`) when the tests drift from the recording, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. See `CLAUDE.md` in this directory for the bundle format and the transport seam
Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass
## What a complete test looks like

62
tests/e2e/conftest.py
View file

@ -15,19 +15,27 @@ shared fixtures build on it.
import functools
import os
from collections.abc import Iterator
from collections.abc import Generator, Iterator
from datetime import datetime, timezone
import pytest
import requests
from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL
from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL
from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup
from fixture_transport import (
fixture_mode_collection_error,
fixture_report_lines,
parse_fixture_mode,
replay_leftover_error,
)
from junit_properties import attach_result_properties
from lifecycle import ProxyClientProvider, ResourceManager
from proxy_client import ProxyClient, build_proxy_client
_E2E_TEST_RAN = pytest.StashKey[bool]()
_CALL_PASSED = pytest.StashKey[bool]()
def pytest_configure(config: pytest.Config) -> None:
@ -49,6 +57,21 @@ def pytest_configure(config: pytest.Config) -> None:
)
def pytest_sessionstart(session: pytest.Session) -> None:
"""Abort before collection when E2E_FIXTURE_MODE can never work: an unknown
mode value, or replay against a missing, unreadable, or stale bundle (the
stale message names the bundle's age). Live and record modes pass through."""
reason = fixture_mode_collection_error(
FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)
)
if reason is not None:
raise pytest.UsageError(reason)
def pytest_report_header(config: pytest.Config) -> list[str]:
return fixture_report_lines(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc))
def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
"""Attach the two custom signals (suite package and covered cell ids) to every
test's user_properties so the standard JUnit report (`--junitxml`) records them
@ -91,9 +114,12 @@ def _proxy_fail_reason() -> str | None:
def pytest_runtest_setup(item: pytest.Item) -> None:
"""Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe.
Unmarked tests (unit coverage of the harness) don't touch the proxy, so they
run even when none is up. Never skip for a missing proxy."""
run even when none is up. Never skip for a missing proxy. Replay mode serves
every call from the fixture bundle, so it needs no live proxy either."""
if item.get_closest_marker("e2e") is None:
return
if parse_fixture_mode(FIXTURE_MODE_RAW) == "replay":
return
reason = _proxy_fail_reason()
if reason is not None:
pytest.fail(reason)
@ -110,6 +136,36 @@ def pytest_runtest_call(item: pytest.Item) -> None:
item.session.stash[_E2E_TEST_RAN] = True
@pytest.hookimpl(wrapper=True)
def pytest_runtest_makereport(
item: pytest.Item, call: pytest.CallInfo[None]
) -> Generator[None, pytest.TestReport, pytest.TestReport]:
"""Stash the call-phase outcome so teardown can tell a passed test from a
failed one without re-deriving it."""
report = yield
if report.when == "call":
item.stash[_CALL_PASSED] = report.passed
return report
@pytest.hookimpl(wrapper=True)
def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]:
"""In replay mode a passing test must consume its whole recording: leftover
interactions mean the test now makes fewer calls than it did at record time,
so the replay proved less than the bundle claims. The check runs after the
yield so fixture finalizers replay their recorded calls first. Failed tests
are left alone - their own failure already explains any unconsumed tail."""
result = yield
if not item.stash.get(_CALL_PASSED, False):
return result
reason = replay_leftover_error(
mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid
)
if reason is not None:
pytest.fail(reason)
return result
def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
"""Once the whole e2e session is done (all suites), optionally truncate the
spend logs so the DB doesn't accumulate test rows. The truncate is destructive

17
tests/e2e/e2e_config.py
View file

@ -13,6 +13,8 @@ from pathlib import Path
from dotenv import load_dotenv
from fixture_transport import deterministic_marker, parse_fixture_mode
# Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md).
# Compose injects them into the proxy container, but pytest on the host does not
# inherit that file unless we load it. override=False so a real shell export wins.
@ -90,6 +92,15 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15"))
EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes")
# Record/replay fixture selection (see fixture_transport.py). The raw mode value
# is parsed and validated there; "live" (the default, also for empty values)
# means the harness behaves exactly as before this knob existed.
FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live")
FIXTURE_DIR = Path(
os.environ.get("E2E_FIXTURE_DIR", "").strip()
or str(Path(__file__).resolve().parent / ".fixtures")
)
# Deliberately modest concurrency. The suite shares its proxy with every other
# suite in the run, and 750 users at spawn rate 50 saturated the request path hard
# enough to distort latency-sensitive neighbours (and to spend real provider money
@ -148,7 +159,11 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str:
def unique_marker() -> str:
"""A short unique token per call/run, so concurrent runs and the shared
response cache never collide on prompts, tags, or customer ids."""
response cache never collide on prompts, tags, or customer ids. In record
and replay modes the token is deterministic per test instead, so a replay
run regenerates the exact requests the record run sent."""
if parse_fixture_mode(FIXTURE_MODE_RAW) in ("record", "replay"):
return deterministic_marker()
return uuid.uuid4().hex[:12]

314
tests/e2e/fixture_bundle.py Normal file
View file

@ -0,0 +1,314 @@
"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729).
A bundle is a directory: one ``manifest.json`` (record timestamp + harness
version + format version) plus one subdirectory per test, holding one JSON file
per transport interaction in call order. Bundles older than
``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a
green replay run can never certify against fixtures that have drifted more than
a week from the live proxy.
This module owns the format only. The transports that produce and consume it
live in fixture_transport.py; canonical request matching, streaming chunk
fidelity, and provider-scoping are follow-ups (LIT-5741/5742/5745) and are
deliberately absent here, which is why every interaction file stores the full
redacted request even though replay today matches by call order.
"""
from __future__ import annotations
import hashlib
import re
import shutil
import subprocess
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Annotated, Final, Literal
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
from e2e_http import (
BinaryStream,
NetworkError,
ProbeResult,
RateLimitedError,
Result,
StreamingResponse,
Success,
UnauthorizedError,
UnknownApiError,
ValidationError,
)
BUNDLE_FORMAT_VERSION: Final = 1
MAX_BUNDLE_AGE: Final = timedelta(days=7)
MANIFEST_FILENAME: Final = "manifest.json"
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
class Manifest(BaseModel):
format_version: int
recorded_at: datetime
harness_version: str
class RecordedRequest(BaseModel):
"""The request as the transport saw it, auth header values redacted.
Replay today only matches ``method`` (the transport verb, not the HTTP verb)
and ``path`` in call order; the rest is stored so LIT-5741 can move to
content-based match keys without re-recording. File uploads store a content
digest instead of the bytes."""
method: str
path: str
headers: dict[str, str]
params: dict[str, str] = {}
body: JsonValue | None = None
form: dict[str, str] | None = None
file_name: str | None = None
file_sha256: str | None = None
file_bytes: int | None = None
class RecordedResult(BaseModel):
"""A ``Result[R]`` flattened for disk. ``data`` holds the success payload as
raw JSON; replay re-validates it against the ``response_type`` the caller
passes, exactly like a live response body."""
shape: Literal["result"] = "result"
kind: Literal["success", "network", "unauthorized", "rate_limited", "validation", "unknown"]
status_code: int | None = None
data: JsonValue | None = None
message: str | None = None
body: str | None = None
retry_after_seconds: int | None = None
class RecordedStreaming(BaseModel):
shape: Literal["streaming"] = "streaming"
payload: StreamingResponse
class RecordedBinary(BaseModel):
shape: Literal["binary"] = "binary"
payload: BinaryStream
class RecordedProbe(BaseModel):
shape: Literal["probe"] = "probe"
payload: ProbeResult
type RecordedResponse = RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe
class Interaction(BaseModel):
request: RecordedRequest
response: Annotated[
RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe,
Field(discriminator="shape"),
]
def to_json_value(model: BaseModel) -> JsonValue:
return _JSON.validate_json(model.model_dump_json(by_alias=True))
def from_result[R: BaseModel](result: Result[R]) -> RecordedResult:
match result:
case Success(status_code=status_code, data=data):
return RecordedResult(kind="success", status_code=status_code, data=to_json_value(data))
case NetworkError(message=message):
return RecordedResult(kind="network", message=message)
case UnauthorizedError():
return RecordedResult(kind="unauthorized")
case RateLimitedError(retry_after_seconds=retry_after_seconds, body=body):
return RecordedResult(kind="rate_limited", retry_after_seconds=retry_after_seconds, body=body)
case ValidationError(message=message):
return RecordedResult(kind="validation", message=message)
case UnknownApiError(status_code=status_code, body=body):
return RecordedResult(kind="unknown", status_code=status_code, body=body)
def to_result[R: BaseModel](recorded: RecordedResult, response_type: type[R]) -> Result[R]:
match recorded.kind:
case "success":
return Success(
status_code=recorded.status_code or 200,
data=response_type.model_validate(recorded.data),
)
case "network":
return NetworkError(message=recorded.message or "")
case "unauthorized":
return UnauthorizedError()
case "rate_limited":
return RateLimitedError(
retry_after_seconds=recorded.retry_after_seconds, body=recorded.body or ""
)
case "validation":
return ValidationError(message=recorded.message or "")
case "unknown":
return UnknownApiError(status_code=recorded.status_code or 0, body=recorded.body or "")
def slugify(raw: str, *, limit: int = 60) -> str:
clean = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-")
return clean[:limit].rstrip("-")
def slug_for_test(test_key: str) -> str:
"""Directory name for one test's interactions: a readable tail plus a short
digest of the full node id, so same-named methods in different classes or
files never collide."""
digest = hashlib.sha1(test_key.encode()).hexdigest()[:8]
tail = slugify(test_key.rsplit("::", 1)[-1])
return f"{tail}-{digest}" if tail else digest
def interaction_filename(ordinal: int, request: RecordedRequest) -> str:
path_part = slugify(request.path, limit=40) or "root"
return f"{ordinal:04d}-{request.method}-{path_part}.json"
def harness_version() -> str:
try:
proc = subprocess.run(
("git", "rev-parse", "--short", "HEAD"),
cwd=Path(__file__).resolve().parent,
capture_output=True,
text=True,
timeout=10,
check=False,
)
except (OSError, subprocess.SubprocessError):
return "unknown"
return proc.stdout.strip() or "unknown"
@dataclass(slots=True)
class BundleRecorder:
"""Appends interaction files under ``root``, one subdirectory per test, with
a per-test ordinal that fixes replay order. ``prepare_bundle`` is the only
constructor: it guarantees the directory started empty with a fresh
manifest, so record mode never reads (or merges into) an existing bundle."""
root: Path
_ordinals: dict[str, int] = field(default_factory=dict)
def record(self, *, test_key: str, request: RecordedRequest, response: RecordedResponse) -> None:
slug = slug_for_test(test_key)
ordinal = self._ordinals.get(slug, 0)
self._ordinals[slug] = ordinal + 1
directory = self.root / slug
directory.mkdir(parents=True, exist_ok=True)
interaction = Interaction(request=request, response=response)
target = directory / interaction_filename(ordinal, request)
target.write_text(interaction.model_dump_json(indent=2), encoding="utf-8")
@dataclass(frozen=True, slots=True)
class UnsafeBundleDir:
path: Path
reason: str
def prepare_bundle(root: Path) -> BundleRecorder | UnsafeBundleDir:
"""Start a fresh bundle at ``root`` for record mode: wipe whatever bundle is
there and write a new manifest. Refuses to wipe a directory that is neither
empty nor a bundle (no manifest.json), so a mistyped E2E_FIXTURE_DIR can
never delete unrelated files."""
if root.exists():
if not root.is_dir():
return UnsafeBundleDir(path=root, reason="exists and is not a directory")
entries = tuple(root.iterdir())
if entries and not (root / MANIFEST_FILENAME).is_file():
return UnsafeBundleDir(
path=root,
reason=f"is not empty and has no {MANIFEST_FILENAME}; refusing to wipe a non-bundle directory",
)
shutil.rmtree(root)
root.mkdir(parents=True)
manifest = Manifest(
format_version=BUNDLE_FORMAT_VERSION,
recorded_at=datetime.now(timezone.utc),
harness_version=harness_version(),
)
(root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
return BundleRecorder(root=root)
@dataclass(frozen=True, slots=True)
class FreshBundle:
manifest: Manifest
@dataclass(frozen=True, slots=True)
class StaleBundle:
recorded_at: datetime
age: timedelta
limit: timedelta
@dataclass(frozen=True, slots=True)
class UnreadableBundle:
reason: str
type BundleFreshness = FreshBundle | StaleBundle | UnreadableBundle
def _read_manifest(root: Path) -> Manifest | UnreadableBundle:
manifest_path = root / MANIFEST_FILENAME
if not manifest_path.is_file():
return UnreadableBundle(reason=f"no {MANIFEST_FILENAME} found (record one with E2E_FIXTURE_MODE=record)")
try:
return Manifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
except ValueError as exc:
return UnreadableBundle(reason=f"{MANIFEST_FILENAME} is invalid: {exc}")
def check_freshness(root: Path, *, now: datetime) -> BundleFreshness:
manifest = _read_manifest(root)
if isinstance(manifest, UnreadableBundle):
return manifest
if manifest.format_version != BUNDLE_FORMAT_VERSION:
return UnreadableBundle(
reason=f"format_version {manifest.format_version} != supported {BUNDLE_FORMAT_VERSION}"
)
recorded_at = (
manifest.recorded_at
if manifest.recorded_at.tzinfo is not None
else manifest.recorded_at.replace(tzinfo=timezone.utc)
)
age = now - recorded_at
if age > MAX_BUNDLE_AGE:
return StaleBundle(recorded_at=recorded_at, age=age, limit=MAX_BUNDLE_AGE)
return FreshBundle(manifest=manifest)
def format_age(age: timedelta) -> str:
total_hours = int(age.total_seconds()) // 3600
return f"{total_hours // 24}d{total_hours % 24}h"
@dataclass(frozen=True, slots=True)
class LoadedBundle:
manifest: Manifest
interactions: dict[str, tuple[Interaction, ...]]
def load_bundle(root: Path) -> LoadedBundle | UnreadableBundle:
manifest = _read_manifest(root)
if isinstance(manifest, UnreadableBundle):
return manifest
interactions = {
directory.name: tuple(
Interaction.model_validate_json(file.read_text(encoding="utf-8"))
for file in sorted(directory.glob("*.json"))
)
for directory in sorted(root.iterdir())
if directory.is_dir()
}
return LoadedBundle(manifest=manifest, interactions=interactions)

574
tests/e2e/fixture_transport.py Normal file
View file

@ -0,0 +1,574 @@
"""Record/replay transports behind the same ``Transport`` protocol (LIT-5729).
``RecordingTransport`` decorates the live transport: every call passes through
unchanged and its request/response pair is appended to the fixture bundle.
``ReplayTransport`` implements the protocol from a recorded bundle alone: no
HTTP, no proxy, no provider spend. Because both fulfil ``Transport``, no test
or client changes shape; ``build_proxy_client`` picks the transport from
``E2E_FIXTURE_MODE`` (live | record | replay, default live).
Replay matches each call by test node id and call order, verifying transport
verb + path and failing hard on any drift (``ReplayMiss``). Canonical
content-based match keys are LIT-5741; streaming chunk fidelity is LIT-5742;
scoping record/replay to provider-bound traffic is LIT-5745.
"""
from __future__ import annotations
import functools
import hashlib
import os
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Final, Literal, assert_never
from pydantic import BaseModel
from e2e_http import AuthHeaders, BinaryStream, ProbeResult, Result, StreamingResponse
from fixture_bundle import (
BundleRecorder,
FreshBundle,
Interaction,
LoadedBundle,
RecordedBinary,
RecordedProbe,
RecordedRequest,
RecordedResponse,
RecordedResult,
RecordedStreaming,
StaleBundle,
UnreadableBundle,
UnsafeBundleDir,
check_freshness,
format_age,
from_result,
load_bundle,
prepare_bundle,
slug_for_test,
to_json_value,
to_result,
)
from transport import Transport
type FixtureMode = Literal["live", "record", "replay"]
FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay")
SESSION_TEST_KEY: Final = "session"
REDACTED_HEADER_NAMES: Final[frozenset[str]] = frozenset({"authorization", "x-litellm-api-key"})
REDACTED_VALUE: Final = "<redacted>"
@dataclass(frozen=True, slots=True)
class InvalidFixtureMode:
value: str
def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode:
normalized = raw.strip().lower() or "live"
match normalized:
case "live" | "record" | "replay":
return normalized
case _:
return InvalidFixtureMode(value=raw)
def current_test_key() -> str:
"""The pytest node id of the running test, from the PYTEST_CURRENT_TEST env
var pytest maintains (``<nodeid> (setup|call|teardown)``); ``session`` for
calls outside any test (e.g. session-finish cleanup)."""
raw = os.environ.get("PYTEST_CURRENT_TEST", "")
if not raw:
return SESSION_TEST_KEY
return raw.rsplit(" (", 1)[0]
class ReplayMiss(AssertionError):
"""Replay had no recorded interaction for a call the suite made. The test
drifted from the bundle (or the bundle from the suite): re-record."""
_marker_ordinals: Final[dict[str, int]] = {}
def deterministic_marker() -> str:
"""Stable stand-in for uuid-based unique markers in record and replay modes:
the Nth marker of a test is a pure function of the test's node id and N, so a
replay run regenerates exactly the model names, prompts, and tags the record
run sent and every recorded poll response still satisfies its predicate."""
test_key = current_test_key()
ordinal = _marker_ordinals.get(test_key, 0)
_marker_ordinals[test_key] = ordinal + 1
return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12]
def _dump_flat(model: BaseModel | None) -> dict[str, str]:
if model is None:
return {}
dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True)
return {key: str(value) for key, value in dumped.items()}
def _redact(headers: dict[str, str]) -> dict[str, str]:
return {
name: REDACTED_VALUE if name.lower() in REDACTED_HEADER_NAMES else value
for name, value in headers.items()
}
def recorded_request(
method: str,
path: str,
*,
headers: BaseModel,
body: BaseModel | None = None,
params: BaseModel | None = None,
form: BaseModel | None = None,
file_name: str | None = None,
file_content: bytes | None = None,
) -> RecordedRequest:
return RecordedRequest(
method=method,
path=path,
headers=_redact(_dump_flat(headers)),
params=_dump_flat(params),
body=None if body is None else to_json_value(body),
form=None if form is None else _dump_flat(form),
file_name=file_name,
file_sha256=None if file_content is None else hashlib.sha256(file_content).hexdigest(),
file_bytes=None if file_content is None else len(file_content),
)
@dataclass(frozen=True, slots=True)
class RecordingTransport:
"""Decorator over the live transport: forwards every call and appends the
interaction to the bundle, so a green live run leaves behind exactly the
traffic replay needs."""
inner: Transport
recorder: BundleRecorder
def _record(self, request: RecordedRequest, response: RecordedResponse) -> None:
self.recorder.record(test_key=current_test_key(), request=request, response=response)
def bearer(self, key: str) -> AuthHeaders:
return self.inner.bearer(key)
@property
def master(self) -> AuthHeaders:
return self.inner.master
def post[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
result = self.inner.post(path, headers=headers, json=json, response_type=response_type)
self._record(recorded_request("post", path, headers=headers, body=json), from_result(result))
return result
def get[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
params: BaseModel,
response_type: type[R],
timeout: float | None = None,
) -> Result[R]:
result = self.inner.get(
path, headers=headers, params=params, response_type=response_type, timeout=timeout
)
self._record(recorded_request("get", path, headers=headers, params=params), from_result(result))
return result
def delete[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
response_type: type[R],
params: BaseModel | None = None,
) -> Result[R]:
result = self.inner.delete(
path, headers=headers, json=json, response_type=response_type, params=params
)
self._record(
recorded_request("delete", path, headers=headers, body=json, params=params),
from_result(result),
)
return result
def patch[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
result = self.inner.patch(path, headers=headers, json=json, response_type=response_type)
self._record(recorded_request("patch", path, headers=headers, body=json), from_result(result))
return result
def put[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
result = self.inner.put(path, headers=headers, json=json, response_type=response_type)
self._record(recorded_request("put", path, headers=headers, body=json), from_result(result))
return result
def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse:
response = self.inner.stream(path, headers=headers, json=json)
self._record(
recorded_request("stream", path, headers=headers, body=json),
RecordedStreaming(payload=response),
)
return response
def stream_binary(
self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192
) -> BinaryStream:
response = self.inner.stream_binary(path, headers=headers, json=json, chunk_size=chunk_size)
self._record(
recorded_request("stream_binary", path, headers=headers, body=json),
RecordedBinary(payload=response),
)
return response
def send(
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
params: BaseModel | None = None,
stream: bool = False,
) -> StreamingResponse:
response = self.inner.send(path, headers=headers, json=json, params=params, stream=stream)
self._record(
recorded_request("send", path, headers=headers, body=json, params=params),
RecordedStreaming(payload=response),
)
return response
def probe(self, path: str, *, params: BaseModel) -> ProbeResult:
response = self.inner.probe(path, params=params)
self._record(
recorded_request("probe", path, headers=self.master, params=params),
RecordedProbe(payload=response),
)
return response
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: BaseModel,
filename: str,
content: bytes,
file_content_type: str = "application/jsonl",
file_field: str = "file",
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]:
result = self.inner.upload(
path,
headers=headers,
form=form,
filename=filename,
content=content,
file_content_type=file_content_type,
file_field=file_field,
params=params,
response_type=response_type,
)
self._record(
recorded_request(
"upload",
path,
headers=headers,
params=params,
form=form,
file_name=filename,
file_content=content,
),
from_result(result),
)
return result
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse:
response = self.inner.download(path, headers=headers)
self._record(
recorded_request("download", path, headers=headers),
RecordedStreaming(payload=response),
)
return response
@dataclass(slots=True)
class ReplaySource:
"""One shared cursor set over a loaded bundle, so every client built in the
session consumes the same recorded sequence per test."""
bundle: LoadedBundle
_cursors: dict[str, int] = field(default_factory=dict)
def next_interaction(self, method: str, path: str) -> Interaction:
test_key = current_test_key()
slug = slug_for_test(test_key)
recorded = self.bundle.interactions.get(slug, ())
index = self._cursors.get(slug, 0)
if index >= len(recorded):
raise ReplayMiss(
f"replay exhausted for {test_key}: call #{index + 1} ({method} {path}) has no recorded "
f"interaction ({len(recorded)} recorded under {slug}); re-record with E2E_FIXTURE_MODE=record"
)
interaction = recorded[index]
if interaction.request.method != method or interaction.request.path != path:
raise ReplayMiss(
f"replay mismatch for {test_key} at call #{index + 1}: recorded "
f"{interaction.request.method} {interaction.request.path}, test made {method} {path}; "
"re-record with E2E_FIXTURE_MODE=record"
)
self._cursors[slug] = index + 1
return interaction
def leftover_error(self, test_key: str) -> str | None:
"""Non-None when the test consumed fewer interactions than were recorded,
meaning a passing replay proved less than the bundle claims."""
slug = slug_for_test(test_key)
recorded = self.bundle.interactions.get(slug, ())
consumed = self._cursors.get(slug, 0)
if consumed >= len(recorded):
return None
pending = recorded[consumed]
return (
f"replay incomplete for {test_key}: {len(recorded) - consumed} of {len(recorded)} recorded "
f"interactions never consumed, next is {pending.request.method} {pending.request.path}; "
"re-record with E2E_FIXTURE_MODE=record"
)
def _expect_result(interaction: Interaction) -> RecordedResult:
match interaction.response:
case RecordedResult() as recorded:
return recorded
case RecordedStreaming() | RecordedBinary() | RecordedProbe():
raise ReplayMiss(
f"recorded {interaction.request.method} {interaction.request.path} is not a typed result"
)
def _expect_streaming(interaction: Interaction) -> StreamingResponse:
match interaction.response:
case RecordedStreaming(payload=payload):
return payload
case RecordedResult() | RecordedBinary() | RecordedProbe():
raise ReplayMiss(
f"recorded {interaction.request.method} {interaction.request.path} is not a streaming response"
)
@dataclass(frozen=True, slots=True)
class ReplayTransport:
"""A ``Transport`` served entirely from a recorded bundle: never opens a
connection, so a replay run cannot bill a provider."""
source: ReplaySource
master_key: str
def bearer(self, key: str) -> AuthHeaders:
return AuthHeaders(authorization=f"Bearer {key}")
@property
def master(self) -> AuthHeaders:
return self.bearer(self.master_key)
def post[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("post", path)), response_type)
def get[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
params: BaseModel,
response_type: type[R],
timeout: float | None = None,
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("get", path)), response_type)
def delete[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
response_type: type[R],
params: BaseModel | None = None,
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("delete", path)), response_type)
def patch[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("patch", path)), response_type)
def put[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("put", path)), response_type)
def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse:
return _expect_streaming(self.source.next_interaction("stream", path))
def stream_binary(
self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192
) -> BinaryStream:
interaction = self.source.next_interaction("stream_binary", path)
match interaction.response:
case RecordedBinary(payload=payload):
return payload
case RecordedResult() | RecordedStreaming() | RecordedProbe():
raise ReplayMiss(
f"recorded stream_binary {interaction.request.path} is not a binary stream"
)
def send(
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
params: BaseModel | None = None,
stream: bool = False,
) -> StreamingResponse:
return _expect_streaming(self.source.next_interaction("send", path))
def probe(self, path: str, *, params: BaseModel) -> ProbeResult:
interaction = self.source.next_interaction("probe", path)
match interaction.response:
case RecordedProbe(payload=payload):
return payload
case RecordedResult() | RecordedStreaming() | RecordedBinary():
raise ReplayMiss(f"recorded probe {interaction.request.path} is not a probe result")
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: BaseModel,
filename: str,
content: bytes,
file_content_type: str = "application/jsonl",
file_field: str = "file",
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]:
return to_result(_expect_result(self.source.next_interaction("upload", path)), response_type)
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse:
return _expect_streaming(self.source.next_interaction("download", path))
@functools.lru_cache(maxsize=8)
def _shared_recorder(root: Path) -> BundleRecorder:
prepared = prepare_bundle(root)
if isinstance(prepared, UnsafeBundleDir):
raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}")
return prepared
@functools.lru_cache(maxsize=8)
def _shared_replay_source(root: Path) -> ReplaySource:
loaded = load_bundle(root)
if isinstance(loaded, UnreadableBundle):
raise ValueError(f"cannot replay from {root}: {loaded.reason}")
return ReplaySource(bundle=loaded)
def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None:
"""Teardown-time completeness check: in replay mode a passed test with
unconsumed recorded interactions must fail instead of passing against a
recording it no longer matches. Inert in every other mode."""
if parse_fixture_mode(mode_raw) != "replay":
return None
return _shared_replay_source(bundle_dir).leftover_error(test_key)
def select_transport(
live: Transport, *, mode_raw: str, bundle_dir: Path, master_key: str
) -> Transport:
"""The one seam every client build goes through: wraps (record), replaces
(replay), or passes through (live) the transport per E2E_FIXTURE_MODE. The
recorder and replay cursors are process-wide singletons per bundle dir, so
every client in a session shares one bundle and one recorded sequence."""
mode = parse_fixture_mode(mode_raw)
match mode:
case InvalidFixtureMode(value=value):
raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}")
case "live":
return live
case "record":
return RecordingTransport(inner=live, recorder=_shared_recorder(bundle_dir))
case "replay":
return ReplayTransport(source=_shared_replay_source(bundle_dir), master_key=master_key)
case _:
assert_never(mode)
def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None:
"""Session-abort reason for a fixture-mode setup that can never work, or None.
Called at collection time (conftest pytest_sessionstart) so a stale or missing
bundle fails the whole run up front, naming the bundle age, instead of failing
every test individually."""
mode = parse_fixture_mode(mode_raw)
match mode:
case InvalidFixtureMode(value=value):
return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}"
case "live" | "record":
return None
case "replay":
freshness = check_freshness(bundle_dir, now=now)
match freshness:
case FreshBundle():
return None
case StaleBundle(recorded_at=recorded_at, age=age, limit=limit):
return (
f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, "
f"age {format_age(age)} exceeds the {limit.days}-day limit; "
"re-record with E2E_FIXTURE_MODE=record"
)
case UnreadableBundle(reason=reason):
return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}"
case _:
assert_never(freshness)
case _:
assert_never(mode)
def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]:
"""pytest report-header lines; empty in live mode so an unset
E2E_FIXTURE_MODE keeps today's output byte-identical."""
mode = parse_fixture_mode(mode_raw)
match mode:
case InvalidFixtureMode() | "live":
return []
case "record":
return [f"e2e fixture mode: record -> {bundle_dir}"]
case "replay":
freshness = check_freshness(bundle_dir, now=now)
match freshness:
case FreshBundle(manifest=manifest):
return [
f"e2e fixture mode: replay <- {bundle_dir} "
f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})"
]
case StaleBundle() | UnreadableBundle():
return [f"e2e fixture mode: replay <- {bundle_dir}"]
case _:
assert_never(freshness)
case _:
assert_never(mode)

37
tests/e2e/proxy_client.py
View file

@ -65,6 +65,8 @@ from models import (
)
from e2e_config import (
CONTROL_PLANE_BASE_URL,
FIXTURE_DIR,
FIXTURE_MODE_RAW,
MASTER_KEY,
POLL_INTERVAL,
POLL_TIMEOUT,
@ -72,6 +74,7 @@ from e2e_config import (
REQUEST_TIMEOUT,
settle_propagation,
)
from fixture_transport import select_transport
from transport import HttpTransport, SplitTransport, Transport
RowsPredicate = Callable[[list[SpendLogRow]], bool]
@ -531,19 +534,29 @@ def build_proxy_client(
The endpoints are injectable for callers that resolve the proxy some other
way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must
pass all three together, since a caller that overrides only the data plane
would leave management calls pointed at the env default."""
would leave management calls pointed at the env default.
E2E_FIXTURE_MODE wraps (record) or replaces (replay) the transport here, so
every client built from this seam records or replays without changing shape;
unset it stays the plain SplitTransport (see fixture_transport.py)."""
split = SplitTransport(
data=HttpTransport(
base_url=base_url,
master_key=master_key,
request_timeout=REQUEST_TIMEOUT,
),
control=HttpTransport(
base_url=control_plane_base_url,
master_key=master_key,
request_timeout=REQUEST_TIMEOUT,
),
)
return ProxyClient(
transport=SplitTransport(
data=HttpTransport(
base_url=base_url,
master_key=master_key,
request_timeout=REQUEST_TIMEOUT,
),
control=HttpTransport(
base_url=control_plane_base_url,
master_key=master_key,
request_timeout=REQUEST_TIMEOUT,
),
transport=select_transport(
split,
mode_raw=FIXTURE_MODE_RAW,
bundle_dir=FIXTURE_DIR,
master_key=master_key,
),
poll_timeout=POLL_TIMEOUT,
poll_interval=POLL_INTERVAL,

218
tests/e2e/test_fixture_bundle.py Normal file
View file

@ -0,0 +1,218 @@
"""Harness coverage for the on-disk fixture bundle format (LIT-5729).
No proxy and no ``e2e`` marker: these pin the bundle CONTRACT - the seven-day
freshness gate that names the bundle's age, record mode's wipe safety (never
delete a directory that is not a bundle), collision-free per-test slugs, and
lossless Result round-trips - so replay can never silently drift from what
record wrote.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
from pydantic import BaseModel
from e2e_http import (
NetworkError,
RateLimitedError,
Result,
Success,
UnauthorizedError,
UnknownApiError,
ValidationError,
)
from fixture_bundle import (
BUNDLE_FORMAT_VERSION,
MANIFEST_FILENAME,
MAX_BUNDLE_AGE,
BundleRecorder,
FreshBundle,
LoadedBundle,
Manifest,
RecordedRequest,
RecordedResult,
StaleBundle,
UnreadableBundle,
UnsafeBundleDir,
check_freshness,
format_age,
from_result,
interaction_filename,
load_bundle,
prepare_bundle,
slug_for_test,
to_result,
)
NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc)
class Payload(BaseModel):
value: str
def write_manifest(
root: Path, recorded_at: datetime, *, format_version: int = BUNDLE_FORMAT_VERSION
) -> None:
root.mkdir(parents=True, exist_ok=True)
manifest = Manifest(
format_version=format_version, recorded_at=recorded_at, harness_version="abc1234"
)
(root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8")
def prepared(root: Path) -> BundleRecorder:
recorder = prepare_bundle(root)
assert isinstance(recorder, BundleRecorder)
return recorder
def plain_request(path: str) -> RecordedRequest:
return RecordedRequest(method="post", path=path, headers={})
class TestResultRoundTrip:
@pytest.mark.parametrize(
"result",
[
Success(status_code=201, data=Payload(value="ok")),
NetworkError(message="connection refused"),
UnauthorizedError(),
RateLimitedError(retry_after_seconds=7, body="slow down"),
ValidationError(message="bad shape"),
UnknownApiError(status_code=502, body="upstream exploded"),
],
)
def test_every_result_kind_survives_disk_and_back(self, result: Result[Payload]) -> None:
assert to_result(from_result(result), Payload) == result
class TestFreshness:
def test_bundle_at_the_limit_is_still_fresh(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, NOW - MAX_BUNDLE_AGE)
assert isinstance(check_freshness(root, now=NOW), FreshBundle)
def test_stale_bundle_reports_age_and_limit(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, NOW - timedelta(days=8, hours=3))
freshness = check_freshness(root, now=NOW)
assert isinstance(freshness, StaleBundle)
assert freshness.age == timedelta(days=8, hours=3)
assert format_age(freshness.age) == "8d3h"
assert freshness.limit == MAX_BUNDLE_AGE
def test_naive_recorded_at_is_read_as_utc(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, (NOW - timedelta(days=1)).replace(tzinfo=None))
assert isinstance(check_freshness(root, now=NOW), FreshBundle)
def test_missing_manifest_is_unreadable_with_recording_hint(self, tmp_path: Path) -> None:
freshness = check_freshness(tmp_path / "absent", now=NOW)
assert isinstance(freshness, UnreadableBundle)
assert MANIFEST_FILENAME in freshness.reason
assert "E2E_FIXTURE_MODE=record" in freshness.reason
def test_corrupt_manifest_is_unreadable(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
root.mkdir()
(root / MANIFEST_FILENAME).write_text("{not json", encoding="utf-8")
assert isinstance(check_freshness(root, now=NOW), UnreadableBundle)
def test_unknown_format_version_is_unreadable(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, NOW, format_version=BUNDLE_FORMAT_VERSION + 1)
freshness = check_freshness(root, now=NOW)
assert isinstance(freshness, UnreadableBundle)
assert f"format_version {BUNDLE_FORMAT_VERSION + 1}" in freshness.reason
class TestPrepareBundle:
def test_fresh_directory_gets_a_fresh_manifest(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
prepared(root)
freshness = check_freshness(root, now=datetime.now(timezone.utc))
assert isinstance(freshness, FreshBundle)
assert freshness.manifest.format_version == BUNDLE_FORMAT_VERSION
assert freshness.manifest.harness_version
def test_record_wipes_the_previous_bundle_instead_of_reading_it(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
prepared(root).record(
test_key="old.py::test_old",
request=plain_request("/stale"),
response=RecordedResult(kind="unauthorized"),
)
assert any(entry.is_dir() for entry in root.iterdir())
prepared(root)
assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME}
def test_refuses_to_wipe_a_directory_that_is_not_a_bundle(self, tmp_path: Path) -> None:
root = tmp_path / "precious"
root.mkdir()
(root / "notes.txt").write_text("keep me", encoding="utf-8")
outcome = prepare_bundle(root)
assert isinstance(outcome, UnsafeBundleDir)
assert MANIFEST_FILENAME in outcome.reason
assert (root / "notes.txt").read_text(encoding="utf-8") == "keep me"
def test_refuses_a_path_that_is_a_file(self, tmp_path: Path) -> None:
target = tmp_path / "not-a-dir"
target.write_text("x", encoding="utf-8")
outcome = prepare_bundle(target)
assert isinstance(outcome, UnsafeBundleDir)
assert "not a directory" in outcome.reason
class TestSlugs:
def test_slug_for_test_is_deterministic(self) -> None:
key = "tests/e2e/suite/test_mod.py::TestX::test_case"
assert slug_for_test(key) == slug_for_test(key)
def test_same_tail_in_different_files_never_collides(self) -> None:
first = slug_for_test("tests/e2e/a/test_a.py::test_case")
second = slug_for_test("tests/e2e/b/test_b.py::test_case")
assert first != second
assert first.startswith("test_case-")
assert second.startswith("test_case-")
def test_interaction_filename_orders_and_slugs(self) -> None:
request = RecordedRequest(method="post", path="/chat/completions", headers={})
assert interaction_filename(3, request) == "0003-post-chat-completions.json"
class TestRecordAndLoad:
def test_load_returns_interactions_in_recorded_order(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
recorder = prepared(root)
key = "suite/test_mod.py::test_ordered"
for path in ("/first", "/second", "/third"):
recorder.record(
test_key=key,
request=plain_request(path),
response=RecordedResult(kind="unauthorized"),
)
loaded = load_bundle(root)
assert isinstance(loaded, LoadedBundle)
assert [
interaction.request.path for interaction in loaded.interactions[slug_for_test(key)]
] == ["/first", "/second", "/third"]
def test_interactions_group_per_test(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
recorder = prepared(root)
for key in ("suite/test_a.py::test_one", "suite/test_b.py::test_two"):
recorder.record(
test_key=key,
request=plain_request(f"/{key[-3:]}"),
response=RecordedResult(kind="unauthorized"),
)
loaded = load_bundle(root)
assert isinstance(loaded, LoadedBundle)
assert set(loaded.interactions) == {
slug_for_test("suite/test_a.py::test_one"),
slug_for_test("suite/test_b.py::test_two"),
}

485
tests/e2e/test_fixture_transport.py Normal file
View file

@ -0,0 +1,485 @@
"""Harness coverage for the record/replay transports (LIT-5729).
No proxy and no ``e2e`` marker. A fake in-memory ``Transport`` stands in for
the live one (dependency injection, no monkeypatching): recording must pass
every value through unchanged while writing one redacted interaction file per
call, and replay must serve identical values from the bundle alone - the
fake's call log proves nothing reaches the inner transport - failing hard
(``ReplayMiss``) on any drift in order, verb, or path. The collection-time
gate and report header are pinned here too, including the stale message that
names the bundle's age.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
from pydantic import BaseModel
from e2e_http import (
AuthHeaders,
BinaryStream,
ProbeResult,
Result,
StreamingResponse,
Success,
)
from fixture_bundle import (
BUNDLE_FORMAT_VERSION,
MANIFEST_FILENAME,
BundleRecorder,
Interaction,
LoadedBundle,
Manifest,
load_bundle,
prepare_bundle,
slug_for_test,
)
from fixture_transport import (
InvalidFixtureMode,
RecordingTransport,
ReplayMiss,
ReplaySource,
ReplayTransport,
current_test_key,
deterministic_marker,
fixture_mode_collection_error,
fixture_report_lines,
parse_fixture_mode,
replay_leftover_error,
select_transport,
)
from transport import Transport
NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc)
class Payload(BaseModel):
value: str
class Body(BaseModel):
prompt: str
class Query(BaseModel):
q: str
STREAMING = StreamingResponse(
status_code=200,
body="",
content_type="text/event-stream",
chunks=2,
stream_events=["one", "two"],
stream_done=True,
)
BINARY = BinaryStream(status_code=200, content_type="audio/mpeg", chunk_count=3, total_bytes=42)
PROBE = ProbeResult(status_code=200, body="alive")
@dataclass
class FakeTransport:
calls: list[str] = field(default_factory=list)
def bearer(self, key: str) -> AuthHeaders:
return AuthHeaders(authorization=f"Bearer {key}")
@property
def master(self) -> AuthHeaders:
return self.bearer("sk-fake-master")
def _success[R: BaseModel](self, response_type: type[R]) -> Result[R]:
return Success(status_code=200, data=response_type.model_validate({"value": "live"}))
def post[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
self.calls.append(f"post {path}")
return self._success(response_type)
def get[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
params: BaseModel,
response_type: type[R],
timeout: float | None = None,
) -> Result[R]:
self.calls.append(f"get {path}")
return self._success(response_type)
def delete[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
response_type: type[R],
params: BaseModel | None = None,
) -> Result[R]:
self.calls.append(f"delete {path}")
return self._success(response_type)
def patch[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
self.calls.append(f"patch {path}")
return self._success(response_type)
def put[R: BaseModel](
self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R]
) -> Result[R]:
self.calls.append(f"put {path}")
return self._success(response_type)
def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse:
self.calls.append(f"stream {path}")
return STREAMING
def stream_binary(
self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192
) -> BinaryStream:
self.calls.append(f"stream_binary {path}")
return BINARY
def send(
self,
path: str,
*,
headers: BaseModel,
json: BaseModel,
params: BaseModel | None = None,
stream: bool = False,
) -> StreamingResponse:
self.calls.append(f"send {path}")
return STREAMING
def probe(self, path: str, *, params: BaseModel) -> ProbeResult:
self.calls.append(f"probe {path}")
return PROBE
def upload[R: BaseModel](
self,
path: str,
*,
headers: BaseModel,
form: BaseModel,
filename: str,
content: bytes,
file_content_type: str = "application/jsonl",
file_field: str = "file",
params: BaseModel | None = None,
response_type: type[R],
) -> Result[R]:
self.calls.append(f"upload {path}")
return self._success(response_type)
def download(self, path: str, *, headers: BaseModel) -> StreamingResponse:
self.calls.append(f"download {path}")
return STREAMING
def make_recorder(root: Path) -> BundleRecorder:
recorder = prepare_bundle(root)
assert isinstance(recorder, BundleRecorder)
return recorder
def replay_source(root: Path) -> ReplaySource:
loaded = load_bundle(root)
assert isinstance(loaded, LoadedBundle)
return ReplaySource(bundle=loaded)
def this_tests_files(root: Path) -> list[Path]:
slug_dir = root / slug_for_test(current_test_key())
return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else []
def write_manifest(root: Path, recorded_at: datetime) -> None:
root.mkdir(parents=True, exist_ok=True)
manifest = Manifest(
format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234"
)
(root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8")
class TestParseFixtureMode:
@pytest.mark.parametrize(
("raw", "expected"),
[("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")],
)
def test_known_values_normalize(self, raw: str, expected: str) -> None:
assert parse_fixture_mode(raw) == expected
def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None:
assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached")
class TestDeterministicMarker:
def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None:
"""A replay process must regenerate exactly the markers the record
process generated, so the Nth marker of a test is pinned to a pure
function of the node id and N."""
key = current_test_key()
assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12]
assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12]
class TestCurrentTestKey:
def test_names_this_test_and_strips_the_phase(self) -> None:
key = current_test_key()
assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase")
assert "(call)" not in key
class TestRecordingTransport:
def test_passes_the_result_through_and_writes_one_file_per_call(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
result = recording.post(
"/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload
)
assert result == Success(status_code=200, data=Payload(value="live"))
assert fake.calls == ["post /model/new"]
files = this_tests_files(root)
assert [file.name for file in files] == ["0000-post-model-new.json"]
interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8"))
assert interaction.request.method == "post"
assert interaction.request.path == "/model/new"
def test_redacts_auth_header_values_in_the_recorded_request(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
headers = AuthHeaders.model_validate(
{"authorization": "Bearer sk-secret", "x-litellm-api-key": "sk-other"}
)
recording.post("/key/generate", headers=headers, json=Body(prompt="x"), response_type=Payload)
interaction = Interaction.model_validate_json(
this_tests_files(root)[0].read_text(encoding="utf-8")
)
assert interaction.request.headers == {
"authorization": "<redacted>",
"x-litellm-api-key": "<redacted>",
}
assert "sk-secret" not in this_tests_files(root)[0].read_text(encoding="utf-8")
def test_upload_records_a_content_digest_not_the_bytes(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.upload(
"/v1/files",
headers=fake.master,
form=Query(q="batch"),
filename="batch.jsonl",
content=b'{"custom_id": "1"}',
response_type=Payload,
)
interaction = Interaction.model_validate_json(
this_tests_files(root)[0].read_text(encoding="utf-8")
)
assert interaction.request.file_name == "batch.jsonl"
assert interaction.request.file_bytes == len(b'{"custom_id": "1"}')
assert interaction.request.file_sha256 is not None
assert "custom_id" not in interaction.request.model_dump_json()
class TestReplayTransport:
def test_serves_recorded_values_without_touching_the_inner_transport(
self, tmp_path: Path
) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recorded_post = recording.post(
"/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload
)
recorded_get = recording.get(
"/v1/models", headers=fake.master, params=Query(q="all"), response_type=Payload
)
recorded_stream = recording.stream(
"/chat/completions", headers=fake.master, json=Body(prompt="hi")
)
recorded_probe = recording.probe("/health/liveliness", params=Query(q="1"))
recorded_binary = recording.stream_binary(
"/v1/audio/speech", headers=fake.master, json=Body(prompt="say")
)
calls_after_record = list(fake.calls)
replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234")
assert (
replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload)
== recorded_post
)
assert (
replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload)
== recorded_get
)
assert (
replay.stream("/chat/completions", headers=replay.master, json=Body(prompt="hi"))
== recorded_stream
)
assert replay.probe("/health/liveliness", params=Query(q="1")) == recorded_probe
assert (
replay.stream_binary("/v1/audio/speech", headers=replay.master, json=Body(prompt="say"))
== recorded_binary
)
assert fake.calls == calls_after_record
def test_mismatched_call_names_recorded_and_actual(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload)
replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234")
with pytest.raises(ReplayMiss, match=r"recorded post /model/new, test made get /v1/models"):
replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload)
def test_exhausted_recording_names_the_call_count(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload)
replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234")
replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload)
with pytest.raises(ReplayMiss, match=r"call #2 \(post /model/new\) has no recorded interaction \(1 recorded"):
replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload)
class TestReplayLeftover:
def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload)
source = replay_source(root)
replay: Transport = ReplayTransport(source=source, master_key="sk-1234")
replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload)
assert source.leftover_error(current_test_key()) is None
def test_unconsumed_trailing_interactions_name_the_next_call(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload)
recording.probe("/health/liveliness", params=Query(q="1"))
source = replay_source(root)
replay: Transport = ReplayTransport(source=source, master_key="sk-1234")
replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload)
error = source.leftover_error(current_test_key())
assert error is not None
assert "1 of 2 recorded interactions never consumed" in error
assert "next is probe /health/liveliness" in error
assert "re-record with E2E_FIXTURE_MODE=record" in error
def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
make_recorder(root)
assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None
def test_inert_outside_replay_mode(self, tmp_path: Path) -> None:
missing = tmp_path / "missing"
assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None
assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None
def test_replay_mode_reads_the_shared_bundle(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root))
recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload)
error = replay_leftover_error(mode_raw="replay", bundle_dir=root, test_key=current_test_key())
assert error is not None
assert "1 of 1 recorded interactions never consumed" in error
class TestSelectTransport:
def test_live_returns_the_live_transport_untouched(self, tmp_path: Path) -> None:
fake = FakeTransport()
for mode_raw in ("live", ""):
assert (
select_transport(fake, mode_raw=mode_raw, bundle_dir=tmp_path / "b", master_key="sk")
is fake
)
def test_record_wraps_live_and_starts_a_fresh_bundle(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
write_manifest(root, NOW - timedelta(days=30))
(root / "old-test-slug").mkdir()
(root / "old-test-slug" / "0000-post-old.json").write_text("{}", encoding="utf-8")
selected = select_transport(fake, mode_raw="record", bundle_dir=root, master_key="sk")
assert isinstance(selected, RecordingTransport)
assert selected.inner is fake
assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME}
def test_replay_builds_a_transport_from_the_bundle_alone(self, tmp_path: Path) -> None:
fake = FakeTransport()
root = tmp_path / "bundle"
make_recorder(root)
selected = select_transport(fake, mode_raw="replay", bundle_dir=root, master_key="sk-master")
assert isinstance(selected, ReplayTransport)
assert selected.master == AuthHeaders(authorization="Bearer sk-master")
def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None:
with pytest.raises(ValueError, match="cached"):
select_transport(
FakeTransport(), mode_raw="cached", bundle_dir=tmp_path / "b", master_key="sk"
)
class TestCollectionGate:
def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None:
assert (
fixture_mode_collection_error("cached", tmp_path, now=NOW)
== "E2E_FIXTURE_MODE='cached' is not one of live, record, replay"
)
@pytest.mark.parametrize("mode_raw", ["live", "", "record"])
def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None:
assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None
def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None:
reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW)
assert reason is not None
assert f"no {MANIFEST_FILENAME}" in reason
assert "E2E_FIXTURE_MODE=record" in reason
def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, NOW - timedelta(days=9, hours=5))
reason = fixture_mode_collection_error("replay", root, now=NOW)
assert reason is not None
assert "age 9d5h exceeds the 7-day limit" in reason
assert "re-record with E2E_FIXTURE_MODE=record" in reason
def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
write_manifest(root, NOW - timedelta(days=2))
assert fixture_mode_collection_error("replay", root, now=NOW) is None
class TestReportHeader:
def test_live_mode_prints_nothing(self, tmp_path: Path) -> None:
assert fixture_report_lines("live", tmp_path, now=NOW) == []
assert fixture_report_lines("", tmp_path, now=NOW) == []
def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
recorded_at = NOW - timedelta(days=1)
write_manifest(root, recorded_at)
assert fixture_report_lines("record", root, now=NOW) == [
f"e2e fixture mode: record -> {root}"
]
replay_lines = fixture_report_lines("replay", root, now=NOW)
assert len(replay_lines) == 1
assert "replay" in replay_lines[0]
assert recorded_at.isoformat() in replay_lines[0]

74
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -1774,6 +1774,80 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params():
assert provider_cost == 0.00025
def test_perplexity_streaming_dict_cost_propagates_to_hidden_params():
"""
Regression: Perplexity reports usage.cost as a breakdown object, which used to
blow up the end of the stream with
`float() argument must be a string or a real number, not 'dict'`.
"""
import litellm
from litellm.cost_calculator import get_response_cost_from_hidden_params
chunks = [
ModelResponseStream(
id="chatcmpl-pplx",
created=1742056047,
model="perplexity/sonar",
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content="Hi", role="assistant"),
)
],
usage=None,
),
ModelResponseStream(
id="chatcmpl-pplx",
created=1742056048,
model="perplexity/sonar",
choices=[
StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=""))
],
usage=None,
),
ModelResponseStream(
id="chatcmpl-pplx",
created=1742056049,
model="perplexity/sonar",
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content=""))
],
usage=Usage(
completion_tokens=18,
prompt_tokens=12,
total_tokens=30,
cost={
"input_tokens_cost": 0.000012,
"output_tokens_cost": 0.000018,
"request_cost": 0.005,
"total_cost": 0.00503,
},
),
),
]
complete_response = litellm.stream_chunk_builder(
chunks=chunks, messages=[{"role": "user", "content": "test"}]
)
assert complete_response is not None
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response)
assert (
get_response_cost_from_hidden_params(complete_response._hidden_params)
== 0.00503
)
def test_provider_reported_cost_ignores_unusable_shapes():
assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None
assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None
assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None
assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5
def test_handle_special_delta_attributes(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):

69
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
View file

@ -4395,8 +4395,12 @@ class TestMCPServerManager:
captured: dict = {}
def fake_create_tool_function(path, method, operation, base_url, headers=None):
def fake_create_tool_function(
path, method, operation, base_url, headers=None, server_label=None, relays_upstream_auth=False
):
captured["headers"] = headers
captured["server_label"] = server_label
captured["relays_upstream_auth"] = relays_upstream_auth
async def tool_func(**kwargs):
return "ok"
@ -4425,6 +4429,11 @@ class TestMCPServerManager:
assert captured["headers"] is not None
assert captured["headers"]["Authorization"] == "STATIC token"
# The label names the server in an upstream-failure error, so registration must thread it;
# without this the fake would simply tolerate the argument and prove nothing about it.
assert captured["server_label"] == "openapi-server"
# auth_type is none here, so a 401 from this upstream must not be dressed up as a re-auth signal
assert captured["relays_upstream_auth"] is False
@pytest.mark.asyncio
async def test_pre_call_tool_check_allowed_tools_list_allows_tool(self):
@ -10765,3 +10774,61 @@ class TestResolveOpenapiToolAuth:
)
assert "Authorization" not in (forwarded or {})
class TestOpenApiHandlerRelaysUpstreamAuth:
"""`_call_openapi_tool_handler` must not flatten a re-auth signal into a generic message.
Its catch-all turned every exception into "Error calling OpenAPI tool ...", which is an isError
result but loses the status, so the REST surface could no longer relay a 401 with the upstream's
WWW-Authenticate and the streamable surface could not name the status the caller must act on.
"""
@staticmethod
def _server() -> MCPServer:
return MCPServer(
server_id="srv-openapi",
name="report_api",
server_name="report_api",
url="https://api.example.com",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth_delegate,
spec_path="https://api.example.com/openapi.json",
)
@pytest.mark.asyncio
async def test_upstream_auth_error_keeps_its_type(self):
from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry
manager = MCPServerManager()
server = self._server()
async def raising_handler(**_kwargs):
raise MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer realm=x", server_name="report_api")
tool = MagicMock()
tool.handler = raising_handler
with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool):
with pytest.raises(MCPUpstreamAuthError) as exc:
await manager._call_openapi_tool_handler(server, "list_reports", {})
assert exc.value.status_code == 401
assert exc.value.www_authenticate == "Bearer realm=x"
@pytest.mark.asyncio
async def test_other_failures_still_become_an_error_result(self):
from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry
manager = MCPServerManager()
async def raising_handler(**_kwargs):
raise RuntimeError("upstream returned HTTP 503")
tool = MagicMock()
tool.handler = raising_handler
with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool):
result = await manager._call_openapi_tool_handler(self._server(), "list_reports", {})
assert result.isError is True
assert "upstream returned HTTP 503" in result.content[0].text

125
tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py
View file

@ -27,12 +27,21 @@ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
resolve_operation_params,
)
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPOpenApiUpstreamError,
MCPUpstreamAuthError,
)
GET_ASYNC_CLIENT_TARGET = "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.get_async_httpx_client"
def _create_mock_client(method: str, response_text: str) -> AsyncMock:
"""Utility to create a mocked async httpx client for the given method."""
response = SimpleNamespace(text=response_text)
def _create_mock_client(method: str, response_text: str, status_code: int = 200) -> AsyncMock:
"""Utility to create a mocked async httpx client for the given method.
``status_code`` and ``headers`` are part of the real response the tool function reads, so the
fake carries them too; a fake that omits them cannot observe whether the status is checked.
"""
response = SimpleNamespace(text=response_text, status_code=status_code, headers={})
client = AsyncMock()
setattr(client, method, AsyncMock(return_value=response))
return client
@ -1259,3 +1268,113 @@ class TestRequestExtraHeaders:
headers_sent = async_client.get.call_args[1]["headers"]
assert "Authorization" not in headers_sent
class TestUpstreamStatusIsClassified:
"""A non-2xx upstream must never be returned as tool output.
The body used to be returned verbatim whatever the status, so an upstream rejection arrived as a
successful tool result and the request logged as a success. 401 is singled out because it is the
only status the caller can act on by re-authenticating, matching `_call_regular_mcp_tool` where a
403 deliberately does not produce a challenge.
"""
@staticmethod
def _tool(status_code: int, text: str = "body", headers: dict | None = None, relays_upstream_auth: bool = True):
response = SimpleNamespace(text=text, status_code=status_code, headers=headers or {})
client = AsyncMock()
client.get = AsyncMock(return_value=response)
return create_tool_function(
"/reports",
"get",
{"operationId": "list_reports"},
"https://api.example.com",
server_label="report_api",
relays_upstream_auth=relays_upstream_auth,
), client
@pytest.mark.asyncio
async def test_success_still_returns_the_body(self):
tool, client = self._tool(200, text='{"reports": []}')
with patch(GET_ASYNC_CLIENT_TARGET, return_value=client):
assert await tool() == '{"reports": []}'
@pytest.mark.asyncio
async def test_401_raises_the_reauth_signal_carrying_the_challenge(self):
tool, client = self._tool(401, text='{"error":"invalid_token"}', headers={"www-authenticate": 'Bearer realm="x"'})
with patch(GET_ASYNC_CLIENT_TARGET, return_value=client):
with pytest.raises(MCPUpstreamAuthError) as exc:
await tool()
assert exc.value.status_code == 401
assert exc.value.www_authenticate == 'Bearer realm="x"'
assert exc.value.server_name == "report_api"
@pytest.mark.asyncio
async def test_401_on_a_non_forwarding_server_is_not_a_reauth_signal(self):
"""Only the client-forwarded modes carry the caller's own upstream token, so only they can act
on a 401. `_call_regular_mcp_tool` gates its signal the same way, and without the gate an
api_key server rejecting a token would push clients into an OAuth flow that does not apply."""
tool, client = self._tool(401, text='{"error":"bad key"}', relays_upstream_auth=False)
with patch(GET_ASYNC_CLIENT_TARGET, return_value=client):
with pytest.raises(MCPOpenApiUpstreamError) as exc:
await tool()
assert exc.value.status_code == 401
@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"])
@pytest.mark.parametrize("status_code, expected", [(401, "auth"), (500, "other")])
@pytest.mark.asyncio
async def test_non_get_methods_are_classified_too(self, method: str, status_code: int, expected: str):
"""post/put/patch/delete call raise_for_status inside the HTTP handler.
Only `get` hands a 4xx back to the caller; the others raise `MaskedHTTPStatusError` before any
status check the tool function could do, so classifying the returned response alone would
leave every non-GET tool still serving an upstream error body as successful tool output. A
fake client that simply returns a response cannot observe this, which is why this test builds
the error the real handler raises.
"""
import httpx
from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError
request = httpx.Request(method.upper(), "https://api.example.com/reports")
raw = httpx.Response(
status_code,
headers={"www-authenticate": 'Bearer realm="x"'},
text="internal hostname db-prod-7.corp.example.com",
request=request,
)
masked = MaskedHTTPStatusError(httpx.HTTPStatusError("boom", request=request, response=raw))
client = AsyncMock()
setattr(client, method, AsyncMock(side_effect=masked))
tool = create_tool_function(
"/reports",
method,
{"operationId": "list_reports"},
"https://api.example.com",
server_label="report_api",
relays_upstream_auth=True,
)
expected_type = MCPUpstreamAuthError if expected == "auth" else MCPOpenApiUpstreamError
with patch(GET_ASYNC_CLIENT_TARGET, return_value=client):
with pytest.raises(expected_type) as exc:
await tool()
assert exc.value.status_code == status_code
assert "db-prod-7" not in str(exc.value)
@pytest.mark.parametrize("status_code", [403, 404, 429, 500, 503])
@pytest.mark.asyncio
async def test_other_failures_raise_without_leaking_the_upstream_body(self, status_code: int):
secret_body = "internal hostname db-prod-7.corp.example.com and a stack trace"
tool, client = self._tool(status_code, text=secret_body)
with patch(GET_ASYNC_CLIENT_TARGET, return_value=client):
with pytest.raises(MCPOpenApiUpstreamError) as exc:
await tool()
assert exc.value.status_code == status_code
assert secret_body not in str(exc.value)
assert str(exc.value) == f"upstream returned HTTP {status_code}"

77
tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py
View file

@ -652,3 +652,80 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc
assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN}
assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN
assert _request_auth_header.get() is None
@pytest.mark.parametrize("failure", ["auth", "other"])
@pytest.mark.asyncio
async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: str):
"""A failing local handler must never be reported as a successful tool result, and only an auth
failure may propagate.
`_handle_local_mcp_tool` used to catch every exception and return it as TextContent, and both of
its callers then stamped `isError=False`, so an upstream rejection was served as tool output and
`extract_mcp_tool_result_error_message` logged the request as a success.
The two kinds are split by consequence. `MCPUpstreamAuthError` propagates because both renderers
know it: the streamable path names the status and the REST path relays a real 401 with the
upstream's WWW-Authenticate. Anything else is reported as `isError=True` right here, because
`call_tool_rest_api` turns an unrecognized exception into HTTP 500 and an upstream 403 or 429 is
not a gateway crash.
"""
from litellm.proxy._experimental.mcp_server import server as mcp_module
from litellm.proxy._experimental.mcp_server.exceptions import (
MCPOpenApiUpstreamError,
MCPUpstreamAuthError,
)
error = (
MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name="report_api")
if failure == "auth"
else MCPOpenApiUpstreamError(429, "report_api")
)
async def raising_handler(**_kwargs):
raise error
fake_tool = MagicMock()
fake_tool.name = "list_reports"
fake_tool.handler = raising_handler
server = MCPServer(
server_id="srv-openapi",
name="report_api",
server_name="report_api",
url="https://api.example.com",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth_delegate,
spec_path="https://api.example.com/openapi.json",
)
user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value)
with (
patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server),
patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})),
patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool),
patch.object(
mcp_module.global_mcp_server_manager,
"resolve_openapi_upstream_auth",
new=AsyncMock(return_value=(None, None)),
),
patch(
"litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed",
return_value=True,
),
):
call = mcp_module.execute_mcp_tool(
name="list_reports",
arguments={},
allowed_mcp_servers=[server],
start_time=datetime.now(timezone.utc),
user_api_key_auth=user,
)
if failure == "auth":
with pytest.raises(MCPUpstreamAuthError):
await call
return
result = await call
# A non-auth upstream failure stays a 200 with isError, so REST does not report it as a gateway 500
assert result.isError is True
assert "upstream returned HTTP 429" in result.content[0].text

717
tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
View file

@ -4,6 +4,7 @@ Unit tests for auto router management endpoints
import os
import sys
from pathlib import Path
import pytest
from fastapi import HTTPException
@ -325,9 +326,7 @@ class TestAutoRouterBenchmarks:
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
totals = _benchmark_totals(self.ROW)
bucket_hits = (
totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits
)
bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits
assert bucket_hits == 27
assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1)
@ -490,7 +489,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import (
start_shadow_eval,
stop_shadow_eval_job,
)
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalJobResponse, StartShadowEvalRequest
from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest
VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer")
NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user")
@ -507,19 +506,23 @@ def _shadow_router() -> MagicMock:
return router
def _job_record(**overrides: object) -> MagicMock:
"""Spec'd like a real prisma row: only the table's columns exist as attributes, so
from_attributes validation falls back to model defaults for everything else."""
def _leg_record(**overrides: object) -> MagicMock:
"""Spec'd like a real prisma row: only the table's columns exist as attributes. One
row is one key's leg of a job; legs sharing group_id are one job."""
defaults = {
"id": "job-1",
"id": "leg-1",
"group_id": "job-1",
"api_key_id": "key-hash",
"router_name": "my-router",
"direction": "forward",
"baseline_model": None,
"judge_model": "anthropic/claude-sonnet-5",
"shadow_percentage": 10.0,
"max_turns": 200,
"created_at": datetime(2026, 8, 11, tzinfo=timezone.utc),
"ends_at": datetime.now(timezone.utc) + timedelta(days=7),
"stopped_at": None,
"stopped_by": None,
}
fields = {**defaults, **overrides}
record = MagicMock(spec=list(fields))
@ -538,23 +541,90 @@ def _key_record(
return record
def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock:
def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2")) -> MagicMock:
"""The job-table fake honours the filters it is handed, so a read that forgets
stopped_at sees rows the partial index would have released, one that forgets
direction sees the opposite-direction legs a key may hold at the same time, and a
group read that matched on a leg id would come back empty."""
prisma = MagicMock()
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=_key_record())
prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record()])
prisma.db.execute_raw = AsyncMock(return_value=0)
prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job)
prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None)
prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[])
prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=_job_record())
prisma.db.litellm_shadowevaljob.update = AsyncMock(
return_value=_job_record(stopped_at=datetime.now(timezone.utc))
)
prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys])
async def execute_raw(sql: str, *params: object):
if "SET stopped_by" in sql:
group = [row for row in stored if row.group_id == params[0]]
counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows}
sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group)
window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc)
claimable = [row for row in group if row.stopped_by is None]
if not (claimable and sampling and window_open):
return 0
for row in claimable:
row.stopped_by = params[1]
if row.stopped_at is None:
row.stopped_at = datetime.fromisoformat(str(params[2])).replace(tzinfo=timezone.utc)
return len(claimable)
return 0
prisma.db.execute_raw = AsyncMock(side_effect=execute_raw)
stored = legs if isinstance(legs, list) else list(legs)
async def find_many_legs(where=None, **_: object):
current = list(stored)
w = dict(where or {})
if "api_key_id" in w:
wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]]
current = [row for row in current if row.api_key_id in wanted]
if "direction" in w:
current = [row for row in current if row.direction == w["direction"]]
if "stopped_at" in w:
current = [row for row in current if row.stopped_at is w["stopped_at"]]
if "group_id" in w:
wanted = w["group_id"]["in"] if isinstance(w["group_id"], dict) else [w["group_id"]]
current = [row for row in current if row.group_id in wanted]
return current
def newest_groups(rows, limit):
latest: dict = {}
for row in rows:
if row.group_id not in latest or row.created_at > latest[row.group_id]:
latest[row.group_id] = row.created_at
ordered = sorted(latest, key=lambda group_id: latest[group_id], reverse=True)
return ordered[: int(limit)]
def leg_dict(row):
fields = (
"id",
"group_id",
"api_key_id",
"router_name",
"direction",
"baseline_model",
"judge_model",
"shadow_percentage",
"max_turns",
"created_at",
"ends_at",
"stopped_at",
"stopped_by",
)
return {field: getattr(row, field) for field in fields}
prisma.db.litellm_shadowevaljob.find_many = AsyncMock(side_effect=find_many_legs)
prisma.db.litellm_shadowevaljob.create_many = AsyncMock(return_value=1)
prisma.db.litellm_shadowevaljob.update_many = AsyncMock(return_value=1)
prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=None)
prisma.attempt_rows = []
async def query_raw(sql: str, *params: object):
if "AS attempt_count" in sql:
return prisma.attempt_rows
if "GROUP BY group_id" in sql:
scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]]
keep = set(newest_groups(scoped, params[0]))
return [leg_dict(row) for row in stored if row.group_id in keep]
if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql:
return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}]
if "SELECT job_id AS grp" in sql:
return by_leg_rows if by_leg_rows is not None else []
return agg_rows if agg_rows is not None else []
prisma.db.query_raw = AsyncMock(side_effect=query_raw)
@ -563,7 +633,7 @@ def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock:
def _start_request(**overrides: object) -> StartShadowEvalRequest:
payload = {
"api_key_id": "key-hash",
"api_key_ids": ("key-hash",),
"router_name": "my-router",
"shadow_percentage": 10.0,
"judge_model": "anthropic/claude-sonnet-5",
@ -575,44 +645,55 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest:
@pytest.mark.asyncio
async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones(monkeypatch: pytest.MonkeyPatch):
"""Expiry and turn-budget exhaustion both end sampling on their own; either must
release the key's slot in the active-job index so a new eval can start."""
async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeypatch: pytest.MonkeyPatch):
"""N keys become N sibling rows sharing group_id and identical config, written by a
single create_many so a unique-index loser rolls back the whole claim, and expiry or
budget exhaustion frees every requested key's slot first."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
response = await start_shadow_eval(_start_request(), ADMIN)
response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN)
assert response.status == "running"
assert response.max_turns == 200
assert response.judged_count is None
sweep_sql, sweep_key = prisma.db.execute_raw.call_args.args
sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args
assert "stopped_at IS NULL" in sweep_sql
assert "ends_at <= NOW()" in sweep_sql
assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql
assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql
assert ">= j.max_turns" in sweep_sql
assert sweep_key == "key-hash"
create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"]
assert create_data["api_key_id"] == "key-hash"
assert create_data["created_by"] == "admin"
assert "status" not in create_data
assert "j.api_key_id = ANY($1::text[])" in sweep_sql
assert sweep_keys == ["key-hash", "key-hash-2"]
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"]
assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1
assert len({row["group_id"] for row in rows}) == 1
assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows)
assert all("status" not in row and "id" not in row for row in rows)
assert response.job_id == rows[0]["group_id"]
assert response.status == "running"
assert response.judged_count is None
assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [
("key-hash", 200, "prod-alpha"),
("key-hash-2", 200, "prod-alpha"),
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"caller,request_overrides,active,expected_status",
"caller,request_overrides,claimed,expected_status",
[
(NON_ADMIN, {}, None, 403),
(VIEWER, {}, None, 403),
(ADMIN, {"router_name": "not-a-router"}, None, 400),
(ADMIN, {"judge_model": "not/a real model!"}, None, 400),
(ADMIN, {"judge_model": "my-router"}, None, 400),
(ADMIN, {}, "active", 409),
(ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, None, 400),
(ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, None, 400),
(ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, None, 400),
(NON_ADMIN, {}, (), 403),
(VIEWER, {}, (), 403),
(ADMIN, {"router_name": "not-a-router"}, (), 400),
(ADMIN, {"judge_model": "not/a real model!"}, (), 400),
(ADMIN, {"judge_model": "my-router"}, (), 400),
(ADMIN, {}, ("key-hash",), 409),
(ADMIN, {"api_key_ids": ("key-hash", "key-hash-2")}, ("key-hash-2",), 409),
(ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, (), 400),
(ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, (), 400),
(ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, (), 400),
],
ids=[
"non-admin",
@ -621,23 +702,143 @@ async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones
"unresolvable-judge",
"router-as-judge",
"already-active",
"one-of-several-keys-already-active",
"router-as-baseline",
"unresolvable-baseline",
"reverse-still-needs-an-auto-router",
],
)
async def test_start_shadow_eval_rejections(
monkeypatch: pytest.MonkeyPatch, caller, request_overrides, active, expected_status
monkeypatch: pytest.MonkeyPatch, caller, request_overrides, claimed, expected_status
):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(active_job=_job_record() if active else None)
prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(**request_overrides), caller)
assert exc.value.status_code == expected_status
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
@pytest.mark.asyncio
async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pytest.MonkeyPatch):
"""A key busy elsewhere blocks the whole start rather than being silently dropped from
it, and the 409 names which key and which job so the caller can stop or drop it."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN)
assert exc.value.status_code == 409
assert "key-hash-2 (job job-7)" in exc.value.detail
@pytest.mark.asyncio
async def test_start_shadow_eval_reuses_a_key_whose_previous_job_already_stopped(monkeypatch: pytest.MonkeyPatch):
"""The claim is held by unstopped legs only, matching the partial unique index. A read
that forgets that would strand every key that has ever finished a job."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(group_id="job-7", stopped_at=datetime.now(timezone.utc))])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
job = await start_shadow_eval(_start_request(), ADMIN)
assert job.status == "running"
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
@pytest.mark.asyncio
async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch):
"""The two directions ask opposite questions of the same key, so a forward job holding
the slot must not block a reverse one. The second reverse start still 409s."""
import litellm.proxy.proxy_server as proxy_server
legs = [_leg_record(group_id="job-fwd")]
prisma = _shadow_prisma(legs=legs)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o")
response = await start_shadow_eval(reverse, ADMIN)
assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o")
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert rows[0]["direction"] == "reverse"
assert rows[0]["baseline_model"] == "openai/gpt-4o"
legs.append(_leg_record(id="leg-2", group_id="job-rev", direction="reverse"))
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(reverse, ADMIN)
assert exc.value.status_code == 409
@pytest.mark.asyncio
async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
await start_shadow_eval(_start_request(), ADMIN)
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert rows[0]["direction"] == "forward"
assert rows[0]["baseline_model"] is None
@pytest.mark.asyncio
async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch):
"""A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every
unknown key is named at once, so a caller passing several fixes them in one round."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(known_keys=("key-hash",))
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(api_key_ids=("key-hash", "typo-a", "typo-b")), ADMIN)
assert exc.value.status_code == 400
assert "typo-a, typo-b" in exc.value.detail
assert "key-hash," not in exc.value.detail
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set():
"""A key named twice would collide with itself on the one-active-per-key index, a job
scoping no key samples nothing, and the key-count cap bounds every downstream read."""
assert _start_request(api_key_ids=("a", "b", "a")).api_key_ids == ("a", "b")
assert len(_start_request(api_key_ids=tuple(f"k{i}" for i in range(100))).api_key_ids) == 100
with pytest.raises(ValidationError):
_start_request(api_key_ids=())
with pytest.raises(ValidationError):
_start_request(api_key_ids=tuple(f"k{i}" for i in range(101)))
@pytest.mark.asyncio
async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
from prisma.errors import UniqueViolationError
prisma = _shadow_prisma()
prisma.db.litellm_shadowevaljob.create_many = AsyncMock(
side_effect=UniqueViolationError(MagicMock(message="unique constraint"))
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(), ADMIN)
assert exc.value.status_code == 409
@pytest.mark.parametrize(
@ -657,97 +858,25 @@ def test_start_request_pins_baseline_model_to_reverse(overrides):
@pytest.mark.asyncio
async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch):
"""The two directions ask opposite questions of the same key, so a forward job holding
the slot must not block a reverse one. The second reverse start still 409s."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
active = {"forward": _job_record()}
prisma.db.litellm_shadowevaljob.find_first = AsyncMock(
side_effect=lambda where, **_: active.get(str(where.get("direction")))
)
prisma.db.litellm_shadowevaljob.create = AsyncMock(
return_value=_job_record(direction="reverse", baseline_model="openai/gpt-4o")
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o")
response = await start_shadow_eval(reverse, ADMIN)
assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o")
create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"]
assert create_data["direction"] == "reverse"
assert create_data["baseline_model"] == "openai/gpt-4o"
active["reverse"] = _job_record(id="job-2", direction="reverse")
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(reverse, ADMIN)
assert exc.value.status_code == 409
@pytest.mark.asyncio
async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
await start_shadow_eval(_start_request(), ADMIN)
create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"]
assert create_data["direction"] == "forward"
assert create_data["baseline_model"] is None
@pytest.mark.asyncio
async def test_start_shadow_eval_rejects_a_key_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch):
"""A typo'd api_key_id would otherwise create a job no traffic can ever match."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(), ADMIN)
assert exc.value.status_code == 400
assert "not a key on this proxy" in exc.value.detail
@pytest.mark.asyncio
async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
from prisma.errors import UniqueViolationError
prisma = _shadow_prisma()
prisma.db.litellm_shadowevaljob.create = AsyncMock(
side_effect=UniqueViolationError(MagicMock(message="unique constraint"))
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval(_start_request(), ADMIN)
assert exc.value.status_code == 409
@pytest.mark.asyncio
async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(monkeypatch: pytest.MonkeyPatch):
async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monkeypatch: pytest.MonkeyPatch):
"""One read answers for every leg: totals and stratifications aggregate over the
group's leg ids, and the by-key slice maps each leg id back to its key hash."""
import litellm.proxy.proxy_server as proxy_server
tier_rows = [
{"grp": "SIMPLE", "turn_count": 8, "real_wins": 2, "shadow_wins": 4, "ties": 2, "avg_confidence": 0.8},
{"grp": "REASONING", "turn_count": 2, "real_wins": 2, "shadow_wins": 0, "ties": 0, "avg_confidence": 0.9},
]
prisma = _shadow_prisma(agg_rows=tier_rows)
prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record())
prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(
return_value=MagicMock(error="judge call failed: boom")
leg_rows = [
{"grp": "leg-1", "turn_count": 6, "real_wins": 1, "shadow_wins": 4, "ties": 1, "avg_confidence": 0.7},
{"grp": "leg-2", "turn_count": 4, "real_wins": 3, "shadow_wins": 0, "ties": 1, "avg_confidence": 0.6},
]
prisma = _shadow_prisma(
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)],
agg_rows=tier_rows,
by_leg_rows=leg_rows,
)
prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=MagicMock(error="judge call failed: boom"))
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
response = await get_shadow_eval_job("job-1", VIEWER)
@ -762,6 +891,13 @@ async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(m
assert response.results.by_tier[0].shadow_win_rate_pct == 50.0
assert response.results.overall_shadow_win_rate_pct == 40.0
assert response.results.overall_tie_rate_pct == 20.0
assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)]
assert response.results.by_key[0].shadow_win_rate_pct == 66.7
assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)]
totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]]
assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])]
error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"]
assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"}
@pytest.mark.asyncio
@ -780,79 +916,326 @@ async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.Mo
@pytest.mark.asyncio
async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(monkeypatch: pytest.MonkeyPatch):
async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monkeypatch: pytest.MonkeyPatch):
"""A job over two keys is one list entry with both keys, not two entries, and a job
whose keys all stopped reads stopped while a half-stopped one still runs."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
prisma.db.litellm_shadowevaljob.find_many = AsyncMock(
return_value=[
_job_record(),
_job_record(id="job-2", ends_at=datetime.now(timezone.utc) - timedelta(days=1)),
_job_record(id="job-3", stopped_at=datetime.now(timezone.utc)),
stamp = datetime.now(timezone.utc)
prisma = _shadow_prisma(
legs=[
_leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)),
_leg_record(
id="leg-2",
api_key_id="key-hash-2",
stopped_at=stamp,
created_at=datetime(2026, 8, 13, tzinfo=timezone.utc),
),
_leg_record(
id="leg-3",
group_id="job-2",
stopped_at=stamp,
created_at=datetime(2026, 8, 12, tzinfo=timezone.utc),
),
_leg_record(
id="leg-4",
group_id="job-3",
ends_at=datetime.now(timezone.utc) - timedelta(days=1),
created_at=datetime(2026, 8, 11, tzinfo=timezone.utc),
),
]
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert [job.status for job in jobs] == ["running", "completed", "stopped"]
swept = ShadowEvalJobResponse.model_validate(
_job_record(
id="job-4",
ends_at=datetime.now(timezone.utc) - timedelta(days=1),
stopped_at=datetime.now(timezone.utc),
),
from_attributes=True,
)
assert swept.status == "completed"
assert [(job.job_id, job.status) for job in jobs] == [
("job-1", "running"),
("job-2", "stopped"),
("job-3", "completed"),
]
assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"]
assert all(job.judged_count is None and job.results is None for job in jobs)
assert prisma.db.query_raw.await_count == 0
legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args
assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql
assert legs_limit == 50
counts_sql, _ = prisma.db.query_raw.await_args_list[1].args
assert "AS attempt_count" in counts_sql
assert "j.stopped_at IS NULL OR a.created_at <= j.stopped_at" in counts_sql
assert prisma.db.query_raw.await_count == 2
prisma.db.litellm_shadowevaljob.find_many.assert_not_called()
@pytest.mark.asyncio
async def test_shadow_eval_responses_name_the_shadowed_key(monkeypatch: pytest.MonkeyPatch):
async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypatch: pytest.MonkeyPatch):
"""The filter matches a key anywhere in a job's key set and still returns the whole
job, sibling keys included."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
prisma.db.litellm_shadowevaljob.find_many = AsyncMock(
return_value=[_job_record(), _job_record(id="job-2", api_key_id="deleted-key-hash")]
prisma = _shadow_prisma(
legs=[
_leg_record(),
_leg_record(id="leg-2", api_key_id="key-hash-2"),
_leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"),
_leg_record(id="leg-4", group_id="job-3"),
]
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50)
assert [job.job_id for job in jobs] == ["job-1", "job-2"]
assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"]
@pytest.mark.parametrize(
("stopped_flags", "days_left", "expected"),
[
((False, False), 7, "running"),
((True, False), 7, "running"),
((True, True), 7, "stopped"),
((True, True), -1, "completed"),
((False, False), -1, "completed"),
],
)
@pytest.mark.asyncio
async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stopped(
monkeypatch: pytest.MonkeyPatch, stopped_flags: tuple[bool, ...], days_left: int, expected: str
):
import litellm.proxy.proxy_server as proxy_server
stamp = datetime.now(timezone.utc)
prisma = _shadow_prisma(
legs=[
_leg_record(
id=f"leg-{index}",
api_key_id=f"key-{index}",
stopped_at=stamp if stopped else None,
ends_at=datetime.now(timezone.utc) + timedelta(days=days_left),
)
for index, stopped in enumerate(stopped_flags)
]
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert [job.status for job in jobs] == [expected]
@pytest.mark.asyncio
async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch: pytest.MonkeyPatch):
"""A job whose keys all exhausted their turn budgets stopped sampling on its own, so
it must read completed on the very next list, before any sweep stamps its legs; one
key under budget keeps the whole job running. An operator starting an unrelated eval
must never look like it terminated a finished one."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(
legs=[
_leg_record(max_turns=5),
_leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5),
_leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5),
_leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5),
]
)
prisma.attempt_rows = [
{"job_id": "leg-1", "attempt_count": 5},
{"job_id": "leg-2", "attempt_count": 6},
{"job_id": "leg-3", "attempt_count": 5},
{"job_id": "leg-4", "attempt_count": 3},
]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
by_id = {job.job_id: job for job in jobs}
assert by_id["job-1"].status == "completed"
assert all(key.stopped_at is None for key in by_id["job-1"].keys)
assert by_id["job-2"].status == "running"
assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3}
@pytest.mark.asyncio
async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: pytest.MonkeyPatch):
"""A detached attempt can land around the stop and push the raw count past the
budget; the recorded stopped_by must keep the job reading stopped regardless."""
import litellm.proxy.proxy_server as proxy_server
stamp = datetime.now(timezone.utc)
prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert jobs[0].status == "stopped"
assert jobs[0].stopped_by == "admin"
detail = await get_shadow_eval_job("job-1", VIEWER)
assert detail.status == "stopped"
@pytest.mark.asyncio
async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pytest.MonkeyPatch):
"""Jobs stopped before stopped_by existed are backfilled with 'unknown' by the
migration, so even one whose stray attempts crossed the budget stays stopped."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(
legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")]
)
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert jobs[0].status == "stopped"
def test_stopped_by_migration_backfills_every_job_that_displayed_stopped():
"""The migration must close the pre-column population: without the backfill, a
legacy stop whose stray attempts crossed the budget would read completed."""
import litellm_proxy_extras
sql = (
Path(litellm_proxy_extras.__file__).parent
/ "migrations"
/ "20260818224500_add_shadow_eval_stopped_by"
/ "migration.sql"
).read_text()
assert 'ADD COLUMN "stopped_by" TEXT' in sql
assert "SET stopped_by = 'unknown'" in sql
assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql
@pytest.mark.asyncio
async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exhausted:
await stop_shadow_eval_job("job-1", ADMIN)
assert exhausted.value.status_code == 400
assert "completed" in exhausted.value.detail
prisma.db.litellm_shadowevaljob.update_many.assert_not_called()
@pytest.mark.asyncio
async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")],
known_keys=("key-hash", "key-hash-2"),
)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
started = await start_shadow_eval(_start_request(), ADMIN)
assert (started.key_alias, started.key_name) == ("prod-alpha", "sk-...lpha")
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert [(job.key_alias, job.key_name) for job in jobs] == [("prod-alpha", "sk-...lpha"), (None, None)]
assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [
(None, None),
("prod-alpha", "sk-...lpha"),
]
batched_where = prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"]
assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}}
prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record())
detail = await get_shadow_eval_job("job-1", VIEWER)
assert detail.key_alias == "prod-alpha"
assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"]
@pytest.mark.asyncio
async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch):
async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_running(
monkeypatch: pytest.MonkeyPatch,
):
"""One stop ends sampling for the whole job, while a leg that already stopped on its
own budget keeps the stopped_at it earned."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma()
prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record())
earned = datetime.now(timezone.utc) - timedelta(hours=1)
prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
stopped = await stop_shadow_eval_job("job-1", ADMIN)
assert stopped.status == "stopped"
update = prisma.db.litellm_shadowevaljob.update.call_args.kwargs
assert set(update["data"]) == {"stopped_at"}
prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(
return_value=_job_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1))
)
assert stopped.status == "stopped"
assert stopped.stopped_by == "admin"
stop_sql, stop_group, stop_operator, stop_stamp = prisma.db.execute_raw.call_args.args
assert "SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)" in stop_sql
assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql
assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql
assert ") < k.max_turns" in stop_sql
assert (stop_group, stop_operator) == ("job-1", "admin")
assert datetime.fromisoformat(stop_stamp).tzinfo is None
assert prisma.db.execute_raw.await_count == 1
prisma.db.litellm_shadowevaljob.update_many.assert_not_called()
by_key = {key.api_key_id: key.stopped_at for key in stopped.keys}
assert by_key["key-hash-2"] == earned
assert by_key["key-hash"] is not None and by_key["key-hash"] != earned
done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1))
prisma_done = _shadow_prisma(legs=[done_leg])
monkeypatch.setattr(proxy_server, "prisma_client", prisma_done)
with pytest.raises(HTTPException) as exc:
await stop_shadow_eval_job("job-1", ADMIN)
assert exc.value.status_code == 400
assert "already completed" in exc.value.detail
assert done_leg.stopped_by is None
with pytest.raises(HTTPException) as forbidden:
await stop_shadow_eval_job("job-1", VIEWER)
assert forbidden.value.status_code == 403
def test_every_shadow_eval_sql_constant_speaks_naive_utc():
"""The tables store naive UTC wall time (prisma's convention), so SQL-side time must be
NOW() AT TIME ZONE 'utc' and python-side params must cast ::timestamp; a bare NOW() or a
timestamptz cast writes session-local wall time into the naive column and skews every
comparison against prisma-written stamps."""
import litellm.proxy.management_endpoints.auto_router_endpoints as module
sql_constants = {name: value for name, value in vars(module).items() if name.endswith("_SQL")}
assert sql_constants
for name, sql in sql_constants.items():
assert "::timestamptz" not in sql, name
for occurrence in sql.split("NOW()")[1:]:
assert occurrence.startswith(" AT TIME ZONE 'utc'"), name
@pytest.mark.asyncio
async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_stopped(
monkeypatch: pytest.MonkeyPatch,
):
"""The statement claims the job only while a leg still samples, so a stop landing in
the same instant the budget spends records nothing and the job keeps reading
completed; stamping it would misreport a self-ended job as operator-stopped forever."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exc:
await stop_shadow_eval_job("job-1", ADMIN)
assert exc.value.status_code == 400
assert "already completed" in exc.value.detail
assert prisma.db.litellm_shadowevaljob.find_many.await_args.kwargs["where"] == {"group_id": "job-1"}
@pytest.mark.asyncio
async def test_two_racing_stops_produce_exactly_one_winner(monkeypatch: pytest.MonkeyPatch):
"""The statement's stopped_by IS NULL predicate lets only one racer claim rows; the
loser reads the stamped state and gets the same answer a late caller gets."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record()])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
first = await stop_shadow_eval_job("job-1", ADMIN)
assert first.status == "stopped"
with pytest.raises(HTTPException) as exc:
await stop_shadow_eval_job("job-1", ADMIN)
assert exc.value.status_code == 400
assert "already stopped" in exc.value.detail

50
tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
View file

@ -22,6 +22,7 @@ from litellm.proxy._types import (
NewUserRequest,
LiteLLM_BudgetTable,
LiteLLM_OrganizationTable,
LiteLLM_ProjectTableCachedObj,
LiteLLM_TeamTableCachedObj,
LiteLLM_UserTable,
LiteLLM_VerificationToken,
@ -31,9 +32,12 @@ from litellm.proxy._types import (
ResetSpendRequest,
UpdateKeyRequest,
)
from litellm.proxy.auth.auth_checks import _project_cache_key
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.management_endpoints.key_management_endpoints import (
_check_org_key_limits,
_check_project_key_limits,
_check_team_key_limits,
_common_key_generation_helper,
_enforce_upperbound_key_params,
@ -16444,3 +16448,49 @@ async def test_regenerate_key_repoints_live_membership_not_the_key_row_it_read(
assert await _authorized_models_for_key(
access_groups, new_token_hash, ["ag-revoked-since", "ag-attached-since"]
) == ["attached-model"]
async def _cache_with_project(project_id: str, project_models: list[str]) -> UserApiKeyCache:
user_api_key_cache = UserApiKeyCache()
await user_api_key_cache.async_set_cache(
key=_project_cache_key(project_id),
value=LiteLLM_ProjectTableCachedObj(project_id=project_id, team_id="team-lit-5823", models=project_models),
model_type=LiteLLM_ProjectTableCachedObj,
)
return user_api_key_cache
@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest])
@pytest.mark.parametrize("sentinel", ["all-team-models", "all-proxy-models"])
@pytest.mark.asyncio
async def test_check_project_key_limits_accepts_inherited_model_sentinels(request_cls, sentinel):
"""LIT-5823: the sentinels inherit a parent scope, so a project allowlist must not treat them as model names."""
user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"])
await _check_project_key_limits(
project_id="proj-lit-5823",
data=request_cls(key="sk-lit-5823", models=[sentinel]),
prisma_client=MagicMock(),
user_api_key_cache=user_api_key_cache,
)
@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest])
@pytest.mark.parametrize(
"key_models",
[["gpt-5.4-mini"], ["all-team-models", "gpt-5.4-mini"], ["gpt-5.4-nano", "all-proxy-models", "gpt-5.4-mini"]],
)
@pytest.mark.asyncio
async def test_check_project_key_limits_still_rejects_real_model_outside_project(request_cls, key_models):
user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"])
with pytest.raises(HTTPException) as exc_info:
await _check_project_key_limits(
project_id="proj-lit-5823",
data=request_cls(key="sk-lit-5823", models=key_models),
prisma_client=MagicMock(),
user_api_key_cache=user_api_key_cache,
)
assert exc_info.value.status_code == 400
assert "Model 'gpt-5.4-mini' not in project's allowed models" in exc_info.value.detail["error"]

177
tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py
View file

@ -1,5 +1,6 @@
"""Tests for the per-model PTU flat-cost daily rollup."""
import json
import types
from datetime import date, datetime, timedelta, timezone
from unittest.mock import AsyncMock, MagicMock
@ -373,6 +374,182 @@ def test_parse_ptu_model_rejects_an_inverted_window():
assert parsed is None
def _router_entry(model_id="dep-1", model_name="gpt-4o-ptu", model_info=None, with_start=True):
"""A deployment as the router stores one: a plain dict whose id lives in model_info."""
info = dict(model_info or {})
if (
with_start
and info.get("ptu_count") is not None
and info.get("cost_per_ptu_per_hour") is not None
and "ptu_effective_from" not in info
):
info["ptu_effective_from"] = _DEFAULT_PTU_START
if model_id is not None:
info["id"] = model_id
return {
"model_name": model_name,
"litellm_params": {"model": "azure/gpt-4o"},
"model_info": info,
}
# A team-scoped deployment is stored under a synthetic routing key with the operator's name
# in team_public_model_name, while the same deployment in config.yaml carries the operator's
# name directly. Both must resolve to the same PTUModel or the two sources bill differently.
_PARITY_CASES = (
(
"an iso string start against the datetime pydantic coerces it to",
{"ptu_effective_from": "2026-07-30T23:00:00Z"},
{"ptu_effective_from": datetime(2026, 7, 30, 23, 0)},
datetime(2026, 7, 30, 23, 0, tzinfo=timezone.utc),
None,
),
(
"an open window, stored as null and dropped by exclude_none",
{"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": None},
{"ptu_effective_from": datetime(2026, 7, 1, 0, 0)},
datetime(2026, 7, 1, tzinfo=timezone.utc),
None,
),
(
"a closed window",
{"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": "2026-07-31T00:00:00Z"},
{
"ptu_effective_from": datetime(2026, 7, 1, 0, 0),
"ptu_effective_to": datetime(2026, 7, 31, 0, 0),
},
datetime(2026, 7, 1, tzinfo=timezone.utc),
datetime(2026, 7, 31, tzinfo=timezone.utc),
),
)
@pytest.mark.parametrize(
"db_window, router_window, expected_from, expected_to",
[case[1:] for case in _PARITY_CASES],
ids=[case[0] for case in _PARITY_CASES],
)
def test_a_deployment_parses_identically_from_the_db_and_from_config_yaml(
db_window, router_window, expected_from, expected_to
):
# The contract the router union is built on. If these ever diverge, a PTU deployment
# declared in config.yaml is billed differently from the identical one in the database.
from_db = _parse_ptu_model(
_model_row(
model_id="dep-1",
model_name="model_name_t_9f3c2b",
model_info={**_VALID_PTU, **db_window, "team_public_model_name": "gpt-4o-ptu"},
with_start=False,
)
)
from_router = _parse_ptu_model(
ptu_rollup._router_deployment(
_router_entry(model_id="dep-1", model_info={**_VALID_PTU, **router_window}, with_start=False)
)
)
expected = PTUModel(
model_id="dep-1",
model_name="gpt-4o-ptu",
team_id="t",
ptu_count=5,
cost_per_ptu_per_hour=2.0,
effective_from=expected_from,
effective_to=expected_to,
)
assert from_db == from_router == expected
@pytest.mark.parametrize("model_id", [None, "", 12345], ids=["absent", "blank", "not a string"])
def test_a_router_deployment_without_a_usable_id_is_dropped(model_id):
# model_id keys the sentinel row, so an unusable one would file every such deployment
# in a team onto one row and bill for a single reservation.
entry = _router_entry(model_id=None, model_info=dict(_VALID_PTU))
if model_id is not None:
entry["model_info"]["id"] = model_id
assert ptu_rollup._router_deployment(entry) is None
def test_a_router_deployment_keeps_the_name_the_operator_wrote():
priced = _parse_ptu_model(
ptu_rollup._router_deployment(_router_entry(model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU)))
)
assert priced is not None
assert priced.model_name == "gpt-4o-ptu"
def test_a_team_alias_on_a_router_deployment_still_wins_over_the_routing_name():
# config.yaml can carry team_public_model_name to expose a team-facing alias, and the
# charge has to file under the name that team calls rather than the routing one.
priced = _parse_ptu_model(
ptu_rollup._router_deployment(
_router_entry(
model_name="routing-name",
model_info={**_VALID_PTU, "team_public_model_name": "public-alias"},
)
)
)
assert priced is not None
assert priced.model_name == "public-alias"
def test_a_router_deployment_with_a_stringified_model_info_still_decodes():
entry = _router_entry(model_info=dict(_VALID_PTU))
entry["model_info"] = json.dumps(entry["model_info"])
parsed = _parse_ptu_model(ptu_rollup._router_deployment(entry))
assert parsed is not None
assert parsed.model_id == "dep-1"
@pytest.mark.parametrize(
"model_info",
[None, "not json", 42, "[1, 2, 3]", '"a string"', "42", "true", "null"],
ids=[
"absent",
"unparseable",
"not a string or dict",
"json array",
"json string",
"json number",
"json bool",
"json null",
],
)
def test_a_router_deployment_without_usable_model_info_is_dropped(model_info):
# Valid JSON that is not an object decodes to a list or a scalar, and reading fields
# off one raises rather than dropping the single bad deployment the rollup expects
assert ptu_rollup._router_deployment({"model_name": "x", "model_info": model_info}) is None
@pytest.mark.parametrize(
"model_info",
["[1, 2, 3]", '"a string"', "42", "true", "null"],
ids=["json array", "json string", "json number", "json bool", "json null"],
)
def test_a_db_row_holding_non_object_json_is_dropped(model_info):
assert _parse_ptu_model(_model_row(model_info=model_info, with_start=False)) is None
def test_a_router_deployment_does_not_alias_the_routers_own_model_info():
# The rollup runs on a cron while requests are in flight, and the router rewrites
# model_info in place, so a held record must neither observe nor cause those writes.
live = {"id": "dep-1", **_VALID_PTU}
record = ptu_rollup._router_deployment({"model_name": "x", "model_info": live})
assert record is not None
live["ptu_count"] = 999
assert record.model_info["ptu_count"] == _VALID_PTU["ptu_count"]
with pytest.raises(TypeError):
record.model_info["ptu_count"] = 1
def test_a_raw_router_dict_is_not_a_deployment_record():
# The parser reads attributes, so a router dict passed straight to it returns None
# instead of raising. Skipping the factory would silently drop every config.yaml
# deployment while leaving the suite green.
assert _parse_ptu_model(_router_entry(model_info=dict(_VALID_PTU))) is None
@pytest.mark.asyncio
async def test_rollup_returns_empty_when_prisma_client_is_none():
result = await run_ptu_flat_cost_rollup(None, target_date=DAY)

18
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -267,6 +267,24 @@ class TestReasoningMarkerScoring:
# 2+ reasoning markers should force REASONING tier
assert tier == ComplexityTier.REASONING
def test_reasoning_override_does_not_rescue_a_simple_score(self, complexity_router):
"""Reasoning markers on an otherwise trivial prompt must not reach REASONING."""
prompt = "hi, step by step, pros and cons"
tier, score, signals = complexity_router.classify(prompt)
assert score < complexity_router.config.tier_boundaries["simple_medium"]
assert any("step by step" in s and "pros and cons" in s for s in signals)
assert tier == ComplexityTier.SIMPLE
def test_reasoning_override_applies_at_the_simple_medium_boundary(self, complexity_router):
"""A score sitting exactly on simple_medium is not SIMPLE, so the override still promotes it."""
prompt = (
"Give me the pros and cons, step by step, of moving our checkout service "
"to an event-driven architecture."
)
tier, score, signals = complexity_router.classify(prompt)
assert score == complexity_router.config.tier_boundaries["simple_medium"]
assert tier == ComplexityTier.REASONING
def test_system_prompt_reasoning_not_counted(self, complexity_router):
"""Reasoning markers in system prompt should not count for override."""
user_prompt = "What is 2+2?"

18
ui/litellm-dashboard/eslint-suppressions.json
View file

@ -1404,11 +1404,6 @@
"count": 2
}
},
"src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": {
"no-nested-ternary": {
"count": 2
@ -1453,9 +1448,6 @@
"local/no-complex-jsx-arrow": {
"count": 1
},
"no-restricted-imports": {
"count": 1
},
"react-hooks/set-state-in-effect": {
"count": 2
}
@ -1470,11 +1462,6 @@
"count": 1
}
},
"src/app/onboarding/OnboardingFormBody.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/AIHub/ModelHubTable.test.tsx": {
"max-params": {
"count": 1
@ -1998,11 +1985,6 @@
"count": 1
}
},
"src/components/common_components/PassThroughGuardrailsSection.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/components/common_components/RateLimitTypeFormItem.test.tsx": {
"no-restricted-imports": {
"count": 1

157
ui/litellm-dashboard/package-lock.json generated
View file

@ -32,9 +32,9 @@
"openapi-fetch": "^0.17.0",
"openapi-react-query": "^0.5.4",
"papaparse": "5.5.3",
"react": "18.3.1",
"react": "19.2.8",
"react-copy-to-clipboard": "5.1.1",
"react-dom": "18.3.1",
"react-dom": "19.2.8",
"react-hook-form": "7.82.0",
"react-json-view-lite": "2.5.0",
"react-markdown": "9.1.0",
@ -55,9 +55,9 @@
"@testing-library/react": "16.3.2",
"@testing-library/user-event": "14.6.1",
"@types/node": "20.19.37",
"@types/react": "18.2.48",
"@types/react": "19.2.18",
"@types/react-copy-to-clipboard": "5.0.7",
"@types/react-dom": "18.3.7",
"@types/react-dom": "19.2.4",
"@types/react-syntax-highlighter": "15.5.13",
"@vitest/coverage-v8": "3.2.6",
"@vitest/ui": "3.2.6",
@ -200,9 +200,9 @@
}
},
"node_modules/@ant-design/icons-svg": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.4.2.tgz",
"integrity": "sha512-vHbT+zJEVzllwP+CM+ul7reTEfBR0vgxFe7+lREAsAA7YGsYpboiq2sQNeQeRvh09GfQgs/GyFEvZpJ9cLXpXA==",
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.5.0.tgz",
"integrity": "sha512-1BTUFyKPTBZ53MuTP8s0k5SFEXL7o3VHEOwLgzaoWKwnBeqIcqUtVshc4SKzhI6uACfqhJqBwBUE9FsWR3uULA==",
"license": "MIT"
},
"node_modules/@ant-design/react-slick": {
@ -606,19 +606,6 @@
}
}
},
"node_modules/@base-ui/react/node_modules/@floating-ui/react-dom": {
"version": "2.1.8",
"resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz",
"integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==",
"license": "MIT",
"dependencies": {
"@floating-ui/dom": "^1.7.6"
},
"peerDependencies": {
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
}
},
"node_modules/@base-ui/utils": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/@base-ui/utils/-/utils-0.3.1.tgz",
@ -1441,28 +1428,41 @@
}
},
"node_modules/@floating-ui/core": {
"version": "1.7.5",
"resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz",
"integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==",
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz",
"integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==",
"license": "MIT",
"dependencies": {
"@floating-ui/utils": "^0.2.11"
"@floating-ui/utils": "^0.2.12"
}
},
"node_modules/@floating-ui/dom": {
"version": "1.7.6",
"resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz",
"integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==",
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz",
"integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==",
"license": "MIT",
"dependencies": {
"@floating-ui/core": "^1.7.5",
"@floating-ui/utils": "^0.2.11"
"@floating-ui/core": "^1.8.0",
"@floating-ui/utils": "^0.2.12"
}
},
"node_modules/@floating-ui/react-dom": {
"version": "2.1.9",
"resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz",
"integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==",
"license": "MIT",
"dependencies": {
"@floating-ui/dom": "^1.8.0"
},
"peerDependencies": {
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
}
},
"node_modules/@floating-ui/utils": {
"version": "0.2.11",
"resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz",
"integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==",
"version": "0.2.12",
"resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz",
"integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==",
"license": "MIT"
},
"node_modules/@headlessui/tailwindcss": {
@ -2627,9 +2627,9 @@
"license": "MIT"
},
"node_modules/@rc-component/async-validator": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.0.tgz",
"integrity": "sha512-n4HcR5siNUXRX23nDizbZBQPO0ZM/5oTtmKZ6/eqL0L2bo747cklFdZGRN2f+c9qWGICwDzrhW0H7tE9PptdcA==",
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.2.tgz",
"integrity": "sha512-WYbrZSjzznU1ekD0qFq2qRxt309VoS61MTG5npnFQlKYcoy9IzU8T+ZCIhq5bGAXRbXysABFWTspicMfmWFwow==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.24.4"
@ -2669,9 +2669,9 @@
}
},
"node_modules/@rc-component/mini-decimal": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.3.tgz",
"integrity": "sha512-bk/FJ09fLf+NLODMAFll6CfYrHPBioTedhW6lxDBuuWucJEqFUd4l/D/5JgIi3dina6sYahB8iuPAZTNz2pMxw==",
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.4.tgz",
"integrity": "sha512-xiuXcaCwyOWpD8a8scdExFl+bntNphAW8XeenL1ig2en0AAZY0Pcp4pC0dI22qJ+NvxKn9RoNIoRdqYU3BLH4w==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.18.0"
@ -2717,9 +2717,9 @@
}
},
"node_modules/@rc-component/qrcode": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.1.tgz",
"integrity": "sha512-LfLGNymzKdUPjXUbRP+xOhIWY4jQ+YMj5MmWAcgcAq1Ij8XP7tRmAXqyuv96XvLUBE/5cA8hLFl9eO1JQMujrA==",
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.3.tgz",
"integrity": "sha512-aGv6alnn4HbDEsURzKP+jv13rbi1VxmAYfBNZr5GKF1iohMNWy5tAVoJ1E3cOvzMB1kbUPvCXchM6zSFlRGPhA==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.24.7"
@ -3661,12 +3661,12 @@
}
},
"node_modules/@tanstack/react-store": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.0.tgz",
"integrity": "sha512-tX4YXh3PDkmpvGQWkWqKpzs/MSqbtuwY9dWdWhtV9Q50PmO+jOkUKIWIX4G85dwt7lxdHLXsiaEKPdKmC8F41w==",
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.1.tgz",
"integrity": "sha512-HaIGKI3YLmjBYIvy5DFDY23oNaYZIsTZfngey07Uh5iLVJgM3bIGCnZeOFOqzjFld9JHWcaHJnasD/bKoGKwJQ==",
"license": "MIT",
"dependencies": {
"@tanstack/store": "0.11.0",
"@tanstack/store": "0.11.1",
"use-sync-external-store": "^1.6.0"
},
"funding": {
@ -3699,9 +3699,9 @@
}
},
"node_modules/@tanstack/store": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.0.tgz",
"integrity": "sha512-WlzzCt3xi0G6pCAJu1U+2jiECwabETDpQDi3hfkFZvJii9AuZqEKbOiVarX1/bWhTNjU486yQtJCCasi/0q+Cw==",
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.1.tgz",
"integrity": "sha512-mzTOBhypOuDJAy/D8n2MfUZ1HFkXnmSETviRyhqEC8LUE7/IZQExOTxMANj3KjTofYTkFNpBY67qaVrT41YccA==",
"license": "MIT",
"funding": {
"type": "github",
@ -3999,21 +3999,13 @@
"@types/node": "*"
}
},
"node_modules/@types/prop-types": {
"version": "15.7.15",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz",
"integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==",
"license": "MIT"
},
"node_modules/@types/react": {
"version": "18.2.48",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz",
"integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==",
"version": "19.2.18",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz",
"integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==",
"license": "MIT",
"dependencies": {
"@types/prop-types": "*",
"@types/scheduler": "*",
"csstype": "^3.0.2"
"csstype": "^3.2.2"
}
},
"node_modules/@types/react-copy-to-clipboard": {
@ -4027,13 +4019,13 @@
}
},
"node_modules/@types/react-dom": {
"version": "18.3.7",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz",
"integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==",
"version": "19.2.4",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz",
"integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"@types/react": "^18.0.0"
"@types/react": "^19.2.0"
}
},
"node_modules/@types/react-syntax-highlighter": {
@ -4046,12 +4038,6 @@
"@types/react": "*"
}
},
"node_modules/@types/scheduler": {
"version": "0.26.0",
"resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz",
"integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==",
"license": "MIT"
},
"node_modules/@types/unist": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@ -11565,13 +11551,10 @@
}
},
"node_modules/react": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
"integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz",
"integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==",
"license": "MIT",
"dependencies": {
"loose-envify": "^1.1.0"
},
"engines": {
"node": ">=0.10.0"
}
@ -11590,16 +11573,15 @@
}
},
"node_modules/react-dom": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
"integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
"version": "19.2.8",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz",
"integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==",
"license": "MIT",
"dependencies": {
"loose-envify": "^1.1.0",
"scheduler": "^0.23.2"
"scheduler": "^0.27.0"
},
"peerDependencies": {
"react": "^18.3.1"
"react": "^19.2.8"
}
},
"node_modules/react-hook-form": {
@ -12204,13 +12186,10 @@
}
},
"node_modules/scheduler": {
"version": "0.23.2",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
"integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==",
"license": "MIT",
"dependencies": {
"loose-envify": "^1.1.0"
}
"version": "0.27.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz",
"integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==",
"license": "MIT"
},
"node_modules/scroll-into-view-if-needed": {
"version": "3.1.0",

13
ui/litellm-dashboard/package.json
View file

@ -9,8 +9,11 @@
"start": "next start",
"lint": "eslint .",
"test": "vitest",
"test:unit": "vitest run --project unit",
"test:component": "vitest run --project component",
"test:integration": "vitest run --project integration",
"test:dot": "vitest --reporter=dot",
"test:types": "vitest --run --typecheck.only",
"test:types": "vitest run --project types",
"test:watch": "vitest -w",
"test:coverage": "vitest run --coverage",
"format": "prettier --write .",
@ -45,9 +48,9 @@
"openapi-fetch": "^0.17.0",
"openapi-react-query": "^0.5.4",
"papaparse": "5.5.3",
"react": "18.3.1",
"react": "19.2.8",
"react-copy-to-clipboard": "5.1.1",
"react-dom": "18.3.1",
"react-dom": "19.2.8",
"react-hook-form": "7.82.0",
"react-json-view-lite": "2.5.0",
"react-markdown": "9.1.0",
@ -68,9 +71,9 @@
"@testing-library/react": "16.3.2",
"@testing-library/user-event": "14.6.1",
"@types/node": "20.19.37",
"@types/react": "18.2.48",
"@types/react": "19.2.18",
"@types/react-copy-to-clipboard": "5.0.7",
"@types/react-dom": "18.3.7",
"@types/react-dom": "19.2.4",
"@types/react-syntax-highlighter": "15.5.13",
"@vitest/coverage-v8": "3.2.6",
"@vitest/ui": "3.2.6",

18
ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx
View file

@ -7,8 +7,8 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { Card } from "@/components/ui/card";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { Alert as AntdAlert, Modal, Space, Tabs, Typography } from "antd";
import { Info } from "lucide-react";
import { Modal, Space, Tabs, Typography } from "antd";
import { Info, TriangleAlert } from "lucide-react";
import React, { useEffect, useState } from "react";
import NewBadge from "@/components/common_components/NewBadge";
import { useBaseUrl } from "@/components/constants";
@ -223,12 +223,14 @@ const AdminPanel: React.FC<AdminPanelProps> = ({ proxySettings }) => {
<>
<Card className="block p-6">
<Title level={4}> ✨ Security Settings</Title>
<AntdAlert
message="SSO Configuration Deprecated"
description="Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the SSO Settings tab for SSO configuration."
type="warning"
showIcon
/>
<Alert variant="warning">
<TriangleAlert />
<AlertTitle>SSO Configuration Deprecated</AlertTitle>
<AlertDescription>
Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the
SSO Settings tab for SSO configuration.
</AlertDescription>
</Alert>
<div
style={{
display: "flex",

43
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
View file

@ -77,7 +77,15 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
baseline_model: null,
judge_model: "anthropic/claude-sonnet-5",
shadow_percentage: 10,
max_turns: 200,
keys: [
{
api_key_id: "hashed-key-abc",
max_turns: 200,
stopped_at: null,
key_alias: "prod-alpha",
key_name: "sk-...alpha",
},
],
judged_count: 42,
error_count: 1,
judge_spend: 3.21,
@ -110,19 +118,28 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
avg_judge_confidence: 0.8,
},
],
by_key: [],
overall_shadow_win_rate_pct: 48.0,
overall_tie_rate_pct: 22.0,
},
created_at: "2026-08-07T00:00:00Z",
ends_at: "2026-09-07T00:00:00Z",
stopped_at: null,
api_key_id: "hashed-key-abc",
key_alias: "prod-alpha",
key_name: "sk-...alpha",
last_error: null,
...overrides,
});
const keyEntry = (
api_key_id: string,
overrides: Partial<ShadowEvalJob["keys"][number]> = {},
): ShadowEvalJob["keys"][number] => ({
api_key_id,
max_turns: 200,
stopped_at: null,
key_alias: null,
key_name: null,
...overrides,
});
const mockHooks = ({
jobs = [],
detailsById = {},
@ -199,8 +216,8 @@ describe("ShadowEvalSection", () => {
it("gives every active job its own card with a stop button, with the form still offered", () => {
mockHooks({
jobs: [
job({ job_id: "job-a", status: "running", api_key_id: "key-a" }),
job({ job_id: "job-b", status: "running", api_key_id: "key-b" }),
job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }),
job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }),
],
});
render(<ShadowEvalSection />);
@ -342,7 +359,7 @@ describe("ShadowEvalSection", () => {
expect(container).toBeEmptyDOMElement();
});
it("keeps the start button disabled until key, router, and judge model are picked, then submits them", async () => {
it("keeps the start button disabled until key, router, and judge model are picked, then submits the key as a list", async () => {
const user = userEvent.setup();
const { start } = mockHooks({});
render(<ShadowEvalSection />);
@ -361,7 +378,7 @@ describe("ShadowEvalSection", () => {
await user.click(screen.getByText("Start shadow eval"));
const expectedBody = {
api_key_id: "hash-alpha",
api_key_ids: ["hash-alpha"],
router_name: "gpt-auto",
direction: "forward",
shadow_percentage: 10,
@ -396,7 +413,7 @@ describe("ShadowEvalSection", () => {
await user.click(screen.getByText("Start shadow eval"));
const expectedBody = {
api_key_id: "hash-alpha",
api_key_ids: ["hash-alpha"],
router_name: "gpt-auto",
direction: "reverse",
baseline_model: "prod-claude",
@ -429,9 +446,9 @@ describe("ShadowEvalSection", () => {
});
it("labels the shadowed key by alias, then masked name, then truncated hash", () => {
expect(shadowedKeyLabel(job())).toBe("prod-alpha");
expect(shadowedKeyLabel(job({ key_alias: null }))).toBe("sk-...alpha");
expect(shadowedKeyLabel(job({ key_alias: null, key_name: null }))).toBe("hashed-key…");
expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha");
expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha");
expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…");
});
it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {

26
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
View file

@ -24,6 +24,7 @@ import {
useStartShadowEval,
useStopShadowEval,
type ShadowEvalJob,
type ShadowEvalJobKey,
type ShadowEvalSlice,
} from "./useShadowEval";
@ -50,19 +51,24 @@ const routerMatchedOrBeatPct = (
? 100 - results.overall_shadow_win_rate_pct
: results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct;
export const shadowedKeyLabel = (job: ShadowEvalJob): string =>
job.key_alias || job.key_name || `${job.api_key_id.slice(0, 10)}…`;
export const shadowedKeyLabel = (key: ShadowEvalJobKey): string =>
key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`;
const shadowedKeysLabel = (job: ShadowEvalJob): string =>
job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`;
const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0);
const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
job.direction === "reverse" ? (
<>
Comparing <span className="font-mono text-xs">{job.router_name}</span> to{" "}
<span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of{" "}
<span className="font-mono text-xs">{shadowedKeyLabel(job)}</span> traffic
<span className="font-mono text-xs">{shadowedKeysLabel(job)}</span> traffic
</>
) : (
<>
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedKeyLabel(job)}</span> traffic
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedKeysLabel(job)}</span> traffic
via <span className="font-mono text-xs">{job.router_name}</span>
</>
);
@ -178,7 +184,8 @@ const emptyResultsText = (job: ShadowEvalJob, resultsError: boolean): string =>
const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ job, resultsError = false }) => {
const results = job.results;
if (!results || (results.by_tier.length === 0 && results.by_current_model.length === 0)) {
const stratifications = results ? [results.by_tier, results.by_current_model, results.by_key] : [];
if (!results || stratifications.every((slices) => slices.length === 0)) {
return <p className="px-6 py-8 text-center text-sm text-muted-foreground">{emptyResultsText(job, resultsError)}</p>;
}
return (
@ -224,7 +231,7 @@ const JobResults: React.FC<{
<div>
<p className="text-sm font-medium text-foreground">{jobHeadline(job)}</p>
<p className="text-xs text-muted-foreground">
{(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "}
{(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "}
{(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend
{active && remaining ? ` · ${remaining}` : ""}
</p>
@ -386,14 +393,13 @@ const StartForm: React.FC = () => {
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
const parsedMaxTurns = Number.parseInt(maxTurns, 10);
const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000;
const filled =
[apiKeyId, routerName, judgeModel].every((field) => field !== "") &&
(direction === "forward" || baselineModel !== "");
const baselinePicked = direction === "forward" || baselineModel !== "";
const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "") && baselinePicked;
const boundsValid = percentageValid && maxTurnsValid;
const valid = Boolean(accessToken) && filled && boundsValid;
const handleStart = () => {
const startBody = {
api_key_id: apiKeyId,
api_key_ids: [apiKeyId],
router_name: routerName,
direction,
...(direction === "reverse" ? { baseline_model: baselineModel } : {}),

1
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts
View file

@ -7,6 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api";
import type { components } from "@/lib/http/schema";
export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"];
export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"];
export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"];
export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"];

2
ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { exportMultiToPDF, exportMultiToCSV } from "./multi_export_utils";
import type { MultiModelResult } from "./types";

20
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
View file

@ -1,5 +1,7 @@
import React, { useEffect } from "react";
import { Alert, Select, Tooltip, Collapse, Input, Space, Switch } from "antd";
import { Select, Tooltip, Collapse, Input, Space, Switch } from "antd";
import { TriangleAlert } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
@ -271,13 +273,15 @@ const MCPPermissionManagement: React.FC<MCPPermissionManagementProps> = ({
)}
{showInternalDelegatePkceWarning && (
<Alert
type="warning"
showIcon
className="mb-2"
message="Internal server with upstream OAuth delegation"
description="This MCP server is configured as internal-only but delegates auth to upstream. Anonymous users will be able to reach the upstream OAuth2 /authorize flow without a LiteLLM session. Ensure your upstream provider and network enforce access controls."
/>
<Alert variant="warning" className="mb-2">
<TriangleAlert />
<AlertTitle>Internal server with upstream OAuth delegation</AlertTitle>
<AlertDescription>
This MCP server is configured as internal-only but delegates auth to upstream. Anonymous users will be
able to reach the upstream OAuth2 /authorize flow without a LiteLLM session. Ensure your upstream
provider and network enforce access controls.
</AlertDescription>
</Alert>
)}
<MountedFormField

14
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/UserEnvVarsModal.tsx
View file

@ -1,5 +1,7 @@
import React from "react";
import { Modal, Alert, Spin, Tag, Typography } from "antd";
import { Modal, Spin, Tag, Typography } from "antd";
import { CircleAlert, Info } from "lucide-react";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { useMutation, useQuery } from "@tanstack/react-query";
import { z } from "zod/v4";
import { MCPServer, MCPUserEnvVarsStatus, MCPUserEnvVarSpec } from "@/components/mcp_tools/types";
@ -158,9 +160,15 @@ const UserEnvVarsModal: React.FC<UserEnvVarsModalProps> = ({ server, open, acces
<Spin />
</div>
) : isError ? (
<Alert type="error" showIcon message="Failed to load env vars" />
<Alert variant="error">
<CircleAlert />
<AlertTitle>Failed to load env vars</AlertTitle>
</Alert>
) : required.length === 0 ? (
<Alert type="info" showIcon message="No per-user fields configured for this server." />
<Alert variant="info">
<Info />
<AlertTitle>No per-user fields configured for this server.</AlertTitle>
</Alert>
) : (
<>
<Text className="text-sm text-muted-foreground block">

2
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { beforeEach, describe, expect, it, vi } from "vitest";
import { setSecureItem } from "@/utils/secureStorage";
import { CreateUiSnapshot, readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState";

50
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx
View file

@ -1,10 +1,22 @@
/* eslint-disable react/no-unescaped-entities */
import React, { useState } from "react";
import { Card, Typography, Space, Alert, Switch } from "antd";
import { Card, Typography, Space, Switch } from "antd";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { CopyIcon, Code, Terminal, Globe, CheckIcon, ExternalLinkIcon, KeyIcon, ServerIcon, Zap } from "lucide-react";
import {
CopyIcon,
Code,
Terminal,
Globe,
CheckIcon,
ExternalLinkIcon,
Info,
KeyIcon,
ServerIcon,
Zap,
} from "lucide-react";
import { getProxyBaseUrl } from "@/components/networking";
import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils";
@ -69,25 +81,21 @@ const FeatureCard: React.FC<FeatureCardProps> = ({
</Text>
</div>
{useServerHeader && (
<Alert
className="mt-2"
type="info"
showIcon
message="Two Options"
description={
<div>
<p>
<strong>Option 1:</strong> Get a specific server: <code>"{serverName.replace(/\s+/g, "_")}"</code>
</p>
<p>
<strong>Option 2:</strong> Get a group of MCPs: <code>"dev-group"</code>
</p>
<p className="mt-2 text-sm text-muted-foreground">
You can also mix both: <code>"Server1,dev-group"</code>
</p>
</div>
}
/>
<Alert className="mt-2" variant="info">
<Info />
<AlertTitle>Two Options</AlertTitle>
<AlertDescription>
<p>
<strong>Option 1:</strong> Get a specific server: <code>"{serverName.replace(/\s+/g, "_")}"</code>
</p>
<p>
<strong>Option 2:</strong> Get a group of MCPs: <code>"dev-group"</code>
</p>
<p className="mt-2 text-sm text-muted-foreground">
You can also mix both: <code>"Server1,dev-group"</code>
</p>
</AlertDescription>
</Alert>
)}
</div>
)}

18
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx
View file

@ -1939,6 +1939,24 @@ describe("MCPServerEdit OAuth flow prefill display", () => {
expect(screen.queryByText("This server has no OAuth flow set")).not.toBeInTheDocument();
});
it("never flashes the warning while mounting a server that already has a flow", async () => {
const flashes: Node[] = [];
const observer = new MutationObserver((records) => {
for (const record of records) {
for (const node of record.addedNodes) {
if (node.textContent?.includes("This server has no OAuth flow set")) flashes.push(node);
}
}
});
observer.observe(document.body, { childList: true, subtree: true });
renderEdit({ oauth2_flow: "client_credentials" });
await screen.findByText("Machine-to-Machine (M2M)");
observer.disconnect();
expect(flashes).toHaveLength(0);
});
it("does not warn for a delegate (PKCE passthrough) server even with no flow set", () => {
renderEdit({ oauth2_flow: null, delegate_auth_to_upstream: true });

25
ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx
View file

@ -1,5 +1,7 @@
import React, { useState, useEffect } from "react";
import { Select, Tooltip, Input, InputNumber, Alert } from "antd";
import { Select, Tooltip, Input, InputNumber } from "antd";
import { TriangleAlert } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { FormProvider, useForm } from "react-hook-form";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Button } from "@/components/ui/button";
@ -171,7 +173,10 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
const isTokenExchangeAuthType = authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE;
const isIdJagAuthType = authType === AUTH_TYPE.OAUTH2_ID_JAG;
const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4;
const oauthFlowTypeValue = mountedValues.oauth_flow_type as string | undefined;
// Same fallback as the delegate switch below: the value is undefined until the field mounts, so
// reading it alone flashes the "no OAuth flow set" warning at a server that already has one.
const oauthFlowTypeValue =
(mountedValues.oauth_flow_type as string | undefined) ?? oauth2FlowToFormValue(mcpServer.oauth2_flow);
const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M;
// Watch reflects a live toggle when the delegate switch is mounted; fall back to
// the stored value otherwise (useWatch returns undefined for an unmounted field,
@ -1041,13 +1046,15 @@ const MCPServerEdit: React.FC<MCPServerEditProps> = ({
{!isStdioTransport && isOAuthAuthType && (
<>
{!oauthFlowTypeValue && !isDelegateAuth && (
<Alert
type="warning"
showIcon
className="mb-4 rounded-lg"
message="This server has no OAuth flow set"
description="Choose Machine-to-Machine (M2M) or Interactive (PKCE) so LiteLLM authenticates it the way you intend, then save. Until it is set, LiteLLM falls back to interactive per-user auth and treats a machine-to-machine credential shape conservatively."
/>
<Alert variant="warning" className="mb-4 rounded-lg">
<TriangleAlert />
<AlertTitle>This server has no OAuth flow set</AlertTitle>
<AlertDescription>
Choose Machine-to-Machine (M2M) or Interactive (PKCE) so LiteLLM authenticates it the way you
intend, then save. Until it is set, LiteLLM falls back to interactive per-user auth and treats
a machine-to-machine credential shape conservatively.
</AlertDescription>
</Alert>
)}
<OAuthFormFields
isM2M={isM2MFlow}

60
ui/litellm-dashboard/src/app/(dashboard)/policies/_components/add_policy_form.tsx
View file

@ -1,5 +1,5 @@
import React, { useState, useEffect } from "react";
import { Modal, Alert, Tag } from "antd";
import { Modal, Tag } from "antd";
import { z } from "zod/v4";
import { Policy, PolicyCreateRequest, PolicyUpdateRequest } from "@/components/policies/types";
import { Guardrail } from "@/components/guardrails/types";
@ -19,7 +19,8 @@ import { Textarea } from "@/components/ui/textarea";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import { useZodForm } from "@/lib/forms/useZodForm";
import { CircleHelp } from "lucide-react";
import { CircleHelp, Info } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
interface AddPolicyFormProps {
visible: boolean;
@ -340,10 +341,13 @@ const AddPolicyForm: React.FC<AddPolicyFormProps> = ({
{selectedMode === "flow_builder" && (
<Alert
message="You'll be redirected to the full-screen Flow Builder to design your policy logic visually."
type="info"
variant="info"
className="mt-4 border border-indigo-200 bg-indigo-50 dark:border-indigo-800 dark:bg-indigo-950"
/>
>
<AlertTitle>
You&apos;ll be redirected to the full-screen Flow Builder to design your policy logic visually.
</AlertTitle>
</Alert>
)}
<div className="mt-6 flex justify-end gap-2">
@ -457,35 +461,33 @@ const AddPolicyForm: React.FC<AddPolicyFormProps> = ({
</FormField>
{resolvedGuardrails.length > 0 && (
<Alert
message="Resolved Guardrails"
description={
<div>
<span className="mb-2 block text-muted-foreground">
These are the final guardrails that will be applied (including inheritance):
</span>
<div className="flex flex-wrap gap-1">
{resolvedGuardrails.map((g) => (
<Tag key={g} color="blue">
{g}
</Tag>
))}
</div>
<Alert variant="info">
<Info />
<AlertTitle>Resolved Guardrails</AlertTitle>
<AlertDescription>
<span className="mb-2 block text-muted-foreground">
These are the final guardrails that will be applied (including inheritance):
</span>
<div className="flex flex-wrap gap-1">
{resolvedGuardrails.map((g) => (
<Tag key={g} color="blue">
{g}
</Tag>
))}
</div>
}
type="info"
showIcon
/>
</AlertDescription>
</Alert>
)}
<SectionHeading label="Conditions (Optional)" />
<Alert
message="Model Scope"
description="By default, this policy will run on all models. You can optionally restrict it to specific models below."
type="info"
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>Model Scope</AlertTitle>
<AlertDescription>
By default, this policy will run on all models. You can optionally restrict it to specific models below.
</AlertDescription>
</Alert>
<div role="group" className="flex w-full flex-col gap-3">
<span className="text-sm leading-snug font-medium text-foreground">Model Condition Type</span>

10
ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx
View file

@ -1,6 +1,8 @@
import React, { useState, useEffect } from "react";
import { useForm } from "react-hook-form";
import { Alert, Empty } from "antd";
import { Empty } from "antd";
import { CircleAlert } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { resolvePoliciesCall, teamListCall, keyListCall, modelAvailableCall } from "@/components/networking";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { FieldGroup } from "@/components/shared/form/field";
@ -325,7 +327,11 @@ const PolicyTestPanel: React.FC<PolicyTestPanelProps> = ({ accessToken }) => {
)}
{hasSearched && !result && !isLoading && (
<Alert message="Error" description="Failed to resolve policies. Check the proxy logs." type="error" showIcon />
<Alert variant="error">
<CircleAlert />
<AlertTitle>Error</AlertTitle>
<AlertDescription>Failed to resolve policies. Check the proxy logs.</AlertDescription>
</Alert>
)}
</div>
);

2
ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/conversation_panel/MessageList.tsx
View file

@ -8,7 +8,7 @@ interface MessageListProps {
messages: Message[];
isLoading: boolean;
hasVariables: boolean;
messagesEndRef: React.RefObject<HTMLDivElement>;
messagesEndRef: React.RefObject<HTMLDivElement | null>;
}
const MessageList: React.FC<MessageListProps> = ({ messages, isLoading, hasVariables, messagesEndRef }) => {

5
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageViewSelect/UsageViewSelect.test.tsx
View file

@ -1,6 +1,7 @@
import { render, screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { chooseSelectOption } from "@/../tests/test-utils";
import { UsageViewSelect } from "./UsageViewSelect";
const openMenu = async (user: ReturnType<typeof userEvent.setup>) => {
@ -35,9 +36,7 @@ describe("UsageViewSelect", () => {
const user = userEvent.setup();
render(<UsageViewSelect value="global" onChange={mockOnChange} userRole="Admin" />);
await openMenu(user);
const matches = screen.getAllByText("Team Usage");
await user.click(matches[matches.length - 1]);
await chooseSelectOption(user, screen.getByRole("combobox"), /^Team Usage/);
expect(mockOnChange).toHaveBeenCalled();
expect(mockOnChange.mock.calls[0][0]).toBe("team");

38
ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx
View file

@ -1,9 +1,10 @@
import React, { useState } from "react";
import { Upload, Alert } from "antd";
import { Upload } from "antd";
import { toast } from "@/lib/toast";
import { InboxOutlined } from "@ant-design/icons";
import type { UploadProps } from "antd";
import { CircleHelp } from "lucide-react";
import { CircleCheck, CircleHelp, X } from "lucide-react";
import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { ragIngestCall } from "@/components/networking";
import { DocumentUpload, RAGIngestResponse } from "@/components/vector_store_management/types";
import DocumentsTable from "./DocumentsTable";
@ -359,22 +360,23 @@ const CreateVectorStore: React.FC<CreateVectorStoreProps> = ({ accessToken, onSu
{/* Success Message */}
{ingestResults.length > 0 && (
<Alert
message="Vector Store Created Successfully"
description={
<div>
<p>
<strong>Vector Store ID:</strong> {ingestResults[0]?.vector_store_id}
</p>
<p>
<strong>Documents Ingested:</strong> {ingestResults.length}
</p>
</div>
}
type="success"
showIcon
closable
/>
<Alert>
<CircleCheck />
<AlertTitle>Vector Store Created Successfully</AlertTitle>
<AlertDescription>
<p>
<strong>Vector Store ID:</strong> {ingestResults[0]?.vector_store_id}
</p>
<p>
<strong>Documents Ingested:</strong> {ingestResults.length}
</p>
</AlertDescription>
<AlertAction>
<Button variant="ghost" size="icon-sm" aria-label="Close" onClick={() => setIngestResults([])}>
<X />
</Button>
</AlertAction>
</Alert>
)}
</div>
</TooltipProvider>

52
ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx
View file

@ -1,6 +1,6 @@
import React, { useState, useEffect } from "react";
import { Alert } from "antd";
import { CircleHelp } from "lucide-react";
import { CircleHelp, Info } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
import { Field, FieldError, FieldLabel } from "@/components/shared/form/field";
import {
@ -72,32 +72,28 @@ const S3VectorsConfig: React.FC<S3VectorsConfigProps> = ({ accessToken, provider
return (
<TooltipProvider>
<Alert
message="AWS S3 Vectors Setup"
description={
<div>
<p>AWS S3 Vectors allows you to store and query vector embeddings directly in S3:</p>
<ul style={{ marginLeft: "16px", marginTop: "8px" }}>
<li>Vector buckets and indexes will be automatically created if they don&apos;t exist</li>
<li>Vector dimensions are auto-detected from your selected embedding model</li>
<li>Ensure your AWS credentials have permissions for S3 Vectors operations</li>
<li>
Learn more:{" "}
<a
href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vector-buckets.html"
target="_blank"
rel="noopener noreferrer"
>
AWS S3 Vectors Documentation
</a>
</li>
</ul>
</div>
}
type="info"
showIcon
style={{ marginBottom: "16px" }}
/>
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>AWS S3 Vectors Setup</AlertTitle>
<AlertDescription>
<p>AWS S3 Vectors allows you to store and query vector embeddings directly in S3:</p>
<ul style={{ marginLeft: "16px", marginTop: "8px" }}>
<li>Vector buckets and indexes will be automatically created if they don&apos;t exist</li>
<li>Vector dimensions are auto-detected from your selected embedding model</li>
<li>Ensure your AWS credentials have permissions for S3 Vectors operations</li>
<li>
Learn more:{" "}
<a
href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-vector-buckets.html"
target="_blank"
rel="noopener noreferrer"
>
AWS S3 Vectors Documentation
</a>
</li>
</ul>
</AlertDescription>
</Alert>
<Field data-invalid={bucketNameError !== undefined || undefined}>
<FieldLabel htmlFor="s3-vector-bucket-name">

246
ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
View file

@ -1,6 +1,7 @@
import React, { useState, useEffect } from "react";
import { Modal, Alert } from "antd";
import { CircleHelp, Eye, EyeOff } from "lucide-react";
import { Modal } from "antd";
import { CircleHelp, Eye, EyeOff, Info } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { useWatch } from "react-hook-form";
import { z } from "zod/v4";
import { CredentialItem, vectorStoreCreateCall } from "@/components/networking";
@ -310,142 +311,129 @@ const VectorStoreForm: React.FC<VectorStoreFormProps> = ({
</FormField>
{selectedProvider === "pg_vector" && (
<Alert
message="PG Vector Setup Required"
description={
<div>
<p>LiteLLM provides a server to connect to PG Vector. To use this provider:</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Deploy the litellm-pgvector server from:{" "}
<a href="https://github.com/BerriAI/litellm-pgvector" target="_blank" rel="noopener noreferrer">
https://github.com/BerriAI/litellm-pgvector
</a>
</li>
<li>Configure your PostgreSQL database with pgvector extension</li>
<li>Start the server and note the API base URL and API key</li>
<li>Enter those details in the fields below</li>
</ol>
</div>
}
type="info"
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>PG Vector Setup Required</AlertTitle>
<AlertDescription>
<p>LiteLLM provides a server to connect to PG Vector. To use this provider:</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Deploy the litellm-pgvector server from:{" "}
<a href="https://github.com/BerriAI/litellm-pgvector" target="_blank" rel="noopener noreferrer">
https://github.com/BerriAI/litellm-pgvector
</a>
</li>
<li>Configure your PostgreSQL database with pgvector extension</li>
<li>Start the server and note the API base URL and API key</li>
<li>Enter those details in the fields below</li>
</ol>
</AlertDescription>
</Alert>
)}
{selectedProvider === "valkey" && (
<Alert
message="Valkey Setup Required"
description={
<div>
<p>
LiteLLM searches documents you have already stored in Valkey. It does not create the index or
upload documents for you. Before creating this vector store, make sure:
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Your Valkey server has vector search enabled (the valkey-search module, included in the
valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey)
</li>
<li>
You have already created a search index and loaded your documents and their embeddings into it.
Enter that index name as the Vector Store ID
</li>
<li>
You know which embedding model created those stored embeddings. That model must be added to this
proxy under Models so you can pick it below. Using a different model returns wrong results
</li>
<li>
You know the field names your documents use for their text and their embedding. If they are not
&quot;text&quot; and &quot;embedding&quot;, set them below
</li>
</ol>
<p style={{ marginTop: "8px" }}>
When a query comes in, LiteLLM converts it to an embedding with the model below and returns the
closest matching documents from your index.
</p>
</div>
}
type="info"
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>Valkey Setup Required</AlertTitle>
<AlertDescription>
<p>
LiteLLM searches documents you have already stored in Valkey. It does not create the index or upload
documents for you. Before creating this vector store, make sure:
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Your Valkey server has vector search enabled (the valkey-search module, included in the
valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey)
</li>
<li>
You have already created a search index and loaded your documents and their embeddings into it.
Enter that index name as the Vector Store ID
</li>
<li>
You know which embedding model created those stored embeddings. That model must be added to this
proxy under Models so you can pick it below. Using a different model returns wrong results
</li>
<li>
You know the field names your documents use for their text and their embedding. If they are not
&quot;text&quot; and &quot;embedding&quot;, set them below
</li>
</ol>
<p style={{ marginTop: "8px" }}>
When a query comes in, LiteLLM converts it to an embedding with the model below and returns the
closest matching documents from your index.
</p>
</AlertDescription>
</Alert>
)}
{selectedProvider === "vertex_rag_engine" && (
<Alert
message="Vertex AI RAG Engine Setup"
description={
<div>
<p>To use Vertex AI RAG Engine:</p>
<p style={{ marginTop: "4px", fontStyle: "italic" }}>
Note: Google Cloud has renamed this to &quot;RAG Engine&quot; in its console — the steps below
still apply.
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Set up your Vertex AI RAG Engine corpus following the guide:{" "}
<a
href="https://cloud.google.com/vertex-ai/generative-ai/docs/rag-engine/rag-overview"
target="_blank"
rel="noopener noreferrer"
>
Vertex AI RAG Engine Overview
</a>
</li>
<li>Create a corpus in your Google Cloud project</li>
<li>
Note the corpus ID from the Vertex AI console (now labeled &quot;RAG Engine&quot; in Google
Cloud)
</li>
<li>Enter the corpus ID in the Vector Store ID field below</li>
</ol>
</div>
}
type="info"
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>Vertex AI RAG Engine Setup</AlertTitle>
<AlertDescription>
<p>To use Vertex AI RAG Engine:</p>
<p style={{ marginTop: "4px", fontStyle: "italic" }}>
Note: Google Cloud has renamed this to &quot;RAG Engine&quot; in its console — the steps below still
apply.
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Set up your Vertex AI RAG Engine corpus following the guide:{" "}
<a
href="https://cloud.google.com/vertex-ai/generative-ai/docs/rag-engine/rag-overview"
target="_blank"
rel="noopener noreferrer"
>
Vertex AI RAG Engine Overview
</a>
</li>
<li>Create a corpus in your Google Cloud project</li>
<li>
Note the corpus ID from the Vertex AI console (now labeled &quot;RAG Engine&quot; in Google Cloud)
</li>
<li>Enter the corpus ID in the Vector Store ID field below</li>
</ol>
</AlertDescription>
</Alert>
)}
{selectedProvider === "vertex_ai/search_api" && (
<Alert
message="Vertex AI Search Setup"
description={
<div>
<p>To use Vertex AI Search (Discovery Engine):</p>
<p style={{ marginTop: "4px", fontStyle: "italic" }}>
Note: Google Cloud has renamed this to &quot;Agent Search&quot; in its console — the steps below
still apply.
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Enable the Discovery Engine API on your Google Cloud project and create a data store following
the guide:{" "}
<a
href="https://cloud.google.com/generative-ai-app-builder/docs/create-data-store-es"
target="_blank"
rel="noopener noreferrer"
style={{ textDecoration: "underline" }}
>
Create a Vertex AI Search data store
</a>
</li>
<li>Pick a supported location: global, us, or eu</li>
<li>
For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it
in the Vector Store ID field below.
</li>
<li>
For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a
search app on top of the data store, then copy the <strong>Engine ID</strong> and enter it in
the Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this
record, but it isn&apos;t used in the GCP URL when Engine ID is set.
</li>
</ol>
</div>
}
type="info"
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>Vertex AI Search Setup</AlertTitle>
<AlertDescription>
<p>To use Vertex AI Search (Discovery Engine):</p>
<p style={{ marginTop: "4px", fontStyle: "italic" }}>
Note: Google Cloud has renamed this to &quot;Agent Search&quot; in its console — the steps below
still apply.
</p>
<ol style={{ marginLeft: "16px", marginTop: "8px", listStyleType: "decimal" }}>
<li>
Enable the Discovery Engine API on your Google Cloud project and create a data store following the
guide:{" "}
<a
href="https://cloud.google.com/generative-ai-app-builder/docs/create-data-store-es"
target="_blank"
rel="noopener noreferrer"
style={{ textDecoration: "underline" }}
>
Create a Vertex AI Search data store
</a>
</li>
<li>Pick a supported location: global, us, or eu</li>
<li>
For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it in
the Vector Store ID field below.
</li>
<li>
For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a
search app on top of the data store, then copy the <strong>Engine ID</strong> and enter it in the
Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this record,
but it isn&apos;t used in the GCP URL when Engine ID is set.
</li>
</ol>
</AlertDescription>
</Alert>
)}
<FormField

119
ui/litellm-dashboard/src/app/login/LoginPage.tsx
View file

@ -4,6 +4,7 @@ import { useLogin } from "@/app/(dashboard)/hooks/login/useLogin";
import { useUIConfig } from "@/app/(dashboard)/hooks/uiConfig/useUIConfig";
import LoadingScreen from "@/components/common_components/LoadingScreen";
import { exchangeLoginCode, getProxyBaseUrl, switchToWorkerUrl } from "@/components/networking";
import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { Field, FieldGroup, FieldLabel } from "@/components/shared/form/field";
import { FormField } from "@/components/shared/form/FormField";
@ -17,8 +18,7 @@ import { useZodForm } from "@/lib/forms/useZodForm";
import { clearTokenCookies, getCookieFromDocument } from "@/utils/cookieUtils";
import { isJwtExpired } from "@/utils/jwtUtils";
import { consumeReturnUrl, getLoginUrl, getReturnUrl, isValidReturnUrl } from "@/utils/returnUrlUtils";
import { InfoCircleOutlined } from "@ant-design/icons";
import { Alert } from "antd";
import { CircleAlert, Info, TriangleAlert, X } from "lucide-react";
import { useRouter } from "next/navigation";
import { useEffect, useId, useState } from "react";
import { z } from "zod/v4";
@ -31,6 +31,28 @@ const loginSchema = z.object({
type LoginFormValues = z.infer<typeof loginSchema>;
function SsoEnabledNotice() {
const [isDismissed, setIsDismissed] = useState(false);
if (isDismissed) return null;
return (
<Alert variant="info" className="mt-4">
<Info />
<AlertTitle>
Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading
this page. To re-enable auto-redirect-to-SSO, set{" "}
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">AUTO_REDIRECT_UI_LOGIN_TO_SSO=true</code> in your
environment configuration.
</AlertTitle>
<AlertAction>
<Button variant="ghost" size="icon-sm" aria-label="Close" onClick={() => setIsDismissed(true)}>
<X />
</Button>
</AlertAction>
</Alert>
);
}
function LoginPageContent() {
const [isLoading, setIsLoading] = useState(true);
const { data: uiConfig, isLoading: isConfigLoading } = useUIConfig();
@ -170,22 +192,19 @@ function LoginPageContent() {
<h2 className="text-3xl font-semibold text-foreground">🚅 LiteLLM</h2>
</div>
<Alert
message="Admin UI Disabled"
description={
<>
<p className="text-sm">
The Admin UI has been disabled by the administrator. To re-enable it, please update the following
environment variable:
</p>
<p className="mt-2 text-sm">
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">DISABLE_ADMIN_UI=False</code>
</p>
</>
}
type="warning"
showIcon
/>
<Alert variant="warning">
<TriangleAlert />
<AlertTitle>Admin UI Disabled</AlertTitle>
<AlertDescription>
<p className="text-sm">
The Admin UI has been disabled by the administrator. To re-enable it, please update the following
environment variable:
</p>
<p className="mt-2 text-sm">
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">DISABLE_ADMIN_UI=False</code>
</p>
</AlertDescription>
</Alert>
</div>
</CardContent>
</Card>
@ -209,31 +228,32 @@ function LoginPageContent() {
</div>
{!uiConfig?.hide_default_credentials_hint && (
<Alert
message="Default Credentials"
description={
<>
<p className="text-sm">
By default, Username is <code className="bg-muted px-1 py-0.5 rounded-sm text-xs">admin</code>{" "}
and Password is your set LiteLLM Proxy
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">MASTER_KEY</code>.
</p>
<p className="mt-2 text-sm">
Need to set UI credentials or SSO?{" "}
<a href="https://docs.litellm.ai/docs/proxy/ui" target="_blank" rel="noopener noreferrer">
Check the documentation
</a>
.
</p>
</>
}
type="info"
icon={<InfoCircleOutlined />}
showIcon
/>
<Alert variant="info">
<Info />
<AlertTitle>Default Credentials</AlertTitle>
<AlertDescription>
<p className="text-sm">
By default, Username is <code className="bg-muted px-1 py-0.5 rounded-sm text-xs">admin</code> and
Password is your set LiteLLM Proxy
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">MASTER_KEY</code>.
</p>
<p className="mt-2 text-sm">
Need to set UI credentials or SSO?{" "}
<a href="https://docs.litellm.ai/docs/proxy/ui" target="_blank" rel="noopener noreferrer">
Check the documentation
</a>
.
</p>
</AlertDescription>
</Alert>
)}
{error && <Alert message={error} type="error" showIcon />}
{error && (
<Alert variant="error">
<CircleAlert />
<AlertTitle>{error}</AlertTitle>
</Alert>
)}
<form onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
@ -326,22 +346,7 @@ function LoginPageContent() {
</FieldGroup>
</form>
</div>
{uiConfig?.sso_configured && (
<Alert
type="info"
showIcon
closable
className="mt-4"
message={
<span>
Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow
upon loading this page. To re-enable auto-redirect-to-SSO, set{" "}
<code className="bg-muted px-1 py-0.5 rounded-sm text-xs">AUTO_REDIRECT_UI_LOGIN_TO_SSO=true</code>{" "}
in your environment configuration.
</span>
}
/>
)}
{uiConfig?.sso_configured && <SsoEnabledNotice />}
</TooltipProvider>
</CardContent>
</Card>

24
ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx
View file

@ -1,6 +1,7 @@
import { Alert } from "antd";
import { CircleAlert, Info } from "lucide-react";
import React from "react";
import { z } from "zod/v4";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { Field, FieldLabel, FieldGroup } from "@/components/shared/form/field";
import { FormField } from "@/components/shared/form/FormField";
@ -46,11 +47,10 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
</p>
{variant === "signup" && (
<Alert
className="mt-4"
type="info"
message="SSO"
description={
<Alert className="mt-4" variant="info">
<Info />
<AlertTitle>SSO</AlertTitle>
<AlertDescription>
<div className="flex justify-between items-center">
<span>SSO is under the Enterprise Tier.</span>
<a
@ -62,9 +62,8 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
Get Free Trial
</a>
</div>
}
showIcon
/>
</AlertDescription>
</Alert>
)}
<form className="mt-10 mb-5" onSubmit={form.handleSubmit(handleSubmit)}>
@ -84,7 +83,12 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,
</FormField>
</FieldGroup>
{claimError && <Alert type="error" message={claimError} showIcon className="mt-6 mb-4" />}
{claimError && (
<Alert variant="error" className="mt-6 mb-4">
<CircleAlert />
<AlertTitle>{claimError}</AlertTitle>
</Alert>
)}
<div className="mt-10">
<Button type="submit" variant="outline" disabled={isPending}>

29
ui/litellm-dashboard/src/components/CreateUserButton.tsx
View file

@ -10,8 +10,9 @@ import { Input } from "@/components/ui/input";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { Alert, Modal, Typography } from "antd";
import { ChevronRight, CircleHelp, UserPlus } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Modal, Typography } from "antd";
import { ChevronRight, CircleHelp, Info, UserPlus } from "lucide-react";
import React, { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import TeamDropdown from "./common_components/team_dropdown";
@ -134,20 +135,16 @@ const labelWithHint = (label: string, hint: string): React.ReactNode => (
);
const EmailInvitationsNotice: React.FC = () => (
<Alert
message="Email invitations"
description={
<>
New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured.{" "}
<Link href="https://docs.litellm.ai/docs/proxy/email" target="_blank">
Learn how to set up email notifications
</Link>
</>
}
type="info"
showIcon
className="mb-4"
/>
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>Email invitations</AlertTitle>
<AlertDescription>
New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured.{" "}
<Link href="https://docs.litellm.ai/docs/proxy/email" target="_blank">
Learn how to set up email notifications
</Link>
</AlertDescription>
</Alert>
);
export const CreateUserButton: React.FC<CreateuserProps> = ({

2
ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import type { DateRangePickerValue } from "@/components/shared/date_picker_types";
import Papa from "papaparse";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

59
ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx
View file

@ -3,9 +3,9 @@
import { useMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings";
import { useUpdateMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings";
import { toast } from "@/lib/toast";
import { Alert, Card, Col, Row, Skeleton } from "antd";
import { CheckCircleOutlined } from "@ant-design/icons";
import { CircleHelp, Save } from "lucide-react";
import { Card, Col, Row, Skeleton } from "antd";
import { CircleCheck, CircleHelp, Info, Save, X } from "lucide-react";
import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
@ -177,40 +177,39 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi
{isLoading ? (
<Skeleton active />
) : isError ? (
<Alert
type="error"
message="Could not load MCP Semantic Filter settings"
description={error instanceof Error ? error.message : undefined}
style={{ marginBottom: 24 }}
/>
<Alert variant="error" className="mb-6">
<AlertTitle>Could not load MCP Semantic Filter settings</AlertTitle>
{error instanceof Error && <AlertDescription>{error.message}</AlertDescription>}
</Alert>
) : (
<>
<Alert
type="info"
message="Semantic Tool Filtering"
description="Filter MCP tools semantically based on query relevance. This reduces context window size and improves tool selection accuracy. Click 'Save Settings' to apply changes across all pods (takes effect within 10 seconds)."
showIcon
style={{ marginBottom: 24 }}
/>
<Alert variant="info" className="mb-6">
<Info />
<AlertTitle>Semantic Tool Filtering</AlertTitle>
<AlertDescription>
Filter MCP tools semantically based on query relevance. This reduces context window size and improves tool
selection accuracy. Click &apos;Save Settings&apos; to apply changes across all pods (takes effect within
10 seconds).
</AlertDescription>
</Alert>
{saveSuccess && (
<Alert
type="success"
message="Settings saved successfully"
icon={<CheckCircleOutlined />}
showIcon
closable
style={{ marginBottom: 16 }}
/>
<Alert className="mb-4">
<CircleCheck />
<AlertTitle>Settings saved successfully</AlertTitle>
<AlertAction>
<Button variant="ghost" size="icon-sm" aria-label="Close" onClick={() => setSaveSuccess(false)}>
<X />
</Button>
</AlertAction>
</Alert>
)}
{updateError && (
<Alert
type="error"
message="Could not update settings"
description={updateError instanceof Error ? updateError.message : undefined}
style={{ marginBottom: 16 }}
/>
<Alert variant="error" className="mb-4">
<AlertTitle>Could not update settings</AlertTitle>
{updateError instanceof Error && <AlertDescription>{updateError.message}</AlertDescription>}
</Alert>
)}
<Row gutter={24}>

31
ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx
View file

@ -4,7 +4,7 @@ import { act, render, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { focusManager, QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import type { ToolRow } from "@/components/networking";
import { ToolPoliciesPanel } from "./ToolPoliciesPanel";
import { toast } from "@/lib/toast";
@ -84,17 +84,6 @@ const policyValue = (toolId: string, kind: "input" | "output"): string => {
const isSaving = (toolId: string, kind: "input" | "output"): boolean =>
policySelect(toolId, kind).hasAttribute("disabled");
const chooseOption = async (user: ReturnType<typeof userEvent.setup>, trigger: HTMLElement, label: string) => {
await user.click(trigger);
// The label also renders in the trigger once selected, so take the last match:
// the popup is portalled after the table in document order.
const option = await waitFor(() => {
const matches = screen.getAllByText(label);
return matches[matches.length - 1];
});
await user.click(option);
};
const renderPanel = (onSelectTool = vi.fn()) =>
renderWithProviders(<ToolPoliciesPanel accessToken="sk-token" onSelectTool={onSelectTool} />);
@ -202,7 +191,7 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { input_policy: "trusted" });
await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted"));
@ -214,7 +203,7 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "output"), "trusted");
await chooseSelectOption(user, policySelect("tool-1", "output"), "trusted");
expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { output_policy: "trusted" });
});
@ -225,8 +214,8 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseOption(user, policySelect("tool-2", "input"), "blocked");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-2", "input"), "blocked");
expect(isSaving("tool-2", "input")).toBe(true);
expect(isSaving("tool-1", "input")).toBe(true);
@ -241,8 +230,8 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseOption(user, policySelect("tool-2", "input"), "blocked");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-2", "input"), "blocked");
await act(async () => {
finishFirst();
});
@ -262,7 +251,7 @@ describe("ToolPoliciesPanel inline policy editing", () => {
() => new Promise<ToolRow[]>((resolve) => (landStaleRefresh = () => resolve(TOOLS))),
);
await user.click(screen.getByTestId("datatable-refresh"));
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted"));
await act(async () => {
@ -281,7 +270,7 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
await waitFor(() => expect(fromBackend).toHaveBeenCalledWith("Failed to update input policy: nope"));
expect(policyValue("tool-1", "input")).toBe("untrusted");
@ -293,7 +282,7 @@ describe("ToolPoliciesPanel inline policy editing", () => {
renderPanel();
await waitForRows();
await chooseOption(user, policySelect("tool-1", "input"), "trusted");
await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted");
await waitFor(() => expect(isSaving("tool-1", "input")).toBe(true));
expect(isSaving("tool-1", "output")).toBe(false);

18
ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx
View file

@ -5,7 +5,9 @@ import { all_admin_roles, isUserTeamAdminForAnyTeam } from "@/utils/roles";
import { modelCreationScope } from "@/utils/modelPermissions";
import { Switch } from "@/components/ui/switch";
import { Field, FieldLabel } from "@/components/shared/form/field";
import { Select as AntdSelect, Card, Col, Modal, Row, Tooltip, Typography, Alert } from "antd";
import { Select as AntdSelect, Card, Col, Modal, Row, Tooltip, Typography } from "antd";
import { Info } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { Button } from "@/components/ui/button";
import type { UploadProps } from "antd/es/upload";
import React, { useEffect, useMemo, useState } from "react";
@ -168,13 +170,13 @@ const AddModelForm: React.FC<AddModelFormProps> = ({
)}
</MountedFormField>
{!teamAdminSelectedTeam && (
<Alert
message="Team Selection Required"
description="As a team admin, you need to select your team first before adding models."
type="info"
showIcon
className="mb-4"
/>
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>Team Selection Required</AlertTitle>
<AlertDescription>
As a team admin, you need to select your team first before adding models.
</AlertDescription>
</Alert>
)}
</>
)}

2
ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
View file

@ -93,7 +93,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
</li>
<li>
<strong>{effectiveTierLabel("REASONING", value.tier_labels)}</strong>: Score &gt; {ranges.complexReasoning}{" "}
(or 2+ reasoning markers)
(or 2+ reasoning markers with a score of at least {ranges.simpleMedium})
</li>
</ul>
)}

20
ui/litellm-dashboard/src/components/add_pass_through.tsx
View file

@ -1,8 +1,9 @@
"use client";
import React, { useState } from "react";
import { Modal, Alert } from "antd";
import { CircleHelp, Plug } from "lucide-react";
import { Modal } from "antd";
import { CircleHelp, Info, Plug } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import { useWatch } from "react-hook-form";
import { z } from "zod/v4";
@ -156,13 +157,14 @@ const AddPassThroughEndpoint: React.FC<AddFallbacksProps> = ({
}}
>
<div className="mt-6">
<Alert
message="What is a Pass-Through Endpoint?"
description="Route requests from your LiteLLM proxy to any external API. Perfect for custom models, image generation APIs, or any service you want to proxy through LiteLLM."
type="info"
showIcon
className="mb-6"
/>
<Alert variant="info" className="mb-6">
<Info />
<AlertTitle>What is a Pass-Through Endpoint?</AlertTitle>
<AlertDescription>
Route requests from your LiteLLM proxy to any external API. Perfect for custom models, image generation
APIs, or any service you want to proxy through LiteLLM.
</AlertDescription>
</Alert>
<form onSubmit={form.handleSubmit(addPassThrough)} className="space-y-6">
<Card className="block p-5">

40
ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx
View file

@ -1,6 +1,6 @@
import React from "react";
import { Alert } from "antd";
import { CircleHelp } from "lucide-react";
import { CircleHelp, Info } from "lucide-react";
import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
import GuardrailSelector from "../guardrails/GuardrailSelector";
import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput";
@ -67,21 +67,20 @@ const PassThroughGuardrailsSection: React.FC<PassThroughGuardrailsSectionProps>
endpoints.
</p>
<Alert
message={
<span>
Field-Level Targeting{" "}
<a
href="https://docs.litellm.ai/docs/proxy/pass_through_guardrails#field-level-targeting"
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 underline hover:text-blue-800"
>
(Learn More)
</a>
</span>
}
description={
<Alert variant="info" className="mb-4">
<Info />
<AlertTitle>
Field-Level Targeting{" "}
<a
href="https://docs.litellm.ai/docs/proxy/pass_through_guardrails#field-level-targeting"
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 underline hover:text-blue-800"
>
(Learn More)
</a>
</AlertTitle>
<AlertDescription>
<div className="space-y-2">
<div>
Optionally specify which fields to check. If left empty, the entire request/response is sent to the
@ -100,11 +99,8 @@ const PassThroughGuardrailsSection: React.FC<PassThroughGuardrailsSectionProps>
</div>
</div>
</div>
}
type="info"
showIcon
className="mb-4"
/>
</AlertDescription>
</Alert>
<Field>
<FieldLabel htmlFor="pass-through-guardrails">

2
ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx
View file

@ -74,7 +74,7 @@ describe("UserSearchModal", () => {
expect(notice).toHaveTextContent(/users that already exist/i);
expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i);
// info, not warning: a warning here would read as an error state on an empty form
expect(notice.className).toMatch(/ant-alert-info/);
expect(notice.className).toMatch(/text-info/);
});
});

18
ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx
View file

@ -1,5 +1,7 @@
import { useState } from "react";
import { Modal, Alert } from "antd";
import { Modal } from "antd";
import { Info } from "lucide-react";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { UserAddOutlined } from "@ant-design/icons";
import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer";
import { useForm } from "react-hook-form";
@ -202,13 +204,13 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
<Modal title={title} open={isVisible} onCancel={handleClose} footer={null} width={800} maskClosable={!isSubmitting}>
<TooltipProvider>
<form onSubmit={form.handleSubmit(handleSubmit)} noValidate>
<Alert
type="info"
showIcon
className="mb-4"
message="Search selects from users that already exist. To add someone new, ask a proxy admin to create their account first."
data-testid="member-existing-users-notice"
/>
<Alert variant="info" className="mb-4" data-testid="member-existing-users-notice">
<Info />
<AlertTitle>
Search selects from users that already exist. To add someone new, ask a proxy admin to create their
account first.
</AlertTitle>
</Alert>
<FieldGroup>
<FormField control={form.control} name="user_email" label="Email">

10
ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx
View file

@ -1,8 +1,9 @@
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { CheckOutlined, CopyOutlined, SyncOutlined } from "@ant-design/icons";
import { Alert, Modal, Space } from "antd";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { Modal, Space } from "antd";
import { Button } from "@/components/ui/button";
import { CircleHelp } from "lucide-react";
import { CircleHelp, TriangleAlert } from "lucide-react";
import React, { useEffect, useMemo, useState } from "react";
import { useWatch } from "react-hook-form";
import { CopyToClipboard } from "react-copy-to-clipboard";
@ -184,7 +185,10 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat
>
{regeneratedKey ? (
<div className="flex flex-col gap-4">
<Alert type="warning" showIcon message="Save it now, you will not see it again" />
<Alert variant="warning">
<TriangleAlert />
<AlertTitle>Save it now, you will not see it again</AlertTitle>
</Alert>
<div className="flex flex-col gap-0.5">
<span className="text-xs text-muted-foreground">Key Alias</span>

8
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -3,7 +3,7 @@ import * as networking from "@/components/networking";
import { fireEvent, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import TeamInfoView from "./TeamInfo";
const authState = vi.hoisted(() => ({ userRole: "Admin" }));
@ -993,8 +993,7 @@ describe("TeamInfoView", () => {
const user = userEvent.setup({ delay: null });
const resetBudgetSelect = await openSettingsEditorForTeam(user, { budget_duration: "30d" });
await user.click(resetBudgetSelect);
await user.click(await screen.findByText("Never resets"));
await chooseSelectOption(user, resetBudgetSelect, "Never resets");
await waitFor(() => {
expect(resetBudgetSelect).toHaveTextContent("Never resets");
@ -1028,8 +1027,7 @@ describe("TeamInfoView", () => {
const user = userEvent.setup({ delay: null });
const resetBudgetSelect = await openSettingsEditorForTeam(user, { budget_duration: null });
await user.click(resetBudgetSelect);
await user.click(await screen.findByText("weekly"));
await chooseSelectOption(user, resetBudgetSelect, "weekly");
await user.click(screen.getByRole("button", { name: /save changes/i }));

16
ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx
View file

@ -138,15 +138,13 @@ describe("ui primitives forward refs to their DOM node", () => {
});
});
describe("setupTests ref tripwire", () => {
it("records a violation when a ref is passed to a plain function component", () => {
const Plain = (props: React.ComponentPropsWithoutRef<"span">) => <span {...props} />;
describe("plain function components", () => {
it("receives a ref as a prop instead of dropping it", () => {
const Plain = (props: React.ComponentPropsWithoutRef<"span"> & { ref?: React.Ref<HTMLSpanElement> }) => (
<span {...props} />
);
const ref = React.createRef<HTMLSpanElement>();
render(React.createElement(Plain as never, { ref }));
const consume = (globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings;
expect(consume).toBeDefined();
const violations = consume!();
expect(violations).toHaveLength(1);
expect(violations[0]).toContain("Function components cannot be given refs");
render(<Plain ref={ref}>ok</Plain>);
expect(ref.current).toBeInstanceOf(HTMLSpanElement);
});
});

18
ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx
View file

@ -1,10 +1,12 @@
import { Alert, Modal, Typography } from "antd";
import { Modal, Typography } from "antd";
import { TriangleAlert } from "lucide-react";
import { useState } from "react";
import { z } from "zod/v4";
import { modelPatchUpdateCall } from "./networking";
import { toast } from "@/lib/toast";
import { FieldGroup } from "@/components/shared/form/field";
import { FormField } from "@/components/shared/form/FormField";
import { Alert, AlertTitle } from "@/components/shared/Alert";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { Button } from "@/components/ui/button";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
@ -74,12 +76,14 @@ export default function UpdateModelCredentialsModal({
Update this model&apos;s API key. Only the new key is sent; the rest of the deployment configuration is left
untouched.
</Text>
<Alert
type="warning"
showIcon
className="mb-4"
message="Only the API key is rotated here. Models that authenticate with an Azure AD token, AWS credentials, or a Vertex service-account JSON aren't supported yet; update those from the model's LiteLLM Params for now."
/>
<Alert variant="warning" className="mb-4">
<TriangleAlert />
<AlertTitle>
Only the API key is rotated here. Models that authenticate with an Azure AD token, AWS credentials, or a
Vertex service-account JSON aren&apos;t supported yet; update those from the model&apos;s LiteLLM Params for
now.
</AlertTitle>
</Alert>
<form onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
<FormField control={form.control} name="api_key" label="New API Key">

12
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx
View file

@ -59,7 +59,9 @@ describe("RoutingDecisionCard", () => {
}}
/>,
);
expect(screen.getByText("Heuristic, REASONING override (2 or more reasoning markers)")).toBeInTheDocument();
expect(
screen.getByText("Heuristic, REASONING override (2 or more reasoning markers, score above the lowest tier)"),
).toBeInTheDocument();
expect(screen.getByText("0.20")).toBeInTheDocument();
// The score did not decide this tier, so NO band explanation may render at all.
// Asserting the absence of one specific band would pass vacuously: 0.20 sits in
@ -188,7 +190,9 @@ describe("RoutingDecisionCard", () => {
// `signals` is gone under redaction; the cause alone must suppress the band.
render(<RoutingDecisionCard decision={{ ...heuristic, cause: "reasoning_override", signals: undefined }} />);
expect(screen.queryByText(/SIMPLE|MEDIUM|COMPLEX|at or above/)).not.toBeInTheDocument();
expect(screen.getByText("Heuristic, REASONING override (2 or more reasoning markers)")).toBeInTheDocument();
expect(
screen.getByText("Heuristic, REASONING override (2 or more reasoning markers, score above the lowest tier)"),
).toBeInTheDocument();
});
it("shows the operator's tier name on the badge instead of the canonical one", () => {
@ -212,7 +216,9 @@ describe("RoutingDecisionCard", () => {
render(
<RoutingDecisionCard decision={{ ...heuristic, cause: "reasoning_override", score: 0.2, tier_label: "Deep" }} />,
);
expect(screen.getByText("Heuristic, Deep override (2 or more reasoning markers)")).toBeInTheDocument();
expect(
screen.getByText("Heuristic, Deep override (2 or more reasoning markers, score above the lowest tier)"),
).toBeInTheDocument();
});
it("falls back to the raw cause for a value this build does not know", () => {

2
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
View file

@ -72,7 +72,7 @@ function describeCause(decision: RoutingDecision): string {
case "heuristic_scorer":
return "Heuristic scorer";
case "reasoning_override":
return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers)`;
return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers, score above the lowest tier)`;
case "llm_classifier":
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
case "literal_keyword_match":

118
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -816,7 +816,8 @@ export interface paths {
};
/**
* List Shadow Eval Jobs
* @description List shadow eval jobs, newest first. Counts and results ride the detail endpoint only.
* @description List shadow eval jobs, newest first, each key with its attempt count so status is
* accurate. Judged counts, spend, and results ride the detail endpoint only.
*/
get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"];
put?: never;
@ -838,20 +839,21 @@ export interface paths {
put?: never;
/**
* Start Shadow Eval
* @description Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second
* arm, judge the two responses blind, and stratify win rates by tier and by the model that
* served the real arm.
* @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against
* a second arm, judge the two responses blind, and stratify win rates by tier, by the model
* that served the real arm, and by key.
*
* A forward job answers whether the key should adopt router_name: it samples the requests
* A forward job answers whether the keys should adopt router_name: it samples the requests
* the router did not serve and duplicates them through it. A reverse job answers whether a
* key already on the router still gains from it: it samples the requests the router did
* serve and duplicates them against baseline_model. A key can hold one active job per
* direction, so both questions can run at once.
*
* Shadow responses are never served to users. The job samples until it has judged
* max_turns turns, reaches the end of its window, or is stopped; sampling changes
* propagate to pods within about 10 seconds. Shadow and judge calls bill to the
* shadowed key but are excluded from request counts and auto-router adoption metrics.
* Shadow responses are never served to users. Each key samples until it has judged
* max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one
* key running out of budget does not end sampling for the others; sampling changes
* propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed
* key but are excluded from request counts and auto-router adoption metrics.
*/
post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"];
delete?: never;
@ -891,7 +893,12 @@ export interface paths {
put?: never;
/**
* Stop Shadow Eval Job
* @description Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s.
* @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept;
* sampling halts within ~10s. Keys that already stopped on their own budget keep the
* stopped_at they earned. The statement is the whole state machine: it claims the job
* only while a leg still samples inside the window with no stop recorded, so a racing
* operator, a same-instant budget spend, and a repeat stop all read the same 400 with
* the status the job actually holds.
*/
post: operations["stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post"];
delete?: never;
@ -33201,18 +33208,49 @@ export interface components {
timeout?: number | null;
};
/**
* ShadowEvalJobResponse
* @description A shadow-eval job. Validates directly from the prisma record (job_id reads the
* row's id); status is derived from stopped_at and ends_at, never stored, so no writer
* anywhere can produce an inconsistent one. Aggregate fields are populated by the
* detail endpoint only and stay None on list responses.
* ShadowEvalJobKeyResponse
* @description One key a job shadows, with its own budget and stop state.
*/
ShadowEvalJobResponse: {
ShadowEvalJobKeyResponse: {
/**
* Api Key Id
* @description The hashed virtual key whose traffic this job evaluates, and only that key's
* @description The hashed virtual key whose traffic this entry scopes
*/
api_key_id: string;
/**
* Attempt Count
* @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it
*/
attempt_count?: number | null;
/**
* Key Alias
* @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted
*/
key_alias?: string | null;
/**
* Key Name
* @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias
*/
key_name?: string | null;
/**
* Max Turns
* @description This key's own sample budget, independent of its siblings'
*/
max_turns: number;
/**
* Stopped At
* @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset
*/
stopped_at?: string | null;
};
/**
* ShadowEvalJobResponse
* @description A shadow-eval job over one or more keys, each with its own budget and stop state;
* status is derived from stopped_by, the keys' stop and budget state, and ends_at,
* never stored, so no writer anywhere can produce an inconsistent one. Aggregate
* fields are populated by the detail endpoint only and stay None on list responses.
*/
ShadowEvalJobResponse: {
/** Baseline Model */
baseline_model?: string | null;
/**
@ -33251,22 +33289,15 @@ export interface components {
*/
judged_count?: number | null;
/**
* Key Alias
* @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted
* Keys
* @description The keys whose traffic this job evaluates, and only those keys', each with its own budget
*/
key_alias?: string | null;
/**
* Key Name
* @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias
*/
key_name?: string | null;
keys: components["schemas"]["ShadowEvalJobKeyResponse"][];
/**
* Last Error
* @description Most recent attempt error; detail endpoint only
*/
last_error?: string | null;
/** Max Turns */
max_turns: number;
/** @description Stratified verdicts; detail endpoint only */
results?: components["schemas"]["ShadowEvalResult"] | null;
/** Router Name */
@ -33275,13 +33306,19 @@ export interface components {
shadow_percentage: number;
/**
* Status
* @description A job whose window has passed reads completed even if a later sweep stamped
* stopped_at; stopped means sampling ended before the window did.
* @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration
* backfills it for every job that displayed stopped when the column arrived, so the
* pre-column population is closed), completion is the window passing or every key
* spending its budget, and anything else is running. The all-keys-stamped fallback
* covers only stops written by pre-column pods during a rolling deploy.
* @enum {string}
*/
readonly status: "running" | "completed" | "stopped";
/** Stopped At */
stopped_at?: string | null;
/**
* Stopped By
* @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed
*/
stopped_by?: string | null;
};
/**
* ShadowEvalResult
@ -33290,9 +33327,14 @@ export interface components {
ShadowEvalResult: {
/**
* By Current Model
* @description Sliced by the model that served the real arm: the key's incumbent models in forward mode, and in reverse the models the router itself picked
* @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked
*/
by_current_model: components["schemas"]["ShadowEvalSlice"][];
/**
* By Key
* @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero
*/
by_key: components["schemas"]["ShadowEvalSlice"][];
/** By Tier */
by_tier: components["schemas"]["ShadowEvalSlice"][];
/** Overall Shadow Win Rate Pct */
@ -33500,14 +33542,14 @@ export interface components {
};
/**
* StartShadowEvalRequest
* @description Start duplicating a key's traffic for blind comparison against an auto-router.
* @description Start duplicating one or more keys' traffic for blind comparison against an auto-router.
*/
StartShadowEvalRequest: {
/**
* Api Key Id
* @description The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this key's traffic; requests made with any other key are not sampled.
* Api Key Ids
* @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make.
*/
api_key_id: string;
api_key_ids: string[];
/**
* Baseline Model
* @description Required when direction is reverse and rejected otherwise: the fixed model the router's own responses are judged against. Must be a plain model rather than another auto-router
@ -33534,7 +33576,7 @@ export interface components {
judge_model: string;
/**
* Max Turns
* @description Sample budget: the job judges at most this many turns, then completes. This is also the spend bound; expected judge cost is roughly max_turns times one judge call
* @description Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, so a job over N keys judges at most N times max_turns turns. This is also the spend bound; expected judge cost is roughly that turn ceiling times one judge call
* @default 200
*/
max_turns: number;
@ -37931,7 +37973,7 @@ export interface operations {
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
parameters: {
query?: {
/** @description Filter to jobs shadowing this key */
/** @description Filter to jobs that shadow this key, alone or alongside others */
api_key_id?: string | null;
/** @description Newest jobs to return */
limit?: number;

2
ui/litellm-dashboard/src/utils/cookieUtils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { describe, it, expect, beforeEach, vi } from "vitest";
import { clearTokenCookies, getCookie, storeLoginToken } from "./cookieUtils";
import { getToken, setToken } from "./mcpTokenStore";

2
ui/litellm-dashboard/src/utils/dataUtils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { describe, it, expect, beforeEach, vi, afterEach } from "vitest";
import { copyToClipboard, formatNumberWithCommas, getSpendString, updateExistingKeys } from "./dataUtils";

2
ui/litellm-dashboard/src/utils/localStorageUtils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { describe, it, expect, vi, beforeEach } from "vitest";
import {
emitLocalStorageChange,

2
ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { clearAllMcpTokens, getToken, isTokenValid, removeToken, setToken } from "./mcpTokenStore";

2
ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts
View file

@ -1,3 +1,5 @@
// @vitest-environment jsdom
import {
buildLoginUrlWithReturn,
clearStoredReturnUrl,

12
ui/litellm-dashboard/tests/setup.unit.ts Normal file
View file

@ -0,0 +1,12 @@
import { vi } from "vitest";
vi.mock("@/lib/toast", () => ({
toast: {
success: vi.fn(),
info: vi.fn(),
warning: vi.fn(),
error: vi.fn(),
fromError: vi.fn(),
dismiss: vi.fn(),
},
}));

26
ui/litellm-dashboard/tests/setupTests.ts
View file

@ -113,31 +113,7 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
}),
}));
const pendingRefWarnings: string[] = [];
const consumePendingRefWarnings = (): string[] => pendingRefWarnings.splice(0, pendingRefWarnings.length);
(globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings =
consumePendingRefWarnings;
const originalConsoleError = console.error.bind(console);
vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => {
originalConsoleError(...args);
if (typeof args[0] === "string" && args[0].includes("Function components cannot be given refs")) {
pendingRefWarnings.push(args.map(String).join(" "));
}
});
afterEach(() => {
cleanup();
const refWarnings = consumePendingRefWarnings();
if (refWarnings.length > 0) {
throw new Error(
"A ref was passed to a plain function component and silently dropped under React 18, which breaks " +
"ref-based composition (Base UI render triggers, tooltips, focus). Wrap the component in React.forwardRef. " +
"This tripwire lives in tests/setupTests.ts and can be removed after the React 19 upgrade.\n\n" +
refWarnings.join("\n\n"),
);
}
});
afterEach(cleanup);
// Make toLocaleString deterministic in tests; individual tests can override
// This returns ISO-like strings to keep assertions stable.

29
ui/litellm-dashboard/tests/test-utils.tsx
View file

@ -1,7 +1,9 @@
import React, { PropsWithChildren } from "react";
import { render, RenderOptions } from "@testing-library/react";
import { render, RenderOptions, screen, waitFor } from "@testing-library/react";
import type userEvent from "@testing-library/user-event";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { NuqsTestingAdapter, OnUrlUpdateFunction } from "nuqs/adapters/testing";
import { expect } from "vitest";
// Create a client for testing
export const testQueryClient = new QueryClient({
@ -35,4 +37,29 @@ export const renderWithProviders = (ui: React.ReactElement, options?: RenderOpti
return render(ui, { wrapper: Providers, ...renderOptions });
};
const pointerBlocked = (element: HTMLElement): boolean => {
for (let node: HTMLElement | null = element; node !== null; node = node.parentElement) {
if (node.style.pointerEvents === "none") return true;
}
return false;
};
/**
* Opens a Base UI Select and picks an option by its accessible name.
*
* The option is in the DOM one render before the popup finishes entering, and until then its
* positioner still carries `pointer-events: none`, which user-event refuses to click. Waiting on
* the option text alone is a race that React 19's flush timing loses.
*/
export const chooseSelectOption = async (
user: ReturnType<typeof userEvent.setup>,
trigger: HTMLElement,
optionName: string | RegExp,
) => {
await user.click(trigger);
const option = await screen.findByRole("option", { name: optionName });
await waitFor(() => expect(pointerBlocked(option)).toBe(false));
await user.click(option);
};
export * from "@testing-library/react";

101
ui/litellm-dashboard/vitest.config.ts
View file

@ -12,17 +12,81 @@ const staticImageData: Plugin = {
},
};
const config: ViteUserConfig = {
const sharedViteConfig = {
plugins: [staticImageData],
resolve: { alias: { "@": resolve(__dirname, "src") } },
define: { "import.meta.vitest": "undefined" },
esbuild: { jsx: "automatic", jsxImportSource: "react" } as const,
};
const TEST_TS_FILES_THAT_RENDER_REACT: readonly string[] = [
"src/**/hooks/**/*.test.ts",
"src/**/cost-tracking/_components/**/use_*.test.ts",
"src/**/models-and-endpoints/detailNavigation.test.ts",
"src/**/models-and-endpoints/vertexCredentialsUpload.test.ts",
"src/components/chat/useChatHistory.test.ts",
"src/lib/forms/pickDirty.test.ts",
];
const jsdomTier = {
environment: "./tests/jsdomFetchEnv.ts",
setupFiles: ["tests/setupTests.ts"],
globals: true,
css: true,
testTimeout: 60_000,
hookTimeout: 30_000,
};
const config: ViteUserConfig = {
...sharedViteConfig,
test: {
environment: "./tests/jsdomFetchEnv.ts",
setupFiles: ["tests/setupTests.ts"],
globals: true,
css: true, // lets you import CSS/modules without extra mocks
testTimeout: 60000,
hookTimeout: 30000,
projects: [
{
...sharedViteConfig,
test: {
name: "unit",
environment: "node",
setupFiles: ["tests/setup.unit.ts"],
globals: true,
testTimeout: 60_000,
hookTimeout: 30_000,
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
exclude: ["node_modules/**", ...TEST_TS_FILES_THAT_RENDER_REACT],
},
},
{
...sharedViteConfig,
test: {
...jsdomTier,
name: "component",
include: ["src/**/*.test.tsx", "tests/**/*.test.tsx", ...TEST_TS_FILES_THAT_RENDER_REACT],
exclude: ["node_modules/**", "**/*.integration.test.tsx"],
},
},
{
...sharedViteConfig,
test: {
...jsdomTier,
name: "integration",
include: ["src/**/*.integration.test.tsx", "tests/**/*.integration.test.tsx"],
exclude: ["node_modules/**"],
},
},
{
...sharedViteConfig,
test: {
name: "types",
include: [],
typecheck: {
enabled: true,
include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"],
ignoreSourceErrors: true,
},
},
},
],
silent: process.env.CI ? "passed-only" : false,
teardownTimeout: 60000,
teardownTimeout: 60_000,
coverage: {
provider: "v8",
reporter: ["text", "lcov"],
@ -32,37 +96,16 @@ const config: ViteUserConfig = {
"**/*.test.*",
"**/*.test-d.*",
"**/*.spec.*",
"tests/**",
"node_modules/**",
".next/**",
"out/**",
"**/*.config.*",
"postcss.config.*",
"tailwind.config.*",
"next.config.*",
],
},
exclude: ["node_modules/**"],
include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"],
typecheck: {
include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"],
ignoreSourceErrors: true,
},
},
resolve: {
alias: {
"@": resolve(__dirname, "src"),
},
},
define: {
"import.meta.vitest": "undefined",
},
esbuild: {
jsx: "automatic",
jsxImportSource: "react",
},
};