diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 2e5e7d14d14..b876cf2d69a 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 13431 + "limit": 13429 }, "reportArgumentType": { "limit": 2198 @@ -24,7 +24,7 @@ "limit": 19 }, "reportExplicitAny": { - "limit": 3372 + "limit": 3369 }, "reportFunctionMemberAccess": { "limit": 7 @@ -108,7 +108,7 @@ "limit": 38283 }, "reportUnknownParameterType": { - "limit": 19585 + "limit": 19584 }, "reportUnknownVariableType": { "limit": 29829 diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260904000000_shadow_eval_model_scope/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260904000000_shadow_eval_model_scope/migration.sql new file mode 100644 index 00000000000..937f0de2569 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260904000000_shadow_eval_model_scope/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "models" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index e638ad68b6f..1c43668f227 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob { target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // first (often only) auto-router under evaluation; router_names is the full set router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) + models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index df67ba08416..ec17cc1d809 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -257,7 +257,7 @@ class DualCache(BaseCache): self, current_time: float, keys: list[str], - result: Sequence[Any], + result: Sequence[object], ) -> tuple[list[str], dict[str, float | None]]: """ Atomically choose keys to fetch from Redis and reserve their access time. diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 338afe04e5e..023caf06d12 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -35,6 +35,15 @@ span orphaned into its own trace). The anchor — a contextvar inherited by thos child tasks — gives a stable parent in both cases. DB/service spans keep ambient parenting so an auth DB lookup still nests under `auth`. +The anchor is also what `litellm.request.route` is read from: `request_root_http_route` +returns the server span's own `http.route`, so the LLM call span cannot disagree with +its parent about which endpoint served the request. That means the route template on a +normal route and the literal path on a passthrough prefix, because the passthrough hook +rewrote the attribute; an MCP call anchors the same server span, so it reports the +`/mcp` mount point. Attributes stay readable after a span ends, so the async close +callback reads the same value. Where no server span was anchored at all, the route the +proxy recorded at auth (`metadata.user_api_key_request_route`) is the backstop. + **Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's service-logging layer instruments many internal functions, but only some are traceable units of work: diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index a550dca6cc8..5519896a961 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -49,6 +49,7 @@ from litellm.integrations.otel.model.utils import to_ns from litellm.integrations.otel.plumbing.context import ( is_recordable_span, mcp_message_transport_span, + request_root_http_route, request_root_span, resolve_mcp_span_context, resolve_parent_context, @@ -541,6 +542,7 @@ class OpenTelemetryV2(CustomLogger): payload, capture_content=self.config.capture_span_content, time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, + request_route=request_root_http_route(), ) end_time_ns: Final = to_ns(end_time) if carrier is not None and carrier.span is not None: diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 3ac92b04c27..33457f5de16 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -89,6 +89,7 @@ class GenAIMapper: f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent, f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount, LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming, + LiteLLM.REQUEST_ROUTE: lambda d: d.request_route, } _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = { diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 062b2ca20b4..ee116aca46b 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -64,6 +64,7 @@ class RequestIdentity: # completes (routing has picked a deployment), so it's absent from the # auth-time seed and filled only from the payload. provider_model: str | None = None + request_route: str | None = None metadata: Mapping[str, str] = field(default_factory=dict) @classmethod @@ -87,6 +88,7 @@ class RequestIdentity: key_hash=as_str(raw_meta.get("user_api_key_hash")), end_user=as_str(payload.get("end_user")) or as_str(raw_meta.get("user_api_key_end_user_id")), provider_model=resolve_provider_model(payload), + request_route=as_str(raw_meta.get("user_api_key_request_route")), metadata=metadata, ) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index e8ed269f6cb..d0959a6c2e9 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -386,6 +386,7 @@ class LLMCallSpanData: # keeps routes the convention folds into one operation distinguishable. output_type: GenAIOutputType | None = None call_type: str | None = None + request_route: str | None = None @classmethod def from_standard_logging_payload( @@ -393,6 +394,7 @@ class LLMCallSpanData: payload: StandardLoggingPayload, capture_content: bool = False, time_to_first_chunk_seconds: float | None = None, + request_route: str | None = None, ) -> LLMCallSpanData: params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {}) # The single parse of the request's metadata — the request-vs-provider @@ -433,6 +435,7 @@ class LLMCallSpanData: time_to_first_chunk_seconds=time_to_first_chunk_seconds, output_type=resolve_output_type(call_type), call_type=call_type or None, + request_route=request_route or context.identity.request_route, ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index f7a6280f95b..af5327cbd41 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -295,6 +295,7 @@ class LiteLLM: # ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``. PROVIDER_MODEL: Final = "litellm.provider.model" REQUEST_STREAMING: Final = "litellm.request.streaming" + REQUEST_ROUTE: Final = "litellm.request.route" TOOLS_DECLARED: Final = "litellm.request.tools.declared" GUARDRAIL_NAME: Final = "litellm.guardrail.name" GUARDRAIL_MODE: Final = "litellm.guardrail.mode" diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 159a84b121f..aa7cc8e2afd 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -6,6 +6,7 @@ from typing import Final from opentelemetry import baggage from opentelemetry.context import Context, get_current +from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import ( Link, NonRecordingSpan, @@ -18,6 +19,8 @@ from opentelemetry.trace.propagation.tracecontext import ( TraceContextTextMapPropagator, ) +from litellm.integrations.otel.model.semconv import HTTP + _PROPAGATOR: Final = TraceContextTextMapPropagator() # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the @@ -55,6 +58,25 @@ def request_root_span() -> "Span | None": return span if is_recordable_span(span) else None +def request_root_http_route() -> str | None: + """``http.route`` exactly as the request's root SERVER span reports it. + + Read off the span rather than re-derived, so the LLM call span cannot disagree + with its own parent about which endpoint served the request: the template the + instrumentation matched, or the literal path where + ``mount._passthrough_span_name_hook`` rewrote it, are already in the attribute. + An MCP call anchors that same server span, so it reports the ``/mcp`` mount + point the instrumentation matched. Attributes stay readable after a span ends, + so this answers just as well from the async logging callback. + + None when no server span is anchored, which is the SDK path and any deployment + where the FastAPI instrumentation did not mount. + """ + span: Final = request_root_span() + route: Final = span.attributes.get(HTTP.ROUTE) if isinstance(span, ReadableSpan) and span.attributes else None + return route if isinstance(route, str) and route else None + + # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the # MCP client propagated in the current request's ``params._meta``. The MCP gateway # sets it per message so the MCP span can record the client's span as a span diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index b28d21ba1ca..59403874eb0 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_judge import ( ) from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.llms.base_llm.base_utils import type_to_response_format_param +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN @@ -650,6 +651,7 @@ class ActiveShadowEvalJob(BaseModel): id: str router_name: str router_names: tuple[str, ...] = () + models: frozenset[str] = frozenset() direction: ShadowEvalDirection = "forward" baseline_model: str | None = None shadow_percentage: float @@ -692,6 +694,21 @@ class ActiveShadowEvalJob(BaseModel): return self.baseline_model or arm_router +def _canonical_group(router: "Router | None", model_group: str) -> str: + """A model group in the one spelling both a job's scope and a request's model compare + under: an alias resolves to its target so the two never fail to match on spelling.""" + return ( + resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None + ) or model_group + + +def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool: + """Whether the request's group is in the job's model scope. Both sides resolve through + the router's alias map at match time, so a re-pointed alias applies to the next request + rather than after the jobs cache rolls.""" + return not job.models or any(_canonical_group(router, name) == model_group for name in job.models) + + def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None: """The sampling path's view of one job row, or None for a row it cannot sample: an unknown direction, or a reverse job with no baseline model to duplicate against. @@ -714,7 +731,8 @@ class ShadowEvalLogger(CustomLogger): A job targets a virtual key, a team, or a user; a request qualifies for a job when any of its resolved identities (key hash, team id, user id) matches the job's target, so team and user jobs cover JWT-authenticated traffic, which carries no - key hash at all.""" + key hash at all. A job scoped to model groups further requires the request's + requested group to be one of them.""" def __init__( self, @@ -801,19 +819,24 @@ class ShadowEvalLogger(CustomLogger): active_jobs: Sequence[ActiveShadowEvalJob], request_metadata: Mapping[str, object], request_id: str, + model_group: str, ) -> tuple[ActiveShadowEvalJob, ...]: """The jobs that sample this request. A key can hold one job per direction, and a request routed by one job's router while bypassing the other's qualifies for both; each is separately budgeted, so both fire. An admitting job that loses the sampling - dice is counted, so results can weigh judged rows against the traffic they stand for.""" + dice is counted, so results can weigh judged rows against the traffic they stand for. + A request outside a job's direction or model scope is not that job's traffic and + goes uncounted, so the funnel stays a fraction of the traffic the job admits.""" eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission now: Final = datetime.now(timezone.utc) + router: Final = self._router_provider() for job in active_jobs: if ( now >= job.ends_at or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns or (job.max_budget is not None and job.spend >= job.max_budget) or not _direction_admits(request_metadata, job) + or not _scope_admits(router, job, model_group) ): continue if not _sample_hits(request_id, job.id, job.shadow_percentage): @@ -868,6 +891,7 @@ class ShadowEvalLogger(CustomLogger): tuple(job for target in targets for job in active_jobs.get(target, ())), request_metadata, request_id, + _canonical_group(self._router_provider(), str(payload.get("model_group") or "")), ) if not eligible: return diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index df579f6df5b..a175ca1c3f6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3821,7 +3821,7 @@ class Logging(LiteLLMLoggingBaseClass): def record_streamed_anthropic_message_id(self, message_id: str) -> None: self.streamed_anthropic_message_id = message_id - def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse: + def _anthropic_messages_logged_response(self, result: object) -> ModelResponse: """ The ModelResponse a /v1/messages spend_logs row is built from. diff --git a/litellm/proxy/client/cli/commands/debug.py b/litellm/proxy/client/cli/commands/debug.py new file mode 100644 index 00000000000..4e3914143d8 --- /dev/null +++ b/litellm/proxy/client/cli/commands/debug.py @@ -0,0 +1,372 @@ +"""`lite debug claude`: one-shot debug report for a Claude Code session routed through the proxy. + +Claude Code puts its session id in `metadata.user_id`, which the proxy lifts into +`LiteLLM_SpendLogs.session_id`. This command pulls every turn of that session, plus +the request / response bodies for failures and the most recent turns, and renders a +single markdown report that can be pasted into a bug report or handed to another agent. +""" + +import json +import os +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import click +import requests +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator + +from ...http_client import HTTPClient +from ._cli_context import cli_context_values + +CLAUDE_DIR: Final = Path.home() / ".claude" +REPORT_DIR: Final = Path.home() / ".litellm" / "debug" +SESSION_ID_ENV: Final = "CLAUDE_CODE_SESSION_ID" +SLASH_COMMAND_NAME: Final = "debug-lite" +SLASH_COMMAND_BODY: Final = """--- +description: Pull the LiteLLM debug report (spend, request, response, error) for this Claude Code session +allowed-tools: Bash(lite debug claude:*) +--- +Below is the LiteLLM debug report for this Claude Code session. Summarize the failing +request(s) in a few sentences (model, error, request id) and tell me the path the full +report was saved to so I can hand it off. If nothing failed, say so. + +!`lite debug claude $ARGUMENTS` +""" + + +@dataclass(frozen=True, slots=True) +class DebugFailure: + message: str + + +class ErrorInformation(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + error_code: str | None = None + error_class: str | None = None + error_message: str | None = None + llm_provider: str | None = None + + +class SpendLogMetadata(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + status: str | None = None + error_information: ErrorInformation | None = None + + +class SpendLogRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True) + + request_id: str + start_time: str | None = Field(default=None, alias="startTime") + end_time: str | None = Field(default=None, alias="endTime") + model: str | None = None + model_group: str | None = None + custom_llm_provider: str | None = None + api_base: str | None = None + call_type: str | None = None + status: str | None = None + spend: float = 0.0 + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + metadata: SpendLogMetadata = SpendLogMetadata() + + @field_validator("metadata", mode="before") + @classmethod + def _parse_metadata(cls, value: object) -> object: + if value is None: + return SpendLogMetadata() + if isinstance(value, str): + return json.loads(value) if value else SpendLogMetadata() + return value + + @property + def failed(self) -> bool: + return (self.status or self.metadata.status) == "failure" + + @property + def error(self) -> ErrorInformation | None: + return self.metadata.error_information + + +class SessionLogsPage(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + data: tuple[SpendLogRow, ...] + total: int + total_pages: int + + +class RequestResponsePayload(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + proxy_server_request: JsonValue = None + response: JsonValue = None + messages: JsonValue = None + + +_SESSION_PAGE: Final = TypeAdapter(SessionLogsPage) +_PAYLOAD: Final[TypeAdapter[RequestResponsePayload | None]] = TypeAdapter(RequestResponsePayload | None) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + +_SESSION_PAGE_SIZE: Final = 100 +_TRANSPORT_BODY_CHARS: Final = 500 +_SESSION_TRANSCRIPT_STEM: Final = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}") + + +def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | None: + explicit: Final = env.get(SESSION_ID_ENV) + if explicit: + return explicit + transcripts: Final = tuple( + path for path in claude_dir.glob("projects/*/*.jsonl") if _SESSION_TRANSCRIPT_STEM.fullmatch(path.stem) + ) + if not transcripts: + return None + newest: Final = max(transcripts, key=lambda p: p.stat().st_mtime) + return newest.stem + + +def _transport_failure(uri: str, error: requests.exceptions.RequestException) -> DebugFailure: + body: Final = error.response.text[:_TRANSPORT_BODY_CHARS] if error.response is not None else "" + detail: Final = f"\n{body}" if body else "" + return DebugFailure(f"GET {uri} failed: {error}{detail}") + + +class SpendLogsFetcher: + def __init__(self, http: HTTPClient) -> None: + self._http = http + + def session_rows(self, session_id: str) -> tuple[SpendLogRow, ...] | DebugFailure: + first: Final = self._page(session_id, 1) + if isinstance(first, DebugFailure): + return first + rest: Final = tuple(self._page(session_id, page) for page in range(2, first.total_pages + 1)) + failed_page: Final = next((page for page in rest if isinstance(page, DebugFailure)), None) + if failed_page is not None: + return failed_page + rows: Final = first.data + tuple(row for page in rest if isinstance(page, SessionLogsPage) for row in page.data) + return tuple(sorted(rows, key=lambda r: r.start_time or "")) + + def _get(self, uri: str, params: Mapping[str, str | int] | None = None) -> JsonValue | DebugFailure: + try: + return _JSON.validate_python(self._http.request("GET", uri, params=params)) # pyright: ignore[reportUnknownMemberType] # HTTPClient.request is untyped + except requests.exceptions.RequestException as e: + return _transport_failure(uri, e) + + def _page(self, session_id: str, page: int) -> SessionLogsPage | DebugFailure: + uri: Final = "/spend/logs/session/ui" + raw: Final = self._get( + uri, MappingProxyType({"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE}) + ) + if isinstance(raw, DebugFailure): + return raw + try: + return _SESSION_PAGE.validate_python(raw) + except ValidationError as e: + return DebugFailure(f"Unexpected {uri} response: {e}") + + def payload(self, request_id: str) -> RequestResponsePayload | None | DebugFailure: + uri: Final = f"/spend/logs/ui/{request_id}" + raw: Final = self._get(uri) + if isinstance(raw, DebugFailure): + return raw + try: + return _PAYLOAD.validate_python(raw) + except ValidationError as e: + return DebugFailure(f"Unexpected {uri} response: {e}") + + +def _fmt_json(value: JsonValue, max_chars: int) -> str: + text: Final = value if isinstance(value, str) else json.dumps(value, indent=2, default=str) + if len(text) <= max_chars: + return text + return f"{text[:max_chars]}\n... (truncated, {len(text) - max_chars} more chars)" + + +def _fenced(text: str, info: str = "") -> tuple[str, str, str]: + longest_run: Final = max((len(run) for run in re.findall(r"`+", text)), default=0) + fence: Final = "`" * max(3, longest_run + 1) + return (f"{fence}{info}", text, fence) + + +def _row_section(row: SpendLogRow, index: int, payload: RequestResponsePayload | None, max_chars: int) -> str: + err: Final = row.error + error_lines: Final = ( + ( + f"- error: `{err.error_code or '?'}` {err.error_class or ''}".rstrip(), + "", + *_fenced(err.error_message or ""), + ) + if err is not None and row.failed + else () + ) + body_lines: Final = ( + ( + "", + "
request body", + "", + *_fenced(_fmt_json(payload.proxy_server_request, max_chars), "json"), + "
", + "", + "
response", + "", + *_fenced(_fmt_json(payload.response, max_chars), "json"), + "
", + ) + if payload is not None + else () + ) + header: Final = f"### {index}. {'FAILED' if row.failed else 'ok'} {row.model or row.model_group or '?'}" + facts: Final = ( + f"- request_id: `{row.request_id}`", + f"- time: {row.start_time} -> {row.end_time}", + f"- provider: {row.custom_llm_provider or '?'} ({row.api_base or 'n/a'}), call_type: {row.call_type or '?'}", + f"- spend: ${row.spend:.6f}, tokens: {row.prompt_tokens} in / {row.completion_tokens} out", + ) + return "\n".join((header, *facts, *error_lines, *body_lines)) + + +def render_report( + *, + session_id: str, + base_url: str, + rows: Sequence[SpendLogRow], + payloads: Mapping[str, RequestResponsePayload | None], + max_chars: int, +) -> str: + failures: Final = tuple(r for r in rows if r.failed) + summary: Final = ( + f"# LiteLLM debug report: Claude Code session `{session_id}`", + "", + f"- proxy: {base_url}", + f"- generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + f"- turns: {len(rows)}, failed: {len(failures)}", + f"- total spend: ${sum(r.spend for r in rows):.6f}", + f"- models: {', '.join(sorted(frozenset(r.model or r.model_group or '?' for r in rows))) or 'n/a'}", + "", + "Bodies are included for failed turns and the most recent turns. " + "Bodies are empty unless the proxy runs with `general_settings.store_prompts_in_spend_logs: true`.", + "", + "## Turns", + "", + ) + sections: Final = tuple( + _row_section(row, i, payloads.get(row.request_id), max_chars) for i, row in enumerate(rows, start=1) + ) + return "\n".join(summary) + "\n\n".join(sections) + "\n" + + +def build_report( + *, + fetcher: SpendLogsFetcher, + session_id: str, + base_url: str, + recent_bodies: int, + max_chars: int, +) -> str | DebugFailure: + rows: Final = fetcher.session_rows(session_id) + if isinstance(rows, DebugFailure): + return rows + if not rows: + return DebugFailure( + f"No spend logs found for session {session_id!r} on {base_url}. " + "Is Claude Code routed through this proxy (`lite up`), and does your key have log access?" + ) + wanted: Final = frozenset(r.request_id for r in rows if r.failed) | frozenset( + r.request_id for r in rows[-recent_bodies:] if recent_bodies > 0 + ) + fetched: Final = MappingProxyType({rid: fetcher.payload(rid) for rid in sorted(wanted)}) + failed_payload: Final = next((p for p in fetched.values() if isinstance(p, DebugFailure)), None) + if failed_payload is not None: + return failed_payload + payloads: Final = MappingProxyType({rid: p for rid, p in fetched.items() if not isinstance(p, DebugFailure)}) + return render_report(session_id=session_id, base_url=base_url, rows=rows, payloads=payloads, max_chars=max_chars) + + +def write_report(report: str, session_id: str, report_dir: Path) -> Path: + report_dir.mkdir(parents=True, exist_ok=True) + path: Final = report_dir / f"claude-{session_id}.md" + path.write_text(report, encoding="utf-8") + path.chmod(0o600) + return path + + +def install_slash_command(claude_dir: Path) -> Path: + commands_dir: Final = claude_dir / "commands" + commands_dir.mkdir(parents=True, exist_ok=True) + path: Final = commands_dir / f"{SLASH_COMMAND_NAME}.md" + path.write_text(SLASH_COMMAND_BODY, encoding="utf-8") + return path + + +@click.group() +def debug() -> None: + """Pull debug reports (spend, request, response, error) for coding-agent sessions""" + + +@debug.command("claude") +@click.option( + "--session-id", + default=None, + help=f"Claude Code session id. Defaults to ${SESSION_ID_ENV}, else the most recently used transcript in ~/.claude", +) +@click.option( + "--recent-bodies", + default=3, + show_default=True, + type=click.IntRange(min=0), + help="Also include request/response bodies for the N most recent turns (failed turns always get bodies)", +) +@click.option( + "--max-body-chars", + default=20_000, + show_default=True, + type=click.IntRange(min=100), + help="Truncate each request/response body to this many characters", +) +@click.option("--no-save", is_flag=True, help="Print only, do not write the report under ~/.litellm/debug") +@click.pass_context +def debug_claude( + ctx: click.Context, session_id: str | None, recent_bodies: int, max_body_chars: int, no_save: bool +) -> None: + """Render a markdown debug report for one Claude Code session routed through the proxy + + Examples: + lite debug claude + lite debug claude --session-id e96634a3-fa28-4083-b354-55542e2dca01 + """ + resolved: Final = session_id or detect_claude_session_id(os.environ, CLAUDE_DIR) + if resolved is None: + raise click.ClickException(f"Could not find a Claude Code session. Pass --session-id or set ${SESSION_ID_ENV}.") + values: Final = cli_context_values(ctx) + base_url: Final = values["base_url"] + fetcher: Final = SpendLogsFetcher(HTTPClient(base_url, values["api_key"])) + outcome: Final = build_report( + fetcher=fetcher, + session_id=resolved, + base_url=base_url, + recent_bodies=recent_bodies, + max_chars=max_body_chars, + ) + if isinstance(outcome, DebugFailure): + raise click.ClickException(outcome.message) + click.echo(outcome) + if not no_save: + path: Final = write_report(outcome, resolved, REPORT_DIR) + click.echo(f"Saved to {path}", err=True) + + +@debug.command("install-claude-command") +def debug_install_claude_command() -> None: + """Install the /debug-lite slash command into ~/.claude/commands so Claude Code can run `lite debug claude`""" + path: Final = install_slash_command(CLAUDE_DIR) + click.echo(f"Installed /{SLASH_COMMAND_NAME}: {path}") + click.echo("Restart Claude Code (or start a new session), then type /debug-lite.") diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 2674bf49ff0..eae1b0f5bc9 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -14,6 +14,7 @@ from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.config import config_commands, get_config_value, hidden_command_names from .commands.credentials import credentials +from .commands.debug import debug from .commands.encryption import encryption from .commands.http import http from .commands.keys import keys @@ -143,6 +144,7 @@ cli.add_command(encryption) cli.add_command(chat) # Add the http command group cli.add_command(http) +cli.add_command(debug) # Add the keys command group cli.add_command(keys) # Add the teams command group diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 50b225ddc4c..fde40111d49 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -1096,10 +1096,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): add_guardrail_to_applied_guardrails_header, ) - # Collect all chunks - all_chunks: Final[list[Any]] = [] - async for chunk in response: - all_chunks.append(chunk) + all_chunks: Final[Sequence[object]] = tuple([chunk async for chunk in response]) if not all_chunks or self._is_terminal_error_stream(all_chunks): for chunk in all_chunks: diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 21e652114bc..2a7813bc140 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -789,6 +789,26 @@ def _for_teams(team_ids: Sequence[str | None]) -> str: return f" for team {', '.join(named)}" if named else "" +def _validate_model_scope(llm_router: "Router | None", models: Sequence[str]) -> None: + """Reject a scope naming a model no request on this proxy could carry, at start rather + than as a job that silently samples nothing. The question is "could any caller ask for + this name", not "does it resolve for the job's teams": a user target's traffic can arrive + on any team's key, so a team-public name is a legitimate scope for it, and an auto-router + is one too (a forward job on router A scoped to router B samples what B serves today). + Nothing here is ever dispatched to.""" + unreachable: Final = tuple( + model + for model in models + if judge_target(llm_router, model).via == "nothing" + and (llm_router is None or model not in llm_router.team_public_model_names) + ) + if unreachable: + raise HTTPException( + status_code=400, + detail="models not served by this proxy: " + ", ".join(f"'{model}'" for model in unreachable), + ) + + _JUDGED_ROLES: Final[frozenset[StrategyRouterDependencyRole]] = frozenset({"tier", "default"}) @@ -1080,6 +1100,7 @@ class _LegRow(BaseModel): target_id: str router_name: str router_names: tuple[str, ...] = () + models: tuple[str, ...] = () direction: ShadowEvalDirection baseline_model: str | None = None judge_model: str @@ -1150,6 +1171,7 @@ def _group_response( for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id)) ), router_names=first.arm_router_names, + models=first.models, direction=first.direction, baseline_model=first.baseline_model, judge_model=first.judge_model, @@ -1322,7 +1344,10 @@ async def start_shadow_eval( A target is a virtual key, a team, or a user. Team and user targets match on the identity every request resolves to at auth time, so they cover JWT-authenticated traffic, which presents no virtual key; a user target samples that user's traffic - across all their teams, whether it arrives on a JWT or a key they own. + across all their teams, whether it arrives on a JWT or a key they own. models narrows + every target to requests for those model groups, so a user plus one model samples that + user's traffic on that model across every key they own; it is forward-only, since a + reverse job already samples exactly the traffic its own router served. A forward job answers whether the targets should adopt router_name: it samples the requests the router did not serve and duplicates them through it. A reverse job @@ -1411,6 +1436,7 @@ async def start_shadow_eval( if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids) _validate_judge_is_not_a_candidate(llm_router, data, team_ids) + _validate_model_scope(llm_router, data.models) requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = ( *(("key", key) for key in data.api_key_ids), @@ -1456,6 +1482,7 @@ async def start_shadow_eval( # a pre-router_names pod samples router_name alone, so it must be a real arm "router_name": data.router_names[0], "router_names": list(data.router_names), # mutable-ok: Prisma payload + "models": list(data.models), # mutable-ok: Prisma payload "direction": data.direction, "baseline_model": data.baseline_model, "judge_model": data.judge_model, @@ -1517,6 +1544,7 @@ async def start_shadow_eval( for target_type, target_id in sorted(requested_targets) ), router_names=data.router_names, + models=data.models, direction=data.direction, baseline_model=data.baseline_model, judge_model=data.judge_model, diff --git a/litellm/proxy/middleware/admission_control_middleware.py b/litellm/proxy/middleware/admission_control_middleware.py index aa62ef9e3bf..e347428be83 100644 --- a/litellm/proxy/middleware/admission_control_middleware.py +++ b/litellm/proxy/middleware/admission_control_middleware.py @@ -32,9 +32,6 @@ class AdmissionControlSettings: queue_timeout_seconds: float -AdmissionControlSettingsGetter: TypeAlias = Callable[[], AdmissionControlSettings | None] # mutable-ok: Callable params - - @dataclass(frozen=True, slots=True) class AdmissionControlStats: admitted: int @@ -66,13 +63,10 @@ class AdmissionControlMetrics: rejected_counter: _Counter -AdmissionControlMetricsFactory: TypeAlias = Callable[[], AdmissionControlMetrics | None] # mutable-ok: Callable params - - class AdmissionControlState: """Per-process admission counters and the in-flight semaphore shared by one worker's requests.""" - def __init__(self, metrics_factory: AdmissionControlMetricsFactory) -> None: + def __init__(self, metrics_factory: Callable[[], AdmissionControlMetrics | None]) -> None: self._metrics_factory = metrics_factory self._metrics: AdmissionControlMetrics | None = None self._metrics_init_attempted = False @@ -140,7 +134,7 @@ class AdmissionControlMiddleware: def __init__( self, app: ASGIApp, - get_settings: AdmissionControlSettingsGetter, + get_settings: Callable[[], AdmissionControlSettings | None], state: AdmissionControlState, ) -> None: self.app = app diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index e638ad68b6f..1c43668f227 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob { target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // first (often only) auto-router under evaluation; router_names is the full set router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) + models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index dff100bdea7..f8831ca4152 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3365,23 +3365,24 @@ async def view_spend_logs( ) sql_query, params = summary_sql_and_params rows: Final[Sequence[_SpendDailySummaryRow]] = await _query_raw(prisma_client, sql_query, *params) - if len(rows) == 0: - return [] # pyright: ignore[reportUnknownVariableType] # empty summary has no element type - summary_items: Final = tuple( _daily_summary_item(date.fromisoformat(day), tuple(day_rows)) for day, day_rows in groupby(rows, key=lambda row: row["day"]) ) - final_date: Final = date.fromisoformat(rows[-1]["day"]) + final_date: Final = date.fromisoformat(rows[-1]["day"]) if len(rows) > 0 else None end_date_date: Final = end_date_obj.date() - padding: Final[tuple[Mapping[str, object], ...]] = tuple( - { - "startTime": final_date + timedelta(days=offset), - "spend": 0, - "users": {}, - "models": {}, - } - for offset in range(1, (end_date_date - final_date).days + 1) + padding: Final[tuple[Mapping[str, object], ...]] = ( + () + if final_date is None + else tuple( + { + "startTime": final_date + timedelta(days=offset), + "spend": 0, + "users": {}, + "models": {}, + } + for offset in range(1, (end_date_date - final_date).days + 1) + ) ) return [*summary_items, *padding] diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index f271655f5e3..9f9016c5a7f 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -496,10 +496,9 @@ class BaseResponsesAPIStreamingIterator: if logging_response is self.completed_response: return target: Final[object] = getattr(logging_response, "response", None) - existing_hidden: Final[object] = getattr(target, "_hidden_params", None) - if not isinstance(existing_hidden, Mapping): + if not isinstance(target, ResponsesAPIResponse): return - existing: Final[Mapping[str, object]] = existing_hidden + existing: Final[Mapping[str, object]] = target._hidden_params source_hidden: Final[object] = getattr( getattr(self.completed_response, "response", None), "_hidden_params", None ) @@ -510,15 +509,11 @@ class BaseResponsesAPIStreamingIterator: raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy # splats into the client's HTTP headers, and copying non-header keys would carry response_cost - setattr( # noqa: B010 # target is typed object here, so a plain attribute store does not type check - target, - "_hidden_params", - { # mutable-ok: the cost calculator writes optional_params into _hidden_params - "additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it - "headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it - **existing, - }, - ) + target._hidden_params = { # mutable-ok: the cost calculator writes optional_params into _hidden_params + "additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + "headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it + **existing, + } def _handle_logging_completed_response(self): """Base implementation - should be overridden by subclasses""" diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index 00f06c046a2..d411673439f 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -116,28 +116,6 @@ async def aattempt( return RustHandled(adapt(value)) -def call(operation: Callable[[], ResultT], context: BridgeErrorContext) -> ResultT: - exceptions: Final = native_exception_types() - if exceptions is None: - return operation() - upstream: Final = exceptions[1] - try: - return operation() - except upstream as error: - _raise_upstream(error, context) - - -async def acall(operation: Callable[[], Awaitable[ResultT]], context: BridgeErrorContext) -> ResultT: - exceptions: Final = native_exception_types() - if exceptions is None: - return await operation() - upstream: Final = exceptions[1] - try: - return await operation() - except upstream as error: - _raise_upstream(error, context) - - def _decline_reason(error: BaseException) -> str: reason: Final[object] = error.args[0] if error.args else str(error) return reason if isinstance(reason, str) else str(reason) @@ -170,11 +148,3 @@ def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoRetu llm_provider=context.provider, model=context.model, ) from error - - -def identity(value: ResultT) -> ResultT: - return value - - -async def async_none() -> None: - return None diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 88869a1edfb..50c3515cf01 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -292,6 +292,18 @@ class StartShadowEvalRequest(BaseModel): "to across all their teams: JWT requests carrying their subject claim and virtual keys they own" ), ) + models: tuple[str, ...] = Field( + default=(), + max_length=100, + description=( + "Model groups to narrow the sampled traffic to, matched on the group the caller " + "requested and resolved through model_group_alias, so an alias and its target are one " + "name. Empty samples every model the targets use. This ANDs with the targets: a job " + "over a user and one model samples that user's requests on that model across every key " + "they own, and none of their other traffic. Forward jobs only: a reverse job samples " + "exactly the traffic its own router served, which no other model group can name" + ), + ) router_name: str | None = Field( default=None, description=( @@ -372,12 +384,20 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) - @field_validator("api_key_ids", "team_ids", "user_ids") + @field_validator("api_key_ids", "team_ids", "user_ids", "models") @classmethod def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]: - """A target named twice would collide with itself on the one-active-per-(target, direction) index.""" + """A target named twice would collide with itself on the one-active-per-(target, direction) + index; a model named twice is one scope entry.""" return tuple(dict.fromkeys(value)) + @field_validator("models") + @classmethod + def _models_are_names(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if not all(name.strip() for name in value): + raise ValueError("models must be non-empty model group names") + return value + @model_validator(mode="after") def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest": total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids) @@ -387,6 +407,18 @@ class StartShadowEvalRequest(BaseModel): raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids") return self + @model_validator(mode="after") + def _model_scope_is_forward_only(self) -> "StartShadowEvalRequest": + """A reverse job admits exactly the requests its own router served, so every one of + them names that router and nothing else; any other scope would sample nothing and + the router itself is a no-op. Both readings are rejected rather than shipped as a + job that silently never samples.""" + if self.models and self.direction == "reverse": + raise ValueError( + "models is only meaningful for a forward job; a reverse job samples its own router's traffic" + ) + return self + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -599,6 +631,10 @@ class ShadowEvalJobResponse(BaseModel): "traffic and judge every arm against the same real responses" ), ) + models: tuple[str, ...] = Field( + default=(), + description="Model groups the sampled traffic is narrowed to; empty means every model the targets use", + ) direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 3710b55ca0f..fd7b30bc314 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -78,7 +78,7 @@ "limit": 1 }, "C901": { - "limit": 311 + "limit": 306 }, "D419": { "limit": 6 diff --git a/schema.prisma b/schema.prisma index e638ad68b6f..1c43668f227 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob { target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // first (often only) auto-router under evaluation; router_names is the full set router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) + models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index ff553be6461..d38a0eee3de 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -142,6 +142,8 @@ fi lint_dashboard() { ( + trap 'exit 143' TERM + trap 'rm -f "${report:-}"' EXIT rc=0 prettier_rel=() eslint_rel=() @@ -168,7 +170,6 @@ EOF report=$(mktemp) npx eslint . -f json -o "$report" || true node scripts/check-lint-budgets.mjs "$report" eslint-budgets.json || rc=1 - rm -f "$report" exit $rc ) } diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 6d50cb436e2..8a7b68511ec 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -19,11 +19,13 @@ failures are hard test failures (see `tests/e2e/CLAUDE.md`). | OpenAI | yes | yes | yes | yes | yes (lifecycle + terminal output) | OpenAI Files | | Azure | yes | yes | yes | yes | yes (byte-verbatim) | Azure Files | | Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) | -| Bedrock | yes (unified only) | yes | no (limited upstream) | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | +| Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | -Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off -(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix; -flipping those gates is tracked in LIT-4774 and deliberately not part of this suite. +Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the +lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`). +Bedrock has no provider-side list, so list is the proxy's DB-backed managed view: the +unified lifecycle lists with the plain `GET /v1/batches` and the batch must appear +there. Both were gated off until LIT-5730, after LIT-4774 landed cancel support. A batch that completes inside the 2 s pre-cancel window skips the cancel assertion (a documented vacuous pass for the cancel cell, same as OpenAI); the list assertion runs either way. Bedrock file upload requires a model on the request (`encoded` / `unified` scenarios only); `model_param` and `provider_fallback` are omitted because `POST /bedrock/v1/files` has no model-less passthrough path. @@ -148,6 +150,6 @@ never landed. Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a terminal DB status short-circuits retrieve for those ids, so the terminal-state cell uses the encoded path; poller timing does not fit an e2e gate and belongs in a -DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Bedrock -cancel/list stay gated pending LIT-4774. Gemini (non-Vertex) file content raises -`NotImplementedError` upstream and is not a coverage cell. +DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Gemini +(non-Vertex) file content raises `NotImplementedError` upstream and is not a +coverage cell. diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index ee44a50d215..1bcea0a61ee 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -143,8 +143,8 @@ PROVIDERS: tuple[Provider, ...] = ( "bedrock", batch_model_name("bedrock-batch"), "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - can_cancel=False, - can_list=False, + can_cancel=True, + can_list=True, ), ) @@ -248,8 +248,9 @@ def coverage_cells_for_lifecycle(cap: Capability) -> tuple[str, ...]: """Registry cell ids that the parametrized lifecycle test covers for one capability. OpenAI has per-scenario cells plus granular create/retrieve/cancel/list/file - cells. Other providers have one basic cell each. File-upload cells for the - batch-backing path are included when the lifecycle uploads for that provider. + cells. Bedrock adds cancel and list cells behind its gates. Other providers + have one basic cell each. File-upload cells for the batch-backing path are + included when the lifecycle uploads for that provider. """ match cap.provider: case "openai": @@ -279,6 +280,8 @@ def coverage_cells_for_lifecycle(cap: Capability) -> tuple[str, ...]: return ( "llm.batches.bedrock.basic.nonstream.works", "llm.files.bedrock.upload.nonstream.works", + *(("llm.batches.bedrock.cancel.nonstream.works",) if cap.can_cancel else ()), + *(("llm.batches.bedrock.list.nonstream.works",) if cap.can_list else ()), ) case _: return () diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 7af064b1fdd..ed7cf656d01 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -77,7 +77,7 @@ BATCH_OP_RETRIES = 5 # (connection refused, brief 500s) and the registry only has one basic cell per # provider (shared across scenarios). Create + retrieve already prove routing; # cancel is still deferred for cleanup, just not asserted for these two. -_CANCEL_ASSERTED_PROVIDERS = frozenset({"openai"}) +_CANCEL_ASSERTED_PROVIDERS = frozenset({"openai", "bedrock"}) def _transient_status(status_code: int) -> bool: @@ -293,14 +293,13 @@ def test_batch_lifecycle( f"batch reached {pre_cancel.status!r} before cancel; " "provider likely rejected the input" ) - if pre_cancel.status == "completed": - return - cancelled = cancel_batch(client, batch.id, key=key, provider=provider) - assert cancelled.id == batch.id - assert cancelled.object == "batch" - assert cancelled.status in {"cancelling", "cancelled"}, ( - f"unexpected post-cancel status {cancelled.status!r}" - ) + if pre_cancel.status != "completed": + cancelled = cancel_batch(client, batch.id, key=key, provider=provider) + assert cancelled.id == batch.id + assert cancelled.object == "batch" + assert cancelled.status in {"cancelling", "cancelled"}, ( + f"unexpected post-cancel status {cancelled.status!r}" + ) if cap.can_list: list_result = client.list_batches(key=key, provider=provider) diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 47d296e61f3..635ea3f7ea5 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -23,6 +23,8 @@ - {id: llm.batches.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Vertex batches"} - {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"} - {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"} +- {id: llm.batches.bedrock.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch cancel (StopModelInvocationJob) returns the same id with a cancelling/cancelled status"} +- {id: llm.batches.bedrock.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "A Bedrock managed batch is present in the GET /v1/batches list envelope"} - {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"} - {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"} - {id: llm.batches.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.18 / LIT-4778", rationale: "Missing input_file_id and invalid batch id rejected"} diff --git a/tests/store_model_in_db_tests/test_openai_error_handling.py b/tests/store_model_in_db_tests/test_openai_error_handling.py index 9a18d7f3420..d3f38f93bf3 100644 --- a/tests/store_model_in_db_tests/test_openai_error_handling.py +++ b/tests/store_model_in_db_tests/test_openai_error_handling.py @@ -157,6 +157,9 @@ async def test_chat_completion_bad_model_with_spend_logs(): except json.JSONDecodeError: print(f"Could not parse response body as JSON: {response.text}") + assert ( + response.status_code == 400 + ), f"expected HTTP 400, got {response.status_code}: {response.text}" assert ( litellm_call_id is not None ), "Failed to get LiteLLM Call ID from response headers" @@ -191,7 +194,7 @@ async def test_chat_completion_bad_model_with_spend_logs(): # Verify the structure of the log entry assert log_entry["request_id"] == litellm_call_id assert log_entry["model"] == "non-existent-model" - assert log_entry["model_group"] == "non-existent-model" + assert log_entry["model_group"] in ("", "non-existent-model") assert log_entry["spend"] == 0.0 assert log_entry["total_tokens"] == 0 assert log_entry["prompt_tokens"] == 0 @@ -206,8 +209,7 @@ async def test_chat_completion_bad_model_with_spend_logs(): error_info = log_entry["metadata"]["error_information"] assert "traceback" in error_info assert error_info["error_code"] == "400" - assert error_info["error_class"] == "BadRequestError" - assert "litellm.BadRequestError" in error_info["error_message"] + assert error_info["error_class"] in ("ProxyModelNotFoundError", "BadRequestError") assert "non-existent-model" in error_info["error_message"] # Verify request details diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 4973bda29e0..b735abaf7bf 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -188,6 +188,62 @@ def test_streaming_span_carries_time_to_first_chunk(): assert span.attributes[GenAI.RESPONSE_TIME_TO_FIRST_CHUNK] == pytest.approx(0.75) +def test_llm_call_span_reports_the_server_spans_route(): + """``litellm.request.route`` is the anchored server span's own ``http.route``, + so an operator can group LLM spans by endpoint without joining to the parent.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /engines/{model:path}/chat/completions") + root.set_attribute("http.route", "/engines/{model:path}/chat/completions") + set_request_root_span(root) + + _emit_llm(logger, ambient=root) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/engines/{model:path}/chat/completions" + + +def test_llm_call_span_omits_the_route_without_a_server_span(): + """An SDK call has no server span, so the key is absent rather than empty.""" + logger, exporter = _logger() + _emit_llm(logger) + (span,) = exporter.get_finished_spans() + assert LiteLLM.REQUEST_ROUTE not in span.attributes + + +def test_failed_llm_call_span_reports_the_server_spans_route(): + """The failure leg builds the same span data, so an errored call is still + attributable to the endpoint it came in on.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /v1/responses/{response_id}") + root.set_attribute("http.route", "/v1/responses/{response_id}") + set_request_root_span(root) + + _emit_llm(logger, ambient=root, fail=True) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}" + + +def test_deferred_llm_call_span_reports_the_server_spans_route(): + """``pre_call`` driven from a thread pool sees no recordable parent, so the span + is created in the close callback instead. That branch has to carry the route + too, and it can: the worker context still holds the anchor.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /v1/messages") + root.set_attribute("http.route", "/v1/messages") + set_request_root_span(root) + + # no ``ambient``: pre_call runs with no recordable span active, which is what + # defers creation to the close callback + _emit_llm(logger) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/messages" + + def test_non_streaming_span_has_no_time_to_first_chunk(): logger, exporter = _logger() kwargs = { diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py index 0cd71db4ae1..e2007486a40 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py @@ -5,6 +5,8 @@ surface and the server-span + shared-provider behavior it produces. """ +from datetime import datetime, timezone + import pytest @@ -18,6 +20,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, ) +from opentelemetry import trace # noqa: E402 from opentelemetry.trace import SpanKind # noqa: E402 from litellm.integrations.otel.model.config import ( # noqa: E402 @@ -30,6 +33,23 @@ from litellm.integrations.otel.mount import ( # noqa: E402 _passthrough_span_name_hook, instrument_fastapi_app, ) +from litellm.integrations.otel.plumbing.context import ( # noqa: E402 + request_root_http_route, + set_request_root_span, +) + + +@pytest.fixture(autouse=True) +def _reset_request_root_span(): + """Clear the root-span anchor around every test. Production gets a fresh + contextvar copy per request task; the test process shares one context.""" + from litellm.integrations.otel.plumbing import context as _otel_context + + _otel_context._request_root_span.set(None) + _otel_context._mcp_message_transport_span.set(None) + yield + _otel_context._request_root_span.set(None) + _otel_context._mcp_message_transport_span.set(None) @pytest.fixture(autouse=True) @@ -128,6 +148,85 @@ def test_passthrough_hook_ignores_non_recording_span(): assert span.name is None +def test_llm_span_route_is_read_off_the_server_span(monkeypatch): + """``request_root_http_route`` answers with the SERVER span's own ``http.route``. + + Driven through ``instrument_fastapi_app`` and the same + ``create_litellm_proxy_request_started_span`` call the proxy makes per request, + so breaking either the mount or the anchor capture fails this.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "1") + is_otel_v2_enabled.cache_clear() + app = fastapi.FastAPI() + seen = {} + + def _anchor_then_read(key): + logger.create_litellm_proxy_request_started_span(start_time=datetime.now(timezone.utc), headers=None) + seen[key] = request_root_http_route() + + @app.post("/engines/{model:path}/chat/completions") + async def engines(model: str): + _anchor_then_read("templated") + return {} + + @app.post("/openai/{endpoint:path}") + async def openai_passthrough(endpoint: str): + _anchor_then_read("passthrough") + return {} + + logger = OpenTelemetryV2(config=OpenTelemetryV2Config(exporter="in_memory")) + exporter = InMemorySpanExporter() + logger._tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + # instrument_fastapi_app passes no provider, so it binds to the OTel global the + # way the proxy does once proxy_startup_event publishes one. set_tracer_provider + # is a once-per-process door, so place it directly and let monkeypatch undo it. + monkeypatch.setattr(trace, "_TRACER_PROVIDER", logger._tracer_provider) + instrument_fastapi_app(app) + + client = TestClient(app) + client.post("/engines/gpt-4o-mini/chat/completions") + client.post("/openai/v1/responses/resp_abc123") + + routes = { + (s.attributes or {})["http.route"] for s in exporter.get_finished_spans() if s.kind is SpanKind.SERVER + } + # a parameterized route keeps its template; the passthrough hook rewrote the + # catch-all to the literal path, and both spans have to follow their own span + assert routes == {"/engines/{model:path}/chat/completions", "/openai/v1/responses/resp_abc123"} + assert seen["templated"] == "/engines/{model:path}/chat/completions" + assert seen["passthrough"] == "/openai/v1/responses/resp_abc123" + + +def test_server_span_route_survives_the_span_ending(): + """The LLM span closes in an async callback that can run after the server span + has ended, so the attribute has to still be readable then.""" + from opentelemetry.sdk.trace import TracerProvider + + span = TracerProvider().get_tracer("t").start_span("POST /v1/responses/{response_id}") + span.set_attribute("http.route", "/v1/responses/{response_id}") + set_request_root_span(span) + span.end() + + assert request_root_http_route() == "/v1/responses/{response_id}" + + +def test_no_server_span_means_no_route(): + """An SDK call has no anchored server span, so the attribute is omitted rather + than reported as empty.""" + assert request_root_http_route() is None + + +def test_blank_route_on_the_server_span_is_omitted(): + """An excluded or unmatched path leaves the server span without a usable route. + Report nothing rather than a span attribute whose value is the empty string.""" + from opentelemetry.sdk.trace import TracerProvider + + span = TracerProvider().get_tracer("t").start_span("GET") + span.set_attribute("http.route", "") + set_request_root_span(span) + + assert request_root_http_route() is None + + def test_known_passthrough_prefixes_present(): """Guard the prefix set against accidental edits.""" assert {"openai", "anthropic", "vertex_ai", "bedrock"} <= PASSTHROUGH_PREFIXES diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 99d706a9c44..addadf8e598 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -722,6 +722,39 @@ def test_request_identity_falls_back_to_legacy_team_keys(): assert ident.team_alias == "legacy" +def test_llm_span_carries_proxy_request_route(): + """The LLM span records the proxy route the request arrived on, so it can be + filtered by endpoint (``/v1/responses`` vs ``/v1/chat/completions``) without + joining back to the root SERVER span's ``http.route``. The value is that + span's ``http.route`` verbatim, so a parameterized route reports the template + the SERVER span reports and not the path the caller happened to send.""" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(metadata={"user_api_key_request_route": "/v1/responses/resp_abc123"}), + request_route="/v1/responses/{response_id}", + ) + attrs: Final = GenAIMapper().map(data) + + assert attrs[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}" + + +def test_llm_span_falls_back_to_the_logged_route_without_a_server_span(): + """The route the proxy recorded at auth is the backstop for a deployment whose + FastAPI instrumentation never mounted: there is no server span to disagree with + there, and an endpoint name is worth more than an absent attribute.""" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(metadata={"user_api_key_request_route": "/v1/responses"}) + ) + + assert GenAIMapper().map(data)[LiteLLM.REQUEST_ROUTE] == "/v1/responses" + + +def test_llm_span_omits_request_route_off_the_proxy(): + """An SDK call has no inbound route, so the key is absent rather than empty.""" + attrs: Final = GenAIMapper().map(LLMCallSpanData.from_standard_logging_payload(_sample_payload(metadata={}))) + + assert LiteLLM.REQUEST_ROUTE not in attrs + + def test_guardrail_span_data_block_carries_verdict_and_error(): from litellm.integrations.otel.model.payloads import GuardrailSpanData diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index f273a285d49..ebfa1d0eb2f 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -72,6 +72,7 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash target_id=target_id, router_name=job.router_name, router_names=job.router_names, + models=sorted(job.models), direction=job.direction, baseline_model=job.baseline_model, shadow_percentage=job.shadow_percentage, @@ -205,6 +206,7 @@ def _success_kwargs( request_metadata=None, call_type="acompletion", model="claude-opus", + model_group="opus-group", response_cost=None, cache_hit=None, ): @@ -213,6 +215,7 @@ def _success_kwargs( "id": request_id, "call_type": call_type, "model": model, + "model_group": model_group, "metadata": {"user_api_key_hash": api_key_hash}, "model_parameters": {"temperature": 0.5, "stream": True}, "response_cost": response_cost, @@ -1007,6 +1010,79 @@ class TestTargetMatching: assert logger._job_starts == {"key-job": 1, "team-job": 1} +@pytest.mark.asyncio +class TestModelScope: + """A job scoped to model groups samples a target's request only when the group the + caller asked for is one of them; an out-of-scope request is not the job's traffic at + all, so it records no funnel event, exactly like a direction mismatch.""" + + @pytest.mark.parametrize( + "requested,sampled", + [("sonnet-group", True), ("opus-group", False), ("", False)], + ids=["in-scope-group-samples", "other-group-skips", "unknown-group-fails-closed"], + ) + async def test_scope_admits_only_the_named_groups_and_counts_nothing_else(self, requested, sampled): + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, jobs=(_job(models=frozenset({"sonnet-group", "haiku-group"})),)) + + await logger.async_log_success_event(_success_kwargs(model_group=requested), RESPONSE, None, None) + await _drain(logger) + + assert prisma.db.litellm_shadowevalattempt.create.await_count == (1 if sampled else 0) + assert logger._test_funnel == [] + + async def test_an_unscoped_job_samples_every_group(self): + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, jobs=(_job(),)) + + await logger.async_log_success_event(_success_kwargs(model_group="anything"), RESPONSE, None, None) + await _drain(logger) + + prisma.db.litellm_shadowevalattempt.create.assert_awaited_once() + + @pytest.mark.parametrize( + "scoped_to,requested", + [("sonnet-group", "fast"), ("fast", "sonnet-group")], + ids=["job-names-the-target-request-uses-the-alias", "job-names-the-alias-request-uses-the-target"], + ) + async def test_an_alias_and_its_target_are_one_group_on_both_sides(self, scoped_to, requested): + """Both the job's scope and the request's group resolve through the router's alias + map at match time, so re-pointing an alias follows config rather than freezing at + job start.""" + router = _router() + router.model_group_alias = {"fast": "sonnet-group"} + prisma = _prisma(jobs=[_job_record(_job(models=frozenset({scoped_to})))]) + logger = _logger(router=router, prisma=prisma) + + await logger.async_log_success_event(_success_kwargs(model_group=requested), RESPONSE, None, None) + await _drain(logger) + + prisma.db.litellm_shadowevalattempt.create.assert_awaited_once() + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 + + async def test_a_repointed_alias_applies_to_the_next_request_without_a_cache_refill(self): + router = _router() + router.model_group_alias = {"fast": "sonnet-group"} + prisma = _prisma(jobs=[_job_record(_job(models=frozenset({"fast"})))]) + logger = _logger(router=router, prisma=prisma) + await logger.async_log_success_event(_success_kwargs(model_group="sonnet-group"), RESPONSE, None, None) + await _drain(logger) + assert prisma.db.litellm_shadowevalattempt.create.await_count == 1 + + router.model_group_alias = {"fast": "haiku-group"} + await logger.async_log_success_event( + _success_kwargs(request_id="req-2", model_group="sonnet-group"), RESPONSE, None, None + ) + await logger.async_log_success_event( + _success_kwargs(request_id="req-3", model_group="haiku-group"), RESPONSE, None, None + ) + await _drain(logger) + + rows = [call.kwargs["data"]["request_id"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list] + assert rows == ["req-1", "req-3"] + assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 + + @pytest.mark.asyncio class TestActiveJobsCache: async def test_cache_miss_reads_db_once_then_serves_from_cache(self): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py index 00ff06ea082..992b1e8632b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_credentials.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from jwt.utils import base64url_decode, base64url_encode from pydantic import SecretStr from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( @@ -52,6 +53,12 @@ def _refresh_token() -> str: return minted.token.get_secret_value() +def _corrupt_signature(token: str) -> str: + unsigned, signature = token.rsplit(".", 1) + raw = base64url_decode(signature) + return f"{unsigned}.{base64url_encode(bytes((raw[0] ^ 0x01,)) + raw[1:]).decode()}" + + def test_kdf_is_deterministic_and_key_length_is_256_bit(): again = session_keys_from_master_key(MASTER_KEY) assert again.signing_key.get_secret_value() == KEYS.signing_key.get_secret_value() @@ -109,8 +116,7 @@ def test_resolve_fails_expired_token_closed_and_flags_expiry(): def test_resolve_fails_tampered_token_closed_without_expiry_flag(): token = _access_token() - tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb") - result = resolve_session_bearer(f"Bearer {tampered}", KEYS, NOW) + result = resolve_session_bearer(f"Bearer {_corrupt_signature(token)}", KEYS, NOW) assert isinstance(result, SessionBearerInvalid) assert result.expired is False diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py index 2a59e6c1baa..321d6d0a1d2 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_session_token.py @@ -6,6 +6,7 @@ import jwt import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from jwt.utils import base64url_decode, base64url_encode from pydantic import SecretStr, ValidationError from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import ( @@ -66,6 +67,12 @@ def _mint_refresh() -> str: return minted.token.get_secret_value() +def _corrupt_signature(token: str) -> str: + unsigned, signature = token.rsplit(".", 1) + raw = base64url_decode(signature) + return f"{unsigned}.{base64url_encode(bytes((raw[0] ^ 0x01,)) + raw[1:]).decode()}" + + def _sign_claims(payload: dict, prefix: str = SESSION_TOKEN_PREFIX, keys: SessionKeys = KEYS) -> str: return prefix + jwt.encode(payload, keys.signing_key.get_secret_value(), algorithm="HS256") @@ -138,8 +145,7 @@ def test_still_valid_one_second_before_expiry(): def test_tampered_signature_is_bad_signature(): token = _mint_access() - tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb") - assert isinstance(open_session_token(tampered, KEYS, NOW), SessionBadSignature) + assert isinstance(open_session_token(_corrupt_signature(token), KEYS, NOW), SessionBadSignature) def test_key_rotation_invalidates_outstanding_tokens(): @@ -329,8 +335,7 @@ def test_rs256_tampered_signature_is_bad_signature(): minted = mint_session_token(PRINCIPAL, RSA_KEYS, NOW) assert isinstance(minted, MintedSessionToken) token = minted.token.get_secret_value() - tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb") - assert isinstance(open_session_token(tampered, RSA_KEYS, NOW), SessionBadSignature) + assert isinstance(open_session_token(_corrupt_signature(token), RSA_KEYS, NOW), SessionBadSignature) def test_rs256_expired_token_is_expired(): @@ -413,8 +418,7 @@ def test_rotation_window_still_enforces_expiry_and_tamper_on_the_previous_key(): ) after = NOW + timedelta(seconds=SESSION_TTL_SECONDS + 1) assert isinstance(open_session_token(token, rotated, after), SessionExpired) - tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb") - assert isinstance(open_session_token(tampered, rotated, NOW), SessionBadSignature) + assert isinstance(open_session_token(_corrupt_signature(token), rotated, NOW), SessionBadSignature) def test_weak_or_garbage_private_key_pem_rejected_at_construction(): diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py index a17fed36f52..fc6de53cb9e 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -1,5 +1,5 @@ import asyncio -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Optional, Tuple from unittest.mock import patch import click @@ -7,7 +7,8 @@ import pytest import yaml from click.testing import CliRunner from InquirerPy.base.control import Choice -from prompt_toolkit.application import create_app_session +from InquirerPy.prompts.fuzzy import InquirerPyFuzzyControl +from prompt_toolkit.application import AppSession, create_app_session from prompt_toolkit.input import create_pipe_input from prompt_toolkit.output import DummyOutput @@ -283,27 +284,45 @@ class TestRunConfigureWizardNotInteractive: assert not config_path.exists() +def _highlighted_choice(session: AppSession) -> Optional[str]: + if session.app is None: + return None + controls = [c for c in session.app.layout.find_all_controls() if isinstance(c, InquirerPyFuzzyControl)] + if not controls or controls[0].choice_count == 0: + return None + return controls[0].selection["name"] + + +async def _wait_until_highlighted(session: AppSession, name: str) -> None: + async def _poll() -> None: + while _highlighted_choice(session) != name: + await asyncio.sleep(0.01) + + await asyncio.wait_for(_poll(), timeout=5) + + def _drive_fuzzy_pick( models: Tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool, - key_events: List[Tuple[str, float]], + key_events: List[Tuple[str, Optional[str]]], ) -> List[str]: """Drives the real InquirerPy fuzzy prompt through prompt_toolkit's own test input/output, exercising the actual widget (filtering, tab-to-toggle, enter-to-confirm) rather than mocking it away. asyncio.to_thread propagates the create_app_session context into the worker thread - running _fuzzy_pick's synchronous .execute() call.""" + running _fuzzy_pick's synchronous .execute() call. Each key event names the choice the widget + must highlight before the next key is sent (None sends the next key immediately).""" async def _run() -> List[str]: with create_pipe_input() as pipe_input: - with create_app_session(input=pipe_input, output=DummyOutput()): + with create_app_session(input=pipe_input, output=DummyOutput()) as session: task = asyncio.ensure_future( asyncio.to_thread(wizard_module._fuzzy_pick, models, prompt_label, multiselect) ) - await asyncio.sleep(0.05) - for text, delay in key_events: + for text, highlighted in key_events: pipe_input.send_text(text) - await asyncio.sleep(delay) + if highlighted is not None: + await _wait_until_highlighted(session, highlighted) return await task return asyncio.run(_run()) @@ -315,13 +334,13 @@ class TestFuzzyPickWidget: def test_single_select_filters_and_returns_highlighted_match(self): result = _drive_fuzzy_pick( - self._models(), "test", multiselect=False, key_events=[("model-13", 0.3), ("\r", 0.1)] + self._models(), "test", multiselect=False, key_events=[("model-13", "model-13"), ("\r", None)] ) assert result == ["model-13"] def test_multiselect_requires_tab_to_toggle_before_enter(self): result = _drive_fuzzy_pick( - self._models(), "test", multiselect=True, key_events=[("model-7", 0.3), ("\t", 0.1), ("\r", 0.1)] + self._models(), "test", multiselect=True, key_events=[("model-7", "model-7"), ("\t", None), ("\r", None)] ) assert result == ["model-7"] @@ -331,12 +350,12 @@ class TestFuzzyPickWidget: "test", multiselect=True, key_events=[ - ("model-3", 0.3), - ("\t", 0.1), - *[("\x7f", 0.02) for _ in range("model-3".__len__())], - ("model-15", 0.3), - ("\t", 0.1), - ("\r", 0.1), + ("model-3", "model-3"), + ("\t", None), + ("\x7f" * len("model-3"), None), + ("model-15", "model-15"), + ("\t", None), + ("\r", None), ], ) assert set(result) == {"model-3", "model-15"} diff --git a/tests/test_litellm/proxy/client/cli/test_debug_commands.py b/tests/test_litellm/proxy/client/cli/test_debug_commands.py new file mode 100644 index 00000000000..419fd821cfb --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_debug_commands.py @@ -0,0 +1,231 @@ +import json +import os +import time + +import pytest +import requests +import responses +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands import debug as debug_module +from litellm.proxy.client.cli.commands.debug import ( + SLASH_COMMAND_NAME, + detect_claude_session_id, + install_slash_command, +) + +SESSION = "e96634a3-fa28-4083-b354-55542e2dca01" + +OK_ROW = { + "request_id": "req-ok", + "startTime": "2026-09-02T10:00:00", + "endTime": "2026-09-02T10:00:02", + "model": "claude-opus-4-1", + "custom_llm_provider": "anthropic", + "status": "success", + "spend": 0.0125, + "prompt_tokens": 100, + "completion_tokens": 20, + "metadata": {"status": "success"}, +} +FAILED_ROW = { + "request_id": "req-failed", + "startTime": "2026-09-02T10:01:00", + "endTime": "2026-09-02T10:01:01", + "model": "claude-opus-4-1", + "custom_llm_provider": "anthropic", + "status": "failure", + "spend": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0, + "metadata": json.dumps( + { + "status": "failure", + "error_information": { + "error_code": "400", + "error_class": "BadRequestError", + "error_message": "`prompt` is required when `stop` is not true.", + }, + } + ), +} + + +PROXY = "http://localhost:4000" + + +def _mock_proxy(rows, payloads): + responses.get( + f"{PROXY}/spend/logs/session/ui", + json={"data": rows, "total": len(rows), "page": 1, "page_size": 100, "total_pages": 1}, + match=[responses.matchers.query_param_matcher({"session_id": SESSION}, strict_match=False)], + ) + for request_id, payload in payloads.items(): + responses.get(f"{PROXY}/spend/logs/ui/{request_id}", json=payload) + + +def _called_paths(): + return [c.request.path_url.split("?")[0] for c in responses.calls] + + +@pytest.fixture(autouse=True) +def env(monkeypatch, tmp_path): + monkeypatch.setenv("LITELLM_PROXY_URL", PROXY) + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") + monkeypatch.setattr(debug_module, "REPORT_DIR", tmp_path / "reports") + monkeypatch.setattr(debug_module, "CLAUDE_DIR", tmp_path / "claude") + + +@responses.activate +def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path): + payloads = { + "req-failed": { + "proxy_server_request": {"body": {"model": "claude-opus-4-1", "messages": [{"role": "user"}]}}, + "response": {"error": {"message": "`prompt` is required"}}, + }, + "req-ok": {"proxy_server_request": {"body": {"model": "claude-opus-4-1"}}, "response": {"id": "msg_1"}}, + } + _mock_proxy([FAILED_ROW, OK_ROW], payloads) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--recent-bodies", "0"]) + + assert result.exit_code == 0, result.output + assert "turns: 2, failed: 1" in result.output + assert "total spend: $0.012500" in result.output + assert "### 1. ok claude-opus-4-1" in result.output + assert "### 2. FAILED claude-opus-4-1" in result.output + assert "`400` BadRequestError" in result.output + assert "`prompt` is required when `stop` is not true." in result.output + assert '"messages"' in result.output + assert "msg_1" not in result.output + assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-failed"] + saved = tmp_path / "reports" / f"claude-{SESSION}.md" + assert result.stdout.startswith(saved.read_text()) + assert "### 2. FAILED" in saved.read_text() + + +@responses.activate +def test_recent_bodies_fetches_latest_turns_even_when_successful(): + _mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": {"x": 1}}, "response": {"id": "msg_1"}}}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + + assert result.exit_code == 0, result.output + assert "msg_1" in result.output + assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"] + + +@responses.activate +def test_bodies_are_truncated_to_max_chars(): + _mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}}) + result = CliRunner().invoke( + cli, ["debug", "claude", "--session-id", SESSION, "--no-save", "--max-body-chars", "200"] + ) + + assert result.exit_code == 0, result.output + assert "truncated" in result.output + assert "a" * 300 not in result.output + + +@responses.activate +def test_no_rows_is_a_clear_error(): + _mock_proxy([], {}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert result.exit_code != 0 + assert "No spend logs found for session" in result.output + + +def test_no_session_id_anywhere_is_a_clear_error(monkeypatch): + monkeypatch.delenv("CLAUDE_CODE_SESSION_ID", raising=False) + result = CliRunner().invoke(cli, ["debug", "claude"]) + assert result.exit_code != 0 + assert "Could not find a Claude Code session" in result.output + + +OLD_SESSION = "0f3c2b1a-1111-4222-8333-444455556666" +NEW_SESSION = "2d79c54d-4644-4708-b03e-95395ef9ecbd" + + +def test_detect_session_id_prefers_env_then_newest_session_transcript(tmp_path): + project = tmp_path / "projects" / "-Users-me-repo" + project.mkdir(parents=True) + old = project / f"{OLD_SESSION}.jsonl" + new = project / f"{NEW_SESSION}.jsonl" + subagent = project / "agent-a1b2c3d4.jsonl" + old.write_text("{}") + new.write_text("{}") + subagent.write_text("{}") + now = time.time() + os.utime(old, (now - 100, now - 100)) + os.utime(new, (now - 50, now - 50)) + os.utime(subagent, (now, now)) + + assert detect_claude_session_id({}, tmp_path) == NEW_SESSION + assert detect_claude_session_id({"CLAUDE_CODE_SESSION_ID": "from-env"}, tmp_path) == "from-env" + assert detect_claude_session_id({"CLAUDE_SESSION_ID": "stale-name"}, tmp_path) == NEW_SESSION + assert detect_claude_session_id({}, tmp_path / "missing") is None + + +def test_install_slash_command_writes_runnable_command_file(tmp_path): + path = install_slash_command(tmp_path) + assert path == tmp_path / "commands" / f"{SLASH_COMMAND_NAME}.md" + body = path.read_text() + assert body.startswith("---\n") + assert "allowed-tools: Bash(lite debug claude:*)" in body + assert "!`lite debug claude $ARGUMENTS`" in body + + result = CliRunner().invoke(cli, ["debug", "install-claude-command"]) + assert result.exit_code == 0, result.output + assert "/debug-lite" in result.output + + +@responses.activate +def test_rejected_key_is_a_clear_error_not_a_traceback(): + responses.get( + f"{PROXY}/spend/logs/session/ui", + status=401, + json={"error": {"message": "Authentication Error, Invalid proxy server token passed", "code": "401"}}, + ) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "401" in result.output + assert "Invalid proxy server token passed" in result.output + + +@responses.activate +def test_unreachable_proxy_is_a_clear_error_not_a_traceback(): + responses.get(f"{PROXY}/spend/logs/session/ui", body=requests.ConnectionError("Connection refused")) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "Connection refused" in result.output + + +@responses.activate +def test_non_json_proxy_response_is_a_clear_error_not_a_traceback(): + responses.get(f"{PROXY}/spend/logs/session/ui", body="502 Bad Gateway") + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION]) + + assert isinstance(result.exception, SystemExit), result.exception + assert result.exit_code == 1 + assert "/spend/logs/session/ui failed" in result.output + + +@responses.activate +def test_logged_content_with_code_fences_stays_inside_its_fence(): + fenced_error_row = { + **FAILED_ROW, + "metadata": { + "status": "failure", + "error_information": {"error_code": "400", "error_message": "bad\n```\nrequest"}, + }, + } + _mock_proxy([fenced_error_row], {"req-failed": {"proxy_server_request": None, "response": "x\n````\ny"}}) + result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"]) + + assert result.exit_code == 0, result.output + assert "````\nbad\n```\nrequest\n````\n" in result.output + assert "`````json\nx\n````\ny\n`````\n" in result.output diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 624d2f00817..e9e58347337 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1,6 +1,7 @@ import asyncio import json import time +from typing import Final from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -1190,27 +1191,23 @@ def test_health_liveliness_endpoint(proxy_client): Test that /health/liveliness endpoint returns 200 OK with "I'm alive!" message. This is a critical orchestration endpoint that must be simple and fast. """ - # Measure the time taken for the health check call - start_time = time.perf_counter() + warm_up: Final = proxy_client.get("/health/liveliness") + assert warm_up.status_code == 200, f"Expected 200 OK, got {warm_up.status_code}: {warm_up.text}" - # Make GET request to /health/liveliness - response = proxy_client.get("/health/liveliness") + def _timed_poll() -> tuple[float, httpx.Response]: + start_time: Final = time.perf_counter() + response: Final = proxy_client.get("/health/liveliness") + return (time.perf_counter() - start_time) * 1000, response - end_time = time.perf_counter() - duration_ms = (end_time - start_time) * 1000 + polls: Final = tuple(_timed_poll() for _ in range(5)) - # Assert response status - assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}" + for _, response in polls: + assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}" + assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}" - # Assert response content (FastAPI JSON-encodes the string) - assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}" - - # Verify response is fast (should be < 100ms for a simple endpoint) - # This is critical for orchestration systems that poll frequently - assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint" - - # Log the duration for visibility (useful for CI/CD monitoring) - print(f"\n/health/liveliness response time: {duration_ms:.2f}ms") + durations_ms: Final = tuple(sorted(duration_ms for duration_ms, _ in polls)) + median_ms: Final = durations_ms[len(durations_ms) // 2] + assert median_ms < 100, f"Median of {len(polls)} health checks took {median_ms:.2f}ms, expected < 100ms" def test_health_liveness_endpoint(proxy_client): diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index c525af84511..21f8d985f22 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -882,6 +882,7 @@ def _leg_record(**overrides: object) -> MagicMock: "target_id": "key-hash", "router_name": "my-router", "router_names": (), + "models": (), "direction": "forward", "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", @@ -1033,6 +1034,7 @@ def _shadow_prisma( "target_id", "router_name", "router_names", + "models", "direction", "baseline_model", "judge_model", @@ -1533,6 +1535,87 @@ async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkey assert rows[0]["baseline_model"] is None +@pytest.mark.asyncio +async def test_start_shadow_eval_writes_the_model_scope_on_every_leg_and_echoes_it(monkeypatch: pytest.MonkeyPatch): + """A model scope is job config, so every leg carries the same copy and both the start + response and a later list read report it; an auto-router is a legitimate scope (a + forward job on one router may sample what another router serves today).""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval( + _start_request(api_key_ids=("key-hash", "key-hash-2"), models=("cheap", "sonnet-router")), ADMIN + ) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [row["models"] for row in rows] == [["cheap", "sonnet-router"], ["cheap", "sonnet-router"]] + assert response.models == ("cheap", "sonnet-router") + + listed = _shadow_prisma(legs=[_leg_record(models=("cheap",)), _leg_record(id="leg-0", group_id="job-0")]) + monkeypatch.setattr(proxy_server, "prisma_client", listed) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) + assert {job.job_id: job.models for job in jobs} == {"job-1": ("cheap",), "job-0": ()} + + +@pytest.mark.asyncio +async def test_start_shadow_eval_accepts_a_team_public_scope_for_a_user_target(monkeypatch: pytest.MonkeyPatch): + """A user's traffic can arrive on any team's key, so a name only one team can ask for + is a legitimate scope for a user target even though it resolves for nobody unscoped.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(known_users={"dev-alice": "alice@example.com"}) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval( + _start_request(api_key_ids=(), user_ids=("dev-alice",), models=("house-judge",)), ADMIN + ) + + assert response.models == ("house-judge",) + assert prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"][0]["models"] == ["house-judge"] + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_a_model_scope_this_proxy_does_not_serve(monkeypatch: pytest.MonkeyPatch): + """A typo'd model name would otherwise start a job that samples nothing. Only the + unresolvable names are reported, so the caller fixes them in one round.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(models=("cheap", "no-such-model-zzz")), ADMIN) + assert exc.value.status_code == 400 + assert "'no-such-model-zzz'" in exc.value.detail + assert "'cheap'" not in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +def test_start_request_dedupes_the_model_scope_and_rejects_blank_names(): + assert _start_request(models=("cheap", "mid", "cheap")).models == ("cheap", "mid") + assert _start_request().models == () + with pytest.raises(ValidationError, match="non-empty model group names"): + _start_request(models=("cheap", " ")) + + +def test_start_request_rejects_a_model_scope_on_a_reverse_job(): + """Reverse admission is the router's own traffic, whose requested group is always the + router, so a plain-model scope would sample nothing and the router itself is a no-op.""" + with pytest.raises(ValidationError, match="only meaningful for a forward job"): + _start_request(direction="reverse", baseline_model="cheap", models=("mid",)) + with pytest.raises(ValidationError, match="only meaningful for a forward job"): + _start_request(direction="reverse", baseline_model="cheap", models=("my-router",)) + assert _start_request(direction="reverse", baseline_model="cheap").models == () + + @pytest.mark.asyncio async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): """A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index b33bd912be9..59a59c7e16d 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -2,15 +2,27 @@ # This tests litellm router -import pytest - import logging +from typing import Final +import pytest import litellm from litellm._logging import verbose_logger +async def _routed_model_ids( + router: litellm.Router, tags: list[str], remaining: frozenset[str], attempts: int = 100 +) -> frozenset[str]: + if not remaining or attempts == 0: + return frozenset() + response: Final = await router.acompletion( + model="gpt-4", messages=[{"role": "user", "content": "hi"}], metadata={"tags": tags}, mock_response="hi" + ) + seen: Final = frozenset({response._hidden_params["model_id"]}) + return seen | await _routed_model_ids(router, tags, remaining - seen, attempts - 1) + + @pytest.mark.asyncio() async def test_router_free_paid_tier(): """ @@ -850,17 +862,10 @@ async def test_negation_regex_pattern_treated_as_literal(): # The regex-like string matches no deployment tag literally, so all # candidates survive and both model IDs are reachable. - seen_ids = set() - for _ in range(10): - response = await router.acompletion( - model="gpt-4", - messages=[{"role": "user", "content": "hi"}], - metadata={"tags": ["!provider:(anthropic|openai)"]}, - mock_response="hi", - ) - seen_ids.add(response._hidden_params["model_id"]) + expected: Final = frozenset({"anthropic-model", "openai-model"}) + routed_ids: Final = await _routed_model_ids(router, ["!provider:(anthropic|openai)"], expected) - assert seen_ids == {"anthropic-model", "openai-model"} + assert routed_ids == expected @pytest.mark.asyncio() @@ -1281,17 +1286,10 @@ async def test_chain_enable_tag_filtering_false_overrides_router_level_true(): enable_tag_filtering=True, ) - seen_ids = set() - for _ in range(10): - response = await router.acompletion( - model="gpt-4", - messages=[{"role": "user", "content": "hi"}], - metadata={"tags": ["teamA"]}, - mock_response="hi", - ) - seen_ids.add(response._hidden_params["model_id"]) + expected: Final = frozenset({"team-a-deployment", "team-b-deployment"}) + routed_ids: Final = await _routed_model_ids(router, ["teamA"], expected) - assert seen_ids == {"team-a-deployment", "team-b-deployment"} + assert routed_ids == expected @pytest.mark.asyncio() diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index 5ea0e79a196..aa2260e89ea 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -53,6 +53,12 @@ case "$*" in "eslint --no-warn-ignored"*) [ "${STUB_FAIL:-}" = "eslint" ] && exit 1 ;; + "eslint . -f json"*) + if [ -n "${STUB_HANG_DIR:-}" ]; then + touch "$STUB_HANG_DIR/eslint_report.started" + sleep 60 + fi + ;; esac exit 0 """ @@ -340,11 +346,12 @@ def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> Non ) try: assert _wait_until((hang_dir / "make.started").exists, 10) + assert _wait_until((hang_dir / "eslint_report.started").exists, 10) os.killpg(proc.pid, signal.SIGINT) assert proc.wait(timeout=10) != 0 make_pid = int((hang_dir / "make.pid").read_text()) assert _wait_until(lambda: _pid_gone(make_pid), 5) - assert list(tmp_dir.iterdir()) == [] + assert _wait_until(lambda: not any(tmp_dir.iterdir()), 5), list(tmp_dir.iterdir()) finally: with suppress(ProcessLookupError, PermissionError): os.killpg(proc.pid, signal.SIGTERM) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 8bbe28d0d09..4a3ca612d41 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22184 + "limit": 22183 }, "LIT002": { - "limit": 26747 + "limit": 26745 }, "LIT003": { "limit": 261 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index a1de608d0bb..64363da9933 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -104,6 +104,7 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ status: "running", router_name: "claude-auto", router_names: ["claude-auto"], + models: [], direction: "forward", baseline_model: null, judge_model: "anthropic/claude-sonnet-5", @@ -450,6 +451,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha", "hash-beta"], team_ids: [], user_ids: [], + models: [], router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, @@ -479,6 +481,7 @@ describe("ShadowEvalSection", () => { api_key_ids: [], team_ids: ["team-eng"], user_ids: [], + models: [], router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, @@ -489,15 +492,41 @@ describe("ShadowEvalSection", () => { expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); + it("narrows a job to the picked model groups and shows the scope on the job headline", async () => { + const user = userEvent.setup(); + const { start } = mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Search teams by alias")); + const teamList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(teamList).getByText("engineering")); + await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "prod-claude"); + await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto"); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByText("Start shadow eval")); + + expect(start.mutate).toHaveBeenCalledWith( + expect.objectContaining({ team_ids: ["team-eng"], models: ["prod-claude"] }), + ); + + const scoped = job({ models: ["prod-claude", "prod-haiku"] }); + mockHooks({ jobs: [scoped], detailsById: { "job-1": scoped } }); + render(); + expect(screen.getByText("prod-claude, prod-haiku")).toBeInTheDocument(); + }); + it("requires a baseline model in reverse mode and submits it, while forward mode never shows the picker", async () => { const user = userEvent.setup(); const { start } = mockHooks({}); render(); expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument(); + expect(screen.getByPlaceholderText("Every model the targets use")).toBeInTheDocument(); await user.click(screen.getByText("Adoption check: key's traffic vs the router")); await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + expect(screen.queryByPlaceholderText("Every model the targets use")).not.toBeInTheDocument(); await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); @@ -516,6 +545,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha"], team_ids: [], user_ids: [], + models: [], router_names: ["gpt-auto"], direction: "reverse", baseline_model: "prod-claude", @@ -551,6 +581,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha"], team_ids: [], user_ids: [], + models: [], router_names: ["gpt-auto", "claude-auto"], direction: "forward", shadow_percentage: 10, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index c66d74074c2..ea11879d971 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -87,17 +87,25 @@ const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string = const jobRouters = (job: ShadowEvalJob): string => (job.router_names ?? [job.router_name]).join(", "); +const jobModelScope = (job: ShadowEvalJob): React.ReactNode => + job.models && job.models.length > 0 ? ( + <> + {" "} + on {job.models.join(", ")} + + ) : null; + const jobHeadline = (job: ShadowEvalJob): React.ReactNode => job.direction === "reverse" ? ( <> Comparing {jobRouters(job)} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} - {shadowedTargetsLabel(job)} traffic + {shadowedTargetsLabel(job)} traffic{jobModelScope(job)} ) : ( <> Shadowing {job.shadow_percentage}% of {shadowedTargetsLabel(job)}{" "} - traffic via {jobRouters(job)} + traffic{jobModelScope(job)} via {jobRouters(job)} ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx index f96910a4ad6..2eb5fa9c945 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -23,6 +23,7 @@ import { useStartShadowEval, type ShadowEvalJob } from "./useShadowEval"; type ShadowEvalDirection = ShadowEvalJob["direction"]; const MAX_ROUTERS = 4; +const MAX_MODELS = 100; const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const; @@ -206,6 +207,7 @@ interface StartFormValidityInputs { apiKeyIds: string[]; teamIds: string[]; userIds: string[]; + models: string[]; routerNames: string[]; direction: ShadowEvalDirection; baselineModel: string; @@ -224,7 +226,8 @@ const startFormValidity = (inputs: StartFormValidityInputs) => { const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS; const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1; const routersValid = routerCountValid && routersMatchDirection; - const modelsPicked = routersValid && inputs.judgeModel !== "" && baselinePicked; + const scopeValid = routersValid && (inputs.direction === "reverse" || inputs.models.length <= MAX_MODELS); + const modelsPicked = scopeValid && inputs.judgeModel !== "" && baselinePicked; const filled = targetsPicked && modelsPicked; const boundsValid = percentageValid && maxBudgetValid; const valid = Boolean(inputs.accessToken) && filled && boundsValid; @@ -235,6 +238,7 @@ interface StartBodyInputs { apiKeyIds: string[]; teamIds: string[]; userIds: string[]; + models: string[]; routerNames: string[]; direction: ShadowEvalDirection; baselineModel: string; @@ -248,6 +252,7 @@ const buildStartBody = (inputs: StartBodyInputs) => ({ api_key_ids: inputs.apiKeyIds, team_ids: inputs.teamIds, user_ids: inputs.userIds, + models: inputs.direction === "forward" ? inputs.models : [], router_names: inputs.routerNames, direction: inputs.direction, ...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}), @@ -262,6 +267,7 @@ export const StartForm: React.FC = () => { const [apiKeyIds, setApiKeyIds] = useState([]); const [teamIds, setTeamIds] = useState([]); const [userIds, setUserIds] = useState([]); + const [models, setModels] = useState([]); const [routerNames, setRouterNames] = useState([]); const [direction, setDirection] = useState("forward"); const [baselineModel, setBaselineModel] = useState(""); @@ -272,6 +278,11 @@ export const StartForm: React.FC = () => { const { data: autoRouters } = useAutoRouters(); const judgeModelOptions = useJudgeModelOptions(); const baselineModelOptions = useBaselineModelOptions(); + const configuredGroups = usePlainModelGroups(); + const modelOptions = useMemo( + () => [...configuredGroups].toSorted((a, b) => a.localeCompare(b)).map((name) => ({ label: name, value: name })), + [configuredGroups], + ); const start = useStartShadowEval(); const routerOptions = useMemo(() => { @@ -286,6 +297,7 @@ export const StartForm: React.FC = () => { apiKeyIds, teamIds, userIds, + models, routerNames, direction, baselineModel, @@ -299,6 +311,7 @@ export const StartForm: React.FC = () => { apiKeyIds, teamIds, userIds, + models, routerNames, direction, baselineModel, @@ -344,6 +357,22 @@ export const StartForm: React.FC = () => { + {direction === "forward" && ( + + + {models.length > MAX_MODELS ? ( +

Pick at most {MAX_MODELS} models

+ ) : ( +

Narrows every target above to requests for these models

+ )} + + )}