mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_decrease_anys_opus5_r4
# Conflicts: # basedpyright-code-budget.json # type-discipline-budget.json
This commit is contained in:
mateo-berri
commit
29dcd0cc2e
48 changed files with 1335 additions and 171 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"reportAny": {
|
||||
"limit": 13431
|
||||
"limit": 13429
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2198
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 19
|
||||
},
|
||||
"reportExplicitAny": {
|
||||
"limit": 3372
|
||||
"limit": 3369
|
||||
},
|
||||
"reportFunctionMemberAccess": {
|
||||
"limit": 7
|
||||
|
|
@ -108,7 +108,7 @@
|
|||
"limit": 38283
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 19585
|
||||
"limit": 19584
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 29829
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "models" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];
|
||||
|
|
@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // first (often only) auto-router under evaluation; router_names is the full set
|
||||
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
|
||||
models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
|
|
|
|||
|
|
@ -257,7 +257,7 @@ class DualCache(BaseCache):
|
|||
self,
|
||||
current_time: float,
|
||||
keys: list[str],
|
||||
result: Sequence[Any],
|
||||
result: Sequence[object],
|
||||
) -> tuple[list[str], dict[str, float | None]]:
|
||||
"""
|
||||
Atomically choose keys to fetch from Redis and reserve their access time.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,15 @@ span orphaned into its own trace). The anchor — a contextvar inherited by thos
|
|||
child tasks — gives a stable parent in both cases. DB/service spans keep ambient
|
||||
parenting so an auth DB lookup still nests under `auth`.
|
||||
|
||||
The anchor is also what `litellm.request.route` is read from: `request_root_http_route`
|
||||
returns the server span's own `http.route`, so the LLM call span cannot disagree with
|
||||
its parent about which endpoint served the request. That means the route template on a
|
||||
normal route and the literal path on a passthrough prefix, because the passthrough hook
|
||||
rewrote the attribute; an MCP call anchors the same server span, so it reports the
|
||||
`/mcp` mount point. Attributes stay readable after a span ends, so the async close
|
||||
callback reads the same value. Where no server span was anchored at all, the route the
|
||||
proxy recorded at auth (`metadata.user_api_key_request_route`) is the backstop.
|
||||
|
||||
**Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's
|
||||
service-logging layer instruments many internal functions, but only some are
|
||||
traceable units of work:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from litellm.integrations.otel.model.utils import to_ns
|
|||
from litellm.integrations.otel.plumbing.context import (
|
||||
is_recordable_span,
|
||||
mcp_message_transport_span,
|
||||
request_root_http_route,
|
||||
request_root_span,
|
||||
resolve_mcp_span_context,
|
||||
resolve_parent_context,
|
||||
|
|
@ -541,6 +542,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
payload,
|
||||
capture_content=self.config.capture_span_content,
|
||||
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
|
||||
request_route=request_root_http_route(),
|
||||
)
|
||||
end_time_ns: Final = to_ns(end_time)
|
||||
if carrier is not None and carrier.span is not None:
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ class GenAIMapper:
|
|||
f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent,
|
||||
f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount,
|
||||
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
|
||||
LiteLLM.REQUEST_ROUTE: lambda d: d.request_route,
|
||||
}
|
||||
|
||||
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ class RequestIdentity:
|
|||
# completes (routing has picked a deployment), so it's absent from the
|
||||
# auth-time seed and filled only from the payload.
|
||||
provider_model: str | None = None
|
||||
request_route: str | None = None
|
||||
metadata: Mapping[str, str] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -87,6 +88,7 @@ class RequestIdentity:
|
|||
key_hash=as_str(raw_meta.get("user_api_key_hash")),
|
||||
end_user=as_str(payload.get("end_user")) or as_str(raw_meta.get("user_api_key_end_user_id")),
|
||||
provider_model=resolve_provider_model(payload),
|
||||
request_route=as_str(raw_meta.get("user_api_key_request_route")),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -386,6 +386,7 @@ class LLMCallSpanData:
|
|||
# keeps routes the convention folds into one operation distinguishable.
|
||||
output_type: GenAIOutputType | None = None
|
||||
call_type: str | None = None
|
||||
request_route: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_standard_logging_payload(
|
||||
|
|
@ -393,6 +394,7 @@ class LLMCallSpanData:
|
|||
payload: StandardLoggingPayload,
|
||||
capture_content: bool = False,
|
||||
time_to_first_chunk_seconds: float | None = None,
|
||||
request_route: str | None = None,
|
||||
) -> LLMCallSpanData:
|
||||
params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {})
|
||||
# The single parse of the request's metadata — the request-vs-provider
|
||||
|
|
@ -433,6 +435,7 @@ class LLMCallSpanData:
|
|||
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
|
||||
output_type=resolve_output_type(call_type),
|
||||
call_type=call_type or None,
|
||||
request_route=request_route or context.identity.request_route,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -295,6 +295,7 @@ class LiteLLM:
|
|||
# ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``.
|
||||
PROVIDER_MODEL: Final = "litellm.provider.model"
|
||||
REQUEST_STREAMING: Final = "litellm.request.streaming"
|
||||
REQUEST_ROUTE: Final = "litellm.request.route"
|
||||
TOOLS_DECLARED: Final = "litellm.request.tools.declared"
|
||||
GUARDRAIL_NAME: Final = "litellm.guardrail.name"
|
||||
GUARDRAIL_MODE: Final = "litellm.guardrail.mode"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final
|
|||
|
||||
from opentelemetry import baggage
|
||||
from opentelemetry.context import Context, get_current
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
from opentelemetry.trace import (
|
||||
Link,
|
||||
NonRecordingSpan,
|
||||
|
|
@ -18,6 +19,8 @@ from opentelemetry.trace.propagation.tracecontext import (
|
|||
TraceContextTextMapPropagator,
|
||||
)
|
||||
|
||||
from litellm.integrations.otel.model.semconv import HTTP
|
||||
|
||||
_PROPAGATOR: Final = TraceContextTextMapPropagator()
|
||||
|
||||
# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
|
||||
|
|
@ -55,6 +58,25 @@ def request_root_span() -> "Span | None":
|
|||
return span if is_recordable_span(span) else None
|
||||
|
||||
|
||||
def request_root_http_route() -> str | None:
|
||||
"""``http.route`` exactly as the request's root SERVER span reports it.
|
||||
|
||||
Read off the span rather than re-derived, so the LLM call span cannot disagree
|
||||
with its own parent about which endpoint served the request: the template the
|
||||
instrumentation matched, or the literal path where
|
||||
``mount._passthrough_span_name_hook`` rewrote it, are already in the attribute.
|
||||
An MCP call anchors that same server span, so it reports the ``/mcp`` mount
|
||||
point the instrumentation matched. Attributes stay readable after a span ends,
|
||||
so this answers just as well from the async logging callback.
|
||||
|
||||
None when no server span is anchored, which is the SDK path and any deployment
|
||||
where the FastAPI instrumentation did not mount.
|
||||
"""
|
||||
span: Final = request_root_span()
|
||||
route: Final = span.attributes.get(HTTP.ROUTE) if isinstance(span, ReadableSpan) and span.attributes else None
|
||||
return route if isinstance(route, str) and route else None
|
||||
|
||||
|
||||
# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
|
||||
# MCP client propagated in the current request's ``params._meta``. The MCP gateway
|
||||
# sets it per message so the MCP span can record the client's span as a span
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_judge import (
|
|||
)
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
|
||||
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
|
||||
|
||||
|
|
@ -650,6 +651,7 @@ class ActiveShadowEvalJob(BaseModel):
|
|||
id: str
|
||||
router_name: str
|
||||
router_names: tuple[str, ...] = ()
|
||||
models: frozenset[str] = frozenset()
|
||||
direction: ShadowEvalDirection = "forward"
|
||||
baseline_model: str | None = None
|
||||
shadow_percentage: float
|
||||
|
|
@ -692,6 +694,21 @@ class ActiveShadowEvalJob(BaseModel):
|
|||
return self.baseline_model or arm_router
|
||||
|
||||
|
||||
def _canonical_group(router: "Router | None", model_group: str) -> str:
|
||||
"""A model group in the one spelling both a job's scope and a request's model compare
|
||||
under: an alias resolves to its target so the two never fail to match on spelling."""
|
||||
return (
|
||||
resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
|
||||
) or model_group
|
||||
|
||||
|
||||
def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:
|
||||
"""Whether the request's group is in the job's model scope. Both sides resolve through
|
||||
the router's alias map at match time, so a re-pointed alias applies to the next request
|
||||
rather than after the jobs cache rolls."""
|
||||
return not job.models or any(_canonical_group(router, name) == model_group for name in job.models)
|
||||
|
||||
|
||||
def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
|
||||
"""The sampling path's view of one job row, or None for a row it cannot sample: an
|
||||
unknown direction, or a reverse job with no baseline model to duplicate against.
|
||||
|
|
@ -714,7 +731,8 @@ class ShadowEvalLogger(CustomLogger):
|
|||
A job targets a virtual key, a team, or a user; a request qualifies for a job when
|
||||
any of its resolved identities (key hash, team id, user id) matches the job's
|
||||
target, so team and user jobs cover JWT-authenticated traffic, which carries no
|
||||
key hash at all."""
|
||||
key hash at all. A job scoped to model groups further requires the request's
|
||||
requested group to be one of them."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -801,19 +819,24 @@ class ShadowEvalLogger(CustomLogger):
|
|||
active_jobs: Sequence[ActiveShadowEvalJob],
|
||||
request_metadata: Mapping[str, object],
|
||||
request_id: str,
|
||||
model_group: str,
|
||||
) -> tuple[ActiveShadowEvalJob, ...]:
|
||||
"""The jobs that sample this request. A key can hold one job per direction, and a
|
||||
request routed by one job's router while bypassing the other's qualifies for both;
|
||||
each is separately budgeted, so both fire. An admitting job that loses the sampling
|
||||
dice is counted, so results can weigh judged rows against the traffic they stand for."""
|
||||
dice is counted, so results can weigh judged rows against the traffic they stand for.
|
||||
A request outside a job's direction or model scope is not that job's traffic and
|
||||
goes uncounted, so the funnel stays a fraction of the traffic the job admits."""
|
||||
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
router: Final = self._router_provider()
|
||||
for job in active_jobs:
|
||||
if (
|
||||
now >= job.ends_at
|
||||
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
|
||||
or (job.max_budget is not None and job.spend >= job.max_budget)
|
||||
or not _direction_admits(request_metadata, job)
|
||||
or not _scope_admits(router, job, model_group)
|
||||
):
|
||||
continue
|
||||
if not _sample_hits(request_id, job.id, job.shadow_percentage):
|
||||
|
|
@ -868,6 +891,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
tuple(job for target in targets for job in active_jobs.get(target, ())),
|
||||
request_metadata,
|
||||
request_id,
|
||||
_canonical_group(self._router_provider(), str(payload.get("model_group") or "")),
|
||||
)
|
||||
if not eligible:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -3821,7 +3821,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
def record_streamed_anthropic_message_id(self, message_id: str) -> None:
|
||||
self.streamed_anthropic_message_id = message_id
|
||||
|
||||
def _anthropic_messages_logged_response(self, result: Any) -> ModelResponse:
|
||||
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
|
||||
"""
|
||||
The ModelResponse a /v1/messages spend_logs row is built from.
|
||||
|
||||
|
|
|
|||
372
litellm/proxy/client/cli/commands/debug.py
Normal file
372
litellm/proxy/client/cli/commands/debug.py
Normal file
|
|
@ -0,0 +1,372 @@
|
|||
"""`lite debug claude`: one-shot debug report for a Claude Code session routed through the proxy.
|
||||
|
||||
Claude Code puts its session id in `metadata.user_id`, which the proxy lifts into
|
||||
`LiteLLM_SpendLogs.session_id`. This command pulls every turn of that session, plus
|
||||
the request / response bodies for failures and the most recent turns, and renders a
|
||||
single markdown report that can be pasted into a bug report or handed to another agent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import click
|
||||
import requests
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
|
||||
|
||||
from ...http_client import HTTPClient
|
||||
from ._cli_context import cli_context_values
|
||||
|
||||
CLAUDE_DIR: Final = Path.home() / ".claude"
|
||||
REPORT_DIR: Final = Path.home() / ".litellm" / "debug"
|
||||
SESSION_ID_ENV: Final = "CLAUDE_CODE_SESSION_ID"
|
||||
SLASH_COMMAND_NAME: Final = "debug-lite"
|
||||
SLASH_COMMAND_BODY: Final = """---
|
||||
description: Pull the LiteLLM debug report (spend, request, response, error) for this Claude Code session
|
||||
allowed-tools: Bash(lite debug claude:*)
|
||||
---
|
||||
Below is the LiteLLM debug report for this Claude Code session. Summarize the failing
|
||||
request(s) in a few sentences (model, error, request id) and tell me the path the full
|
||||
report was saved to so I can hand it off. If nothing failed, say so.
|
||||
|
||||
!`lite debug claude $ARGUMENTS`
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DebugFailure:
|
||||
message: str
|
||||
|
||||
|
||||
class ErrorInformation(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
error_code: str | None = None
|
||||
error_class: str | None = None
|
||||
error_message: str | None = None
|
||||
llm_provider: str | None = None
|
||||
|
||||
|
||||
class SpendLogMetadata(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
status: str | None = None
|
||||
error_information: ErrorInformation | None = None
|
||||
|
||||
|
||||
class SpendLogRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
|
||||
|
||||
request_id: str
|
||||
start_time: str | None = Field(default=None, alias="startTime")
|
||||
end_time: str | None = Field(default=None, alias="endTime")
|
||||
model: str | None = None
|
||||
model_group: str | None = None
|
||||
custom_llm_provider: str | None = None
|
||||
api_base: str | None = None
|
||||
call_type: str | None = None
|
||||
status: str | None = None
|
||||
spend: float = 0.0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
metadata: SpendLogMetadata = SpendLogMetadata()
|
||||
|
||||
@field_validator("metadata", mode="before")
|
||||
@classmethod
|
||||
def _parse_metadata(cls, value: object) -> object:
|
||||
if value is None:
|
||||
return SpendLogMetadata()
|
||||
if isinstance(value, str):
|
||||
return json.loads(value) if value else SpendLogMetadata()
|
||||
return value
|
||||
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
return (self.status or self.metadata.status) == "failure"
|
||||
|
||||
@property
|
||||
def error(self) -> ErrorInformation | None:
|
||||
return self.metadata.error_information
|
||||
|
||||
|
||||
class SessionLogsPage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
data: tuple[SpendLogRow, ...]
|
||||
total: int
|
||||
total_pages: int
|
||||
|
||||
|
||||
class RequestResponsePayload(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
proxy_server_request: JsonValue = None
|
||||
response: JsonValue = None
|
||||
messages: JsonValue = None
|
||||
|
||||
|
||||
_SESSION_PAGE: Final = TypeAdapter(SessionLogsPage)
|
||||
_PAYLOAD: Final[TypeAdapter[RequestResponsePayload | None]] = TypeAdapter(RequestResponsePayload | None)
|
||||
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
_SESSION_PAGE_SIZE: Final = 100
|
||||
_TRANSPORT_BODY_CHARS: Final = 500
|
||||
_SESSION_TRANSCRIPT_STEM: Final = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
|
||||
|
||||
|
||||
def detect_claude_session_id(env: Mapping[str, str], claude_dir: Path) -> str | None:
|
||||
explicit: Final = env.get(SESSION_ID_ENV)
|
||||
if explicit:
|
||||
return explicit
|
||||
transcripts: Final = tuple(
|
||||
path for path in claude_dir.glob("projects/*/*.jsonl") if _SESSION_TRANSCRIPT_STEM.fullmatch(path.stem)
|
||||
)
|
||||
if not transcripts:
|
||||
return None
|
||||
newest: Final = max(transcripts, key=lambda p: p.stat().st_mtime)
|
||||
return newest.stem
|
||||
|
||||
|
||||
def _transport_failure(uri: str, error: requests.exceptions.RequestException) -> DebugFailure:
|
||||
body: Final = error.response.text[:_TRANSPORT_BODY_CHARS] if error.response is not None else ""
|
||||
detail: Final = f"\n{body}" if body else ""
|
||||
return DebugFailure(f"GET {uri} failed: {error}{detail}")
|
||||
|
||||
|
||||
class SpendLogsFetcher:
|
||||
def __init__(self, http: HTTPClient) -> None:
|
||||
self._http = http
|
||||
|
||||
def session_rows(self, session_id: str) -> tuple[SpendLogRow, ...] | DebugFailure:
|
||||
first: Final = self._page(session_id, 1)
|
||||
if isinstance(first, DebugFailure):
|
||||
return first
|
||||
rest: Final = tuple(self._page(session_id, page) for page in range(2, first.total_pages + 1))
|
||||
failed_page: Final = next((page for page in rest if isinstance(page, DebugFailure)), None)
|
||||
if failed_page is not None:
|
||||
return failed_page
|
||||
rows: Final = first.data + tuple(row for page in rest if isinstance(page, SessionLogsPage) for row in page.data)
|
||||
return tuple(sorted(rows, key=lambda r: r.start_time or ""))
|
||||
|
||||
def _get(self, uri: str, params: Mapping[str, str | int] | None = None) -> JsonValue | DebugFailure:
|
||||
try:
|
||||
return _JSON.validate_python(self._http.request("GET", uri, params=params)) # pyright: ignore[reportUnknownMemberType] # HTTPClient.request is untyped
|
||||
except requests.exceptions.RequestException as e:
|
||||
return _transport_failure(uri, e)
|
||||
|
||||
def _page(self, session_id: str, page: int) -> SessionLogsPage | DebugFailure:
|
||||
uri: Final = "/spend/logs/session/ui"
|
||||
raw: Final = self._get(
|
||||
uri, MappingProxyType({"session_id": session_id, "page": page, "page_size": _SESSION_PAGE_SIZE})
|
||||
)
|
||||
if isinstance(raw, DebugFailure):
|
||||
return raw
|
||||
try:
|
||||
return _SESSION_PAGE.validate_python(raw)
|
||||
except ValidationError as e:
|
||||
return DebugFailure(f"Unexpected {uri} response: {e}")
|
||||
|
||||
def payload(self, request_id: str) -> RequestResponsePayload | None | DebugFailure:
|
||||
uri: Final = f"/spend/logs/ui/{request_id}"
|
||||
raw: Final = self._get(uri)
|
||||
if isinstance(raw, DebugFailure):
|
||||
return raw
|
||||
try:
|
||||
return _PAYLOAD.validate_python(raw)
|
||||
except ValidationError as e:
|
||||
return DebugFailure(f"Unexpected {uri} response: {e}")
|
||||
|
||||
|
||||
def _fmt_json(value: JsonValue, max_chars: int) -> str:
|
||||
text: Final = value if isinstance(value, str) else json.dumps(value, indent=2, default=str)
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
return f"{text[:max_chars]}\n... (truncated, {len(text) - max_chars} more chars)"
|
||||
|
||||
|
||||
def _fenced(text: str, info: str = "") -> tuple[str, str, str]:
|
||||
longest_run: Final = max((len(run) for run in re.findall(r"`+", text)), default=0)
|
||||
fence: Final = "`" * max(3, longest_run + 1)
|
||||
return (f"{fence}{info}", text, fence)
|
||||
|
||||
|
||||
def _row_section(row: SpendLogRow, index: int, payload: RequestResponsePayload | None, max_chars: int) -> str:
|
||||
err: Final = row.error
|
||||
error_lines: Final = (
|
||||
(
|
||||
f"- error: `{err.error_code or '?'}` {err.error_class or ''}".rstrip(),
|
||||
"",
|
||||
*_fenced(err.error_message or ""),
|
||||
)
|
||||
if err is not None and row.failed
|
||||
else ()
|
||||
)
|
||||
body_lines: Final = (
|
||||
(
|
||||
"",
|
||||
"<details><summary>request body</summary>",
|
||||
"",
|
||||
*_fenced(_fmt_json(payload.proxy_server_request, max_chars), "json"),
|
||||
"</details>",
|
||||
"",
|
||||
"<details><summary>response</summary>",
|
||||
"",
|
||||
*_fenced(_fmt_json(payload.response, max_chars), "json"),
|
||||
"</details>",
|
||||
)
|
||||
if payload is not None
|
||||
else ()
|
||||
)
|
||||
header: Final = f"### {index}. {'FAILED' if row.failed else 'ok'} {row.model or row.model_group or '?'}"
|
||||
facts: Final = (
|
||||
f"- request_id: `{row.request_id}`",
|
||||
f"- time: {row.start_time} -> {row.end_time}",
|
||||
f"- provider: {row.custom_llm_provider or '?'} ({row.api_base or 'n/a'}), call_type: {row.call_type or '?'}",
|
||||
f"- spend: ${row.spend:.6f}, tokens: {row.prompt_tokens} in / {row.completion_tokens} out",
|
||||
)
|
||||
return "\n".join((header, *facts, *error_lines, *body_lines))
|
||||
|
||||
|
||||
def render_report(
|
||||
*,
|
||||
session_id: str,
|
||||
base_url: str,
|
||||
rows: Sequence[SpendLogRow],
|
||||
payloads: Mapping[str, RequestResponsePayload | None],
|
||||
max_chars: int,
|
||||
) -> str:
|
||||
failures: Final = tuple(r for r in rows if r.failed)
|
||||
summary: Final = (
|
||||
f"# LiteLLM debug report: Claude Code session `{session_id}`",
|
||||
"",
|
||||
f"- proxy: {base_url}",
|
||||
f"- generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}",
|
||||
f"- turns: {len(rows)}, failed: {len(failures)}",
|
||||
f"- total spend: ${sum(r.spend for r in rows):.6f}",
|
||||
f"- models: {', '.join(sorted(frozenset(r.model or r.model_group or '?' for r in rows))) or 'n/a'}",
|
||||
"",
|
||||
"Bodies are included for failed turns and the most recent turns. "
|
||||
"Bodies are empty unless the proxy runs with `general_settings.store_prompts_in_spend_logs: true`.",
|
||||
"",
|
||||
"## Turns",
|
||||
"",
|
||||
)
|
||||
sections: Final = tuple(
|
||||
_row_section(row, i, payloads.get(row.request_id), max_chars) for i, row in enumerate(rows, start=1)
|
||||
)
|
||||
return "\n".join(summary) + "\n\n".join(sections) + "\n"
|
||||
|
||||
|
||||
def build_report(
|
||||
*,
|
||||
fetcher: SpendLogsFetcher,
|
||||
session_id: str,
|
||||
base_url: str,
|
||||
recent_bodies: int,
|
||||
max_chars: int,
|
||||
) -> str | DebugFailure:
|
||||
rows: Final = fetcher.session_rows(session_id)
|
||||
if isinstance(rows, DebugFailure):
|
||||
return rows
|
||||
if not rows:
|
||||
return DebugFailure(
|
||||
f"No spend logs found for session {session_id!r} on {base_url}. "
|
||||
"Is Claude Code routed through this proxy (`lite up`), and does your key have log access?"
|
||||
)
|
||||
wanted: Final = frozenset(r.request_id for r in rows if r.failed) | frozenset(
|
||||
r.request_id for r in rows[-recent_bodies:] if recent_bodies > 0
|
||||
)
|
||||
fetched: Final = MappingProxyType({rid: fetcher.payload(rid) for rid in sorted(wanted)})
|
||||
failed_payload: Final = next((p for p in fetched.values() if isinstance(p, DebugFailure)), None)
|
||||
if failed_payload is not None:
|
||||
return failed_payload
|
||||
payloads: Final = MappingProxyType({rid: p for rid, p in fetched.items() if not isinstance(p, DebugFailure)})
|
||||
return render_report(session_id=session_id, base_url=base_url, rows=rows, payloads=payloads, max_chars=max_chars)
|
||||
|
||||
|
||||
def write_report(report: str, session_id: str, report_dir: Path) -> Path:
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
path: Final = report_dir / f"claude-{session_id}.md"
|
||||
path.write_text(report, encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
|
||||
def install_slash_command(claude_dir: Path) -> Path:
|
||||
commands_dir: Final = claude_dir / "commands"
|
||||
commands_dir.mkdir(parents=True, exist_ok=True)
|
||||
path: Final = commands_dir / f"{SLASH_COMMAND_NAME}.md"
|
||||
path.write_text(SLASH_COMMAND_BODY, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@click.group()
|
||||
def debug() -> None:
|
||||
"""Pull debug reports (spend, request, response, error) for coding-agent sessions"""
|
||||
|
||||
|
||||
@debug.command("claude")
|
||||
@click.option(
|
||||
"--session-id",
|
||||
default=None,
|
||||
help=f"Claude Code session id. Defaults to ${SESSION_ID_ENV}, else the most recently used transcript in ~/.claude",
|
||||
)
|
||||
@click.option(
|
||||
"--recent-bodies",
|
||||
default=3,
|
||||
show_default=True,
|
||||
type=click.IntRange(min=0),
|
||||
help="Also include request/response bodies for the N most recent turns (failed turns always get bodies)",
|
||||
)
|
||||
@click.option(
|
||||
"--max-body-chars",
|
||||
default=20_000,
|
||||
show_default=True,
|
||||
type=click.IntRange(min=100),
|
||||
help="Truncate each request/response body to this many characters",
|
||||
)
|
||||
@click.option("--no-save", is_flag=True, help="Print only, do not write the report under ~/.litellm/debug")
|
||||
@click.pass_context
|
||||
def debug_claude(
|
||||
ctx: click.Context, session_id: str | None, recent_bodies: int, max_body_chars: int, no_save: bool
|
||||
) -> None:
|
||||
"""Render a markdown debug report for one Claude Code session routed through the proxy
|
||||
|
||||
Examples:
|
||||
lite debug claude
|
||||
lite debug claude --session-id e96634a3-fa28-4083-b354-55542e2dca01
|
||||
"""
|
||||
resolved: Final = session_id or detect_claude_session_id(os.environ, CLAUDE_DIR)
|
||||
if resolved is None:
|
||||
raise click.ClickException(f"Could not find a Claude Code session. Pass --session-id or set ${SESSION_ID_ENV}.")
|
||||
values: Final = cli_context_values(ctx)
|
||||
base_url: Final = values["base_url"]
|
||||
fetcher: Final = SpendLogsFetcher(HTTPClient(base_url, values["api_key"]))
|
||||
outcome: Final = build_report(
|
||||
fetcher=fetcher,
|
||||
session_id=resolved,
|
||||
base_url=base_url,
|
||||
recent_bodies=recent_bodies,
|
||||
max_chars=max_body_chars,
|
||||
)
|
||||
if isinstance(outcome, DebugFailure):
|
||||
raise click.ClickException(outcome.message)
|
||||
click.echo(outcome)
|
||||
if not no_save:
|
||||
path: Final = write_report(outcome, resolved, REPORT_DIR)
|
||||
click.echo(f"Saved to {path}", err=True)
|
||||
|
||||
|
||||
@debug.command("install-claude-command")
|
||||
def debug_install_claude_command() -> None:
|
||||
"""Install the /debug-lite slash command into ~/.claude/commands so Claude Code can run `lite debug claude`"""
|
||||
path: Final = install_slash_command(CLAUDE_DIR)
|
||||
click.echo(f"Installed /{SLASH_COMMAND_NAME}: {path}")
|
||||
click.echo("Restart Claude Code (or start a new session), then type /debug-lite.")
|
||||
|
|
@ -14,6 +14,7 @@ from .commands.autoroute.commands import autoroute_group
|
|||
from .commands.chat import chat
|
||||
from .commands.config import config_commands, get_config_value, hidden_command_names
|
||||
from .commands.credentials import credentials
|
||||
from .commands.debug import debug
|
||||
from .commands.encryption import encryption
|
||||
from .commands.http import http
|
||||
from .commands.keys import keys
|
||||
|
|
@ -143,6 +144,7 @@ cli.add_command(encryption)
|
|||
cli.add_command(chat)
|
||||
# Add the http command group
|
||||
cli.add_command(http)
|
||||
cli.add_command(debug)
|
||||
# Add the keys command group
|
||||
cli.add_command(keys)
|
||||
# Add the teams command group
|
||||
|
|
|
|||
|
|
@ -1096,10 +1096,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
|
|||
add_guardrail_to_applied_guardrails_header,
|
||||
)
|
||||
|
||||
# Collect all chunks
|
||||
all_chunks: Final[list[Any]] = []
|
||||
async for chunk in response:
|
||||
all_chunks.append(chunk)
|
||||
all_chunks: Final[Sequence[object]] = tuple([chunk async for chunk in response])
|
||||
|
||||
if not all_chunks or self._is_terminal_error_stream(all_chunks):
|
||||
for chunk in all_chunks:
|
||||
|
|
|
|||
|
|
@ -789,6 +789,26 @@ def _for_teams(team_ids: Sequence[str | None]) -> str:
|
|||
return f" for team {', '.join(named)}" if named else ""
|
||||
|
||||
|
||||
def _validate_model_scope(llm_router: "Router | None", models: Sequence[str]) -> None:
|
||||
"""Reject a scope naming a model no request on this proxy could carry, at start rather
|
||||
than as a job that silently samples nothing. The question is "could any caller ask for
|
||||
this name", not "does it resolve for the job's teams": a user target's traffic can arrive
|
||||
on any team's key, so a team-public name is a legitimate scope for it, and an auto-router
|
||||
is one too (a forward job on router A scoped to router B samples what B serves today).
|
||||
Nothing here is ever dispatched to."""
|
||||
unreachable: Final = tuple(
|
||||
model
|
||||
for model in models
|
||||
if judge_target(llm_router, model).via == "nothing"
|
||||
and (llm_router is None or model not in llm_router.team_public_model_names)
|
||||
)
|
||||
if unreachable:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="models not served by this proxy: " + ", ".join(f"'{model}'" for model in unreachable),
|
||||
)
|
||||
|
||||
|
||||
_JUDGED_ROLES: Final[frozenset[StrategyRouterDependencyRole]] = frozenset({"tier", "default"})
|
||||
|
||||
|
||||
|
|
@ -1080,6 +1100,7 @@ class _LegRow(BaseModel):
|
|||
target_id: str
|
||||
router_name: str
|
||||
router_names: tuple[str, ...] = ()
|
||||
models: tuple[str, ...] = ()
|
||||
direction: ShadowEvalDirection
|
||||
baseline_model: str | None = None
|
||||
judge_model: str
|
||||
|
|
@ -1150,6 +1171,7 @@ def _group_response(
|
|||
for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id))
|
||||
),
|
||||
router_names=first.arm_router_names,
|
||||
models=first.models,
|
||||
direction=first.direction,
|
||||
baseline_model=first.baseline_model,
|
||||
judge_model=first.judge_model,
|
||||
|
|
@ -1322,7 +1344,10 @@ async def start_shadow_eval(
|
|||
A target is a virtual key, a team, or a user. Team and user targets match on the
|
||||
identity every request resolves to at auth time, so they cover JWT-authenticated
|
||||
traffic, which presents no virtual key; a user target samples that user's traffic
|
||||
across all their teams, whether it arrives on a JWT or a key they own.
|
||||
across all their teams, whether it arrives on a JWT or a key they own. models narrows
|
||||
every target to requests for those model groups, so a user plus one model samples that
|
||||
user's traffic on that model across every key they own; it is forward-only, since a
|
||||
reverse job already samples exactly the traffic its own router served.
|
||||
|
||||
A forward job answers whether the targets should adopt router_name: it samples the
|
||||
requests the router did not serve and duplicates them through it. A reverse job
|
||||
|
|
@ -1411,6 +1436,7 @@ async def start_shadow_eval(
|
|||
if data.baseline_model is not None:
|
||||
_validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids)
|
||||
_validate_judge_is_not_a_candidate(llm_router, data, team_ids)
|
||||
_validate_model_scope(llm_router, data.models)
|
||||
|
||||
requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = (
|
||||
*(("key", key) for key in data.api_key_ids),
|
||||
|
|
@ -1456,6 +1482,7 @@ async def start_shadow_eval(
|
|||
# a pre-router_names pod samples router_name alone, so it must be a real arm
|
||||
"router_name": data.router_names[0],
|
||||
"router_names": list(data.router_names), # mutable-ok: Prisma payload
|
||||
"models": list(data.models), # mutable-ok: Prisma payload
|
||||
"direction": data.direction,
|
||||
"baseline_model": data.baseline_model,
|
||||
"judge_model": data.judge_model,
|
||||
|
|
@ -1517,6 +1544,7 @@ async def start_shadow_eval(
|
|||
for target_type, target_id in sorted(requested_targets)
|
||||
),
|
||||
router_names=data.router_names,
|
||||
models=data.models,
|
||||
direction=data.direction,
|
||||
baseline_model=data.baseline_model,
|
||||
judge_model=data.judge_model,
|
||||
|
|
|
|||
|
|
@ -32,9 +32,6 @@ class AdmissionControlSettings:
|
|||
queue_timeout_seconds: float
|
||||
|
||||
|
||||
AdmissionControlSettingsGetter: TypeAlias = Callable[[], AdmissionControlSettings | None] # mutable-ok: Callable params
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AdmissionControlStats:
|
||||
admitted: int
|
||||
|
|
@ -66,13 +63,10 @@ class AdmissionControlMetrics:
|
|||
rejected_counter: _Counter
|
||||
|
||||
|
||||
AdmissionControlMetricsFactory: TypeAlias = Callable[[], AdmissionControlMetrics | None] # mutable-ok: Callable params
|
||||
|
||||
|
||||
class AdmissionControlState:
|
||||
"""Per-process admission counters and the in-flight semaphore shared by one worker's requests."""
|
||||
|
||||
def __init__(self, metrics_factory: AdmissionControlMetricsFactory) -> None:
|
||||
def __init__(self, metrics_factory: Callable[[], AdmissionControlMetrics | None]) -> None:
|
||||
self._metrics_factory = metrics_factory
|
||||
self._metrics: AdmissionControlMetrics | None = None
|
||||
self._metrics_init_attempted = False
|
||||
|
|
@ -140,7 +134,7 @@ class AdmissionControlMiddleware:
|
|||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
get_settings: AdmissionControlSettingsGetter,
|
||||
get_settings: Callable[[], AdmissionControlSettings | None],
|
||||
state: AdmissionControlState,
|
||||
) -> None:
|
||||
self.app = app
|
||||
|
|
|
|||
|
|
@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // first (often only) auto-router under evaluation; router_names is the full set
|
||||
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
|
||||
models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
|
|
|
|||
|
|
@ -3365,23 +3365,24 @@ async def view_spend_logs(
|
|||
)
|
||||
sql_query, params = summary_sql_and_params
|
||||
rows: Final[Sequence[_SpendDailySummaryRow]] = await _query_raw(prisma_client, sql_query, *params)
|
||||
if len(rows) == 0:
|
||||
return [] # pyright: ignore[reportUnknownVariableType] # empty summary has no element type
|
||||
|
||||
summary_items: Final = tuple(
|
||||
_daily_summary_item(date.fromisoformat(day), tuple(day_rows))
|
||||
for day, day_rows in groupby(rows, key=lambda row: row["day"])
|
||||
)
|
||||
final_date: Final = date.fromisoformat(rows[-1]["day"])
|
||||
final_date: Final = date.fromisoformat(rows[-1]["day"]) if len(rows) > 0 else None
|
||||
end_date_date: Final = end_date_obj.date()
|
||||
padding: Final[tuple[Mapping[str, object], ...]] = tuple(
|
||||
{
|
||||
"startTime": final_date + timedelta(days=offset),
|
||||
"spend": 0,
|
||||
"users": {},
|
||||
"models": {},
|
||||
}
|
||||
for offset in range(1, (end_date_date - final_date).days + 1)
|
||||
padding: Final[tuple[Mapping[str, object], ...]] = (
|
||||
()
|
||||
if final_date is None
|
||||
else tuple(
|
||||
{
|
||||
"startTime": final_date + timedelta(days=offset),
|
||||
"spend": 0,
|
||||
"users": {},
|
||||
"models": {},
|
||||
}
|
||||
for offset in range(1, (end_date_date - final_date).days + 1)
|
||||
)
|
||||
)
|
||||
return [*summary_items, *padding]
|
||||
|
||||
|
|
|
|||
|
|
@ -496,10 +496,9 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if logging_response is self.completed_response:
|
||||
return
|
||||
target: Final[object] = getattr(logging_response, "response", None)
|
||||
existing_hidden: Final[object] = getattr(target, "_hidden_params", None)
|
||||
if not isinstance(existing_hidden, Mapping):
|
||||
if not isinstance(target, ResponsesAPIResponse):
|
||||
return
|
||||
existing: Final[Mapping[str, object]] = existing_hidden
|
||||
existing: Final[Mapping[str, object]] = target._hidden_params
|
||||
source_hidden: Final[object] = getattr(
|
||||
getattr(self.completed_response, "response", None), "_hidden_params", None
|
||||
)
|
||||
|
|
@ -510,15 +509,11 @@ class BaseResponsesAPIStreamingIterator:
|
|||
raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING
|
||||
# rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy
|
||||
# splats into the client's HTTP headers, and copying non-header keys would carry response_cost
|
||||
setattr( # noqa: B010 # target is typed object here, so a plain attribute store does not type check
|
||||
target,
|
||||
"_hidden_params",
|
||||
{ # mutable-ok: the cost calculator writes optional_params into _hidden_params
|
||||
"additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it
|
||||
"headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it
|
||||
**existing,
|
||||
},
|
||||
)
|
||||
target._hidden_params = { # mutable-ok: the cost calculator writes optional_params into _hidden_params
|
||||
"additional_headers": {**headers}, # mutable-ok: fresh copy, logging callbacks may mutate it
|
||||
"headers": {**raw_headers}, # mutable-ok: fresh copy, logging callbacks may mutate it
|
||||
**existing,
|
||||
}
|
||||
|
||||
def _handle_logging_completed_response(self):
|
||||
"""Base implementation - should be overridden by subclasses"""
|
||||
|
|
|
|||
|
|
@ -116,28 +116,6 @@ async def aattempt(
|
|||
return RustHandled(adapt(value))
|
||||
|
||||
|
||||
def call(operation: Callable[[], ResultT], context: BridgeErrorContext) -> ResultT:
|
||||
exceptions: Final = native_exception_types()
|
||||
if exceptions is None:
|
||||
return operation()
|
||||
upstream: Final = exceptions[1]
|
||||
try:
|
||||
return operation()
|
||||
except upstream as error:
|
||||
_raise_upstream(error, context)
|
||||
|
||||
|
||||
async def acall(operation: Callable[[], Awaitable[ResultT]], context: BridgeErrorContext) -> ResultT:
|
||||
exceptions: Final = native_exception_types()
|
||||
if exceptions is None:
|
||||
return await operation()
|
||||
upstream: Final = exceptions[1]
|
||||
try:
|
||||
return await operation()
|
||||
except upstream as error:
|
||||
_raise_upstream(error, context)
|
||||
|
||||
|
||||
def _decline_reason(error: BaseException) -> str:
|
||||
reason: Final[object] = error.args[0] if error.args else str(error)
|
||||
return reason if isinstance(reason, str) else str(reason)
|
||||
|
|
@ -170,11 +148,3 @@ def _raise_upstream(error: BaseException, context: BridgeErrorContext) -> NoRetu
|
|||
llm_provider=context.provider,
|
||||
model=context.model,
|
||||
) from error
|
||||
|
||||
|
||||
def identity(value: ResultT) -> ResultT:
|
||||
return value
|
||||
|
||||
|
||||
async def async_none() -> None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -292,6 +292,18 @@ class StartShadowEvalRequest(BaseModel):
|
|||
"to across all their teams: JWT requests carrying their subject claim and virtual keys they own"
|
||||
),
|
||||
)
|
||||
models: tuple[str, ...] = Field(
|
||||
default=(),
|
||||
max_length=100,
|
||||
description=(
|
||||
"Model groups to narrow the sampled traffic to, matched on the group the caller "
|
||||
"requested and resolved through model_group_alias, so an alias and its target are one "
|
||||
"name. Empty samples every model the targets use. This ANDs with the targets: a job "
|
||||
"over a user and one model samples that user's requests on that model across every key "
|
||||
"they own, and none of their other traffic. Forward jobs only: a reverse job samples "
|
||||
"exactly the traffic its own router served, which no other model group can name"
|
||||
),
|
||||
)
|
||||
router_name: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
@ -372,12 +384,20 @@ class StartShadowEvalRequest(BaseModel):
|
|||
def _round_percentage(cls, value: float) -> float:
|
||||
return round(value, 2)
|
||||
|
||||
@field_validator("api_key_ids", "team_ids", "user_ids")
|
||||
@field_validator("api_key_ids", "team_ids", "user_ids", "models")
|
||||
@classmethod
|
||||
def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
"""A target named twice would collide with itself on the one-active-per-(target, direction) index."""
|
||||
"""A target named twice would collide with itself on the one-active-per-(target, direction)
|
||||
index; a model named twice is one scope entry."""
|
||||
return tuple(dict.fromkeys(value))
|
||||
|
||||
@field_validator("models")
|
||||
@classmethod
|
||||
def _models_are_names(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if not all(name.strip() for name in value):
|
||||
raise ValueError("models must be non-empty model group names")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest":
|
||||
total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids)
|
||||
|
|
@ -387,6 +407,18 @@ class StartShadowEvalRequest(BaseModel):
|
|||
raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _model_scope_is_forward_only(self) -> "StartShadowEvalRequest":
|
||||
"""A reverse job admits exactly the requests its own router served, so every one of
|
||||
them names that router and nothing else; any other scope would sample nothing and
|
||||
the router itself is a no-op. Both readings are rejected rather than shipped as a
|
||||
job that silently never samples."""
|
||||
if self.models and self.direction == "reverse":
|
||||
raise ValueError(
|
||||
"models is only meaningful for a forward job; a reverse job samples its own router's traffic"
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest":
|
||||
if self.direction == "reverse" and self.baseline_model is None:
|
||||
|
|
@ -599,6 +631,10 @@ class ShadowEvalJobResponse(BaseModel):
|
|||
"traffic and judge every arm against the same real responses"
|
||||
),
|
||||
)
|
||||
models: tuple[str, ...] = Field(
|
||||
default=(),
|
||||
description="Model groups the sampled traffic is narrowed to; empty means every model the targets use",
|
||||
)
|
||||
direction: ShadowEvalDirection = "forward"
|
||||
baseline_model: str | None = None
|
||||
judge_model: str
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@
|
|||
"limit": 1
|
||||
},
|
||||
"C901": {
|
||||
"limit": 311
|
||||
"limit": 306
|
||||
},
|
||||
"D419": {
|
||||
"limit": 6
|
||||
|
|
|
|||
|
|
@ -1536,6 +1536,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // first (often only) auto-router under evaluation; router_names is the full set
|
||||
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
|
||||
models String[] @default([]) // model groups the sampled traffic is narrowed to; empty samples every model
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
|
|
|
|||
|
|
@ -142,6 +142,8 @@ fi
|
|||
|
||||
lint_dashboard() {
|
||||
(
|
||||
trap 'exit 143' TERM
|
||||
trap 'rm -f "${report:-}"' EXIT
|
||||
rc=0
|
||||
prettier_rel=()
|
||||
eslint_rel=()
|
||||
|
|
@ -168,7 +170,6 @@ EOF
|
|||
report=$(mktemp)
|
||||
npx eslint . -f json -o "$report" || true
|
||||
node scripts/check-lint-budgets.mjs "$report" eslint-budgets.json || rc=1
|
||||
rm -f "$report"
|
||||
exit $rc
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,11 +19,13 @@ failures are hard test failures (see `tests/e2e/CLAUDE.md`).
|
|||
| OpenAI | yes | yes | yes | yes | yes (lifecycle + terminal output) | OpenAI Files |
|
||||
| Azure | yes | yes | yes | yes | yes (byte-verbatim) | Azure Files |
|
||||
| Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) |
|
||||
| Bedrock | yes (unified only) | yes | no (limited upstream) | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
|
||||
| Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
|
||||
|
||||
Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off
|
||||
(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix;
|
||||
flipping those gates is tracked in LIT-4774 and deliberately not part of this suite.
|
||||
Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the
|
||||
lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`).
|
||||
Bedrock has no provider-side list, so list is the proxy's DB-backed managed view: the
|
||||
unified lifecycle lists with the plain `GET /v1/batches` and the batch must appear
|
||||
there. Both were gated off until LIT-5730, after LIT-4774 landed cancel support. A batch that completes inside the 2 s pre-cancel window skips the cancel assertion (a documented vacuous pass for the cancel cell, same as OpenAI); the list assertion runs either way.
|
||||
Bedrock file upload requires a model on the request (`encoded` / `unified` scenarios only);
|
||||
`model_param` and `provider_fallback` are omitted because `POST /bedrock/v1/files` has no
|
||||
model-less passthrough path.
|
||||
|
|
@ -148,6 +150,6 @@ never landed.
|
|||
Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a
|
||||
terminal DB status short-circuits retrieve for those ids, so the terminal-state cell
|
||||
uses the encoded path; poller timing does not fit an e2e gate and belongs in a
|
||||
DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Bedrock
|
||||
cancel/list stay gated pending LIT-4774. Gemini (non-Vertex) file content raises
|
||||
`NotImplementedError` upstream and is not a coverage cell.
|
||||
DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Gemini
|
||||
(non-Vertex) file content raises `NotImplementedError` upstream and is not a
|
||||
coverage cell.
|
||||
|
|
|
|||
|
|
@ -143,8 +143,8 @@ PROVIDERS: tuple[Provider, ...] = (
|
|||
"bedrock",
|
||||
batch_model_name("bedrock-batch"),
|
||||
"bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
can_cancel=False,
|
||||
can_list=False,
|
||||
can_cancel=True,
|
||||
can_list=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -248,8 +248,9 @@ def coverage_cells_for_lifecycle(cap: Capability) -> tuple[str, ...]:
|
|||
"""Registry cell ids that the parametrized lifecycle test covers for one capability.
|
||||
|
||||
OpenAI has per-scenario cells plus granular create/retrieve/cancel/list/file
|
||||
cells. Other providers have one basic cell each. File-upload cells for the
|
||||
batch-backing path are included when the lifecycle uploads for that provider.
|
||||
cells. Bedrock adds cancel and list cells behind its gates. Other providers
|
||||
have one basic cell each. File-upload cells for the batch-backing path are
|
||||
included when the lifecycle uploads for that provider.
|
||||
"""
|
||||
match cap.provider:
|
||||
case "openai":
|
||||
|
|
@ -279,6 +280,8 @@ def coverage_cells_for_lifecycle(cap: Capability) -> tuple[str, ...]:
|
|||
return (
|
||||
"llm.batches.bedrock.basic.nonstream.works",
|
||||
"llm.files.bedrock.upload.nonstream.works",
|
||||
*(("llm.batches.bedrock.cancel.nonstream.works",) if cap.can_cancel else ()),
|
||||
*(("llm.batches.bedrock.list.nonstream.works",) if cap.can_list else ()),
|
||||
)
|
||||
case _:
|
||||
return ()
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ BATCH_OP_RETRIES = 5
|
|||
# (connection refused, brief 500s) and the registry only has one basic cell per
|
||||
# provider (shared across scenarios). Create + retrieve already prove routing;
|
||||
# cancel is still deferred for cleanup, just not asserted for these two.
|
||||
_CANCEL_ASSERTED_PROVIDERS = frozenset({"openai"})
|
||||
_CANCEL_ASSERTED_PROVIDERS = frozenset({"openai", "bedrock"})
|
||||
|
||||
|
||||
def _transient_status(status_code: int) -> bool:
|
||||
|
|
@ -293,14 +293,13 @@ def test_batch_lifecycle(
|
|||
f"batch reached {pre_cancel.status!r} before cancel; "
|
||||
"provider likely rejected the input"
|
||||
)
|
||||
if pre_cancel.status == "completed":
|
||||
return
|
||||
cancelled = cancel_batch(client, batch.id, key=key, provider=provider)
|
||||
assert cancelled.id == batch.id
|
||||
assert cancelled.object == "batch"
|
||||
assert cancelled.status in {"cancelling", "cancelled"}, (
|
||||
f"unexpected post-cancel status {cancelled.status!r}"
|
||||
)
|
||||
if pre_cancel.status != "completed":
|
||||
cancelled = cancel_batch(client, batch.id, key=key, provider=provider)
|
||||
assert cancelled.id == batch.id
|
||||
assert cancelled.object == "batch"
|
||||
assert cancelled.status in {"cancelling", "cancelled"}, (
|
||||
f"unexpected post-cancel status {cancelled.status!r}"
|
||||
)
|
||||
|
||||
if cap.can_list:
|
||||
list_result = client.list_batches(key=key, provider=provider)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@
|
|||
- {id: llm.batches.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Vertex batches"}
|
||||
- {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"}
|
||||
- {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"}
|
||||
- {id: llm.batches.bedrock.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch cancel (StopModelInvocationJob) returns the same id with a cancelling/cancelled status"}
|
||||
- {id: llm.batches.bedrock.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "A Bedrock managed batch is present in the GET /v1/batches list envelope"}
|
||||
- {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"}
|
||||
- {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"}
|
||||
- {id: llm.batches.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.18 / LIT-4778", rationale: "Missing input_file_id and invalid batch id rejected"}
|
||||
|
|
|
|||
|
|
@ -157,6 +157,9 @@ async def test_chat_completion_bad_model_with_spend_logs():
|
|||
except json.JSONDecodeError:
|
||||
print(f"Could not parse response body as JSON: {response.text}")
|
||||
|
||||
assert (
|
||||
response.status_code == 400
|
||||
), f"expected HTTP 400, got {response.status_code}: {response.text}"
|
||||
assert (
|
||||
litellm_call_id is not None
|
||||
), "Failed to get LiteLLM Call ID from response headers"
|
||||
|
|
@ -191,7 +194,7 @@ async def test_chat_completion_bad_model_with_spend_logs():
|
|||
# Verify the structure of the log entry
|
||||
assert log_entry["request_id"] == litellm_call_id
|
||||
assert log_entry["model"] == "non-existent-model"
|
||||
assert log_entry["model_group"] == "non-existent-model"
|
||||
assert log_entry["model_group"] in ("", "non-existent-model")
|
||||
assert log_entry["spend"] == 0.0
|
||||
assert log_entry["total_tokens"] == 0
|
||||
assert log_entry["prompt_tokens"] == 0
|
||||
|
|
@ -206,8 +209,7 @@ async def test_chat_completion_bad_model_with_spend_logs():
|
|||
error_info = log_entry["metadata"]["error_information"]
|
||||
assert "traceback" in error_info
|
||||
assert error_info["error_code"] == "400"
|
||||
assert error_info["error_class"] == "BadRequestError"
|
||||
assert "litellm.BadRequestError" in error_info["error_message"]
|
||||
assert error_info["error_class"] in ("ProxyModelNotFoundError", "BadRequestError")
|
||||
assert "non-existent-model" in error_info["error_message"]
|
||||
|
||||
# Verify request details
|
||||
|
|
|
|||
|
|
@ -188,6 +188,62 @@ def test_streaming_span_carries_time_to_first_chunk():
|
|||
assert span.attributes[GenAI.RESPONSE_TIME_TO_FIRST_CHUNK] == pytest.approx(0.75)
|
||||
|
||||
|
||||
def test_llm_call_span_reports_the_server_spans_route():
|
||||
"""``litellm.request.route`` is the anchored server span's own ``http.route``,
|
||||
so an operator can group LLM spans by endpoint without joining to the parent."""
|
||||
logger, exporter = _logger()
|
||||
root = logger.tracer.start_span("POST /engines/{model:path}/chat/completions")
|
||||
root.set_attribute("http.route", "/engines/{model:path}/chat/completions")
|
||||
set_request_root_span(root)
|
||||
|
||||
_emit_llm(logger, ambient=root)
|
||||
root.end()
|
||||
|
||||
llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT)
|
||||
assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/engines/{model:path}/chat/completions"
|
||||
|
||||
|
||||
def test_llm_call_span_omits_the_route_without_a_server_span():
|
||||
"""An SDK call has no server span, so the key is absent rather than empty."""
|
||||
logger, exporter = _logger()
|
||||
_emit_llm(logger)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert LiteLLM.REQUEST_ROUTE not in span.attributes
|
||||
|
||||
|
||||
def test_failed_llm_call_span_reports_the_server_spans_route():
|
||||
"""The failure leg builds the same span data, so an errored call is still
|
||||
attributable to the endpoint it came in on."""
|
||||
logger, exporter = _logger()
|
||||
root = logger.tracer.start_span("POST /v1/responses/{response_id}")
|
||||
root.set_attribute("http.route", "/v1/responses/{response_id}")
|
||||
set_request_root_span(root)
|
||||
|
||||
_emit_llm(logger, ambient=root, fail=True)
|
||||
root.end()
|
||||
|
||||
llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT)
|
||||
assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}"
|
||||
|
||||
|
||||
def test_deferred_llm_call_span_reports_the_server_spans_route():
|
||||
"""``pre_call`` driven from a thread pool sees no recordable parent, so the span
|
||||
is created in the close callback instead. That branch has to carry the route
|
||||
too, and it can: the worker context still holds the anchor."""
|
||||
logger, exporter = _logger()
|
||||
root = logger.tracer.start_span("POST /v1/messages")
|
||||
root.set_attribute("http.route", "/v1/messages")
|
||||
set_request_root_span(root)
|
||||
|
||||
# no ``ambient``: pre_call runs with no recordable span active, which is what
|
||||
# defers creation to the close callback
|
||||
_emit_llm(logger)
|
||||
root.end()
|
||||
|
||||
llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT)
|
||||
assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/messages"
|
||||
|
||||
|
||||
def test_non_streaming_span_has_no_time_to_first_chunk():
|
||||
logger, exporter = _logger()
|
||||
kwargs = {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ surface and the server-span + shared-provider behavior it produces.
|
|||
"""
|
||||
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
|
|
@ -18,6 +20,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402
|
|||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
|
||||
InMemorySpanExporter,
|
||||
)
|
||||
from opentelemetry import trace # noqa: E402
|
||||
from opentelemetry.trace import SpanKind # noqa: E402
|
||||
|
||||
from litellm.integrations.otel.model.config import ( # noqa: E402
|
||||
|
|
@ -30,6 +33,23 @@ from litellm.integrations.otel.mount import ( # noqa: E402
|
|||
_passthrough_span_name_hook,
|
||||
instrument_fastapi_app,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.context import ( # noqa: E402
|
||||
request_root_http_route,
|
||||
set_request_root_span,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_request_root_span():
|
||||
"""Clear the root-span anchor around every test. Production gets a fresh
|
||||
contextvar copy per request task; the test process shares one context."""
|
||||
from litellm.integrations.otel.plumbing import context as _otel_context
|
||||
|
||||
_otel_context._request_root_span.set(None)
|
||||
_otel_context._mcp_message_transport_span.set(None)
|
||||
yield
|
||||
_otel_context._request_root_span.set(None)
|
||||
_otel_context._mcp_message_transport_span.set(None)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -128,6 +148,85 @@ def test_passthrough_hook_ignores_non_recording_span():
|
|||
assert span.name is None
|
||||
|
||||
|
||||
def test_llm_span_route_is_read_off_the_server_span(monkeypatch):
|
||||
"""``request_root_http_route`` answers with the SERVER span's own ``http.route``.
|
||||
|
||||
Driven through ``instrument_fastapi_app`` and the same
|
||||
``create_litellm_proxy_request_started_span`` call the proxy makes per request,
|
||||
so breaking either the mount or the anchor capture fails this."""
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "1")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
app = fastapi.FastAPI()
|
||||
seen = {}
|
||||
|
||||
def _anchor_then_read(key):
|
||||
logger.create_litellm_proxy_request_started_span(start_time=datetime.now(timezone.utc), headers=None)
|
||||
seen[key] = request_root_http_route()
|
||||
|
||||
@app.post("/engines/{model:path}/chat/completions")
|
||||
async def engines(model: str):
|
||||
_anchor_then_read("templated")
|
||||
return {}
|
||||
|
||||
@app.post("/openai/{endpoint:path}")
|
||||
async def openai_passthrough(endpoint: str):
|
||||
_anchor_then_read("passthrough")
|
||||
return {}
|
||||
|
||||
logger = OpenTelemetryV2(config=OpenTelemetryV2Config(exporter="in_memory"))
|
||||
exporter = InMemorySpanExporter()
|
||||
logger._tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
# instrument_fastapi_app passes no provider, so it binds to the OTel global the
|
||||
# way the proxy does once proxy_startup_event publishes one. set_tracer_provider
|
||||
# is a once-per-process door, so place it directly and let monkeypatch undo it.
|
||||
monkeypatch.setattr(trace, "_TRACER_PROVIDER", logger._tracer_provider)
|
||||
instrument_fastapi_app(app)
|
||||
|
||||
client = TestClient(app)
|
||||
client.post("/engines/gpt-4o-mini/chat/completions")
|
||||
client.post("/openai/v1/responses/resp_abc123")
|
||||
|
||||
routes = {
|
||||
(s.attributes or {})["http.route"] for s in exporter.get_finished_spans() if s.kind is SpanKind.SERVER
|
||||
}
|
||||
# a parameterized route keeps its template; the passthrough hook rewrote the
|
||||
# catch-all to the literal path, and both spans have to follow their own span
|
||||
assert routes == {"/engines/{model:path}/chat/completions", "/openai/v1/responses/resp_abc123"}
|
||||
assert seen["templated"] == "/engines/{model:path}/chat/completions"
|
||||
assert seen["passthrough"] == "/openai/v1/responses/resp_abc123"
|
||||
|
||||
|
||||
def test_server_span_route_survives_the_span_ending():
|
||||
"""The LLM span closes in an async callback that can run after the server span
|
||||
has ended, so the attribute has to still be readable then."""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
span = TracerProvider().get_tracer("t").start_span("POST /v1/responses/{response_id}")
|
||||
span.set_attribute("http.route", "/v1/responses/{response_id}")
|
||||
set_request_root_span(span)
|
||||
span.end()
|
||||
|
||||
assert request_root_http_route() == "/v1/responses/{response_id}"
|
||||
|
||||
|
||||
def test_no_server_span_means_no_route():
|
||||
"""An SDK call has no anchored server span, so the attribute is omitted rather
|
||||
than reported as empty."""
|
||||
assert request_root_http_route() is None
|
||||
|
||||
|
||||
def test_blank_route_on_the_server_span_is_omitted():
|
||||
"""An excluded or unmatched path leaves the server span without a usable route.
|
||||
Report nothing rather than a span attribute whose value is the empty string."""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
span = TracerProvider().get_tracer("t").start_span("GET")
|
||||
span.set_attribute("http.route", "")
|
||||
set_request_root_span(span)
|
||||
|
||||
assert request_root_http_route() is None
|
||||
|
||||
|
||||
def test_known_passthrough_prefixes_present():
|
||||
"""Guard the prefix set against accidental edits."""
|
||||
assert {"openai", "anthropic", "vertex_ai", "bedrock"} <= PASSTHROUGH_PREFIXES
|
||||
|
|
|
|||
|
|
@ -722,6 +722,39 @@ def test_request_identity_falls_back_to_legacy_team_keys():
|
|||
assert ident.team_alias == "legacy"
|
||||
|
||||
|
||||
def test_llm_span_carries_proxy_request_route():
|
||||
"""The LLM span records the proxy route the request arrived on, so it can be
|
||||
filtered by endpoint (``/v1/responses`` vs ``/v1/chat/completions``) without
|
||||
joining back to the root SERVER span's ``http.route``. The value is that
|
||||
span's ``http.route`` verbatim, so a parameterized route reports the template
|
||||
the SERVER span reports and not the path the caller happened to send."""
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_sample_payload(metadata={"user_api_key_request_route": "/v1/responses/resp_abc123"}),
|
||||
request_route="/v1/responses/{response_id}",
|
||||
)
|
||||
attrs: Final = GenAIMapper().map(data)
|
||||
|
||||
assert attrs[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}"
|
||||
|
||||
|
||||
def test_llm_span_falls_back_to_the_logged_route_without_a_server_span():
|
||||
"""The route the proxy recorded at auth is the backstop for a deployment whose
|
||||
FastAPI instrumentation never mounted: there is no server span to disagree with
|
||||
there, and an endpoint name is worth more than an absent attribute."""
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
_sample_payload(metadata={"user_api_key_request_route": "/v1/responses"})
|
||||
)
|
||||
|
||||
assert GenAIMapper().map(data)[LiteLLM.REQUEST_ROUTE] == "/v1/responses"
|
||||
|
||||
|
||||
def test_llm_span_omits_request_route_off_the_proxy():
|
||||
"""An SDK call has no inbound route, so the key is absent rather than empty."""
|
||||
attrs: Final = GenAIMapper().map(LLMCallSpanData.from_standard_logging_payload(_sample_payload(metadata={})))
|
||||
|
||||
assert LiteLLM.REQUEST_ROUTE not in attrs
|
||||
|
||||
|
||||
def test_guardrail_span_data_block_carries_verdict_and_error():
|
||||
from litellm.integrations.otel.model.payloads import GuardrailSpanData
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash
|
|||
target_id=target_id,
|
||||
router_name=job.router_name,
|
||||
router_names=job.router_names,
|
||||
models=sorted(job.models),
|
||||
direction=job.direction,
|
||||
baseline_model=job.baseline_model,
|
||||
shadow_percentage=job.shadow_percentage,
|
||||
|
|
@ -205,6 +206,7 @@ def _success_kwargs(
|
|||
request_metadata=None,
|
||||
call_type="acompletion",
|
||||
model="claude-opus",
|
||||
model_group="opus-group",
|
||||
response_cost=None,
|
||||
cache_hit=None,
|
||||
):
|
||||
|
|
@ -213,6 +215,7 @@ def _success_kwargs(
|
|||
"id": request_id,
|
||||
"call_type": call_type,
|
||||
"model": model,
|
||||
"model_group": model_group,
|
||||
"metadata": {"user_api_key_hash": api_key_hash},
|
||||
"model_parameters": {"temperature": 0.5, "stream": True},
|
||||
"response_cost": response_cost,
|
||||
|
|
@ -1007,6 +1010,79 @@ class TestTargetMatching:
|
|||
assert logger._job_starts == {"key-job": 1, "team-job": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestModelScope:
|
||||
"""A job scoped to model groups samples a target's request only when the group the
|
||||
caller asked for is one of them; an out-of-scope request is not the job's traffic at
|
||||
all, so it records no funnel event, exactly like a direction mismatch."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested,sampled",
|
||||
[("sonnet-group", True), ("opus-group", False), ("", False)],
|
||||
ids=["in-scope-group-samples", "other-group-skips", "unknown-group-fails-closed"],
|
||||
)
|
||||
async def test_scope_admits_only_the_named_groups_and_counts_nothing_else(self, requested, sampled):
|
||||
prisma = _prisma()
|
||||
logger = _logger(router=_router(), prisma=prisma, jobs=(_job(models=frozenset({"sonnet-group", "haiku-group"})),))
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(model_group=requested), RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
|
||||
assert prisma.db.litellm_shadowevalattempt.create.await_count == (1 if sampled else 0)
|
||||
assert logger._test_funnel == []
|
||||
|
||||
async def test_an_unscoped_job_samples_every_group(self):
|
||||
prisma = _prisma()
|
||||
logger = _logger(router=_router(), prisma=prisma, jobs=(_job(),))
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(model_group="anything"), RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
|
||||
prisma.db.litellm_shadowevalattempt.create.assert_awaited_once()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scoped_to,requested",
|
||||
[("sonnet-group", "fast"), ("fast", "sonnet-group")],
|
||||
ids=["job-names-the-target-request-uses-the-alias", "job-names-the-alias-request-uses-the-target"],
|
||||
)
|
||||
async def test_an_alias_and_its_target_are_one_group_on_both_sides(self, scoped_to, requested):
|
||||
"""Both the job's scope and the request's group resolve through the router's alias
|
||||
map at match time, so re-pointing an alias follows config rather than freezing at
|
||||
job start."""
|
||||
router = _router()
|
||||
router.model_group_alias = {"fast": "sonnet-group"}
|
||||
prisma = _prisma(jobs=[_job_record(_job(models=frozenset({scoped_to})))])
|
||||
logger = _logger(router=router, prisma=prisma)
|
||||
|
||||
await logger.async_log_success_event(_success_kwargs(model_group=requested), RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
|
||||
prisma.db.litellm_shadowevalattempt.create.assert_awaited_once()
|
||||
assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1
|
||||
|
||||
async def test_a_repointed_alias_applies_to_the_next_request_without_a_cache_refill(self):
|
||||
router = _router()
|
||||
router.model_group_alias = {"fast": "sonnet-group"}
|
||||
prisma = _prisma(jobs=[_job_record(_job(models=frozenset({"fast"})))])
|
||||
logger = _logger(router=router, prisma=prisma)
|
||||
await logger.async_log_success_event(_success_kwargs(model_group="sonnet-group"), RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
assert prisma.db.litellm_shadowevalattempt.create.await_count == 1
|
||||
|
||||
router.model_group_alias = {"fast": "haiku-group"}
|
||||
await logger.async_log_success_event(
|
||||
_success_kwargs(request_id="req-2", model_group="sonnet-group"), RESPONSE, None, None
|
||||
)
|
||||
await logger.async_log_success_event(
|
||||
_success_kwargs(request_id="req-3", model_group="haiku-group"), RESPONSE, None, None
|
||||
)
|
||||
await _drain(logger)
|
||||
|
||||
rows = [call.kwargs["data"]["request_id"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list]
|
||||
assert rows == ["req-1", "req-3"]
|
||||
assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestActiveJobsCache:
|
||||
async def test_cache_miss_reads_db_once_then_serves_from_cache(self):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone
|
|||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from jwt.utils import base64url_decode, base64url_encode
|
||||
from pydantic import SecretStr
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (
|
||||
|
|
@ -52,6 +53,12 @@ def _refresh_token() -> str:
|
|||
return minted.token.get_secret_value()
|
||||
|
||||
|
||||
def _corrupt_signature(token: str) -> str:
|
||||
unsigned, signature = token.rsplit(".", 1)
|
||||
raw = base64url_decode(signature)
|
||||
return f"{unsigned}.{base64url_encode(bytes((raw[0] ^ 0x01,)) + raw[1:]).decode()}"
|
||||
|
||||
|
||||
def test_kdf_is_deterministic_and_key_length_is_256_bit():
|
||||
again = session_keys_from_master_key(MASTER_KEY)
|
||||
assert again.signing_key.get_secret_value() == KEYS.signing_key.get_secret_value()
|
||||
|
|
@ -109,8 +116,7 @@ def test_resolve_fails_expired_token_closed_and_flags_expiry():
|
|||
|
||||
def test_resolve_fails_tampered_token_closed_without_expiry_flag():
|
||||
token = _access_token()
|
||||
tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb")
|
||||
result = resolve_session_bearer(f"Bearer {tampered}", KEYS, NOW)
|
||||
result = resolve_session_bearer(f"Bearer {_corrupt_signature(token)}", KEYS, NOW)
|
||||
assert isinstance(result, SessionBearerInvalid)
|
||||
assert result.expired is False
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import jwt
|
|||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from jwt.utils import base64url_decode, base64url_encode
|
||||
from pydantic import SecretStr, ValidationError
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
|
||||
|
|
@ -66,6 +67,12 @@ def _mint_refresh() -> str:
|
|||
return minted.token.get_secret_value()
|
||||
|
||||
|
||||
def _corrupt_signature(token: str) -> str:
|
||||
unsigned, signature = token.rsplit(".", 1)
|
||||
raw = base64url_decode(signature)
|
||||
return f"{unsigned}.{base64url_encode(bytes((raw[0] ^ 0x01,)) + raw[1:]).decode()}"
|
||||
|
||||
|
||||
def _sign_claims(payload: dict, prefix: str = SESSION_TOKEN_PREFIX, keys: SessionKeys = KEYS) -> str:
|
||||
return prefix + jwt.encode(payload, keys.signing_key.get_secret_value(), algorithm="HS256")
|
||||
|
||||
|
|
@ -138,8 +145,7 @@ def test_still_valid_one_second_before_expiry():
|
|||
|
||||
def test_tampered_signature_is_bad_signature():
|
||||
token = _mint_access()
|
||||
tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb")
|
||||
assert isinstance(open_session_token(tampered, KEYS, NOW), SessionBadSignature)
|
||||
assert isinstance(open_session_token(_corrupt_signature(token), KEYS, NOW), SessionBadSignature)
|
||||
|
||||
|
||||
def test_key_rotation_invalidates_outstanding_tokens():
|
||||
|
|
@ -329,8 +335,7 @@ def test_rs256_tampered_signature_is_bad_signature():
|
|||
minted = mint_session_token(PRINCIPAL, RSA_KEYS, NOW)
|
||||
assert isinstance(minted, MintedSessionToken)
|
||||
token = minted.token.get_secret_value()
|
||||
tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb")
|
||||
assert isinstance(open_session_token(tampered, RSA_KEYS, NOW), SessionBadSignature)
|
||||
assert isinstance(open_session_token(_corrupt_signature(token), RSA_KEYS, NOW), SessionBadSignature)
|
||||
|
||||
|
||||
def test_rs256_expired_token_is_expired():
|
||||
|
|
@ -413,8 +418,7 @@ def test_rotation_window_still_enforces_expiry_and_tamper_on_the_previous_key():
|
|||
)
|
||||
after = NOW + timedelta(seconds=SESSION_TTL_SECONDS + 1)
|
||||
assert isinstance(open_session_token(token, rotated, after), SessionExpired)
|
||||
tampered = token[:-2] + ("aa" if not token.endswith("aa") else "bb")
|
||||
assert isinstance(open_session_token(tampered, rotated, NOW), SessionBadSignature)
|
||||
assert isinstance(open_session_token(_corrupt_signature(token), rotated, NOW), SessionBadSignature)
|
||||
|
||||
|
||||
def test_weak_or_garbage_private_key_pem_rejected_at_construction():
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import asyncio
|
||||
from typing import Any, Dict, List, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from unittest.mock import patch
|
||||
|
||||
import click
|
||||
|
|
@ -7,7 +7,8 @@ import pytest
|
|||
import yaml
|
||||
from click.testing import CliRunner
|
||||
from InquirerPy.base.control import Choice
|
||||
from prompt_toolkit.application import create_app_session
|
||||
from InquirerPy.prompts.fuzzy import InquirerPyFuzzyControl
|
||||
from prompt_toolkit.application import AppSession, create_app_session
|
||||
from prompt_toolkit.input import create_pipe_input
|
||||
from prompt_toolkit.output import DummyOutput
|
||||
|
||||
|
|
@ -283,27 +284,45 @@ class TestRunConfigureWizardNotInteractive:
|
|||
assert not config_path.exists()
|
||||
|
||||
|
||||
def _highlighted_choice(session: AppSession) -> Optional[str]:
|
||||
if session.app is None:
|
||||
return None
|
||||
controls = [c for c in session.app.layout.find_all_controls() if isinstance(c, InquirerPyFuzzyControl)]
|
||||
if not controls or controls[0].choice_count == 0:
|
||||
return None
|
||||
return controls[0].selection["name"]
|
||||
|
||||
|
||||
async def _wait_until_highlighted(session: AppSession, name: str) -> None:
|
||||
async def _poll() -> None:
|
||||
while _highlighted_choice(session) != name:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
await asyncio.wait_for(_poll(), timeout=5)
|
||||
|
||||
|
||||
def _drive_fuzzy_pick(
|
||||
models: Tuple[DiscoveredModel, ...],
|
||||
prompt_label: str,
|
||||
multiselect: bool,
|
||||
key_events: List[Tuple[str, float]],
|
||||
key_events: List[Tuple[str, Optional[str]]],
|
||||
) -> List[str]:
|
||||
"""Drives the real InquirerPy fuzzy prompt through prompt_toolkit's own test input/output,
|
||||
exercising the actual widget (filtering, tab-to-toggle, enter-to-confirm) rather than mocking
|
||||
it away. asyncio.to_thread propagates the create_app_session context into the worker thread
|
||||
running _fuzzy_pick's synchronous .execute() call."""
|
||||
running _fuzzy_pick's synchronous .execute() call. Each key event names the choice the widget
|
||||
must highlight before the next key is sent (None sends the next key immediately)."""
|
||||
|
||||
async def _run() -> List[str]:
|
||||
with create_pipe_input() as pipe_input:
|
||||
with create_app_session(input=pipe_input, output=DummyOutput()):
|
||||
with create_app_session(input=pipe_input, output=DummyOutput()) as session:
|
||||
task = asyncio.ensure_future(
|
||||
asyncio.to_thread(wizard_module._fuzzy_pick, models, prompt_label, multiselect)
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
for text, delay in key_events:
|
||||
for text, highlighted in key_events:
|
||||
pipe_input.send_text(text)
|
||||
await asyncio.sleep(delay)
|
||||
if highlighted is not None:
|
||||
await _wait_until_highlighted(session, highlighted)
|
||||
return await task
|
||||
|
||||
return asyncio.run(_run())
|
||||
|
|
@ -315,13 +334,13 @@ class TestFuzzyPickWidget:
|
|||
|
||||
def test_single_select_filters_and_returns_highlighted_match(self):
|
||||
result = _drive_fuzzy_pick(
|
||||
self._models(), "test", multiselect=False, key_events=[("model-13", 0.3), ("\r", 0.1)]
|
||||
self._models(), "test", multiselect=False, key_events=[("model-13", "model-13"), ("\r", None)]
|
||||
)
|
||||
assert result == ["model-13"]
|
||||
|
||||
def test_multiselect_requires_tab_to_toggle_before_enter(self):
|
||||
result = _drive_fuzzy_pick(
|
||||
self._models(), "test", multiselect=True, key_events=[("model-7", 0.3), ("\t", 0.1), ("\r", 0.1)]
|
||||
self._models(), "test", multiselect=True, key_events=[("model-7", "model-7"), ("\t", None), ("\r", None)]
|
||||
)
|
||||
assert result == ["model-7"]
|
||||
|
||||
|
|
@ -331,12 +350,12 @@ class TestFuzzyPickWidget:
|
|||
"test",
|
||||
multiselect=True,
|
||||
key_events=[
|
||||
("model-3", 0.3),
|
||||
("\t", 0.1),
|
||||
*[("\x7f", 0.02) for _ in range("model-3".__len__())],
|
||||
("model-15", 0.3),
|
||||
("\t", 0.1),
|
||||
("\r", 0.1),
|
||||
("model-3", "model-3"),
|
||||
("\t", None),
|
||||
("\x7f" * len("model-3"), None),
|
||||
("model-15", "model-15"),
|
||||
("\t", None),
|
||||
("\r", None),
|
||||
],
|
||||
)
|
||||
assert set(result) == {"model-3", "model-15"}
|
||||
|
|
|
|||
231
tests/test_litellm/proxy/client/cli/test_debug_commands.py
Normal file
231
tests/test_litellm/proxy/client/cli/test_debug_commands.py
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
from click.testing import CliRunner
|
||||
|
||||
from litellm.proxy.client.cli import cli
|
||||
from litellm.proxy.client.cli.commands import debug as debug_module
|
||||
from litellm.proxy.client.cli.commands.debug import (
|
||||
SLASH_COMMAND_NAME,
|
||||
detect_claude_session_id,
|
||||
install_slash_command,
|
||||
)
|
||||
|
||||
SESSION = "e96634a3-fa28-4083-b354-55542e2dca01"
|
||||
|
||||
OK_ROW = {
|
||||
"request_id": "req-ok",
|
||||
"startTime": "2026-09-02T10:00:00",
|
||||
"endTime": "2026-09-02T10:00:02",
|
||||
"model": "claude-opus-4-1",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"status": "success",
|
||||
"spend": 0.0125,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 20,
|
||||
"metadata": {"status": "success"},
|
||||
}
|
||||
FAILED_ROW = {
|
||||
"request_id": "req-failed",
|
||||
"startTime": "2026-09-02T10:01:00",
|
||||
"endTime": "2026-09-02T10:01:01",
|
||||
"model": "claude-opus-4-1",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"status": "failure",
|
||||
"spend": 0.0,
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"metadata": json.dumps(
|
||||
{
|
||||
"status": "failure",
|
||||
"error_information": {
|
||||
"error_code": "400",
|
||||
"error_class": "BadRequestError",
|
||||
"error_message": "`prompt` is required when `stop` is not true.",
|
||||
},
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
PROXY = "http://localhost:4000"
|
||||
|
||||
|
||||
def _mock_proxy(rows, payloads):
|
||||
responses.get(
|
||||
f"{PROXY}/spend/logs/session/ui",
|
||||
json={"data": rows, "total": len(rows), "page": 1, "page_size": 100, "total_pages": 1},
|
||||
match=[responses.matchers.query_param_matcher({"session_id": SESSION}, strict_match=False)],
|
||||
)
|
||||
for request_id, payload in payloads.items():
|
||||
responses.get(f"{PROXY}/spend/logs/ui/{request_id}", json=payload)
|
||||
|
||||
|
||||
def _called_paths():
|
||||
return [c.request.path_url.split("?")[0] for c in responses.calls]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def env(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("LITELLM_PROXY_URL", PROXY)
|
||||
monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test")
|
||||
monkeypatch.setattr(debug_module, "REPORT_DIR", tmp_path / "reports")
|
||||
monkeypatch.setattr(debug_module, "CLAUDE_DIR", tmp_path / "claude")
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_report_includes_spend_error_and_bodies_for_failed_turn(tmp_path):
|
||||
payloads = {
|
||||
"req-failed": {
|
||||
"proxy_server_request": {"body": {"model": "claude-opus-4-1", "messages": [{"role": "user"}]}},
|
||||
"response": {"error": {"message": "`prompt` is required"}},
|
||||
},
|
||||
"req-ok": {"proxy_server_request": {"body": {"model": "claude-opus-4-1"}}, "response": {"id": "msg_1"}},
|
||||
}
|
||||
_mock_proxy([FAILED_ROW, OK_ROW], payloads)
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--recent-bodies", "0"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "turns: 2, failed: 1" in result.output
|
||||
assert "total spend: $0.012500" in result.output
|
||||
assert "### 1. ok claude-opus-4-1" in result.output
|
||||
assert "### 2. FAILED claude-opus-4-1" in result.output
|
||||
assert "`400` BadRequestError" in result.output
|
||||
assert "`prompt` is required when `stop` is not true." in result.output
|
||||
assert '"messages"' in result.output
|
||||
assert "msg_1" not in result.output
|
||||
assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-failed"]
|
||||
saved = tmp_path / "reports" / f"claude-{SESSION}.md"
|
||||
assert result.stdout.startswith(saved.read_text())
|
||||
assert "### 2. FAILED" in saved.read_text()
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_recent_bodies_fetches_latest_turns_even_when_successful():
|
||||
_mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": {"x": 1}}, "response": {"id": "msg_1"}}})
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "msg_1" in result.output
|
||||
assert _called_paths() == ["/spend/logs/session/ui", "/spend/logs/ui/req-ok"]
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_bodies_are_truncated_to_max_chars():
|
||||
_mock_proxy([OK_ROW], {"req-ok": {"proxy_server_request": {"body": "a" * 5000}, "response": None}})
|
||||
result = CliRunner().invoke(
|
||||
cli, ["debug", "claude", "--session-id", SESSION, "--no-save", "--max-body-chars", "200"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "truncated" in result.output
|
||||
assert "a" * 300 not in result.output
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_no_rows_is_a_clear_error():
|
||||
_mock_proxy([], {})
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION])
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "No spend logs found for session" in result.output
|
||||
|
||||
|
||||
def test_no_session_id_anywhere_is_a_clear_error(monkeypatch):
|
||||
monkeypatch.delenv("CLAUDE_CODE_SESSION_ID", raising=False)
|
||||
result = CliRunner().invoke(cli, ["debug", "claude"])
|
||||
assert result.exit_code != 0
|
||||
assert "Could not find a Claude Code session" in result.output
|
||||
|
||||
|
||||
OLD_SESSION = "0f3c2b1a-1111-4222-8333-444455556666"
|
||||
NEW_SESSION = "2d79c54d-4644-4708-b03e-95395ef9ecbd"
|
||||
|
||||
|
||||
def test_detect_session_id_prefers_env_then_newest_session_transcript(tmp_path):
|
||||
project = tmp_path / "projects" / "-Users-me-repo"
|
||||
project.mkdir(parents=True)
|
||||
old = project / f"{OLD_SESSION}.jsonl"
|
||||
new = project / f"{NEW_SESSION}.jsonl"
|
||||
subagent = project / "agent-a1b2c3d4.jsonl"
|
||||
old.write_text("{}")
|
||||
new.write_text("{}")
|
||||
subagent.write_text("{}")
|
||||
now = time.time()
|
||||
os.utime(old, (now - 100, now - 100))
|
||||
os.utime(new, (now - 50, now - 50))
|
||||
os.utime(subagent, (now, now))
|
||||
|
||||
assert detect_claude_session_id({}, tmp_path) == NEW_SESSION
|
||||
assert detect_claude_session_id({"CLAUDE_CODE_SESSION_ID": "from-env"}, tmp_path) == "from-env"
|
||||
assert detect_claude_session_id({"CLAUDE_SESSION_ID": "stale-name"}, tmp_path) == NEW_SESSION
|
||||
assert detect_claude_session_id({}, tmp_path / "missing") is None
|
||||
|
||||
|
||||
def test_install_slash_command_writes_runnable_command_file(tmp_path):
|
||||
path = install_slash_command(tmp_path)
|
||||
assert path == tmp_path / "commands" / f"{SLASH_COMMAND_NAME}.md"
|
||||
body = path.read_text()
|
||||
assert body.startswith("---\n")
|
||||
assert "allowed-tools: Bash(lite debug claude:*)" in body
|
||||
assert "!`lite debug claude $ARGUMENTS`" in body
|
||||
|
||||
result = CliRunner().invoke(cli, ["debug", "install-claude-command"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "/debug-lite" in result.output
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_rejected_key_is_a_clear_error_not_a_traceback():
|
||||
responses.get(
|
||||
f"{PROXY}/spend/logs/session/ui",
|
||||
status=401,
|
||||
json={"error": {"message": "Authentication Error, Invalid proxy server token passed", "code": "401"}},
|
||||
)
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION])
|
||||
|
||||
assert isinstance(result.exception, SystemExit), result.exception
|
||||
assert result.exit_code == 1
|
||||
assert "401" in result.output
|
||||
assert "Invalid proxy server token passed" in result.output
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_unreachable_proxy_is_a_clear_error_not_a_traceback():
|
||||
responses.get(f"{PROXY}/spend/logs/session/ui", body=requests.ConnectionError("Connection refused"))
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION])
|
||||
|
||||
assert isinstance(result.exception, SystemExit), result.exception
|
||||
assert result.exit_code == 1
|
||||
assert "Connection refused" in result.output
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_non_json_proxy_response_is_a_clear_error_not_a_traceback():
|
||||
responses.get(f"{PROXY}/spend/logs/session/ui", body="<html>502 Bad Gateway</html>")
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION])
|
||||
|
||||
assert isinstance(result.exception, SystemExit), result.exception
|
||||
assert result.exit_code == 1
|
||||
assert "/spend/logs/session/ui failed" in result.output
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_logged_content_with_code_fences_stays_inside_its_fence():
|
||||
fenced_error_row = {
|
||||
**FAILED_ROW,
|
||||
"metadata": {
|
||||
"status": "failure",
|
||||
"error_information": {"error_code": "400", "error_message": "bad\n```\nrequest"},
|
||||
},
|
||||
}
|
||||
_mock_proxy([fenced_error_row], {"req-failed": {"proxy_server_request": None, "response": "x\n````\ny"}})
|
||||
result = CliRunner().invoke(cli, ["debug", "claude", "--session-id", SESSION, "--no-save"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "````\nbad\n```\nrequest\n````\n" in result.output
|
||||
assert "`````json\nx\n````\ny\n`````\n" in result.output
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import Final
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -1190,27 +1191,23 @@ def test_health_liveliness_endpoint(proxy_client):
|
|||
Test that /health/liveliness endpoint returns 200 OK with "I'm alive!" message.
|
||||
This is a critical orchestration endpoint that must be simple and fast.
|
||||
"""
|
||||
# Measure the time taken for the health check call
|
||||
start_time = time.perf_counter()
|
||||
warm_up: Final = proxy_client.get("/health/liveliness")
|
||||
assert warm_up.status_code == 200, f"Expected 200 OK, got {warm_up.status_code}: {warm_up.text}"
|
||||
|
||||
# Make GET request to /health/liveliness
|
||||
response = proxy_client.get("/health/liveliness")
|
||||
def _timed_poll() -> tuple[float, httpx.Response]:
|
||||
start_time: Final = time.perf_counter()
|
||||
response: Final = proxy_client.get("/health/liveliness")
|
||||
return (time.perf_counter() - start_time) * 1000, response
|
||||
|
||||
end_time = time.perf_counter()
|
||||
duration_ms = (end_time - start_time) * 1000
|
||||
polls: Final = tuple(_timed_poll() for _ in range(5))
|
||||
|
||||
# Assert response status
|
||||
assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
for _, response in polls:
|
||||
assert response.status_code == 200, f"Expected 200 OK, got {response.status_code}: {response.text}"
|
||||
assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
|
||||
# Assert response content (FastAPI JSON-encodes the string)
|
||||
assert response.json() == "I'm alive!", f"Expected 'I'm alive!' message, got: {response.json()}"
|
||||
|
||||
# Verify response is fast (should be < 100ms for a simple endpoint)
|
||||
# This is critical for orchestration systems that poll frequently
|
||||
assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint"
|
||||
|
||||
# Log the duration for visibility (useful for CI/CD monitoring)
|
||||
print(f"\n/health/liveliness response time: {duration_ms:.2f}ms")
|
||||
durations_ms: Final = tuple(sorted(duration_ms for duration_ms, _ in polls))
|
||||
median_ms: Final = durations_ms[len(durations_ms) // 2]
|
||||
assert median_ms < 100, f"Median of {len(polls)} health checks took {median_ms:.2f}ms, expected < 100ms"
|
||||
|
||||
|
||||
def test_health_liveness_endpoint(proxy_client):
|
||||
|
|
|
|||
|
|
@ -882,6 +882,7 @@ def _leg_record(**overrides: object) -> MagicMock:
|
|||
"target_id": "key-hash",
|
||||
"router_name": "my-router",
|
||||
"router_names": (),
|
||||
"models": (),
|
||||
"direction": "forward",
|
||||
"baseline_model": None,
|
||||
"judge_model": "anthropic/claude-sonnet-5",
|
||||
|
|
@ -1033,6 +1034,7 @@ def _shadow_prisma(
|
|||
"target_id",
|
||||
"router_name",
|
||||
"router_names",
|
||||
"models",
|
||||
"direction",
|
||||
"baseline_model",
|
||||
"judge_model",
|
||||
|
|
@ -1533,6 +1535,87 @@ async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkey
|
|||
assert rows[0]["baseline_model"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_writes_the_model_scope_on_every_leg_and_echoes_it(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A model scope is job config, so every leg carries the same copy and both the start
|
||||
response and a later list read report it; an auto-router is a legitimate scope (a
|
||||
forward job on one router may sample what another router serves today)."""
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma()
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
response = await start_shadow_eval(
|
||||
_start_request(api_key_ids=("key-hash", "key-hash-2"), models=("cheap", "sonnet-router")), ADMIN
|
||||
)
|
||||
|
||||
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
|
||||
assert [row["models"] for row in rows] == [["cheap", "sonnet-router"], ["cheap", "sonnet-router"]]
|
||||
assert response.models == ("cheap", "sonnet-router")
|
||||
|
||||
listed = _shadow_prisma(legs=[_leg_record(models=("cheap",)), _leg_record(id="leg-0", group_id="job-0")])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", listed)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
assert {job.job_id: job.models for job in jobs} == {"job-1": ("cheap",), "job-0": ()}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_accepts_a_team_public_scope_for_a_user_target(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A user's traffic can arrive on any team's key, so a name only one team can ask for
|
||||
is a legitimate scope for a user target even though it resolves for nobody unscoped."""
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(known_users={"dev-alice": "alice@example.com"})
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
response = await start_shadow_eval(
|
||||
_start_request(api_key_ids=(), user_ids=("dev-alice",), models=("house-judge",)), ADMIN
|
||||
)
|
||||
|
||||
assert response.models == ("house-judge",)
|
||||
assert prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"][0]["models"] == ["house-judge"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_rejects_a_model_scope_this_proxy_does_not_serve(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A typo'd model name would otherwise start a job that samples nothing. Only the
|
||||
unresolvable names are reported, so the caller fixes them in one round."""
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma()
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await start_shadow_eval(_start_request(models=("cheap", "no-such-model-zzz")), ADMIN)
|
||||
assert exc.value.status_code == 400
|
||||
assert "'no-such-model-zzz'" in exc.value.detail
|
||||
assert "'cheap'" not in exc.value.detail
|
||||
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
|
||||
|
||||
|
||||
def test_start_request_dedupes_the_model_scope_and_rejects_blank_names():
|
||||
assert _start_request(models=("cheap", "mid", "cheap")).models == ("cheap", "mid")
|
||||
assert _start_request().models == ()
|
||||
with pytest.raises(ValidationError, match="non-empty model group names"):
|
||||
_start_request(models=("cheap", " "))
|
||||
|
||||
|
||||
def test_start_request_rejects_a_model_scope_on_a_reverse_job():
|
||||
"""Reverse admission is the router's own traffic, whose requested group is always the
|
||||
router, so a plain-model scope would sample nothing and the router itself is a no-op."""
|
||||
with pytest.raises(ValidationError, match="only meaningful for a forward job"):
|
||||
_start_request(direction="reverse", baseline_model="cheap", models=("mid",))
|
||||
with pytest.raises(ValidationError, match="only meaningful for a forward job"):
|
||||
_start_request(direction="reverse", baseline_model="cheap", models=("my-router",))
|
||||
assert _start_request(direction="reverse", baseline_model="cheap").models == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every
|
||||
|
|
|
|||
|
|
@ -2,15 +2,27 @@
|
|||
# This tests litellm router
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
import logging
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
async def _routed_model_ids(
|
||||
router: litellm.Router, tags: list[str], remaining: frozenset[str], attempts: int = 100
|
||||
) -> frozenset[str]:
|
||||
if not remaining or attempts == 0:
|
||||
return frozenset()
|
||||
response: Final = await router.acompletion(
|
||||
model="gpt-4", messages=[{"role": "user", "content": "hi"}], metadata={"tags": tags}, mock_response="hi"
|
||||
)
|
||||
seen: Final = frozenset({response._hidden_params["model_id"]})
|
||||
return seen | await _routed_model_ids(router, tags, remaining - seen, attempts - 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_router_free_paid_tier():
|
||||
"""
|
||||
|
|
@ -850,17 +862,10 @@ async def test_negation_regex_pattern_treated_as_literal():
|
|||
|
||||
# The regex-like string matches no deployment tag literally, so all
|
||||
# candidates survive and both model IDs are reachable.
|
||||
seen_ids = set()
|
||||
for _ in range(10):
|
||||
response = await router.acompletion(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"tags": ["!provider:(anthropic|openai)"]},
|
||||
mock_response="hi",
|
||||
)
|
||||
seen_ids.add(response._hidden_params["model_id"])
|
||||
expected: Final = frozenset({"anthropic-model", "openai-model"})
|
||||
routed_ids: Final = await _routed_model_ids(router, ["!provider:(anthropic|openai)"], expected)
|
||||
|
||||
assert seen_ids == {"anthropic-model", "openai-model"}
|
||||
assert routed_ids == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
|
|
@ -1281,17 +1286,10 @@ async def test_chain_enable_tag_filtering_false_overrides_router_level_true():
|
|||
enable_tag_filtering=True,
|
||||
)
|
||||
|
||||
seen_ids = set()
|
||||
for _ in range(10):
|
||||
response = await router.acompletion(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata={"tags": ["teamA"]},
|
||||
mock_response="hi",
|
||||
)
|
||||
seen_ids.add(response._hidden_params["model_id"])
|
||||
expected: Final = frozenset({"team-a-deployment", "team-b-deployment"})
|
||||
routed_ids: Final = await _routed_model_ids(router, ["teamA"], expected)
|
||||
|
||||
assert seen_ids == {"team-a-deployment", "team-b-deployment"}
|
||||
assert routed_ids == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@ case "$*" in
|
|||
"eslint --no-warn-ignored"*)
|
||||
[ "${STUB_FAIL:-}" = "eslint" ] && exit 1
|
||||
;;
|
||||
"eslint . -f json"*)
|
||||
if [ -n "${STUB_HANG_DIR:-}" ]; then
|
||||
touch "$STUB_HANG_DIR/eslint_report.started"
|
||||
sleep 60
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
exit 0
|
||||
"""
|
||||
|
|
@ -340,11 +346,12 @@ def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> Non
|
|||
)
|
||||
try:
|
||||
assert _wait_until((hang_dir / "make.started").exists, 10)
|
||||
assert _wait_until((hang_dir / "eslint_report.started").exists, 10)
|
||||
os.killpg(proc.pid, signal.SIGINT)
|
||||
assert proc.wait(timeout=10) != 0
|
||||
make_pid = int((hang_dir / "make.pid").read_text())
|
||||
assert _wait_until(lambda: _pid_gone(make_pid), 5)
|
||||
assert list(tmp_dir.iterdir()) == []
|
||||
assert _wait_until(lambda: not any(tmp_dir.iterdir()), 5), list(tmp_dir.iterdir())
|
||||
finally:
|
||||
with suppress(ProcessLookupError, PermissionError):
|
||||
os.killpg(proc.pid, signal.SIGTERM)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"LIT001": {
|
||||
"limit": 22184
|
||||
"limit": 22183
|
||||
},
|
||||
"LIT002": {
|
||||
"limit": 26747
|
||||
"limit": 26745
|
||||
},
|
||||
"LIT003": {
|
||||
"limit": 261
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
status: "running",
|
||||
router_name: "claude-auto",
|
||||
router_names: ["claude-auto"],
|
||||
models: [],
|
||||
direction: "forward",
|
||||
baseline_model: null,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
|
|
@ -450,6 +451,7 @@ describe("ShadowEvalSection", () => {
|
|||
api_key_ids: ["hash-alpha", "hash-beta"],
|
||||
team_ids: [],
|
||||
user_ids: [],
|
||||
models: [],
|
||||
router_names: ["gpt-auto"],
|
||||
direction: "forward",
|
||||
shadow_percentage: 10,
|
||||
|
|
@ -479,6 +481,7 @@ describe("ShadowEvalSection", () => {
|
|||
api_key_ids: [],
|
||||
team_ids: ["team-eng"],
|
||||
user_ids: [],
|
||||
models: [],
|
||||
router_names: ["gpt-auto"],
|
||||
direction: "forward",
|
||||
shadow_percentage: 10,
|
||||
|
|
@ -489,15 +492,41 @@ describe("ShadowEvalSection", () => {
|
|||
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
|
||||
});
|
||||
|
||||
it("narrows a job to the picked model groups and shows the scope on the job headline", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { start } = mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Search teams by alias"));
|
||||
const teamList = await screen.findByTestId("paginated-multi-select-list");
|
||||
await user.click(within(teamList).getByText("engineering"));
|
||||
await chooseSelectOption(user, screen.getByPlaceholderText("Every model the targets use"), "prod-claude");
|
||||
await chooseSelectOption(user, screen.getByPlaceholderText("Select up to 4 auto-routers"), "gpt-auto");
|
||||
await user.click(screen.getByPlaceholderText("Select a judge model"));
|
||||
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
|
||||
await user.click(screen.getByText("Start shadow eval"));
|
||||
|
||||
expect(start.mutate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ team_ids: ["team-eng"], models: ["prod-claude"] }),
|
||||
);
|
||||
|
||||
const scoped = job({ models: ["prod-claude", "prod-haiku"] });
|
||||
mockHooks({ jobs: [scoped], detailsById: { "job-1": scoped } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText("prod-claude, prod-haiku")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("requires a baseline model in reverse mode and submits it, while forward mode never shows the picker", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { start } = mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.queryByPlaceholderText("Select a baseline model")).not.toBeInTheDocument();
|
||||
expect(screen.getByPlaceholderText("Every model the targets use")).toBeInTheDocument();
|
||||
|
||||
await user.click(screen.getByText("Adoption check: key's traffic vs the router"));
|
||||
await user.click(await screen.findByText("Regression check: router's picks vs a baseline"));
|
||||
expect(screen.queryByPlaceholderText("Every model the targets use")).not.toBeInTheDocument();
|
||||
await user.click(screen.getByPlaceholderText("Search keys by alias"));
|
||||
const keyList = await screen.findByTestId("paginated-multi-select-list");
|
||||
await user.click(within(keyList).getByText("prod-alpha"));
|
||||
|
|
@ -516,6 +545,7 @@ describe("ShadowEvalSection", () => {
|
|||
api_key_ids: ["hash-alpha"],
|
||||
team_ids: [],
|
||||
user_ids: [],
|
||||
models: [],
|
||||
router_names: ["gpt-auto"],
|
||||
direction: "reverse",
|
||||
baseline_model: "prod-claude",
|
||||
|
|
@ -551,6 +581,7 @@ describe("ShadowEvalSection", () => {
|
|||
api_key_ids: ["hash-alpha"],
|
||||
team_ids: [],
|
||||
user_ids: [],
|
||||
models: [],
|
||||
router_names: ["gpt-auto", "claude-auto"],
|
||||
direction: "forward",
|
||||
shadow_percentage: 10,
|
||||
|
|
|
|||
|
|
@ -87,17 +87,25 @@ const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string =
|
|||
|
||||
const jobRouters = (job: ShadowEvalJob): string => (job.router_names ?? [job.router_name]).join(", ");
|
||||
|
||||
const jobModelScope = (job: ShadowEvalJob): React.ReactNode =>
|
||||
job.models && job.models.length > 0 ? (
|
||||
<>
|
||||
{" "}
|
||||
on <span className="font-mono text-xs">{job.models.join(", ")}</span>
|
||||
</>
|
||||
) : null;
|
||||
|
||||
const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
|
||||
job.direction === "reverse" ? (
|
||||
<>
|
||||
Comparing <span className="font-mono text-xs">{jobRouters(job)}</span> to{" "}
|
||||
<span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of{" "}
|
||||
<span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span> traffic
|
||||
<span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span> traffic{jobModelScope(job)}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span>{" "}
|
||||
traffic via <span className="font-mono text-xs">{jobRouters(job)}</span>
|
||||
traffic{jobModelScope(job)} via <span className="font-mono text-xs">{jobRouters(job)}</span>
|
||||
</>
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import { useStartShadowEval, type ShadowEvalJob } from "./useShadowEval";
|
|||
type ShadowEvalDirection = ShadowEvalJob["direction"];
|
||||
|
||||
const MAX_ROUTERS = 4;
|
||||
const MAX_MODELS = 100;
|
||||
|
||||
const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const;
|
||||
|
||||
|
|
@ -206,6 +207,7 @@ interface StartFormValidityInputs {
|
|||
apiKeyIds: string[];
|
||||
teamIds: string[];
|
||||
userIds: string[];
|
||||
models: string[];
|
||||
routerNames: string[];
|
||||
direction: ShadowEvalDirection;
|
||||
baselineModel: string;
|
||||
|
|
@ -224,7 +226,8 @@ const startFormValidity = (inputs: StartFormValidityInputs) => {
|
|||
const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS;
|
||||
const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1;
|
||||
const routersValid = routerCountValid && routersMatchDirection;
|
||||
const modelsPicked = routersValid && inputs.judgeModel !== "" && baselinePicked;
|
||||
const scopeValid = routersValid && (inputs.direction === "reverse" || inputs.models.length <= MAX_MODELS);
|
||||
const modelsPicked = scopeValid && inputs.judgeModel !== "" && baselinePicked;
|
||||
const filled = targetsPicked && modelsPicked;
|
||||
const boundsValid = percentageValid && maxBudgetValid;
|
||||
const valid = Boolean(inputs.accessToken) && filled && boundsValid;
|
||||
|
|
@ -235,6 +238,7 @@ interface StartBodyInputs {
|
|||
apiKeyIds: string[];
|
||||
teamIds: string[];
|
||||
userIds: string[];
|
||||
models: string[];
|
||||
routerNames: string[];
|
||||
direction: ShadowEvalDirection;
|
||||
baselineModel: string;
|
||||
|
|
@ -248,6 +252,7 @@ const buildStartBody = (inputs: StartBodyInputs) => ({
|
|||
api_key_ids: inputs.apiKeyIds,
|
||||
team_ids: inputs.teamIds,
|
||||
user_ids: inputs.userIds,
|
||||
models: inputs.direction === "forward" ? inputs.models : [],
|
||||
router_names: inputs.routerNames,
|
||||
direction: inputs.direction,
|
||||
...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}),
|
||||
|
|
@ -262,6 +267,7 @@ export const StartForm: React.FC = () => {
|
|||
const [apiKeyIds, setApiKeyIds] = useState<string[]>([]);
|
||||
const [teamIds, setTeamIds] = useState<string[]>([]);
|
||||
const [userIds, setUserIds] = useState<string[]>([]);
|
||||
const [models, setModels] = useState<string[]>([]);
|
||||
const [routerNames, setRouterNames] = useState<string[]>([]);
|
||||
const [direction, setDirection] = useState<ShadowEvalDirection>("forward");
|
||||
const [baselineModel, setBaselineModel] = useState("");
|
||||
|
|
@ -272,6 +278,11 @@ export const StartForm: React.FC = () => {
|
|||
const { data: autoRouters } = useAutoRouters();
|
||||
const judgeModelOptions = useJudgeModelOptions();
|
||||
const baselineModelOptions = useBaselineModelOptions();
|
||||
const configuredGroups = usePlainModelGroups();
|
||||
const modelOptions = useMemo<SearchSelectOption[]>(
|
||||
() => [...configuredGroups].toSorted((a, b) => a.localeCompare(b)).map((name) => ({ label: name, value: name })),
|
||||
[configuredGroups],
|
||||
);
|
||||
const start = useStartShadowEval();
|
||||
|
||||
const routerOptions = useMemo<SearchSelectOption[]>(() => {
|
||||
|
|
@ -286,6 +297,7 @@ export const StartForm: React.FC = () => {
|
|||
apiKeyIds,
|
||||
teamIds,
|
||||
userIds,
|
||||
models,
|
||||
routerNames,
|
||||
direction,
|
||||
baselineModel,
|
||||
|
|
@ -299,6 +311,7 @@ export const StartForm: React.FC = () => {
|
|||
apiKeyIds,
|
||||
teamIds,
|
||||
userIds,
|
||||
models,
|
||||
routerNames,
|
||||
direction,
|
||||
baselineModel,
|
||||
|
|
@ -344,6 +357,22 @@ export const StartForm: React.FC = () => {
|
|||
<Field label="Users to shadow" htmlFor="shadow-eval-user">
|
||||
<UserSelect value={userIds} onChange={setUserIds} />
|
||||
</Field>
|
||||
{direction === "forward" && (
|
||||
<Field label="Only on models">
|
||||
<MultiSelect
|
||||
options={modelOptions}
|
||||
value={models}
|
||||
onValueChange={setModels}
|
||||
placeholder="Every model the targets use"
|
||||
emptyText="No models configured"
|
||||
/>
|
||||
{models.length > MAX_MODELS ? (
|
||||
<p className="text-xs text-destructive">Pick at most {MAX_MODELS} models</p>
|
||||
) : (
|
||||
<p className="text-xs text-muted-foreground">Narrows every target above to requests for these models</p>
|
||||
)}
|
||||
</Field>
|
||||
)}
|
||||
<RouterField
|
||||
options={routerOptions}
|
||||
routerNames={routerNames}
|
||||
|
|
|
|||
17
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
17
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1264,7 +1264,10 @@ export interface paths {
|
|||
* A target is a virtual key, a team, or a user. Team and user targets match on the
|
||||
* identity every request resolves to at auth time, so they cover JWT-authenticated
|
||||
* traffic, which presents no virtual key; a user target samples that user's traffic
|
||||
* across all their teams, whether it arrives on a JWT or a key they own.
|
||||
* across all their teams, whether it arrives on a JWT or a key they own. models narrows
|
||||
* every target to requests for those model groups, so a user plus one model samples that
|
||||
* user's traffic on that model across every key they own; it is forward-only, since a
|
||||
* reverse job already samples exactly the traffic its own router served.
|
||||
*
|
||||
* A forward job answers whether the targets should adopt router_name: it samples the
|
||||
* requests the router did not serve and duplicates them through it. A reverse job
|
||||
|
|
@ -35805,6 +35808,12 @@ export interface components {
|
|||
* @description Most recent attempt error; detail endpoint only
|
||||
*/
|
||||
last_error?: string | null;
|
||||
/**
|
||||
* Models
|
||||
* @description Model groups the sampled traffic is narrowed to; empty means every model the targets use
|
||||
* @default []
|
||||
*/
|
||||
models: string[];
|
||||
/** @description Stratified verdicts; detail endpoint only */
|
||||
results?: components["schemas"]["ShadowEvalResult"] | null;
|
||||
/**
|
||||
|
|
@ -36228,6 +36237,12 @@ export interface components {
|
|||
* @default 10
|
||||
*/
|
||||
max_budget: number;
|
||||
/**
|
||||
* Models
|
||||
* @description Model groups to narrow the sampled traffic to, matched on the group the caller requested and resolved through model_group_alias, so an alias and its target are one name. Empty samples every model the targets use. This ANDs with the targets: a job over a user and one model samples that user's requests on that model across every key they own, and none of their other traffic. Forward jobs only: a reverse job samples exactly the traffic its own router served, which no other model group can name
|
||||
* @default []
|
||||
*/
|
||||
models: string[];
|
||||
/**
|
||||
* Router Name
|
||||
* @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue