diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index 36c6c790b84..c3b8ce22c68 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -4,17 +4,16 @@ description: >- so only the first job on a given Cargo.lock compiles the bridge from scratch. litellm builds through maturin, which compiles litellm-rust/crates/python-bridge - in release mode before it can produce a wheel. `uv sync` therefore pays a full - build in every job that installs the workspace: measured at 2m40s per unit shard - on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught - it, because the uv cache holds wheels uv downloads rather than wheels it builds, - and a path dependency whose source moves every commit could never hit that cache - anyway. Cargo rebuilds only what changed when its target directory survives, so a - warm job pays for the bridge crate alone. + in the dev profile for editable installs. `uv sync` therefore pays a full build + in every job that installs the workspace. Nothing caught it, because the uv cache + holds wheels uv downloads rather than wheels it builds, and a path dependency + whose source moves every commit could never hit that cache anyway. Cargo rebuilds + only what changed when its target directory survives, so a warm job pays for the + bridge crate alone. - The key namespace is separate from test-rust.yml's. Both cache the same directory, - but that workflow fills it with debug and clippy artifacts, which a release build - cannot reuse, and a shared key would let whichever ran first deny the other a save. + The key namespace is separate from test-rust.yml's check and release caches. They + cache the same directory for different workloads, and a shared key would let + whichever ran first deny the others a save. runs: using: composite @@ -26,6 +25,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-release- + ${{ runner.os }}-maturin-dev- diff --git a/.github/workflows/test-redis-compat.yml b/.github/workflows/test-redis-compat.yml new file mode 100644 index 00000000000..f29755a74b1 --- /dev/null +++ b/.github/workflows/test-redis-compat.yml @@ -0,0 +1,77 @@ +name: "Unit Tests: Redis Client Version Compatibility" + +on: + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + paths: + - "litellm/_redis.py" + - "litellm/_redis_credential_provider.py" + - "tests/test_litellm/test_redis.py" + - "tests/test_litellm/caching/test_redis_connection_pool.py" + - ".github/workflows/test-redis-compat.yml" + - "pyproject.toml" + - "uv.lock" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + redis-compat: + name: "redis-py ${{ matrix.redis-version }}" + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the + # newer legs prove the inspect.signature introspection in litellm/_redis.py + # keeps extracting kwargs on the redis-py releases people actually run now. + # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra) + # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in + # for the 6.x line. + redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"] + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Pin redis-py to the matrix version + env: + REDIS_VERSION: ${{ matrix.redis-version }} + run: | + uv pip install "redis==${REDIS_VERSION:?}" + uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)" + + - name: Run redis unit tests + run: | + uv run --no-sync pytest \ + tests/test_litellm/test_redis.py \ + tests/test_litellm/caching/test_redis_connection_pool.py \ + --tb=short -vv \ + --reruns 2 \ + --reruns-delay 1 \ + --durations=20 diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql new file mode 100644 index 00000000000..90b21205310 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "router_names" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; + +ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "router_name" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 481ea3f8f66..c17a0605fc7 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -30,3 +30,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "unwind" +debug = false +incremental = false +strip = "symbols" diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 0c4a753f762..d461a483ae0 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,8 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["extension-module"] +default = ["abi3"] +abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] [dependencies] diff --git a/litellm/_logging.py b/litellm/_logging.py index fbb35b72be2..9435562f890 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -264,13 +264,17 @@ def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str: class LevelRoutingStreamHandler(logging.StreamHandler): - """Writes records below WARNING to stdout and WARNING and above to stderr. + """Writes records below WARNING and invalid-key warnings to stdout, others to stderr. Collectors that derive severity from the stream report every stderr line as an error. + Invalid-key warnings route to stdout so LITELLM_LOG=ERROR can suppress them. """ def emit(self, record: logging.LogRecord) -> None: - preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr + is_stdout_record: Final = record.levelno < logging.WARNING or ( + record.levelno == logging.WARNING and record.name == verbose_proxy_stdout_logger.name + ) + preferred: Final = sys.stdout if is_stdout_record else sys.stderr if preferred is None or getattr(preferred, "closed", False): self.stream = sys.stderr # rebind-ok: fall back to the pre-fix stream rather than raising per record else: @@ -508,6 +512,9 @@ else: handler.setFormatter(formatter) verbose_proxy_logger = logging.getLogger("LiteLLM Proxy") +# Malformed virtual key rejections log through this child; LevelRoutingStreamHandler +# writes its WARNING records to stdout. It has no handler or level of its own. +verbose_proxy_stdout_logger: Final = verbose_proxy_logger.getChild("stdout") verbose_router_logger = logging.getLogger("LiteLLM Router") verbose_logger = logging.getLogger("LiteLLM") @@ -520,6 +527,7 @@ verbose_logger.addHandler(handler) # handlers (JSON mode, uvicorn log config, a host app's root handler). verbose_router_logger.addFilter(_stdout_truncation_filter) verbose_proxy_logger.addFilter(_stdout_truncation_filter) +verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter) verbose_logger.addFilter(_stdout_truncation_filter) @@ -683,6 +691,7 @@ def _turn_on_json(): - Adds a JSON formatter to all loggers """ handler: Final = LevelRoutingStreamHandler() + handler.setLevel(numeric_level) handler.setFormatter(JsonFormatter()) _initialize_loggers_with_handler(handler) # Set up exception handlers @@ -700,12 +709,14 @@ def _disable_debugging(): verbose_logger.disabled = True verbose_router_logger.disabled = True verbose_proxy_logger.disabled = True + verbose_proxy_stdout_logger.disabled = True def _enable_debugging(): verbose_logger.disabled = False verbose_router_logger.disabled = False verbose_proxy_logger.disabled = False + verbose_proxy_stdout_logger.disabled = False def print_verbose(print_statement): diff --git a/litellm/_redis.py b/litellm/_redis.py index 9381357931e..3e68d50cf16 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -13,6 +13,7 @@ import json # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation import os from collections.abc import Callable, Mapping +from types import MappingProxyType from typing import Final from urllib.parse import urlsplit, urlunsplit @@ -38,9 +39,25 @@ from ._logging import verbose_logger AZURE_REDIS_SCOPE: Final = "https://redis.azure.com/.default" -def _get_redis_kwargs(): - arg_spec: Final = inspect.getfullargspec(redis.Redis) +def _unwrapped_init_args(cls: type) -> frozenset[str]: + """Every parameter on a single class's own ``__init__``, decorator-unwrapped. + Unlike ``_init_arg_names`` below, this does not walk the MRO: ``redis.Redis`` + and ``redis.RedisCluster`` (sync and async) each declare every real + constructor parameter directly on their own ``__init__``, so MRO-walking is + unnecessary — and it actively breaks the several tests here that mock the + class with ``patch(..., autospec=True)``, since ``inspect.getmro`` needs a + real ``__mro__`` that an autospec'd stand-in for a class does not provide. + + Still unwraps first: redis-py >= 7.4 decorates these ``__init__``s with + ``@deprecated_args`` too, which the same class of bug as ``_init_arg_names`` + would otherwise silently empty this allowlist through (see its docstring). + """ + spec: Final = inspect.getfullargspec(inspect.unwrap(cls.__init__)) + return frozenset(spec.args + spec.kwonlyargs) + + +def _get_redis_kwargs(): # Only allow primitive arguments exclude_args: Final = { "self", @@ -60,7 +77,7 @@ def _get_redis_kwargs(): "azure_client_secret", } - available_args: Final = {x for x in arg_spec.args if x not in exclude_args} | include_args + available_args: Final = {x for x in _unwrapped_init_args(redis.Redis) if x not in exclude_args} | include_args return available_args @@ -120,15 +137,23 @@ def _get_redis_url_kwargs(client: type | None = None) -> tuple[str, ...]: return tuple(x for x in _init_arg_names(connection_cls) if x not in exclude_args) + include_args -def _get_redis_cluster_kwargs(client=None): +def _get_redis_cluster_kwargs(client: type | None = None): + """Config kwargs the target cluster client's constructor actually accepts. + + Defaults to the sync ``redis.RedisCluster``, but the async cluster client + (``redis.asyncio.cluster.RedisCluster``) declares connection settings such as + ``decode_responses`` on its own constructor, where the sync class takes them + through ``**kwargs`` and so never names them in its signature. Introspecting + only the sync class regardless of which client is actually built silently + drops those for every async cluster caller. + """ if client is None: - client = redis.Redis.from_url - arg_spec: Final = inspect.getfullargspec(redis.RedisCluster) + client = redis.RedisCluster # Only allow primitive arguments exclude_args: Final = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"} - available_args = {x for x in arg_spec.args if x not in exclude_args} + available_args = {x for x in _unwrapped_init_args(client) if x not in exclude_args} available_args |= { "password", "username", @@ -161,6 +186,79 @@ def _get_redis_env_kwarg_mapping(): return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment} +def _str_to_bool(value: str) -> bool: + return value.lower() in ("true", "1", "yes") + + +def _coerce_redis_kwargs_types( + redis_kwargs: Mapping[str, object], + client: type | tuple[type, ...] = redis.Redis, +) -> dict[str, object]: # mutable-ok: a caller mutates the returned kwargs before constructing its client + """Coerces string values to the numeric/boolean type ``client``'s constructor + declares for that parameter. ``client`` may be a tuple of client classes; a + parameter's type is taken from the first signature that declares it, which + lets cluster callers coerce cluster-only kwargs such as + ``cluster_error_retry_attempts`` alongside the shared connection kwargs. + + Environment variables are always strings, and Helm ``--set`` stringifies values + too, so a config value like ``health_check_interval`` or ``socket_timeout`` + can arrive as ``"30"``/``"5.5"`` rather than a real number. redis-py's own + connection-health-check arithmetic (``loop.time() + self.health_check_interval``) + then raises ``TypeError`` on every Redis operation instead of connecting. + + ``max_connections``, ``socket_timeout``, and ``socket_connect_timeout`` use an + explicit target type rather than the parameter's own signature default: redis-py + 8.x changed the timeout defaults from ``None`` to int ``5``, so inferring the + type from the default would make a fractional ``"5.5"`` fail ``int()`` and get + silently dropped on 8.x while working on older versions. ``socket_keepalive`` + is explicit too: its signature default is ``None``, which carries no type to + infer from, and leaving it a string makes ``"false"`` truthy. + """ + signatures: Final = tuple(inspect.signature(c) for c in (client if isinstance(client, tuple) else (client,))) + explicit_param_types: Final = MappingProxyType( + { + "max_connections": int, + "socket_timeout": float, + "socket_connect_timeout": float, + "socket_keepalive": bool, + } + ) + result: Final = dict(redis_kwargs) # mutable-ok: per-key try/except coercion below needs to drop individual keys + for key, value in redis_kwargs.items(): + if not isinstance(value, str): + continue + param = next((sig.parameters[key] for sig in signatures if key in sig.parameters), None) + if param is None: + continue + explicit_type = explicit_param_types.get(key) + if explicit_type is bool: + result[key] = _str_to_bool(value) + continue + if explicit_type is not None: + try: + result[key] = explicit_type(value) + except (ValueError, TypeError): + del result[key] + continue + default: object = param.default # pyright: ignore[reportAny] # inspect.Parameter.default is stubbed as Any + if default is inspect.Parameter.empty: + continue + # bool must be checked before int, since bool subclasses int + if isinstance(default, bool): + result[key] = _str_to_bool(value) + elif isinstance(default, int): + try: + result[key] = int(value) + except (ValueError, TypeError): + del result[key] + elif isinstance(default, float): + try: + result[key] = float(value) + except (ValueError, TypeError): + del result[key] + return result + + def _redis_kwargs_from_environment(): mapping: Final = _get_redis_env_kwarg_mapping() @@ -505,7 +603,12 @@ def _get_redis_client_logic(**env_overrides): raise ValueError("Either 'host' or 'url' must be specified for redis.") # litellm.print_verbose(f"redis_kwargs: {redis_kwargs}") - return redis_kwargs + coercion_client: Final = ( + (redis.Redis, redis.RedisCluster, async_redis.RedisCluster) + if redis_kwargs.get("startup_nodes") + else redis.Redis + ) + return _coerce_redis_kwargs_types(redis_kwargs, client=coercion_client) def init_redis_cluster(redis_kwargs) -> redis.RedisCluster: @@ -657,7 +760,9 @@ def get_redis_client(**env_overrides): if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs: return _init_redis_sentinel(redis_kwargs) - return redis.Redis(**redis_kwargs) + return redis.Redis( # pyright: ignore[reportCallIssue] # object-valued kwargs match no overload statically + **redis_kwargs, # pyright: ignore[reportArgumentType] # allow-listed and coerced against this signature + ) def get_redis_async_client( @@ -669,7 +774,7 @@ def get_redis_async_client( if "startup_nodes" in redis_kwargs: from redis.cluster import ClusterNode - args = _get_redis_cluster_kwargs() + args = _get_redis_cluster_kwargs(async_redis.RedisCluster) cluster_kwargs: Final = {} for arg in redis_kwargs: if arg in args: diff --git a/litellm/constants.py b/litellm/constants.py index 1c8a6dc5874..c482ab0e39a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1427,6 +1427,12 @@ DEFAULT_SOFT_BUDGET: Final = float( ) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash" +# Prefix of the 401 raised when a submitted virtual key is not shaped like one. +INVALID_VIRTUAL_KEY_ERROR_MESSAGE: Final = "LiteLLM Virtual Key expected" +# Attribute stamped on that 401 at its raise site so log routing recognises it by +# provenance. Message text is caller-influenceable on other 401s, so it must not +# be used to classify. +INVALID_VIRTUAL_KEY_ERROR_MARKER: Final = "_litellm_invalid_virtual_key_error" # Python garbage collection threshold configuration # Format: "gen0,gen1,gen2" e.g., "1000,50,50" diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 9e949db625a..6c33621ec89 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -115,9 +115,11 @@ class SpeechToCompletionBridgeHandler: **request_data, ) + requested_response_format: Final = optional_params.get("response_format") if isinstance(result, ModelResponse): return self.transformation_handler.transform_response( model_response=result, + response_format=requested_response_format if isinstance(requested_response_format, str) else None, ) else: raise Exception(f"Unmapped response type. Got type: {type(result)}") diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 9b757ce86be..2ed140c0208 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -21,6 +21,8 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None: GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16" +GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm" +GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT}) class ChatAudioParam(TypedDict): @@ -29,6 +31,26 @@ class ChatAudioParam(TypedDict): class SpeechToCompletionBridgeTransformationHandler: + def _validate_response_format( + self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object] + ) -> None: + if not self._is_gemini_tts_model(model): + return + response_format: Final = optional_params.get("response_format") + if not isinstance(response_format, str) or response_format in GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: + return + from litellm.exceptions import BadRequestError + + supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS)) + raise BadRequestError( + message=( + f"Gemini TTS only produces raw PCM16 audio, so response_format='{response_format}'" + f" is not supported. Supported response formats: {supported}." + ), + model=model, + llm_provider=custom_llm_provider, + ) + def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]: return MappingProxyType( { @@ -67,6 +89,7 @@ class SpeechToCompletionBridgeTransformationHandler: litellm_logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, ) -> dict: + self._validate_response_format(model, custom_llm_provider, optional_params) user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input} return_kwargs: Final = { "model": model, @@ -125,7 +148,14 @@ class SpeechToCompletionBridgeTransformationHandler: """Check if the model is a Gemini TTS model that returns PCM16 data.""" return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower()) - def transform_response(self, model_response: "ModelResponse") -> "HttpxBinaryResponseContent": + def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]: + if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT: + return decoded_audio, "audio/pcm" + return self._convert_pcm16_to_wav(decoded_audio), "audio/wav" + + def transform_response( + self, model_response: "ModelResponse", response_format: str | None + ) -> "HttpxBinaryResponseContent": import base64 import httpx @@ -136,23 +166,17 @@ class SpeechToCompletionBridgeTransformationHandler: audio_part: Final = cast(Choices, model_response.choices[0]).message.audio if audio_part is None: raise ValueError("No audio part found in the response") - audio_content: Final = audio_part.data + decoded_audio: Final = base64.b64decode(audio_part.data) - # Decode base64 to get binary content - binary_data = base64.b64decode(audio_content) - - # Check if this is a Gemini TTS model that returns raw PCM16 data model: Final = getattr(model_response, "model", "") - headers: Final = {} - if self._is_gemini_tts_model(model): - # Convert PCM16 to WAV format for proper audio file playback - binary_data = self._convert_pcm16_to_wav(binary_data) - headers["Content-Type"] = "audio/wav" - else: - headers["Content-Type"] = "audio/mpeg" - - # Create an httpx.Response object - response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers) + content, content_type = ( + self._gemini_tts_response_body(decoded_audio, response_format) + if self._is_gemini_tts_model(model) + else (decoded_audio, "audio/mpeg") + ) + response: Final = httpx.Response( + status_code=200, content=content, headers=MappingProxyType({"Content-Type": content_type}) + ) binary_response: Final = HttpxBinaryResponseContent(response) binary_response.set_response_cost(_completion_response_cost(model_response)) return binary_response diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 18bda0a9d55..27da785331a 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -1,8 +1,11 @@ """Shadow Eval Logger: samples a shadowed key's successful LLM requests (chat completions, Anthropic Messages, and Responses API surfaces, each normalized to chat shape), duplicates -each against the job's other arm in a detached task (the auto-router for a forward job, the -fixed baseline model for a reverse one), blind-judges real vs shadow, and appends one -``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write. +each through every shadow arm in one detached task (each candidate auto-router for a +forward job, the fixed baseline model for a reverse one), blind-judges real vs each arm, +and appends one ``LiteLLM_ShadowEvalAttempt`` row per arm (verdict or error) as the +feature's only hot-path write. A multi-router job's arms therefore score the identical +sampled requests against the identical real responses, which is what makes their win +rates comparable head-to-head. Counts, status, and spend derive from those rows at read time, so nothing can disagree across pods or stop races; the hook reads active jobs through a short-TTL cache.""" @@ -498,12 +501,16 @@ def _decision_classifier_cost(metadata: Mapping[str, object]) -> float: return float(raw) if isinstance(raw, (int, float)) else 0.0 -def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool: - """Whether the router under evaluation served this request, which is what decides - the direction it belongs to. A forward job skips its own router's traffic, since - duplicating it would compare the router to itself: guaranteed ties, judge spend for - zero information. A reverse job samples exactly that traffic and nothing else.""" - return _routing_decision(request_metadata).get("router_model_name") == router_name +def _direction_admits(request_metadata: Mapping[str, object], job: "ActiveShadowEvalJob") -> bool: + """Whether this request belongs to the job's direction. A forward job skips traffic + any of its candidate routers served: duplicating a router's own request compares it + to itself (guaranteed ties), and judging a sibling against another candidate's live + response would score candidates against each other instead of against the incumbent. + A reverse job samples exactly its one router's traffic and nothing else.""" + routed_by: Final = _routing_decision(request_metadata).get("router_model_name") + if job.direction == "reverse": + return routed_by == job.router_name + return routed_by not in job.arm_router_names @dataclass(frozen=True, slots=True) @@ -546,6 +553,7 @@ class ActiveShadowEvalJob(BaseModel): id: str router_name: str + router_names: tuple[str, ...] = () direction: ShadowEvalDirection = "forward" baseline_model: str | None = None shadow_percentage: float @@ -567,12 +575,25 @@ class ActiveShadowEvalJob(BaseModel): raise ValueError("baseline_model is set for exactly the reverse jobs") return self + @model_validator(mode="after") + def _reverse_evaluates_one_router(self) -> "ActiveShadowEvalJob": + """A reverse row naming several routers is unsamplable (there is no one traffic + slice they share) and fails closed.""" + if self.direction == "reverse" and len(self.arm_router_names) > 1: + raise ValueError("a reverse job evaluates exactly one router") + return self + @property - def shadow_target(self) -> str: - """The model the duplicated arm calls: the router itself for a forward job, the - fixed baseline for a reverse one. Total because the validator above pins + def arm_router_names(self) -> tuple[str, ...]: + """The job's full router set; rows from before router_names existed hold it in + router_name alone. The one place that reading lives on the sampling side.""" + return self.router_names or (self.router_name,) + + def arm_target(self, arm_router: str) -> str: + """The model one duplicated arm calls: the candidate router itself for a forward + job, the fixed baseline for a reverse one. Total because the validator above pins baseline_model to reverse jobs and only those.""" - return self.baseline_model or self.router_name + return self.baseline_model or arm_router def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None: @@ -696,7 +717,7 @@ class ShadowEvalLogger(CustomLogger): now >= job.ends_at or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns or (job.max_budget is not None and job.spend >= job.max_budget) - or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse") + or not _direction_admits(request_metadata, job) ): continue if not _sample_hits(request_id, job.id, job.shadow_percentage): @@ -773,7 +794,10 @@ class ShadowEvalLogger(CustomLogger): if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS: self._record_funnel(job.id, "shed") continue - self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1 + # One start writes one attempt row per arm, and max_turns is a row + # ceiling, so admission must pre-count every arm or a multi-router + # job overshoots the valve N-fold within a cache generation. + self._job_starts[job.id] = self._job_starts.get(job.id, 0) + len(job.arm_router_names) self._inflight_shadow_tasks += 1 asyncio.create_task( self._run_shadow_eval( @@ -812,32 +836,74 @@ class ShadowEvalLogger(CustomLogger): shadow_params: Mapping[str, object], parent_metadata: Mapping[str, object], ) -> None: - """Budget gate -> shadow call -> blind judge -> one attempt row, and every exit - in exactly one coverage bucket: the gates that decline to spend on an admitted - sample (no DB to record into, an over-budget key, an unverifiable or exhausted - eval budget) count it withheld, so eligible traffic still reconciles as - not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits - above the dispatch so no provider spend happens without a place to record the - outcome, and the budget read lives here rather than in the success hook.""" + """Budget gates once per sampled request, then every router arm in turn: shadow + call -> blind judge -> one attempt row stamped with the arm. The gates that + decline to spend on an admitted sample (no DB to record into, an over-budget key, + an unverifiable or exhausted eval budget) count the REQUEST withheld before any + arm runs, so funnel counters stay per-request and a leg's eligible traffic still + reconciles as not_sampled + unjudgeable + shed + withheld + sampled requests, + where each sampled request writes one attempt row per arm. A budget crossed + mid-loop lets the remaining arms overshoot by one round, the same class of + overshoot as the samples already in flight when the cap is crossed. The prisma + gate sits above the dispatch so no provider spend happens without a place to + record the outcome, and the budget read lives here rather than in the success + hook.""" prisma: Final = self._prisma_provider() + if prisma is None: + self._record_funnel(job.id, "withheld") + return + if await _key_or_team_is_over_budget(parent_metadata): + self._record_funnel(job.id, "withheld") + return + if job.max_budget is not None: + try: + spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget) + except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it + verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e) + self._record_funnel(job.id, "withheld") + return + if spend >= job.max_budget: + self._record_funnel(job.id, "withheld") + return + for arm_router in job.arm_router_names: + await self._run_shadow_arm( + prisma=prisma, + job=job, + arm_router=arm_router, + request_id=request_id, + messages=messages, + real_text=real_text, + real_model=real_model, + real_cost=real_cost, + real_classifier_cost=real_classifier_cost, + real_cache_hit=real_cache_hit, + control_tier=control_tier, + shadow_params=shadow_params, + parent_metadata=parent_metadata, + ) + + async def _run_shadow_arm( + self, + prisma: "PrismaClient", + job: ActiveShadowEvalJob, + arm_router: str, + request_id: str, + messages: Sequence[Mapping[str, object]], + real_text: str, + real_model: str, + real_cost: float, + real_classifier_cost: float, + real_cache_hit: bool, + control_tier: str | None, + shadow_params: Mapping[str, object], + parent_metadata: Mapping[str, object], + ) -> None: + """One arm's pipeline: shadow call -> blind judge -> one attempt row, every exit + recording this arm's outcome, so one arm's fault never silences a sibling arm.""" try: - if prisma is None: - self._record_funnel(job.id, "withheld") - return - if await _key_or_team_is_over_budget(parent_metadata): - self._record_funnel(job.id, "withheld") - return - if job.max_budget is not None: - try: - spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget) - except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it - verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e) - self._record_funnel(job.id, "withheld") - return - if spend >= job.max_budget: - self._record_funnel(job.id, "withheld") - return - shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata) + shadow: Final = await self._call_router_shadow( + job.arm_target(arm_router), messages, shadow_params, parent_metadata + ) except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e) await self._record_attempt( @@ -845,6 +911,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=f"pipeline error: {e}", real_cost=real_cost, @@ -858,6 +925,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=shadow.error, shadow_cost=shadow.cost, @@ -882,6 +950,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=verdict.error, shadow=shadow, @@ -898,6 +967,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome=verdict.preference, shadow=shadow, real_model=real_model, @@ -916,6 +986,7 @@ class ShadowEvalLogger(CustomLogger): job, request_id, control_tier, + router_name=arm_router, outcome="error", error=f"pipeline error: {e}", shadow=shadow, @@ -933,6 +1004,7 @@ class ShadowEvalLogger(CustomLogger): request_id: str, control_tier: str | None, *, + router_name: str, outcome: str, real_cost: float, real_classifier_cost: float, @@ -955,6 +1027,7 @@ class ShadowEvalLogger(CustomLogger): data={ # mutable-ok: Prisma payload "job_id": job.id, "request_id": request_id, + "router_name": router_name, "outcome": outcome, "tier": control_tier if job.direction == "reverse" else (shadow.tier if shadow else None), "real_model": real_model or None, diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 81310b9ddc3..aefd4fa3b47 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -416,15 +416,25 @@ class WebSearchInterceptionLogger(CustomLogger): if not tools: return None - if call_type in (CallTypes.responses, CallTypes.aresponses): - return self._convert_responses_tools(kwargs=kwargs, tools=tools) - - # Check if any tool is a web search tool (native or already LiteLLM standard) - has_websearch: Final = any(is_web_search_tool(t) for t in tools) - + is_responses_call: Final = call_type in (CallTypes.responses, CallTypes.aresponses) + has_websearch: Final = ( + any(is_web_search_tool_responses(tool) for tool in tools) + if is_responses_call + else any(is_web_search_tool(tool) for tool in tools) + ) if not has_websearch: return None + if self.search_tool_name: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + llm_router = None + self._select_search_tool_from_router(llm_router=llm_router) + + if is_responses_call: + return self._convert_responses_tools(kwargs=kwargs, tools=tools) + verbose_logger.debug("WebSearchInterception: Converting native web_search tools to LiteLLM standard") # If the client sent an Anthropic-native web_search_* tool, mark the @@ -1631,9 +1641,7 @@ class WebSearchInterceptionLogger(CustomLogger): return None def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None": - if llm_router is None or not hasattr(llm_router, "search_tools"): - return None - search_tools: Final = list(getattr(llm_router, "search_tools") or []) + search_tools: Final = list(getattr(llm_router, "search_tools", []) or []) return self._select_search_tool_from_list(search_tools=search_tools, source="router") def _select_search_tool_from_list( @@ -1643,20 +1651,26 @@ class WebSearchInterceptionLogger(CustomLogger): ) -> "_SearchToolConfig | None": if self.search_tool_name: matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name] - if matching_tools: - search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider") - verbose_logger.debug( - "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", - self.search_tool_name, - source, - search_provider, + if not matching_tools: + raise ValueError(f"Configured search tool '{self.search_tool_name}' was not found") + + selected_tool: Final = matching_tools[0] + litellm_params: Final = selected_tool.get("litellm_params") + selected_search_provider: Final = ( + litellm_params.get("search_provider") if isinstance(litellm_params, Mapping) else None + ) + if not isinstance(selected_search_provider, str) or not selected_search_provider.strip(): + raise ValueError( + f"Configured search tool '{self.search_tool_name}' does not define a valid search provider" ) - return matching_tools[0] + verbose_logger.debug( - "WebSearchInterception: Search tool '%s' not found in %s, falling back to first available or perplexity", + "WebSearchInterception: Found search tool '%s' from %s with provider '%s'", self.search_tool_name, source, + selected_search_provider, ) + return selected_tool if search_tools: first_tool: Final = search_tools[0] diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 3b3775a8fe6..dab3e48f91a 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -7,7 +7,13 @@ import os from dataclasses import dataclass from typing import Final -from litellm.types.files import get_file_mime_type_from_extension +from litellm.types.files import ( + AUDIO_FILE_TYPES, + FILE_EXTENSIONS, + FILE_MIME_TYPES, + FileType, + get_file_mime_type_from_extension, +) from litellm.types.utils import FileTypes @@ -323,3 +329,75 @@ def calculate_request_duration(file: FileTypes) -> float | None: except Exception: # Silently fail if duration extraction fails return None + + +DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg" + + +def _speech_media_type_for_response_format(response_format: str) -> str | None: + file_type: Final = next( + (candidate for candidate, extensions in FILE_EXTENSIONS.items() if response_format.lower() in extensions), + None, + ) + if file_type is None or file_type not in AUDIO_FILE_TYPES: + return None + return FILE_MIME_TYPES[file_type] + + +def resolve_speech_media_type(upstream_content_type: str | None, response_format: str | None) -> str: + upstream_media_type: Final = (upstream_content_type or "").split(";", 1)[0].strip().lower() + if upstream_media_type.startswith("audio/"): + return upstream_media_type + requested_media_type: Final = ( + None if response_format is None else _speech_media_type_for_response_format(response_format) + ) + return requested_media_type or DEFAULT_SPEECH_MEDIA_TYPE + + +_OGG_OPUS_HEAD_WINDOW: Final = 64 +_ADTS_SYNC_AND_LAYER_MASK: Final = 0xF6 +_ADTS_SYNC_AND_LAYER: Final = 0xF0 +_ADTS_SAMPLE_RATE_INDEX_LIMIT: Final = 13 +_MPEG_SYNC_MASK: Final = 0xE0 +_MPEG_LAYER_MASK: Final = 0x06 +_MPEG_RESERVED_VERSION: Final = 0x01 +_MPEG_INVALID_BITRATE_INDEX: Final = 0x0F +_MPEG_RESERVED_SAMPLE_RATE_INDEX: Final = 0x03 + + +def _adts_aac_frame_media_type(header: bytes) -> str | None: + sample_rate_index: Final = (header[2] >> 2) & 0x0F + return FILE_MIME_TYPES[FileType.AAC] if sample_rate_index < _ADTS_SAMPLE_RATE_INDEX_LIMIT else None + + +def _mpeg_audio_frame_media_type(header: bytes) -> str | None: + version: Final = (header[1] >> 3) & 0x03 + layer: Final = header[1] & _MPEG_LAYER_MASK + bitrate_index: Final = header[2] >> 4 + sample_rate_index: Final = (header[2] >> 2) & 0x03 + if ( + (header[1] & _MPEG_SYNC_MASK) != _MPEG_SYNC_MASK + or version == _MPEG_RESERVED_VERSION + or layer == 0 + or bitrate_index == _MPEG_INVALID_BITRATE_INDEX + or sample_rate_index == _MPEG_RESERVED_SAMPLE_RATE_INDEX + ): + return None + return FILE_MIME_TYPES[FileType.MP3] + + +def speech_media_type_from_audio_bytes(audio: bytes) -> str | None: + if audio[:4] == b"RIFF" and audio[8:12] == b"WAVE": + return FILE_MIME_TYPES[FileType.WAV] + if audio[:4] == b"fLaC": + return FILE_MIME_TYPES[FileType.FLAC] + if audio[:4] == b"OggS": + is_opus: Final = b"OpusHead" in audio[:_OGG_OPUS_HEAD_WINDOW] + return FILE_MIME_TYPES[FileType.OPUS if is_opus else FileType.OGG] + if audio[:3] == b"ID3": + return FILE_MIME_TYPES[FileType.MP3] + if len(audio) < 3 or audio[0] != 0xFF: + return None + if (audio[1] & _ADTS_SYNC_AND_LAYER_MASK) == _ADTS_SYNC_AND_LAYER: + return _adts_aac_frame_media_type(audio) + return _mpeg_audio_frame_media_type(audio) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 1c8f10d3307..bda365102af 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -205,6 +205,41 @@ def is_non_content_values_set(message: AllMessageValues) -> bool: return any(message.get(key, None) is not None for key in message if key not in ignore_keys) +_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"}) +_IMAGE_SCAN_MAX_DEPTH: Final = 4 + + +def _content_parts_contain_image(parts: Sequence[object]) -> bool: + """Depth-bounded frontier walk over nested content lists, iterative because the repo bans + recursion; an Anthropic tool_result nests its image parts exactly one level down.""" + frontier = parts # rebind-ok: depth-bounded frontier walk + for _ in range(_IMAGE_SCAN_MAX_DEPTH): + if any(isinstance(part, Mapping) and part.get("type") in _IMAGE_CONTENT_PART_TYPES for part in frontier): + return True + frontier = tuple( # rebind-ok: depth-bounded frontier walk + nested + for part in frontier + if isinstance(part, Mapping) + for content in (part.get("content"),) + if isinstance(content, list) + for nested in content + ) + if not frontier: + return False + return False + + +def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool: + """Whether any message carries an image content part, across the dialects that reach + pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``, + and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks.""" + return any( + isinstance(content, list) and _content_parts_contain_image(content) + for message in messages + for content in (message.get("content"),) + ) + + def _audio_or_image_in_message_content(message: AllMessageValues) -> bool: """ Checks if message content contains an image or audio diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index cf14ab88751..332f892ae6b 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -7,10 +7,14 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s import base64 from collections.abc import Coroutine +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union import httpx +from litellm.litellm_core_utils.audio_utils.utils import ( + speech_media_type_from_audio_bytes, +) from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -457,12 +461,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): if not response_content: raise ValueError("No audioContent in Vertex AI TTS response") - # Decode base64 to get binary content binary_data: Final = base64.b64decode(response_content) - - # Create an httpx.Response object with the binary data + media_type: Final = speech_media_type_from_audio_bytes(binary_data) response: Final = httpx.Response( status_code=200, + headers=None if media_type is None else MappingProxyType({"content-type": media_type}), content=binary_data, ) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index a42187b3a44..64878a480a7 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -2,13 +2,14 @@ Handles Authentication Errors """ +import logging from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from fastapi import HTTPException, Request, status import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger from litellm.constants import EMPTY_MAPPING from litellm.integrations.otel.runtime import seed_request_identity from litellm.litellm_core_utils.core_helpers import is_expected_client_error @@ -18,7 +19,11 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_utils import _get_request_ip_address +from litellm.proxy.auth.auth_utils import ( + _get_request_ip_address, + is_invalid_virtual_key_error, + mark_invalid_virtual_key_error, +) from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -36,6 +41,41 @@ else: Span = Any +def _as_proxy_exception(e: Exception) -> ProxyException: + """Convert an authentication failure into the ProxyException the client receives.""" + if isinstance(e, litellm.BudgetExceededError): + return ProxyException( + message=e.message, + type=ProxyErrorTypes.budget_exceeded, + param=None, + code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + ) + if isinstance(e, HTTPException): + return ProxyException( + message=getattr(e, "detail", f"Authentication Error({e})"), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), + ) + if isinstance(e, ProxyException): + return e + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + return ProxyException( + message=( + "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly." + ), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + return ProxyException( + message="Authentication Error, " + str(e), + type=ProxyErrorTypes.auth_error, + param=getattr(e, "param", "None"), + code=status.HTTP_401_UNAUTHORIZED, + ) + + def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]: """Auth gate rejections are raised before `add_litellm_data_to_request` records the caller IP, so their failure logs would otherwise carry no IP nor key/user identity.""" @@ -110,16 +150,21 @@ class UserAPIKeyAuthExceptionHandler: request=request, use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True, ) - log_fn: Final = ( - verbose_proxy_logger.error - if is_expected_client_error(e) and not litellm.log_client_error_tracebacks - else verbose_proxy_logger.exception - ) - log_fn( + + # Log authentication failures before identity seeding and callbacks, so the log + # survives a raising callback pipeline. Classify and route malformed virtual-key + # rejections to WARNING on stdout (suppressible via LITELLM_LOG=ERROR). + log_extra: Final = {"requester_ip": requester_ip} + is_invalid_virtual_key: Final = is_invalid_virtual_key_error(e) + is_quiet_log: Final = is_invalid_virtual_key and not litellm.log_client_error_tracebacks + logger: Final = verbose_proxy_stdout_logger if is_quiet_log else verbose_proxy_logger + logger.log( + logging.WARNING if is_quiet_log else logging.ERROR, "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", e, requester_ip, - extra={"requester_ip": requester_ip}, + exc_info=True if litellm.log_client_error_tracebacks or not is_expected_client_error(e) else None, + extra=log_extra, ) # Log this exception to OTEL, Datadog etc. Reuse the identity resolved @@ -167,35 +212,13 @@ class UserAPIKeyAuthExceptionHandler: if transformed_exception is not None: e = transformed_exception - if isinstance(e, litellm.BudgetExceededError): - raise ProxyException( - message=e.message, - type=ProxyErrorTypes.budget_exceeded, - param=None, - code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS), + final_exception: Final = mark_invalid_virtual_key_error(_as_proxy_exception(e), is_invalid_virtual_key) + # If a quiet-logged malformed-key transform yields non-401, escalate to ERROR + if is_quiet_log and str(final_exception.code) != str(status.HTTP_401_UNAUTHORIZED): + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s", + final_exception, + requester_ip, + extra=log_extra, ) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "detail", f"Authentication Error({e})"), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), - ) - elif isinstance(e, ProxyException): - raise e - if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - raise ProxyException( - message=( - "Service Unavailable, the authentication database is " - "temporarily unreachable. Please retry shortly." - ), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) - raise ProxyException( - message="Authentication Error, " + str(e), - type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), - code=status.HTTP_401_UNAUTHORIZED, - ) + raise final_exception diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9b1a6ba5aa7..83ee10b8108 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY, EMPTY_MAPPING, + INVALID_VIRTUAL_KEY_ERROR_MARKER, MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS, ) @@ -34,6 +35,43 @@ from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS from litellm.types.utils import CustomPricingLiteLLMParams +def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: + """True when an authentication error rejects a malformed virtual key. + + Classifies only by the marker stamped where that 401 is raised. Message + content is never inspected: other 401s interpolate caller-supplied values + (vector store ids, organization ids) into their messages, so a phrase + match would let a request body demote an authorization failure to the + quiet log path. + """ + if not isinstance(exception, (HTTPException, ProxyException)): + return False + + code: Final[object] = getattr(exception, "code", None) + status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None) + if str(status_code) != str(status.HTTP_401_UNAUTHORIZED): + return False + + return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True + + +def mark_invalid_virtual_key_error(exception: ProxyException, is_invalid_virtual_key: bool) -> ProxyException: + """Return an independently marked malformed-key exception after callback transformations.""" + if not is_invalid_virtual_key or str(exception.code) != str(status.HTTP_401_UNAUTHORIZED): + return exception + marked_exception: Final = ProxyException( + message=exception.message, + type=exception.type, + param=exception.param, + code=exception.code, + headers=exception.headers.copy(), + openai_code=None if exception.openai_code is None else str(exception.openai_code), + provider_specific_fields=exception.provider_specific_fields, + ) + setattr(marked_exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return marked_exception + + def _get_request_ip_address(request: Request, use_x_forwarded_for: bool | None = False) -> str | None: client_ip = None if use_x_forwarded_for is True and "x-forwarded-for" in request.headers: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e92d090a2fb..5fb6dad0cd7 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -19,12 +19,15 @@ import fastapi import orjson from fastapi import HTTPException, Request, WebSocket, status from fastapi.security.api_key import APIKeyHeader +from starlette.exceptions import WebSocketException import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( GLOBAL_PROXY_SPEND_CACHE_KEY, + INVALID_VIRTUAL_KEY_ERROR_MARKER, + INVALID_VIRTUAL_KEY_ERROR_MESSAGE, LITELLM_PROXY_BUDGET_NAME, LITELLM_PROXY_MASTER_KEY_ALIAS, ) @@ -65,6 +68,7 @@ from litellm.proxy.auth.auth_utils import ( get_model_from_request, get_request_route, get_request_route_template, + is_invalid_virtual_key_error, iter_request_fallback_targets, normalize_request_route, pre_db_read_auth_checks, @@ -539,6 +543,8 @@ async def user_api_key_auth_websocket(websocket: WebSocket): try: return await user_api_key_auth(request=request, api_key=f"Bearer {api_key}") except Exception as e: + if is_invalid_virtual_key_error(e): + raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) verbose_proxy_logger.exception(e) await websocket.close(code=status.WS_1008_POLICY_VIOLATION) raise HTTPException(status_code=403, detail=str(e)) @@ -1867,13 +1873,17 @@ async def _user_api_key_auth_builder( _masked_key: Final = f"{api_key[:4]}****{api_key[-4:]}" if len(api_key) > 8 else "****" if not api_key.startswith("sk-"): _hint = _JWT_AUTH_DISABLED_HINT if not enable_jwt_auth and JWTHandler.is_jwt(token=api_key) else "" - raise HTTPException( + _malformed_key_error = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail=( - f"LiteLLM Virtual Key expected. Received={_masked_key}, " + f"{INVALID_VIRTUAL_KEY_ERROR_MESSAGE}. Received={_masked_key}, " f"expected to start with 'sk-'.{_hint}" ), ) # prevent token hashes from being used + # Stamp provenance here so log routing classifies this 401 by + # where it was raised, never by its message text. + setattr(_malformed_key_error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + raise _malformed_key_error else: verbose_logger.warning( "litellm.proxy.proxy_server.user_api_key_auth(): Warning - Key is not a string. Got type={}".format( diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 44b0cdcca2e..21e652114bc 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -833,7 +833,7 @@ def _judge_collisions_for_team( return tuple( (role, model) for role, model in ( - *_router_arm_models(llm_router, data.router_name), + *(arm for name in data.router_names for arm in _router_arm_models(llm_router, name)), *((("baseline", data.baseline_model),) if data.baseline_model is not None else ()), ) if judge & judge_target(llm_router, model, team_id).models @@ -904,7 +904,7 @@ class _AttemptAggRow(BaseModel): _ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow]) -_ATTEMPT_AGG_SELECT: Final = """ +_ATTEMPT_AGG_COLUMNS: Final = """ COUNT(*)::int AS turn_count, COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins, COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins, @@ -913,15 +913,34 @@ _ATTEMPT_AGG_SELECT: Final = """ COALESCE(SUM(real_cost + real_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS real_spend, COALESCE(SUM(shadow_cost + shadow_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS shadow_spend, COUNT(*) FILTER (WHERE real_cache_hit)::int AS cache_hit_turns +""" + +_ATTEMPT_AGG_SELECT: Final = ( + _ATTEMPT_AGG_COLUMNS + + """ FROM "LiteLLM_ShadowEvalAttempt" WHERE job_id = ANY($1::text[]) AND outcome != 'error' GROUP BY 1 """ +) _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT +# Attempt rows from before arm stamping carry no router_name; they belong to the job's +# own router, which the join reads off the leg. +_ATTEMPT_AGG_BY_ROUTER_SQL: Final = ( + "SELECT COALESCE(a.router_name, j.router_name) AS grp," + + _ATTEMPT_AGG_COLUMNS + + """ +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND a.outcome != 'error' +GROUP BY 1 +""" +) + # These guards derive spend from attempt rows, the cross-pod authority; the sampler also # reads the live counter, so admission can stop before a row-based guard would fire (safe # direction, and mid-deploy rows from old pods price as judge-only until the deploy ends). @@ -1060,6 +1079,7 @@ class _LegRow(BaseModel): target_type: ShadowEvalTargetType target_id: str router_name: str + router_names: tuple[str, ...] = () direction: ShadowEvalDirection baseline_model: str | None = None judge_model: str @@ -1071,6 +1091,12 @@ class _LegRow(BaseModel): stopped_at: datetime | None = None stopped_by: str | None = None + @property + def arm_router_names(self) -> tuple[str, ...]: + """The job's full router set; rows from before router_names existed hold it in + router_name alone. The one place that reading lives on the endpoint side.""" + return self.router_names or (self.router_name,) + @field_validator("created_at", "ends_at", "stopped_at") @classmethod def _as_aware_utc(cls, value: datetime | None) -> datetime | None: @@ -1123,7 +1149,7 @@ def _group_response( ) for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id)) ), - router_name=first.router_name, + router_names=first.arm_router_names, direction=first.direction, baseline_model=first.baseline_model, judge_model=first.judge_model, @@ -1252,6 +1278,9 @@ async def _shadow_eval_results( for slice in _slices(by_leg) } ) + by_router: Final = _ATTEMPT_AGG_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_ROUTER_SQL, leg_ids) or () + ) total_turns: Final = sum(r.turn_count for r in by_tier) funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids) counted: Final = _FunnelTotalsRow.model_validate(funnel_rows[0]) if funnel_rows else None @@ -1261,6 +1290,7 @@ async def _shadow_eval_results( result: Final = ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), + by_router=_slices(by_router), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), sampled_real_spend=sum(r.real_spend for r in by_tier), @@ -1314,8 +1344,15 @@ async def start_shadow_eval( _require_admin_writer(user_api_key_dict, "start a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name): - raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router") + unconfigured: Final = tuple( + name + for name in data.router_names + if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, name) + ) + if unconfigured: + raise HTTPException( + status_code=400, detail=f"Not a configured auto-router: {', '.join(repr(n) for n in unconfigured)}" + ) token_rows: Final = ( await _verification_tokens(prisma_client).find_many( where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter @@ -1416,7 +1453,9 @@ async def start_shadow_eval( ends_at: Final = now + timedelta(days=data.duration_days) shared_config: Final = { # mutable-ok: Prisma payload "group_id": group_id, - "router_name": data.router_name, + # a pre-router_names pod samples router_name alone, so it must be a real arm + "router_name": data.router_names[0], + "router_names": list(data.router_names), # mutable-ok: Prisma payload "direction": data.direction, "baseline_model": data.baseline_model, "judge_model": data.judge_model, @@ -1477,7 +1516,7 @@ async def start_shadow_eval( ) for target_type, target_id in sorted(requested_targets) ), - router_name=data.router_name, + router_names=data.router_names, direction=data.direction, baseline_model=data.baseline_model, judge_model=data.judge_model, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 26b65dd48cf..99e201930c1 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -20,6 +20,7 @@ import secrets import traceback from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeVar, cast import fastapi @@ -111,6 +112,7 @@ from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper +from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start from litellm.proxy.spend_tracking.spend_tracking_utils import _is_master_key from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( get_ui_settings_cached, @@ -3578,6 +3580,63 @@ async def _build_model_max_budget_usage( ) +def _window_max_budget(window: Mapping[str, object]) -> float | None: + """A window's max_budget as a float; None when absent or unparseable.""" + value: Final = window.get("max_budget") + if not isinstance(value, (int, float, str)): + return None + try: + return float(value) + except ValueError: + return None + + +async def _budget_window_usage( + window: Mapping[str, object], api_key_hash: str +) -> tuple[str, Mapping[str, object]] | None: + """ + (budget_duration, usage entry) for one budget window; None when the window + has no budget_duration to key it by. + + Reads the same cross-pod counter (spend:key:{hashed_token}:window:{budget_duration}) + that _virtual_key_multi_budget_check enforces against, passing the same + window_duration + window_start so a stale-low counter is re-checked against + the LiteLLM_BudgetWindowSpend row instead of a spend-log aggregate. + """ + from litellm.proxy.proxy_server import get_current_spend + + duration: Final = window.get("budget_duration") + if not isinstance(duration, str) or not duration: + return None + spend: Final = await get_current_spend( + counter_key=f"spend:key:{api_key_hash}:window:{duration}", + fallback_spend=0.0, + max_budget=_window_max_budget(window), + window_entity_type="Key", + window_entity_id=api_key_hash, + window_duration=duration, + window_start=get_budget_window_start(window), + ) + return duration, MappingProxyType({"current_spend": round(spend, 4)}) + + +async def _build_budget_limits_usage( + budget_limits: Sequence[object] | str | None, api_key_hash: str +) -> Mapping[str, Mapping[str, object]] | None: + """ + Current-window spend per budget window, keyed by budget_duration, reported + next to the stored budget_limits (which is returned untouched). None when + the key has no windows, so the field only appears on keys that have them. + """ + windows: Final = _budget_limit_windows(budget_limits) + if not windows: + return None + usages: Final = await asyncio.gather( + *(_budget_window_usage(window=window, api_key_hash=api_key_hash) for window in windows) + ) + return MappingProxyType({duration: usage for duration, usage in (u for u in usages if u is not None)}) + + @router.post( "/v2/key/info", tags=["key management"], @@ -3620,7 +3679,6 @@ async def info_key_fn_v2( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={"message": "Malformed request. No keys passed in."}, ) - # Resolve key_aliases to tokens so we never pass token=None (unbounded query) tokens_to_query: Final = list(data.keys) if data.keys else [] if data.key_aliases: @@ -3662,6 +3720,13 @@ async def info_key_fn_v2( model_max_budget=model_max_budget, user_api_key_cache=model_max_budget_limiter.dual_cache, ) + if k_token_hash: + budget_limits_usage = await _build_budget_limits_usage( + budget_limits=k_dict.get("budget_limits"), + api_key_hash=k_token_hash, + ) + if budget_limits_usage is not None: + k_dict["budget_limits_usage"] = budget_limits_usage filtered_key_info.append(k_dict) return {"key": data.keys, "info": filtered_key_info} @@ -3698,6 +3763,10 @@ async def info_key_fn( - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} - model_max_budget_usage: dict | None - Current-window spend per model, present only when the key has per-model budgets + - budget_limits: list | None - Concurrent budget windows, exactly as stored + - budget_limits_usage: dict | None - Current-window spend per budget window, e.g. + {"1h": {"current_spend": 0.0009}}, present only when the key has budget windows + (read from the same cross-pod spend counter the budget enforcement uses) - models: list - Model_name's the key is allowed to call - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} @@ -3777,6 +3846,12 @@ async def info_key_fn( model_max_budget=model_max_budget, user_api_key_cache=model_max_budget_limiter.dual_cache, ) + budget_limits_usage: Final = await _build_budget_limits_usage( + budget_limits=key_info.get("budget_limits"), + api_key_hash=key_token_hash, + ) + if budget_limits_usage is not None: + key_info["budget_limits_usage"] = budget_limits_usage # Attach object_permission if object_permission_id is set key_info = await attach_object_permission_to_dict(key_info, prisma_client) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 09d3dedaafa..ddd27f0fc2a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -6,9 +6,10 @@ import posixpath import traceback from base64 import b64encode from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from itertools import groupby -from typing import Any, Final, TypedDict, cast +from typing import TYPE_CHECKING, Any, Final, TypedDict, cast from urllib.parse import urlencode, urlparse import httpx @@ -47,6 +48,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, ) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_no_callback_env_reference from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -78,7 +80,10 @@ from litellm.proxy.common_utils.http_parsing_utils import ( from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above +) from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -90,7 +95,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import Usage +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging @@ -99,6 +104,9 @@ from .upstream_usage_headers import ( apply_upstream_reported_usage, ) +if TYPE_CHECKING: + from litellm.proxy.proxy_server import ProxyConfig + router: Final = APIRouter() pass_through_endpoint_logging: Final = PassThroughEndpointLogging() @@ -752,6 +760,67 @@ def _build_passthrough_failure_request_payload( return request_payload +@dataclass(frozen=True, slots=True) +class _TeamCallbackWiring: + success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg + + +def _resolve_team_callback_wiring( + user_api_key_dict: UserAPIKeyAuth, + proxy_config: "ProxyConfig", + route_description: str, +) -> _TeamCallbackWiring: + """Resolve key/team dynamic logging callbacks for a passthrough request. + + Mirrors add_litellm_data_to_request: callback_vars are unpacked top-level + (read by initialize_standard_callback_dynamic_params) and also stamped on + the proxy-owned trusted-vars field (read by get_trusted_callback_params). + + Fails open: a callback resolution or validation error is logged at error + level and the request proceeds without dynamic callbacks, since a broken + logging config must not fail the customer's upstream call (and the + websocket is already accepted by the time this runs on that path). The + env-reference check runs here because the deprecated callback_settings + branch skips AddTeamCallback validation, and Logging.__init__ would + otherwise reject the vars mid-request. + """ + try: + callback_settings_obj: Final = _get_dynamic_logging_metadata( + user_api_key_dict=user_api_key_dict, proxy_config=proxy_config + ) + if callback_settings_obj and callback_settings_obj.callback_vars: + for ( + item + ) in callback_settings_obj.callback_vars.items(): # rebind-ok: dict.items iteration for env-ref validation + validate_no_callback_env_reference(item[0], item[1], source="key/team callback metadata") + except Exception: # noqa: BLE001 - a broken logging config must never fail the passthrough request + verbose_proxy_logger.exception( + "%s: failed to resolve team logging callbacks, continuing without them", + route_description, + ) + return _TeamCallbackWiring() + if callback_settings_obj is None: + return _TeamCallbackWiring() + callback_vars: Final = callback_settings_obj.callback_vars + success_callbacks: Final = callback_settings_obj.success_callback + failure_callbacks: Final = callback_settings_obj.failure_callback + logging_kwargs: Final = ( + None + if not callback_vars + else { # mutable-ok: Logging arg + **callback_vars, + TRUSTED_CALLBACK_VARS_FIELD: callback_vars, + } + ) + return _TeamCallbackWiring( + success_callbacks=None if success_callbacks is None else [*success_callbacks], # mutable-ok: Logging arg + failure_callbacks=None if failure_callbacks is None else [*failure_callbacks], # mutable-ok: Logging arg + logging_kwargs=logging_kwargs, + ) + + async def _log_passthrough_upstream_failure( response: httpx.Response, user_api_key_dict: UserAPIKeyAuth, @@ -845,7 +914,7 @@ async def pass_through_request( from litellm.proxy.pass_through_endpoints.passthrough_guardrails import ( PassthroughGuardrailHandler, ) - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj ######################################################### # Initialize variables @@ -930,6 +999,11 @@ async def pass_through_request( # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. passthrough_model: Final = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" start_time: Final = datetime.now() + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="pass_through_endpoint", + ) logging_obj = Logging( model=passthrough_model, messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], @@ -938,6 +1012,9 @@ async def pass_through_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="1245", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Store passthrough guardrails config on logging_obj for field targeting @@ -2022,7 +2099,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream """ from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj from litellm.types.passthrough_endpoints.pass_through_endpoints import ( PassthroughStandardLoggingPayload, ) @@ -2055,6 +2132,11 @@ async def websocket_passthrough_request( upstream_headers[header_name] = header_value # Initialize logging object similar to HTTP passthrough + team_callbacks: Final = _resolve_team_callback_wiring( + user_api_key_dict=user_api_key_dict, + proxy_config=proxy_config, + route_description="websocket_passthrough", + ) logging_obj: Final = Logging( model="unknown", messages=[{"role": "user", "content": "WebSocket connection"}], @@ -2063,6 +2145,9 @@ async def websocket_passthrough_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="websocket_passthrough", + dynamic_success_callbacks=team_callbacks.success_callbacks, + dynamic_failure_callbacks=team_callbacks.failure_callbacks, + kwargs=team_callbacks.logging_kwargs, ) # Create passthrough logging payload diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index acdd0f99295..70bf8fd0554 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -263,6 +263,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, @@ -11061,15 +11062,14 @@ async def audio_speech( if callback_headers: custom_headers.update(callback_headers) - # Determine media type based on model type - media_type = "audio/mpeg" # Default for OpenAI TTS - request_model: Final = data.get("model", "") - if request_model: - request_model_lower: Final = request_model.lower() - if "gemini" in request_model_lower and ( - "tts" in request_model_lower or "preview-tts" in request_model_lower - ): - media_type = "audio/wav" # Gemini TTS returns WAV format after conversion + requested_format: Final = data.get("response_format") + upstream_content_type: Final = ( + response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None + ) + media_type: Final = resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=requested_format if isinstance(requested_format, str) else None, + ) return StreamingResponse( _audio_speech_chunk_generator(response), @@ -11085,7 +11085,15 @@ async def audio_speech( ) verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e) verbose_proxy_logger.debug(traceback.format_exc()) - raise e + if isinstance(e, (ProxyException, HTTPException)): + raise e + raise ProxyException( + message=getattr(e, "message", f"{e}"), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + openai_code=getattr(e, "code", None), + code=getattr(e, "status_code", 500), + ) @router.post( diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 63ba760ff66..bc8df67cc28 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -154,6 +154,9 @@ model_list: # Fallback model if tier cannot be determined default_model: gpt-4o + + # Replace a routed model that cannot take image input (default: false) + modality_routing: true ``` ## Usage @@ -178,6 +181,25 @@ response = litellm.completion( ## Special Behaviors +### Modality-based capability routing + +The classifier reads text alone, so a request carrying an image can classify cheap and land on a +text-only model, which rejects it with a provider 400 no fallback catches. With +`modality_routing: true`, one gate inspects every decided placement: when the routed model is +explicitly declared `supports_vision: false` (deployment `model_info` first, the model cost map +otherwise; unmapped names stay routable, and a multi-deployment group must accept on every +deployment), the request is re-placed on the nearest HIGHER tier holding a capable model, with +routing plugins still applied to the re-pick, then on `default_model` (never on plugin routers +and never for a plan-floored decision), and otherwise rejected with a clear 400 naming the +router. The walk only ever goes up, so a plan-mode floor cannot be undercut; a router whose only +vision model sits below the decided tier gets the 400 and an actionable message instead. + +A same-tier re-pick keeps the decision's cause and adds `modality:image` to `signals`; a tier +change or default takeover records `cause: modality_escalation` with the displaced placement +(`modality_escalated_from:` or `modality_displaced_default_model`). Escalations are never +pinned by session affinity, and a KEPT session pin bypasses the gate entirely: a session pinned +to a text-only model keeps it even when an image arrives. + ### Heuristic-first chaining `classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 329da35eab3..be7653902a7 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -30,6 +30,7 @@ from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata +from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.types.utils import ( @@ -738,6 +739,10 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo size shrinks again the moment the client compacts: pinning the escalated tier would hold the session on the big-window model long after the oversized context that forced it is gone. The gate re-fires per request, so leaving these unpinned costs nothing but the classifier call. + + A modality escalation is transient the same way: it describes what this one call carries (an + image), not what the session's traffic looks like, and pinning it would hold every following + text turn on the vision-capable model the image forced. """ return decision is None or ( decision.get("cause") @@ -745,6 +750,7 @@ def _decision_is_pinnable(decision: StandardLoggingRoutingDecision | None) -> bo "default_model_fallback", "plan_mode", "housekeeping", + "modality_escalation", ) and not decision.get("context_escalated") ) @@ -2274,6 +2280,175 @@ class ComplexityRouter(CustomLogger): return pinned_model return self.get_model_for_tier(escalated_tier) + def _model_accepts_image_input(self, model_name: str) -> bool: + """Whether a routed model or pool entry can serve an image request. + + Resolved through the deployments that would actually serve the name; a name with no + deployment on the router is served by the SDK directly and is checked against the model + cost map itself. Only an explicit supports_vision false excludes, a deployment-level + model_info override first and the map otherwise, so unmapped custom names stay routable. + + A multi-deployment group must accept on EVERY deployment: the router picks a deployment + inside the group after this gate runs, so a mixed group marked eligible could still hand + the image to its text-only member and fail with the exact 400 the gate exists to prevent. + """ + from litellm.utils import is_vision_explicitly_disabled + + def deployment_accepts(deployment: Mapping[str, Any]) -> bool: + declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision") + if declared is not None: + return declared is True + litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name + return not is_vision_explicitly_disabled(litellm_model) + + deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name) + if not deployments: + return not is_vision_explicitly_disabled(model_name) + return all(deployment_accepts(deployment) for deployment in deployments) + + def _modality_eligible_models(self) -> frozenset[str]: + """Every configured pool entry, plus default_model, that can serve an image request.""" + names: Final = frozenset(entry for pool in self._tier_pools().values() for entry in pool) | frozenset( + name for name in (self.config.default_model,) if name + ) + return frozenset(name for name in names if self._model_accepts_image_input(name)) + + async def _gate_response_modality( + self, + response: PreRoutingHookResponse, + messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + resolved_messages: Sequence[Mapping[str, object]] | None, + request_kwargs: dict, # mutable-ok: same shape the hook receives + ) -> PreRoutingHookResponse: + """Replace a routed model that cannot accept this request's image input. + + The single modality owner, applied to the decided response at the hook's exits so every + routing path is covered uniformly. A KEPT session pin is exempt by design (its cause); + replacement picks and every other path are just responses. The re-placement walks + UPWARD-ONLY from the decision's tier (so a plan-mode floor can never be undercut), picks + through `_pick_model_for_tier` so routing plugins still apply, then falls to + default_model (never on plugin routers, and never on a plan-floored decision, since + default_model carries no tier guarantee), else raises the clear 400. The rewritten + decision keeps its cause on a same-tier repick and becomes modality_escalation when the + tier moved or default_model took over, with the displaced placement in signals. + """ + decision: Final = response.routing_decision + if ( + not self.config.modality_routing + or not resolved_messages + or response.model is None + or (decision is not None and decision.get("cause") == "session_affinity_pin") + or not request_contains_image_content(resolved_messages) + or self._model_accepts_image_input(response.model) + ): + return response + eligible: Final = self._modality_eligible_models() + names: Final = self.config.tier_names() + pools: Final = self._tier_pools() + decided: Final = decision.get("tier") if decision is not None else None + start: Final = names.index(decided) if isinstance(decided, str) and decided in names else 0 + capable: Final = next( + (name for name in names[start:] if any(entry in eligible for entry in pools.get(name, ()))), None + ) + if capable is not None: + new_tier: ComplexityTier | str | None = capable if self.config.has_custom_tiers else ComplexityTier(capable) + repick_messages: Final = list(resolved_messages) # mutable-ok: the pick's param is list-typed + new_model = await self._pick_model_for_tier( + new_tier, + messages, + repick_messages, # pyright: ignore[reportArgumentType] # hook-resolved message dicts; the pick only reads them + request_kwargs, + allowed_models=tuple(entry for entry in pools.get(capable, ()) if entry in eligible), + ) + elif self._modality_default_model_usable(request_kwargs, resolved_messages, eligible): + new_tier = None + new_model = self._placed_default_model() + else: + import litellm + + raise litellm.BadRequestError( + message=( + f"Auto-router {self.model_name} received a request with image input, but no model " + f"at or above the decided tier accepts images and modality_routing is enabled. " + f"Tiers checked: {', '.join(names[start:])}. Add a vision-capable model to a tier, " + f"or set a vision-capable default_model, or remove the image content." + ), + model=self.model_name, + llm_provider="", + ) + self._restamp_adaptive_choice(request_kwargs, response.model, new_model) + same_tier: Final = capable is not None and decided == capable + base_cause: Final = (decision.get("cause") if decision is not None else None) or "default_fallback" + displaced_default: Final = decided is None and response.model == self.config.default_model + markers: Final = ( + "modality:image", + *((f"modality_escalated_from:{decided}",) if not same_tier and isinstance(decided, str) else ()), + *(("modality_displaced_default_model",) if not same_tier and displaced_default else ()), + ) + old_signals: Final = tuple(decision.get("signals") or ()) if decision is not None else () + new_decision: Final = self._build_routing_decision( + routed_model=new_model, + cause=base_cause if same_tier else "modality_escalation", + tier=new_tier, + score=decision.get("score") if decision is not None else None, + signals=(*old_signals, *markers), + matched_keyword=decision.get("matched_keyword") if decision is not None else None, + escalation_keyword=decision.get("escalation_keyword") if decision is not None else None, + escalated=bool(decision.get("escalated", False)) if decision is not None else False, + classifier_model=decision.get("classifier_model") if decision is not None else None, + classifier_cost=decision.get("classifier_cost") if decision is not None else None, + conversation_continuing=bool(decision.get("conversation_continuing", True)) + if decision is not None + else True, + tier_litellm_params=self._litellm_params_for_model(new_tier, new_model), + context_escalation_original_tier=( + decision.get("context_escalation_original_tier") if decision is not None else None + ), + ) + from litellm.types.router import PreRoutingHookResponse as HookResponse + + return HookResponse( + model=new_model, + messages=response.messages, + litellm_params=self._litellm_params_for_model(new_tier, new_model), + routing_decision=new_decision, + ) + + def _modality_default_model_usable( + self, + request_kwargs: Mapping[str, object], + resolved_messages: Sequence[Mapping[str, object]] | None, + eligible: frozenset[str], + ) -> bool: + """default_model may serve a gated request only when it is configured, plugin-free + (it is never checked against the plugin pipeline), capability-eligible, and the turn + carries no plan-mode sentinel. The sentinel is re-detected here rather than read off + the decision record, because the record only marks turns the floor RAISED; a sentinel + turn already at or above the floor keeps its ordinary cause, and default_model carries + no tier the floor could vouch for on any sentinel turn.""" + return ( + bool(self.config.default_model) + and not self.config.plugins + and self.config.default_model in eligible + and self._matched_plan_mode_signal(request_kwargs, resolved_messages) is None + ) + + def _placed_default_model(self) -> str: + """The default_model behind a usable-default verdict; the raise is the type-level + proof, not a reachable path.""" + model: Final = self.config.default_model + if model is None: + raise ValueError(f"Auto-router {self.model_name}: modality gate routed to an unset default_model") + return model + + @staticmethod + def _restamp_adaptive_choice(request_kwargs: Mapping[str, object], old_model: str, new_model: str) -> None: + """The adaptive feedback loop reads its chosen-model marker from request metadata; a + gate rewrite must move the marker with the model or rewards land on the displaced one.""" + metadata: Final = request_kwargs.get("metadata") + if isinstance(metadata, dict) and metadata.get("adaptive_router_chosen_model") == old_model: + metadata["adaptive_router_chosen_model"] = new_model + def _lexical_tier_override(self, user_message: str) -> KeywordOverride | None: """When keyword_tier_rules match literally, the most-severe matched tier wins. @@ -2655,25 +2830,30 @@ class ComplexityRouter(CustomLogger): session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model) has_original_messages: Final = messages is not None and len(messages) > 0 return self._with_session_deployment_affinity( - PreRoutingHookResponse( - model=routed_model, - messages=messages if has_original_messages else None, - litellm_params=session_tier_litellm_params, - routing_decision=self._build_routing_decision( - routed_model=routed_model, - cause=cause, - tier=routed_pin_tier, - matched_keyword=pin_plan_sentinel if plan_floored else None, - escalation_keyword=pin_escalation_keyword, - escalated=escalated, - conversation_continuing=conversation_continuing, - tier_litellm_params=session_tier_litellm_params, - context_escalation_original_tier=pin_context_original_tier, + await self._gate_response_modality( + PreRoutingHookResponse( + model=routed_model, + messages=messages if has_original_messages else None, + litellm_params=session_tier_litellm_params, + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause=cause, + tier=routed_pin_tier, + matched_keyword=pin_plan_sentinel if plan_floored else None, + escalation_keyword=pin_escalation_keyword, + escalated=escalated, + conversation_continuing=conversation_continuing, + tier_litellm_params=session_tier_litellm_params, + context_escalation_original_tier=pin_context_original_tier, + ), ), + messages, + resolved_messages, + request_kwargs, ) ) - response: Final = await self._classify_and_route( + routed_response: Final = await self._classify_and_route( model=model, request_kwargs=request_kwargs, messages=messages, @@ -2682,6 +2862,11 @@ class ComplexityRouter(CustomLogger): conversation_continuing=conversation_continuing, resolved_messages=resolved_messages, ) + response: Final = ( + await self._gate_response_modality(routed_response, messages, resolved_messages, request_kwargs) + if routed_response is not None + else None + ) # Sentinel presence, not the plan_mode cause, gates the pin write: a plan-mode turn # classified at or above the floor keeps its ordinary cause, yet on an adaptive router # the hard floor constrained its pick, so pinning it would carry a plan-mode-shaped diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 3c5e8aafa18..70aeecb31c6 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -848,6 +848,18 @@ class ComplexityRouterConfig(BaseModel): "drift plus the response tokens." ), ) + modality_routing: bool = Field( + default=False, + description=( + "Route image-bearing requests only to models that can accept image input. The " + "classifier reads text alone, so an image request whose text classifies cheap " + "otherwise lands on a text-only model and fails with a provider 400. When enabled, " + "a routed model explicitly declared supports_vision false (deployment model_info " + "or the model cost map; unmapped names stay routable) is replaced by the nearest " + "HIGHER tier holding a capable model, then default_model, else a clear 400. A kept " + "session-affinity pin still wins even when an image arrives." + ), + ) # Semantic (embedding) matching for keyword_tier_rules instead of literal text matching semantic_keyword_matching: bool = Field( diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index dfc45ccf9bf..88869a1edfb 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -251,8 +251,12 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" # Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that # fails before billing) never consumes spend budget, so it must terminate on count instead. +# A multi-router job writes one attempt row per router arm, so the valve is reached +# proportionally sooner; it is a safety valve, not a sample budget. SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 +SHADOW_EVAL_MAX_ROUTERS: Final[int] = 4 + class StartShadowEvalRequest(BaseModel): """Start duplicating one or more targets' traffic for blind comparison against an auto-router. @@ -288,7 +292,24 @@ class StartShadowEvalRequest(BaseModel): "to across all their teams: JWT requests carrying their subject claim and virtual keys they own" ), ) - router_name: str = Field(description="The auto-router under evaluation, in either direction") + router_name: str | None = Field( + default=None, + description=( + "The auto-router under evaluation, in either direction: the single-router spelling of " + "router_names. Provide exactly one of the two fields" + ), + ) + router_names: tuple[str, ...] = Field( + default=(), + max_length=SHADOW_EVAL_MAX_ROUTERS, + description=( + "The auto-routers under evaluation, at most " + f"{SHADOW_EVAL_MAX_ROUTERS}. Every sampled request runs through every router listed and each " + "arm is judged independently against the same real response, so routers compare head-to-head " + "on identical traffic. More than one router requires direction 'forward'. After validation " + "this field always carries the full deduplicated set, whichever spelling the caller used" + ), + ) direction: ShadowEvalDirection = Field( default="forward", description=( @@ -332,7 +353,8 @@ class StartShadowEvalRequest(BaseModel): "Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " "the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval " "spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight " - "samples can overshoot the cap by one sampling cache window" + "samples can overshoot the cap by one sampling cache window. Every router arm draws from the " + "same per-target budget, so a multi-router job reaches it proportionally sooner" ), ) @@ -373,6 +395,23 @@ class StartShadowEvalRequest(BaseModel): raise ValueError("baseline_model is only meaningful when direction is 'reverse'") return self + @model_validator(mode="after") + def _resolve_router_set(self) -> "StartShadowEvalRequest": + """Whichever spelling the caller used, router_names leaves validation as the full + deduplicated set, so every downstream reader consumes one field.""" + if (self.router_name is None) == (not self.router_names): + raise ValueError("provide exactly one of router_name or router_names") + single: Final = () if self.router_name is None else (self.router_name,) + routers: Final = tuple(dict.fromkeys(self.router_names or single)) + if not all(name.strip() for name in routers): + raise ValueError("router names must be non-empty strings") + if len(routers) > 1 and self.direction == "reverse": + raise ValueError("a reverse job evaluates one router against baseline_model; pass a single router") + # A returned model_copy is ignored on the __init__ construction path, so the + # normalization must land as a self attribute store to hold for every caller. + self.router_names = routers + return self + class ShadowEvalSlice(BaseModel): """Judge outcomes for one slice of a job's verdicts: a router tier, one of the @@ -428,15 +467,28 @@ class ShadowEvalResult(BaseModel): "and in reverse the models the router itself picked" ) ) + by_router: tuple[ShadowEvalSlice, ...] = Field( + default=(), + description=( + "One slice per router arm, grouped on the router name. Every arm of a multi-router job is " + "judged against the same real responses over the same sampled requests, so these slices " + "compare routers head-to-head: like-for-like win rates and spends on identical traffic. " + "Verdicts from before arm stamping existed count toward the job's own router" + ), + ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float sampled_real_spend: float = Field( default=0.0, - description="USD the real arm billed across all judged turns, cache-served turns excluded", + description=( + "USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn " + "is one (request, router arm) verdict, so a multi-router job counts the real response once per " + "arm it was judged against; per-router comparisons read by_router" + ), ) sampled_shadow_spend: float = Field( default=0.0, - description="USD the shadow arm billed across the same turns, judge excluded, like for like", + description="USD the shadow arms billed across the same turns, judge excluded, like for like", ) not_sampled_count: int | None = Field( default=None, @@ -540,7 +592,13 @@ class ShadowEvalJobResponse(BaseModel): min_length=1, description="The targets whose traffic this job evaluates, and only theirs, each with its own budget", ) - router_name: str + router_names: tuple[str, ...] = Field( + min_length=1, + description=( + "Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of " + "traffic and judge every arm against the same real responses" + ), + ) direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str @@ -562,6 +620,13 @@ class ShadowEvalJobResponse(BaseModel): last_error: str | None = Field(default=None, description="Most recent attempt error; detail endpoint only") results: ShadowEvalResult | None = Field(default=None, description="Stratified verdicts; detail endpoint only") + @computed_field + @property + def router_name(self) -> str: + """The first router, kept for callers that predate router_names; derived so the + two fields can never disagree.""" + return self.router_names[0] + @computed_field @property def status(self) -> ShadowEvalStatus: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a1b3523442b..55a32989b1c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2840,6 +2840,10 @@ RoutingDecisionCause = Literal[ # never called. The matched sentinel rides in matched_keyword. Distinct from the keyword causes, # which are operator-authored rules; these sentinels ship with the router. "housekeeping", + # modality_routing replaced the decided placement: the request carries an image and the + # routed model does not accept image input, so the nearest higher capable tier or + # default_model served instead. The displaced placement rides in signals. + "modality_escalation", "session_affinity_pin", "session_affinity_escalation", # classification_mode 'user_turn': the request is an agent loop's continuation turn (no new diff --git a/litellm/utils.py b/litellm/utils.py index d8b19a406e6..b6e8e378e81 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2660,10 +2660,19 @@ def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, ``_supports_factory`` so caching, fallback, and normalisation improvements apply here automatically. """ + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + model = model.removeprefix( + f"{declared}/" + ) # rebind-ok: mirrors get_llm_provider's split without its OAuth flow + custom_llm_provider = declared # rebind-ok: same + else: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) val: Final = model_info.get(key) if val is False: @@ -2751,6 +2760,15 @@ def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> ) +def is_vision_explicitly_disabled(model: str, custom_llm_provider: str | None = None) -> bool: + """True only when supports_vision is explicitly declared false for the model. + + The opt-out mirror of :func:`supports_vision`: a missing declaration reads as not + disabled, so unknown or newly added models stay eligible for image routing. + """ + return _is_explicitly_disabled_factory(model, custom_llm_provider, "supports_vision") + + def supports_vision(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports vision and return a boolean value. @@ -9415,6 +9433,10 @@ class ProviderConfigManager: return RunwayMLTextToSpeechConfig() elif litellm.LlmProviders.VERTEX_AI == provider: + if "gemini" in model: + # Gemini TTS uses the speech_to_completion bridge, and Google Cloud TTS param + # mapping would drop response_format before the bridge sees it (LIT-6501) + return None from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAITextToSpeechConfig, ) diff --git a/pyproject.toml b/pyproject.toml index 96dcf1121bc..f33bf68822a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -267,7 +267,7 @@ healthcheck = [ ] [build-system] -requires = ["maturin==1.9.4"] +requires = ["maturin==1.15.0"] build-backend = "maturin" [tool.maturin] @@ -275,6 +275,9 @@ manifest-path = "litellm-rust/crates/python-bridge/Cargo.toml" module-name = "litellm.rust_bridge._native" python-source = "." bindings = "pyo3" +features = ["extension-module"] +profile = "release" +editable-profile = "dev" include = ["litellm/proxy/_experimental/out/**"] exclude = [ "litellm/proxy/enterprise", diff --git a/schema.prisma b/schema.prisma index 01a607b68a9..7604ceadf7a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob { group_id String // legs of one job share this; the API's job id target_type String @default("key") // key | team | user target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows - router_name String // the auto-router under evaluation, in either direction + router_name String // first (often only) auto-router under evaluation; router_names is the full set + router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name) direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String @@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt { job_id String request_id String // the judged real request outcome String // real | shadow | tie | error + router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router tier String? // router's tier for the prompt, when classified real_model String? shadow_model String? diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 1e6de0c3d6a..d571fb36546 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -64,14 +64,12 @@ - {id: mgmt.budget.list_v1.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "management_v1/budgets.py:129", rationale: "Budget enumeration the Budgets page can page, sort and filter"} - {id: mgmt.budget.list_v1.admin_only, module: mgmt, tier: P1, surface: api, assertions: [admin_only], source: "management_v1/budgets.py:129", rationale: "A caller without admin view is refused, not served an empty page"} - {id: mgmt.callback.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "callback_management_endpoints.py", rationale: "Callback config (smoke)"} -- {id: mgmt.cache_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cache_settings_endpoints.py", rationale: "Cache config (smoke). Deliberately uncovered: the previous test read the live settings and wrote them back, which proves nothing (identical values in, so a no-op POST still passes) while being able to break the deployment. /cache/settings persists what it receives and that row outranks YAML cache_params, re-applied on a timer, so a write that omits ssl or redis_startup_nodes turns a TLS cluster into a plaintext standalone node and every later Redis call hangs. That took out 60 of 72 tests on 2026-07-25. GET cannot round-trip it either: it resolves the stored row overlaid with REDIS_* env and never reads YAML, so on a fresh deploy it cannot see YAML ssl to echo back. A safe test needs an isolated proxy, or LIT-4816 fixed so a partial write cannot downgrade transport. Do not re-add a read-then-write-back test against a shared proxy."} - {id: mgmt.cost_tracking.estimate.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cost_tracking_settings.py", rationale: "Cost estimate (smoke)"} - {id: mgmt.router_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "router_settings_endpoints.py", rationale: "Router config (smoke)"} - {id: mgmt.jwt_key_mapping.new.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "jwt_key_mapping_endpoints.py", rationale: "JWT->key mapping (smoke)"} - {id: mgmt.compliance.gdpr.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "compliance_endpoints.py", rationale: "GDPR ops (smoke)"} - {id: mgmt.tool_management.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "tool_management_endpoints.py", rationale: "Tool inventory (smoke)"} - {id: mgmt.fallback_management.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "fallback_management_endpoints.py", rationale: "Fallback config (smoke)"} -- {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"} - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"} - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} diff --git a/tests/e2e/management/test_config_misc_endpoints_e2e.py b/tests/e2e/management/test_config_misc_endpoints_e2e.py index 195732c0201..099ffa4b3bd 100644 --- a/tests/e2e/management/test_config_misc_endpoints_e2e.py +++ b/tests/e2e/management/test_config_misc_endpoints_e2e.py @@ -7,9 +7,13 @@ so a read-back reflects the change. Router settings, which mutate global proxy state, are exercised with a benign, self-restoring change so a shared proxy is left as it was found. -Cache settings are deliberately not covered here; see the rationale on -mgmt.cache_settings.update.happy_path in coverage_registry/mgmt.yaml before adding -a test for that route. +Cache settings and the Vault config override are deliberately not covered here. +Both routes reconfigure the whole proxy: /cache/settings persists what it receives +into a row that outranks the YAML cache_params and is re-applied on a timer, and +/config_overrides/hashicorp_vault swaps the process-wide secret manager. Neither can +be exercised safely against the shared proxy the suites run on, so they need an +isolated proxy before a test lands. Do not add a read-then-write-back test for +either one. """ from __future__ import annotations diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index c824d3e7a0e..54dbe5361d7 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,15 +1,14 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). - -Tests are pure unit tests — no Redis server required. -""" - from unittest.mock import AsyncMock, MagicMock, patch import pytest -import redis.asyncio as async_redis -from litellm._redis import get_redis_async_client, get_redis_connection_pool +from litellm._redis import ( + _coerce_redis_kwargs_types, + _get_redis_client_logic, + _get_redis_env_kwarg_mapping, + get_redis_async_client, + get_redis_connection_pool, +) def test_url_config_uses_passed_pool(): @@ -60,16 +59,14 @@ def test_max_connections_url_config_string_value(monkeypatch): assert pool.max_connections == 25 -def test_max_connections_url_config_invalid_value(): - """Invalid max_connections should be silently ignored, falling back - to the pool default (50 for BlockingConnectionPool).""" - with patch("litellm._redis._get_redis_client_logic") as mock_logic: - mock_logic.return_value = { - "url": "redis://localhost:6379/0", - "max_connections": "not_a_number", - } +def test_max_connections_url_config_invalid_value(monkeypatch): + """Invalid max_connections from an env var should be silently dropped, + falling back to the pool default (50 for BlockingConnectionPool).""" + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.delenv("REDIS_HOST", raising=False) + monkeypatch.setenv("REDIS_MAX_CONNECTIONS", "not_a_number") - pool = get_redis_connection_pool() + pool = get_redis_connection_pool() # BlockingConnectionPool default is 50 assert pool.max_connections == 50 @@ -128,3 +125,173 @@ async def test_disconnect_idempotent(): await cache.disconnect() await cache.disconnect() # should not raise + + +def test_coerce_redis_kwargs_types_int(): + """String values for int-typed Redis params are coerced to int.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "30", "port": "6380", "db": "1"}) + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + assert result["port"] == 6380 + assert result["db"] == 1 + + +def test_coerce_redis_kwargs_types_bool(): + """String values for bool-typed Redis params are coerced to bool.""" + result = _coerce_redis_kwargs_types({"ssl": "true", "decode_responses": "false"}) + assert result["ssl"] is True + assert result["decode_responses"] is False + + +def test_coerce_redis_kwargs_types_none_default_numeric(): + """String values for known None-default numeric params are coerced.""" + result = _coerce_redis_kwargs_types({"max_connections": "20", "socket_timeout": "5.5"}) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + assert result["socket_timeout"] == 5.5 + assert isinstance(result["socket_timeout"], float) + + +def _redis_signature_pre_8x( + socket_timeout=None, + socket_connect_timeout=None, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py <= 7.x Redis signature, where the timeout defaults are None.""" + + +def _redis_signature_8x( + socket_timeout=5, + socket_connect_timeout=5, + max_connections=None, + health_check_interval=0, +): + """Stand-in for the redis-py 8.x Redis signature, where the timeout defaults became int 5.""" + + +@pytest.mark.parametrize( + "client", + [_redis_signature_pre_8x, _redis_signature_8x], + ids=["redis-py<=7.x", "redis-py-8.x"], +) +def test_coerce_fractional_socket_timeout_survives_signature_default_change(client): + """redis-py 8.x changed socket_timeout's default from None to int 5. Deriving the + target type from the signature default made int("5.5") raise, so the key was dropped + and REDIS_SOCKET_TIMEOUT=5.5 silently disappeared on 8.x.""" + result = _coerce_redis_kwargs_types( + {"socket_timeout": "5.5", "socket_connect_timeout": "2.5", "max_connections": "20"}, + client=client, + ) + + assert result["socket_timeout"] == pytest.approx(5.5) + assert isinstance(result["socket_timeout"], float) + assert result["socket_connect_timeout"] == pytest.approx(2.5) + assert isinstance(result["socket_connect_timeout"], float) + assert result["max_connections"] == 20 + assert isinstance(result["max_connections"], int) + + +def test_coerce_invalid_socket_timeout_is_still_dropped(): + """Garbage must not survive the explicit-type path; Redis falls back to its own default.""" + result = _coerce_redis_kwargs_types({"socket_timeout": "not_a_number"}, client=_redis_signature_8x) + + assert "socket_timeout" not in result + + +def test_coerce_redis_kwargs_types_invalid_drops_key(): + """A string that cannot be coerced to the expected numeric type is dropped.""" + result = _coerce_redis_kwargs_types({"health_check_interval": "not_a_number"}) + assert "health_check_interval" not in result + + +def test_coerce_redis_kwargs_types_non_string_unchanged(): + """Non-string values pass through without modification.""" + result = _coerce_redis_kwargs_types({"health_check_interval": 30, "ssl": True}) + assert result["health_check_interval"] == 30 + assert result["ssl"] is True + + +def test_health_check_interval_from_env_is_int(monkeypatch): + monkeypatch.setenv("REDIS_HOST", "localhost") + monkeypatch.setenv("REDIS_HEALTH_CHECK_INTERVAL", "30") + + pool = get_redis_connection_pool() + + assert pool is not None + interval = pool.connection_kwargs.get("health_check_interval") + assert interval == 30 + assert isinstance(interval, int), f"Expected int, got {type(interval)}: {interval!r}" + + +def _signature_without_defaults(testkey): + """Stand-in for a client whose parameter declares no default at all.""" + + +def _signature_with_float_default(myparam=1.0): + """Stand-in for a client whose parameter declares a float default.""" + + +def test_coerce_redis_kwargs_types_empty_default_param_unchanged(): + """String params whose signature entry has no default (inspect.Parameter.empty) are left as-is.""" + result = _coerce_redis_kwargs_types({"testkey": "some_value"}, client=_signature_without_defaults) + + assert result["testkey"] == "some_value" + assert isinstance(result["testkey"], str) + + +def test_coerce_redis_kwargs_types_float_valid(): + """String values for params whose signature default is a float are coerced to float.""" + result = _coerce_redis_kwargs_types({"myparam": "3.14"}, client=_signature_with_float_default) + + assert result["myparam"] == pytest.approx(3.14) + assert isinstance(result["myparam"], float) + + +def test_coerce_redis_kwargs_types_float_invalid_drops_key(): + """An unconvertible string for a float-default param is dropped from the result.""" + result = _coerce_redis_kwargs_types({"myparam": "not_a_float"}, client=_signature_with_float_default) + + assert "myparam" not in result + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("false", False), ("true", True), ("0", False), ("1", True)], +) +def test_coerce_socket_keepalive_string(raw, expected): + """socket_keepalive's signature default is None, so it needs an explicit bool + coercion: a leftover "false" string is truthy and enables keepalive.""" + result = _coerce_redis_kwargs_types({"socket_keepalive": raw}) + + assert result["socket_keepalive"] is expected + + +def test_get_redis_client_logic_coerces_cluster_only_kwargs(monkeypatch): + """Cluster-only kwargs (absent from redis.Redis's signature) must still be + coerced when routing to a cluster, or Helm-stringified values reach + RedisCluster as strings.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + result = _get_redis_client_logic( + startup_nodes='[{"host": "localhost", "port": 7000}]', + cluster_error_retry_attempts="5", + require_full_coverage="false", + health_check_interval="30", + ) + + assert result["cluster_error_retry_attempts"] == 5 + assert isinstance(result["cluster_error_retry_attempts"], int) + assert result["require_full_coverage"] is False + assert result["health_check_interval"] == 30 + assert isinstance(result["health_check_interval"], int) + + +def test_get_redis_client_logic_raises_without_host_or_url(monkeypatch): + """_get_redis_client_logic raises ValueError when neither host nor url is provided.""" + for envvar in (*_get_redis_env_kwarg_mapping(), "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(envvar, raising=False) + + with pytest.raises(ValueError, match="Either 'host' or 'url' must be specified for redis"): + _get_redis_client_logic() diff --git a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py index c0c720bbaf6..953f028af3c 100644 --- a/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -1,3 +1,4 @@ +import base64 from typing import Final from unittest.mock import MagicMock @@ -8,8 +9,17 @@ from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( SpeechToCompletionBridgeTransformationHandler, ) +from litellm.types.utils import ChatCompletionAudioResponse, Choices, Message, ModelResponse GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview" +PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6 + + +def _model_response(model: str, pcm: bytes) -> ModelResponse: + audio: Final = ChatCompletionAudioResponse( + data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello" + ) + return ModelResponse(model=model, choices=[Choices(message=Message(content=None, audio=audio))]) def _bridge_request(response_format: str | None) -> dict: @@ -28,7 +38,7 @@ def _bridge_request(response_format: str | None) -> dict: ) -@pytest.mark.parametrize("response_format", ["wav", "mp3", "pcm", None]) +@pytest.mark.parametrize("response_format", ["wav", "pcm", None]) def test_gemini_tts_request_keeps_speech_response_format_out_of_chat_params(response_format: str | None) -> None: request: Final = _bridge_request(response_format) @@ -60,3 +70,48 @@ def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> assert "response_format" not in request assert request["audio"] == {"voice": "alloy", "format": "wav"} + + +@pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"]) +def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None: + with pytest.raises(litellm.BadRequestError) as excinfo: + _bridge_request(response_format) + + assert excinfo.value.status_code == 400 + assert response_format in str(excinfo.value) + assert "pcm" in str(excinfo.value) + assert "wav" in str(excinfo.value) + + +def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + response_format="pcm", + ) + + assert response.response.content == PCM_BYTES + assert response.response.headers["content-type"] == "audio/pcm" + + +@pytest.mark.parametrize("response_format", ["wav", None]) +def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: str | None) -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + response_format=response_format, + ) + + body: Final = response.response.content + assert body[:4] == b"RIFF" + assert body[8:12] == b"WAVE" + assert body[44:] == PCM_BYTES + assert response.response.headers["content-type"] == "audio/wav" + + +def test_non_gemini_response_keeps_original_bytes_and_mpeg_content_type() -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response("gpt-4o-audio-preview", PCM_BYTES), + response_format="mp3", + ) + + assert response.response.content == PCM_BYTES + assert response.response.headers["content-type"] == "audio/mpeg" diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 1af3dd3f613..5628d69de26 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -65,6 +65,7 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash target_type=target_type, target_id=target_id, router_name=job.router_name, + router_names=job.router_names, direction=job.direction, baseline_model=job.baseline_model, shadow_percentage=job.shadow_percentage, @@ -81,6 +82,7 @@ def _router( shadow_text="shadow answer", judge_json='{"preference": "A", "confidence": 0.9, "reasoning": "x"}', classifier_cost=None, + sibling_router_texts=None, ): """One mock router serving the shadow call first, the judge call second, told apart by the internal-origin stamp rather than the model, since a reverse job's shadow arm names @@ -100,6 +102,15 @@ def _router( decision["classifier_cost"] = classifier_cost kwargs["metadata"]["routing_decision"] = decision return {"choices": [{"message": {"content": shadow_text}}], "usage": {"completion_tokens": 5}} + if sibling_router_texts and kwargs["model"] in sibling_router_texts: + kwargs["metadata"]["routing_decision"] = { + "tier_label": "MEDIUM", + "routed_model": f"{kwargs['model']}-pick", + } + return { + "choices": [{"message": {"content": sibling_router_texts[kwargs["model"]]}}], + "usage": {"completion_tokens": 5}, + } return ModelResponse( model=kwargs["model"], choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": shadow_text}}], @@ -1210,29 +1221,36 @@ class TestJobValidation: {"direction": "reverse"}, {"baseline_model": "baseline-model"}, {"direction": "sideways", "baseline_model": "baseline-model"}, + {"direction": "reverse", "baseline_model": "baseline-model", "router_names": ("a", "b")}, ], - ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction"], + ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction", "reverse-with-router-set"], ) def test_unsamplable_shapes_are_rejected(self, overrides): with pytest.raises(ValidationError): _job(**overrides) - def test_shadow_target_follows_direction(self): - assert _job().shadow_target == "my-router" - assert _reverse_job().shadow_target == "baseline-model" + def test_arm_target_follows_direction(self): + assert _job().arm_target("my-router") == "my-router" + assert _reverse_job().arm_target("my-router") == "baseline-model" + + def test_rows_from_before_router_names_carry_their_set_in_router_name(self): + assert _job().arm_router_names == ("my-router",) + assert _job(router_names=("my-router", "alt-router")).arm_router_names == ("my-router", "alt-router") @pytest.mark.asyncio class TestDirection: @pytest.mark.parametrize( - "job,routed_by,sampled", + "job,routed_by,attempt_rows", [ - (_job(), None, True), - (_job(), "my-router", False), - (_job(), "other-router", True), - (_reverse_job(), "my-router", True), - (_reverse_job(), None, False), - (_reverse_job(), "other-router", False), + (_job(), None, 1), + (_job(), "my-router", 0), + (_job(), "other-router", 1), + (_reverse_job(), "my-router", 1), + (_reverse_job(), None, 0), + (_reverse_job(), "other-router", 0), + (_job(router_names=("my-router", "alt-router")), "alt-router", 0), + (_job(router_names=("my-router", "alt-router")), "other-router", 2), ], ids=[ "forward-samples-unrouted", @@ -1241,20 +1259,24 @@ class TestDirection: "reverse-samples-its-own-router", "reverse-skips-unrouted", "reverse-skips-another-router", + "forward-skips-any-candidates-own-traffic", + "forward-multi-samples-once-per-arm", ], ) - async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, sampled): + async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, attempt_rows): """The two directions partition the key's traffic: whatever one samples, the other - skips, so a key running both never judges the same turn twice for the same reason.""" + skips, so a key running both never judges the same turn twice for the same reason. + A multi-router job extends the forward skip to every candidate: a request one + candidate served must not be judged as the incumbent against another candidate.""" prisma = _prisma() - logger = _logger(router=_router(), prisma=prisma, jobs=(job,)) + logger = _logger(router=_router(sibling_router_texts={"alt-router": "alt answer"}), prisma=prisma, jobs=(job,)) await logger.async_log_success_event( _success_kwargs(request_metadata=_routed_by(routed_by) if routed_by else {}), RESPONSE, None, None ) await _drain(logger) - assert prisma.db.litellm_shadowevalattempt.create.await_count == int(sampled) + assert prisma.db.litellm_shadowevalattempt.create.await_count == attempt_rows async def test_reverse_duplicates_against_the_baseline_model(self): prisma = _prisma() @@ -1316,6 +1338,134 @@ class TestDirection: assert logger._job_starts == {"forward-job": 1, "reverse-job": 1} +@pytest.mark.asyncio +class TestMultiRouterArms: + async def test_every_arm_judges_the_same_request_and_stamps_its_own_row(self): + """One sampled request, one row per candidate router, both judged against the same + real response: the paired comparison that makes multi-router win rates comparable.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router")), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.001, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert [row["router_name"] for row in rows] == ["my-router", "alt-router"] + assert {row["request_id"] for row in rows} == {"req-1"} + assert [row["shadow_model"] for row in rows] == ["cheap-model", "alt-router-pick"] + assert all(row["outcome"] in ("real", "shadow", "tie") for row in rows) + assert all(row["real_cost"] == 0.001 for row in rows) + + async def test_a_single_router_job_stamps_its_router_on_the_row(self): + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["router_name"] == "my-router" + + async def test_one_arms_failure_never_silences_the_sibling(self): + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + healthy = router.acompletion.side_effect + + async def first_arm_explodes(**kwargs): + if kwargs["model"] == "my-router": + raise RuntimeError("provider exploded") + return await healthy(**kwargs) + + router.acompletion.side_effect = first_arm_explodes + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router")), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert [row["router_name"] for row in rows] == ["my-router", "alt-router"] + assert rows[0]["outcome"] == "error" + assert "provider exploded" in rows[0]["error"] + assert rows[1]["outcome"] in ("real", "shadow", "tie") + + async def test_the_turn_valve_counts_every_arm_a_start_will_write(self): + """max_turns is a row ceiling and one sampled request writes one row per arm, so + admission pre-counts the arms: a two-arm job with two turns of budget admits one + request, not two.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger( + router=router, prisma=prisma, jobs=(_job(router_names=("my-router", "alt-router"), max_turns=2),) + ) + + await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(logger) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + assert {row["request_id"] for row in rows} == {"req-1"} + assert len(rows) == 2 + + async def test_a_withheld_request_runs_no_arm_and_counts_once(self): + """The budget gates run once per sampled request, before any arm: funnel counters + stay per-request, so coverage math is arm-count independent.""" + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + logger = _logger(router=router, prisma=prisma) + + await logger._run_shadow_eval( + job=_job(router_names=("my-router", "alt-router"), max_budget=1.0, spend=2.0), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + assert logger._test_funnel == [("job-1", "withheld")] + + @pytest.mark.asyncio class TestActiveJobsFailClosed: async def test_a_row_the_sampler_cannot_read_is_dropped_not_guessed(self): diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index f39f41a6d12..ec4bc1f49eb 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -14,7 +14,7 @@ from litellm.integrations.websearch_interception.handler import ( ) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth -from litellm.types.utils import LlmProviders +from litellm.types.utils import CallTypes, LlmProviders def test_initialize_from_proxy_config(): @@ -230,6 +230,124 @@ async def test_execute_search_passes_selected_search_tool_litellm_params(monkeyp assert forwarded_kwargs["max_retries"] == 2 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_tools", "error"), + [ + pytest.param(None, "was not found", id="router-not-configured"), + pytest.param( + [{"search_tool_name": "other-search", "litellm_params": {"search_provider": "tavily"}}], + "was not found", + id="requested-tool-not-configured", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": "not-a-mapping"}], + "does not define a valid search provider", + id="invalid-parameters", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {}}], + "does not define a valid search provider", + id="missing-provider", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {"search_provider": " "}}], + "does not define a valid search provider", + id="whitespace-provider", + ), + pytest.param( + [{"search_tool_name": "parallel-search", "litellm_params": {"search_provider": 123}}], + "does not define a valid search provider", + id="invalid-provider", + ), + ], +) +async def test_execute_search_rejects_invalid_explicit_search_tool(monkeypatch, search_tools, error): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(search_tool_name="parallel-search") + router = None if search_tools is None else MagicMock(search_tools=search_tools) + mock_asearch = AsyncMock() + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ValueError, match=f"Configured search tool 'parallel-search' {error}"): + await logger._execute_search("what is litellm") + + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_execute_search_honors_explicit_parallel_search_tool(monkeypatch): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(search_tool_name="parallel-search") + router = MagicMock( + search_tools=[ + { + "search_tool_name": "other-search", + "litellm_params": {"search_provider": "tavily", "api_key": "other-key"}, + }, + { + "search_tool_name": "parallel-search", + "litellm_params": {"search_provider": "parallel_ai", "api_key": "parallel-key"}, + }, + ], + ) + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm") + + mock_asearch.assert_awaited_once_with( + query="what is litellm", + search_provider="parallel_ai", + api_key="parallel-key", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_tools", "expected_search_kwargs"), + [ + pytest.param(None, {"search_provider": "perplexity"}, id="router-not-configured"), + pytest.param( + [ + { + "search_tool_name": "first-search", + "litellm_params": {"search_provider": "tavily", "api_key": "first-key"}, + }, + { + "search_tool_name": "parallel-search", + "litellm_params": {"search_provider": "parallel_ai", "api_key": "parallel-key"}, + }, + ], + {"search_provider": "tavily", "api_key": "first-key"}, + id="first-configured-tool", + ), + ], +) +async def test_execute_search_preserves_implicit_provider_selection(monkeypatch, search_tools, expected_search_kwargs): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger() + router = None if search_tools is None else MagicMock(search_tools=search_tools) + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm") + + mock_asearch.assert_awaited_once_with(query="what is litellm", **expected_search_kwargs) + + @pytest.mark.asyncio async def test_execute_search_attributes_spend_to_the_calling_key(monkeypatch): """An intercepted search is billed and logged against the key that made the LLM request. @@ -397,6 +515,72 @@ async def test_execute_search_enforces_team_search_tool_permission(monkeypatch): mock_asearch.assert_not_awaited() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call_type", "web_search_tool"), + [ + pytest.param( + CallTypes.acompletion, + {"type": "web_search_20250305", "name": "web_search"}, + id="chat-completion", + ), + pytest.param(CallTypes.responses, {"type": "web_search"}, id="responses"), + pytest.param(CallTypes.aresponses, {"type": "web_search"}, id="async-responses"), + pytest.param( + CallTypes.anthropic_messages, + {"type": "web_search_20250305", "name": "web_search"}, + id="anthropic-messages", + ), + ], +) +async def test_deployment_hook_dispatcher_propagates_missing_explicit_search_tool( + monkeypatch, call_type, web_search_tool +): + import litellm + from litellm.proxy import proxy_server + from litellm.utils import async_pre_call_deployment_hook + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="parallel-search") + mock_asearch = AsyncMock() + kwargs = { + "model": "bedrock/claude-sonnet-4", + "tools": [web_search_tool], + "custom_llm_provider": "bedrock", + } + + monkeypatch.setattr( + proxy_server, + "llm_router", + MagicMock(search_tools=[{"search_tool_name": "other-search", "litellm_params": {"search_provider": "tavily"}}]), + ) + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ValueError, match="Configured search tool 'parallel-search' was not found"): + await async_pre_call_deployment_hook(kwargs=kwargs, call_type=call_type.value) + + assert kwargs["tools"] == [web_search_tool] + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_deployment_hook_skips_explicit_tool_validation_for_non_search_responses(monkeypatch): + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="parallel-search") + monkeypatch.setattr(proxy_server, "llm_router", MagicMock(search_tools=[])) + + result = await logger.async_pre_call_deployment_hook( + kwargs={ + "tools": [{"type": "function", "name": "calculator"}], + "custom_llm_provider": "bedrock", + }, + call_type=CallTypes.aresponses, + ) + + assert result is None + + @pytest.mark.asyncio async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs(): """Test that async_pre_call_deployment_hook finds custom_llm_provider at top-level kwargs. diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 772fbf98c57..9ab66d55f3f 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1433,3 +1433,50 @@ class TestFlattenTopLevelSchemaCombinators: flatten_top_level_schema_combinators(schema) assert schema == snapshot + + +class TestRequestContainsImageContent: + """One detector for every dialect that reaches pre-routing hooks untranslated.""" + + @pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + }, + ], + ) + def test_detects_every_image_dialect_including_tool_results(self, part): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}, part]}] + assert request_contains_image_content(messages) is True + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "user", "content": "plain string"}], + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "x"}}]}], + [{"role": "user", "content": [{"type": "tool_result", "content": [{"type": "text", "text": "ok"}]}]}], + [{"role": "user", "content": None}], + [], + ], + ) + def test_ignores_text_audio_and_degenerate_shapes(self, messages): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + assert request_contains_image_content(messages) is False + + def test_hostile_nesting_is_depth_bounded(self): + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + + nested: dict = {"type": "image", "source": {"type": "base64", "data": "aGk="}} + for _ in range(50): + nested = {"type": "tool_result", "content": [nested]} + assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False diff --git a/tests/test_litellm/litellm_core_utils/test_audio_utils.py b/tests/test_litellm/litellm_core_utils/test_audio_utils.py index 0e8176fffce..155f6680416 100644 --- a/tests/test_litellm/litellm_core_utils/test_audio_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_audio_utils.py @@ -347,3 +347,65 @@ class TestNormalizeTranscriptionLanguageToBcp47: ) assert normalize_transcription_language_to_bcp47(language) == expected + + +class TestResolveSpeechMediaType: + @pytest.mark.parametrize( + ("upstream_content_type", "response_format", "expected"), + [ + ("audio/wav", None, "audio/wav"), + ("AUDIO/WAV", None, "audio/wav"), + ("audio/flac; charset=binary", "mp3", "audio/flac"), + ("application/json", "flac", "audio/flac"), + ("application/octet-stream", "pcm", "audio/pcm"), + (None, "wav", "audio/wav"), + (None, "WAV", "audio/wav"), + (None, "opus", "audio/opus"), + (None, "aac", "audio/aac"), + (None, "mp3", "audio/mpeg"), + (None, "mp4", "audio/mpeg"), + (None, "bogus", "audio/mpeg"), + (None, None, "audio/mpeg"), + ("", None, "audio/mpeg"), + ], + ) + def test_resolution(self, upstream_content_type, response_format, expected): + from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type + + resolved = resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=response_format, + ) + assert resolved == expected + + +class TestSpeechMediaTypeFromAudioBytes: + @pytest.mark.parametrize( + ("audio", "expected"), + [ + (b"RIFF\x24\x00\x00\x00WAVEfmt ", "audio/wav"), + (b"fLaC\x00\x00\x00\x22", "audio/flac"), + (b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"), + (b"OggS" + b"\x00" * 24 + b"\x01vorbis", "audio/ogg"), + (b"ID3\x04\x00\x00\x00\x00\x00\x00", "audio/mpeg"), + (b"\xff\xfb\x90\x64", "audio/mpeg"), + (b"\xff\xf3\x80\x00", "audio/mpeg"), + (b"\xff\xf1\x50\x80", "audio/aac"), + (b"\xff\xf9\x50\x80", "audio/aac"), + (b"RIFF\x24\x00\x00\x00AVI LIST", None), + (b"\xff\xff\xff\xff\xff\xff", None), + (b"\xff\xfb\xf0\x00", None), + (b"\xff\xfb\x9c\x00", None), + (b"\xff\xeb\x90\x00", None), + (b"\xff\xf1\xf4\x80", None), + (b"\xff\x00\x00\x00", None), + (b"\x00\x01\x02\x03\x04\x05", None), + (b"\xff\xfb", None), + (b"\xff", None), + (b"", None), + ], + ) + def test_sniffing(self, audio, expected): + from litellm.litellm_core_utils.audio_utils.utils import speech_media_type_from_audio_bytes + + assert speech_media_type_from_audio_bytes(audio) == expected diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 05da22a73fd..fba337b5f2c 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,3 +1,4 @@ +import base64 from unittest.mock import MagicMock, Mock, patch import httpx @@ -126,6 +127,48 @@ class TestVertexAITextToSpeechConfig: assert voice_dict == voice_input +@pytest.mark.parametrize( + ("audio", "expected_content_type"), + [ + (b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00", "audio/wav"), + (b"\xff\xfb\x90\x64\x00\x00\x00\x00", "audio/mpeg"), + (b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"), + (b"fLaC\x00\x00\x00\x22", "audio/flac"), + ], +) +def test_transform_text_to_speech_response_labels_content_type(audio, expected_content_type): + raw_response = httpx.Response( + status_code=200, + json={"audioContent": base64.b64encode(audio).decode()}, + ) + + result = VertexAITextToSpeechConfig().transform_text_to_speech_response( + model="vertex_ai/chirp", + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert result.response.headers["content-type"] == expected_content_type + assert result.response.content == audio + + +def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): + raw_pcm = b"\x00\x01\x02\x03\x04\x05\x06\x07" + raw_response = httpx.Response( + status_code=200, + json={"audioContent": base64.b64encode(raw_pcm).decode()}, + ) + + result = VertexAITextToSpeechConfig().transform_text_to_speech_response( + model="vertex_ai/chirp", + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert "content-type" not in result.response.headers + assert result.response.content == raw_pcm + + @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 90b3b29d919..90be51cfa5b 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -26,6 +26,7 @@ from prisma.errors import ( from litellm._logging import verbose_proxy_logger +from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler @@ -703,23 +704,43 @@ async def test_auth_failure_ip_stamp_does_not_mutate_callers_request_data(): assert request_data == {"model": "gpt-4o"} +def _marked_malformed_key_error() -> HTTPException: + """Build the malformed-key 401 as its raise site does: marker stamped on it.""" + error = HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test") + setattr(error, INVALID_VIRTUAL_KEY_ERROR_MARKER, True) + return error + + @pytest.mark.asyncio @pytest.mark.parametrize( - "auth_error,expect_traceback", + "auth_error,expect_traceback,expect_level", [ pytest.param( ProxyException( message="Authentication Error", type=ProxyErrorTypes.auth_error, param=None, code=401 ), False, + "ERROR", id="expected_401_no_traceback", ), - pytest.param(ValueError("unexpected internal error"), True, id="unexpected_error_keeps_traceback"), + pytest.param(ValueError("unexpected internal error"), True, "ERROR", id="unexpected_error_keeps_traceback"), + pytest.param( + _marked_malformed_key_error(), + False, + "WARNING", + id="malformed_virtual_key_warning_no_traceback", + ), + pytest.param( + HTTPException(status_code=401, detail="LiteLLM Virtual Key expected. Received=test"), + False, + "ERROR", + id="phrase_without_marker_stays_loud", + ), ], ) -async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, caplog): +async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, expect_level, caplog): """Regression for LIT-6043: expected 4xx auth rejections must not format a - traceback via logger.exception; unexpected errors must keep it.""" + traceback via logger.exception; malformed virtual keys log at WARNING.""" handler = UserAPIKeyAuthExceptionHandler() with ( @@ -740,8 +761,8 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( try: try: raise auth_error - except (ProxyException, ValueError) as caught: - with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException): + except (ProxyException, ValueError, HTTPException) as caught: + with caplog.at_level(expect_level, logger="LiteLLM Proxy"), pytest.raises((ProxyException, HTTPException)): await handler._handle_authentication_error( caught, MagicMock(), @@ -756,3 +777,6 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( records = [r for r in caplog.records if "user_api_key_auth(): Exception occured" in r.getMessage()] assert len(records) == 1 assert (records[0].exc_info is not None) is expect_traceback + assert records[0].levelname == expect_level + expected_logger_name = "LiteLLM Proxy.stdout" if expect_level == "WARNING" else "LiteLLM Proxy" + assert records[0].name == expected_logger_name diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index a74aa553449..c525af84511 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -881,6 +881,7 @@ def _leg_record(**overrides: object) -> MagicMock: "target_type": "key", "target_id": "key-hash", "router_name": "my-router", + "router_names": (), "direction": "forward", "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", @@ -931,6 +932,7 @@ def _shadow_prisma( legs=(), agg_rows=None, by_leg_rows=None, + by_router_rows=None, known_keys=("key-hash", "key-hash-2"), key_teams=None, known_teams=None, @@ -1030,6 +1032,7 @@ def _shadow_prisma( "target_type", "target_id", "router_name", + "router_names", "direction", "baseline_model", "judge_model", @@ -1065,6 +1068,8 @@ def _shadow_prisma( return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] if "SELECT job_id AS grp" in sql: return by_leg_rows if by_leg_rows is not None else [] + if "COALESCE(a.router_name" in sql: + return by_router_rows if by_router_rows is not None else [] if 'FROM "LiteLLM_ShadowEvalFunnel"' in sql: return prisma.funnel_rows return agg_rows if agg_rows is not None else [] @@ -1121,7 +1126,15 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")] - assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1 + assert ( + len( + { + frozenset((k, tuple(v) if isinstance(v, list) else v) for k, v in row.items() if k not in ("target_id", "id")) + for row in rows + } + ) + == 1 + ) assert len({row["id"] for row in rows}) == len(rows) assert len({row["group_id"] for row in rows}) == 1 assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows) @@ -1138,6 +1151,63 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets) +@pytest.mark.asyncio +async def test_start_shadow_eval_multi_router_writes_the_set_on_every_leg(monkeypatch: pytest.MonkeyPatch): + """A multi-router job stores the full set in router_names and the first router in + router_name, so a rolling-deploy pod that predates router_names still runs a valid + single-arm eval and its unstamped attempt rows attribute to that first router.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval( + _start_request(router_name=None, router_names=("my-router", "classifier-router")), ADMIN + ) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert all(row["router_name"] == "my-router" for row in rows) + assert all(row["router_names"] == ["my-router", "classifier-router"] for row in rows) + assert response.router_names == ("my-router", "classifier-router") + assert response.router_name == "my-router" + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_an_unconfigured_router_in_the_set(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException, match="not-a-router") as exc: + await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "not-a-router")), ADMIN) + + assert exc.value.status_code == 400 + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_judge_collision_is_found_on_every_router_of_the_set(monkeypatch: pytest.MonkeyPatch): + """The judge-as-candidate guard walks every candidate router: a judge that serves an + arm of the SECOND router still poisons the whole job's win rates.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException, match="also an arm") as exc: + await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "sonnet-router")), ADMIN) + + assert exc.value.status_code == 400 + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + @pytest.mark.asyncio async def test_start_shadow_eval_rejects_an_uncredentialed_sdk_judge(monkeypatch: pytest.MonkeyPatch) -> None: import litellm @@ -1798,6 +1868,63 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"} +@pytest.mark.asyncio +async def test_get_shadow_eval_job_slices_results_per_router(monkeypatch: pytest.MonkeyPatch): + """A multi-router job's detail carries one slice per arm, aggregated by the arm + stamped on each attempt row, with unstamped legacy rows attributed to the job's own + router by the read (the COALESCE against the leg's router_name).""" + import litellm.proxy.proxy_server as proxy_server + + def agg(grp: str, wins: int) -> dict[str, object]: + return { + "grp": grp, + "turn_count": 4, + "real_wins": 4 - wins, + "shadow_wins": wins, + "ties": 0, + "avg_confidence": 0.8, + "real_spend": 0.08, + "shadow_spend": 0.02, + "cache_hit_turns": 0, + } + + prisma = _shadow_prisma( + legs=[_leg_record(router_names=("my-router", "alt-router"))], + agg_rows=[agg("SIMPLE", 3)], + by_router_rows=[agg("my-router", 1), agg("alt-router", 3)], + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + assert response.router_names == ("my-router", "alt-router") + assert response.router_name == "my-router" + assert [(s.group, s.shadow_win_rate_pct) for s in response.results.by_router] == [ + ("my-router", 25.0), + ("alt-router", 75.0), + ] + router_sql = next( + call.args[0] for call in prisma.db.query_raw.await_args_list if "COALESCE(a.router_name" in call.args[0] + ) + assert "COALESCE(a.router_name, j.router_name)" in router_sql + assert 'JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id' in router_sql + assert "a.job_id = ANY($1::text[])" in router_sql + + +@pytest.mark.asyncio +async def test_job_responses_resolve_router_names_with_legacy_fallback(monkeypatch: pytest.MonkeyPatch): + """Rows from before router_names existed carry their whole set in router_name.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(router_names=())]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + assert response.router_names == ("my-router",) + assert response.router_name == "my-router" + + @pytest.mark.asyncio async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 1c80aa5683f..b5a4204f0af 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -14142,6 +14142,311 @@ async def test_info_key_fn_v2_budget_table_fallback(monkeypatch): mock_prisma_client.db.query_raw.assert_not_awaited() +@pytest.mark.asyncio +async def test_info_key_fn_reports_budget_limits_usage(monkeypatch): + """ + /key/info reports current-window spend per budget window under budget_limits_usage, + keyed by budget_duration and read from the same counter enforcement uses, while + budget_limits itself comes back exactly as stored. + """ + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + test_key_token = "hashed_token_window_test" + budget_limits = [ + { + "reset_at": "2026-08-15T18:00:00+00:00", + "max_budget": 2.0, + "budget_duration": "1h", + } + ] + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=0.73) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key_info = MagicMock(spec=LiteLLM_VerificationToken) + mock_key_info.token = test_key_token + mock_key_info.object_permission_id = None + mock_key_info.user_id = "user-w" + mock_key_info.team_id = None + mock_key_info.litellm_budget_table = None + mock_key_info.model_dump.return_value = { + "token": test_key_token, + "budget_limits": [dict(w) for w in budget_limits], + "user_id": "user-w", + "team_id": None, + "object_permission_id": None, + "litellm_budget_table": None, + } + mock_key_info.dict.return_value = mock_key_info.model_dump.return_value + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_info + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test-window-key", + ) + + result = await info_key_fn( + key="sk-test-window-key", + user_api_key_dict=user_api_key_dict, + ) + + assert result["info"]["budget_limits"] == budget_limits + assert result["info"]["budget_limits_usage"] == {"1h": {"current_spend": 0.73}} + + mock_get_current_spend.assert_awaited_once() + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == f"spend:key:{test_key_token}:window:1h" + assert call_kwargs["max_budget"] == 2.0 + assert call_kwargs["window_entity_type"] == "Key" + assert call_kwargs["window_entity_id"] == test_key_token + assert call_kwargs["window_duration"] == "1h" + assert call_kwargs["window_start"] is not None + + +@pytest.mark.asyncio +async def test_info_key_fn_no_budget_limits_skips_spend_lookup(monkeypatch): + """Keys without budget windows get no budget_limits_usage field and trigger no spend lookup.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + test_key_token = "hashed_token_no_windows" + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=0.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key_info = MagicMock(spec=LiteLLM_VerificationToken) + mock_key_info.token = test_key_token + mock_key_info.object_permission_id = None + mock_key_info.user_id = "user-nw" + mock_key_info.team_id = None + mock_key_info.litellm_budget_table = None + mock_key_info.model_dump.return_value = { + "token": test_key_token, + "budget_limits": None, + "user_id": "user-nw", + "team_id": None, + "object_permission_id": None, + "litellm_budget_table": None, + } + mock_key_info.dict.return_value = mock_key_info.model_dump.return_value + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_info + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test-no-window-key", + ) + + result = await info_key_fn( + key="sk-test-no-window-key", + user_api_key_dict=user_api_key_dict, + ) + + assert result["info"]["budget_limits"] is None + assert "budget_limits_usage" not in result["info"] + mock_get_current_spend.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_info_key_fn_v2_reports_budget_limits_usage(monkeypatch): + """/v2/key/info reports budget_limits_usage per window and leaves budget_limits as stored.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy._types import KeyRequest, LiteLLM_VerificationToken + from litellm.proxy.management_endpoints.key_management_endpoints import ( + info_key_fn_v2, + ) + + test_key_token = "hashed_token_v2_window_test" + budget_limits = [ + { + "reset_at": "2026-08-15T18:00:00+00:00", + "max_budget": 2.0, + "budget_duration": "1h", + }, + { + "reset_at": "2026-08-16T00:00:00+00:00", + "max_budget": 20.0, + "budget_duration": "1d", + }, + ] + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_user_api_key_cache = AsyncMock() + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + mock_get_current_spend = AsyncMock(return_value=1.25) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + mock_key = MagicMock(spec=LiteLLM_VerificationToken) + mock_key.token = test_key_token + mock_key.user_id = "user-v2-w" + mock_key.team_id = None + mock_key.model_dump.return_value = { + "token": test_key_token, + "budget_limits": [dict(w) for w in budget_limits], + "user_id": "user-v2-w", + "team_id": None, + "litellm_budget_table": None, + } + mock_key.dict.return_value = mock_key.model_dump.return_value + + mock_prisma_client.get_data = AsyncMock(return_value=[mock_key]) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin-v2-w", + ) + + result = await info_key_fn_v2( + data=KeyRequest(keys=[test_key_token]), + user_api_key_dict=user_api_key_dict, + ) + + assert len(result["info"]) == 1 + assert result["info"][0]["budget_limits"] == budget_limits + assert result["info"][0]["budget_limits_usage"] == { + "1h": {"current_spend": 1.25}, + "1d": {"current_spend": 1.25}, + } + assert mock_get_current_spend.await_count == 2 + counter_keys = { + call.kwargs["counter_key"] for call in mock_get_current_spend.await_args_list + } + assert counter_keys == { + f"spend:key:{test_key_token}:window:1h", + f"spend:key:{test_key_token}:window:1d", + } + assert { + call.kwargs["window_duration"] for call in mock_get_current_spend.await_args_list + } == {"1h", "1d"} + + +@pytest.mark.asyncio +async def test_build_budget_limits_usage_json_string_input(monkeypatch): + """budget_limits stored as a JSON string is parsed and reported per window.""" + import json as json_module + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.5) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + raw = json_module.dumps( + [{"budget_duration": "1h", "max_budget": 2.0, "reset_at": None}] + ) + result = await _build_budget_limits_usage(budget_limits=raw, api_key_hash="hash-1") + + assert result == {"1h": {"current_spend": 0.5}} + mock_get_current_spend.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_build_budget_limits_usage_empty_windows_returns_none(monkeypatch): + """A key with no windows (None, [], or "[]") returns None so the field is left off; no spend lookup runs.""" + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + for stored in (None, [], "[]"): + assert await _build_budget_limits_usage(budget_limits=stored, api_key_hash="hash-1") is None + mock_get_current_spend.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_build_budget_limits_usage_window_without_max_budget(monkeypatch): + """A window with only budget_duration still reports current_spend, read without a budget ceiling.""" + from unittest.mock import AsyncMock + + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=0.75) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + result = await _build_budget_limits_usage( + budget_limits=[{"budget_duration": "2d"}], api_key_hash="hash-no-max" + ) + + assert result == {"2d": {"current_spend": 0.75}} + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == "spend:key:hash-no-max:window:2d" + assert call_kwargs["window_duration"] == "2d" + assert call_kwargs["max_budget"] is None + + +@pytest.mark.asyncio +async def test_build_budget_limits_usage_pydantic_windows(monkeypatch): + """BudgetLimitEntry windows (the shape UserAPIKeyAuth carries) are dumped to dicts and reported.""" + from unittest.mock import AsyncMock + + from litellm.models.team import BudgetLimitEntry + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_budget_limits_usage, + ) + + mock_get_current_spend = AsyncMock(return_value=1.0) + monkeypatch.setattr( + "litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend + ) + + result = await _build_budget_limits_usage( + budget_limits=[BudgetLimitEntry(budget_duration="7d", max_budget=10.0)], + api_key_hash="hash-2", + ) + + assert result == {"7d": {"current_spend": 1.0}} + call_kwargs = mock_get_current_spend.await_args.kwargs + assert call_kwargs["counter_key"] == "spend:key:hash-2:window:7d" + assert call_kwargs["window_duration"] == "7d" + assert call_kwargs["max_budget"] == 10.0 + + @pytest.mark.asyncio async def test_info_key_fn_reads_the_configured_budget_model_key(monkeypatch): """/key/info reads the one counter enforcement reads: the configured budget model. diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a3f56adb86f..f5ae0fe5977 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5462,3 +5462,164 @@ def test_the_marker_check_distinguishes_the_two_route_kinds(): builtin = MagicMock(spec=Request) builtin.scope = {"endpoint": llm_passthrough_endpoints.anthropic_proxy_route} assert request_dispatched_to_pass_through_endpoint(builtin) is False + + +async def _drive_passthrough_request_and_capture_logging(user_api_key_dict: UserAPIKeyAuth) -> tuple[int, object]: + import litellm + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + def transport_handler(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": True}) + + real_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": resolve_pass_through_request_timeout(None)}, + ) + cache_dict = litellm.in_memory_llm_clients_cache.cache_dict + cache_key = next((key for key, cached in cache_dict.items() if cached is real_handler), None) + assert cache_key is not None + cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + mock_request.body = AsyncMock(return_value=b'{"model": "gemini-2.0-flash"}') + + captured_data: dict = {} + + async def capture_pre_call_hook(user_api_key_dict, data, call_type): + captured_data.update(data) + return data + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=capture_pre_call_hook) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + mock_proxy_logging.get_proxy_hook = MagicMock(return_value=None) + + try: + with patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside pass_through_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ): + response = await pass_through_request( + request=mock_request, + target="https://upstream.example.test/v1/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + finally: + cache_dict[cache_key] = real_handler + + return response.status_code, captured_data.get("litellm_logging_obj") + + +@pytest.mark.asyncio +async def test_pass_through_request_wires_team_callbacks(): + """LIT-5152 regression: pass_through_request must resolve team-level logging + callbacks from key/team metadata and wire them into the Logging object, the + same way add_litellm_data_to_request does for normal LLM routes.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "logging": [ + { + "callback_name": "langfuse", + "callback_type": "success_and_failure", + "callback_vars": { + "langfuse_public_key": "pk_test", + "langfuse_secret_key": "sk_test", + "langfuse_host": "https://langfuse.example.test", + }, + } + ] + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert logging_obj.dynamic_success_callbacks, "team success callbacks not wired into Logging" + assert logging_obj.dynamic_failure_callbacks, "team failure callbacks not wired into Logging" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") == "pk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_secret_key") == "sk_test" + assert logging_obj.standard_callback_dynamic_params.get("langfuse_host") == "https://langfuse.example.test" + assert ("langfuse_public_key", "pk_test") in logging_obj._trusted_callback_vars + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_malformed_team_logging_metadata(): + """LIT-5152 fail-open: a malformed team ``logging`` value (here a non-iterable) + raises inside callback resolution; the passthrough request must still succeed, + just without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={"logging": 5}, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + + +@pytest.mark.asyncio +async def test_pass_through_request_survives_env_reference_in_deprecated_callback_settings(): + """LIT-5152 fail-open: the deprecated ``callback_settings`` team metadata skips + AddTeamCallback validation, so an ``os.environ/`` callback var would otherwise + blow up inside ``Logging.__init__`` and fail the request; the passthrough must + instead succeed without dynamic callbacks.""" + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + team_id="test-team", + team_metadata={ + "callback_settings": { + "success_callback": ["langfuse"], + "failure_callback": ["langfuse"], + "callback_vars": { + "langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY", + "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY", + "langfuse_host": "https://langfuse.example.test", + }, + } + }, + ) + + status_code, logging_obj = await _drive_passthrough_request_and_capture_logging(user_api_key_dict) + + assert status_code == 200 + assert logging_obj is not None + assert not logging_obj.dynamic_success_callbacks + assert not logging_obj.dynamic_failure_callbacks + assert not logging_obj.standard_callback_dynamic_params.get("langfuse_public_key") + + +@pytest.mark.asyncio +async def test_resolve_team_callback_wiring_fails_open_on_operational_error(): + """LIT-5152 fail-open: an operational error while resolving callback metadata + (e.g. team config lookup hitting a dead secret manager) must not raise; the + request proceeds without dynamic callbacks and the error is logged.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + _resolve_team_callback_wiring, + ) + from litellm.proxy.proxy_server import ProxyConfig + + class RaisingTeamConfig(ProxyConfig): + def load_team_config(self, team_id: str) -> dict: + raise RuntimeError("secret manager unavailable") + + wiring = _resolve_team_callback_wiring( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", team_id="test-team"), + proxy_config=RaisingTeamConfig(), + route_description="pass_through_endpoint", + ) + + assert wiring.success_callbacks is None + assert wiring.failure_callbacks is None + assert wiring.logging_kwargs is None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index 74542a3eaf6..de76c7257cf 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -12,13 +12,16 @@ from __future__ import annotations import io from unittest.mock import AsyncMock, MagicMock +import httpx import pytest from litellm.proxy import proxy_server +from litellm.types.llms.openai import HttpxBinaryResponseContent @pytest.fixture -def patched_speech(monkeypatch): +def patched_speech(monkeypatch, request): + upstream_content_type = getattr(request, "param", "audio/mpeg") monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) monkeypatch.setattr( proxy_server, @@ -36,15 +39,14 @@ def patched_speech(monkeypatch): monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - class _FakeBinaryResp: - async def aiter_bytes(self, chunk_size: int = 8192): - async def _gen(): - yield b"\x00\x01\x02" - - return _gen() - async def _llm_call(): - return _FakeBinaryResp() + return HttpxBinaryResponseContent( + httpx.Response( + status_code=200, + headers={} if upstream_content_type is None else {"content-type": upstream_content_type}, + content=b"\x00\x01\x02", + ) + ) async def _fake_route_request(*args, **kwargs): return _llm_call() @@ -79,6 +81,24 @@ def patched_speech_error(monkeypatch): yield +@pytest.fixture +def patched_speech_provider_rejection(monkeypatch, patched_speech_error): + import litellm + + async def _raise(*args, **kwargs): + raise litellm.BadRequestError( + message=( + "Gemini TTS only produces raw PCM16 audio, so response_format='mp3' is not supported." + " Supported response formats: pcm, wav." + ), + model="gemini-3.1-flash-tts-preview", + llm_provider="gemini", + ) + + monkeypatch.setattr(proxy_server, "route_request", _raise) + yield + + @pytest.fixture def patched_transcription(monkeypatch): router = MagicMock() @@ -152,6 +172,35 @@ def test_audio_speech_happy_path(client, auth_as, patched_speech, path): } +@pytest.mark.parametrize( + ("patched_speech", "response_format", "expected_content_type"), + [ + ("audio/wav", "wav", "audio/wav"), + ("audio/flac", "flac", "audio/flac"), + ("audio/pcm", "pcm", "audio/pcm"), + ("audio/wav", "mp3", "audio/wav"), + ("application/json", "flac", "audio/flac"), + (None, "wav", "audio/wav"), + (None, None, "audio/mpeg"), + ], + indirect=["patched_speech"], +) +def test_audio_speech_content_type_matches_audio_format( + client, auth_as, patched_speech, response_format, expected_content_type +): + """Regression for LIT-6482: /v1/audio/speech mislabeled wav/flac/pcm as audio/mpeg.""" + payload = { + "model": "tts-1", + "input": "Hi", + "voice": "alloy", + **({} if response_format is None else {"response_format": response_format}), + } + with auth_as(): + response = client.post("/v1/audio/speech", json=payload) + assert response.status_code == 200 + assert response.headers.get("content-type", "").split(";")[0] == expected_content_type + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_error(client, auth_as, patched_speech_error, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (error).""" @@ -162,6 +211,18 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): assert len(response.content) > 0 +def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): + """Regression for LIT-6501: a BadRequestError from the speech path surfaced as a generic 500.""" + payload = {"model": "gemini-tts", "input": "Hi", "voice": "Kore", "response_format": "mp3"} + with auth_as(): + response = client.post("/v1/audio/speech", json=payload) + assert response.status_code == 400 + error = response.json()["error"] + assert "response_format='mp3'" in error["message"] + assert "pcm" in error["message"] + assert "wav" in error["message"] + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_happy_path(client, auth_as, patched_transcription, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (happy).""" diff --git a/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py index 99f6f3a9b72..959cb2b1e89 100644 --- a/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py +++ b/tests/test_litellm/proxy/test_audio_speech_prometheus_hooks.py @@ -2,6 +2,7 @@ import asyncio import os from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi.testclient import TestClient @@ -29,6 +30,7 @@ def _make_mock_tts_response(): inner = MagicMock() inner.aiter_bytes = _aiter_bytes inner._hidden_params = {} + inner.response = httpx.Response(status_code=200, headers={"content-type": "audio/mpeg"}) async def _resolver(): return inner diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3f7844cffba..1ec8be88c9b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -10417,3 +10417,323 @@ class TestContextWindowEscalation: assert oversized["model_name"] == "big-model" assert small["model_name"] == "small-model" + + +IMG_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}} +PLAN_BODY = { + "messages": [{"role": "system", "content": [{"type": "text", "text": "Plan mode is active. Do not execute."}]}] +} + + +class TestModalityRouting: + """modality_routing: the response gate replaces a routed model that cannot take images.""" + + IMAGE_MESSAGE = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, IMG_PART]}] + BASE_TIERS = {"SIMPLE": "text-cheap", "MEDIUM": "vision-mid", "COMPLEX": "vision-big"} + BASE_VISION = {"text-cheap": False, "vision-mid": True, "vision-big": True, "vision-default": True} + + @staticmethod + def _router(mock_router_instance, config, vision_by_model): + """vision_by_model: model name -> True/False (deployment model_info) or None (undeclared).""" + + def get_model_list(model_name=None): + if model_name not in vision_by_model: + return [] + declared = vision_by_model[model_name] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}"}, + "model_info": {} if declared is None else {"supports_vision": declared}, + } + ] + + mock_router_instance.get_model_list = get_model_list + return ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "config_extra, vision, send_image, expected_model, expect_marker", + [ + ({}, {"text-cheap": False}, True, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": False}, False, "text-cheap", False), + ({"modality_routing": True}, {"text-cheap": None}, True, "text-cheap", False), + ], + ids=["flag_off", "no_image", "undeclared_model_stays_routable"], + ) + async def test_gate_leaves_ungated_requests_untouched( + self, mock_router_instance, config_extra, vision, send_image, expected_model, expect_marker + ): + router = self._router(mock_router_instance, {"tiers": dict(self.BASE_TIERS), **config_extra}, vision) + request = self.IMAGE_MESSAGE if send_image else [{"role": "user", "content": "What color is the sky?"}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=request) + assert result.model == expected_model + assert result.routing_decision["cause"] == "heuristic_scorer" + assert ("modality:image" in (result.routing_decision.get("signals") or ())) is expect_marker + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "part", + [ + IMG_PART, + {"type": "input_image", "image_url": "data:image/png;base64,aGk="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}, + {"type": "tool_result", "tool_use_id": "tu_1", "content": [dict(IMG_PART, type="image")]}, + ], + ids=["image_url", "input_image", "anthropic_image", "tool_result_nested"], + ) + async def test_every_image_dialect_escalates(self, mock_router_instance, part): + router = self._router( + mock_router_instance, {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, dict(self.BASE_VISION) + ) + message = [{"role": "user", "content": [{"type": "text", "text": "What color is this?"}, part]}] + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=message) + assert result.model == "vision-mid" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "path, expected_model, expected_cause", + [ + ("classifier_escalates", "vision-mid", "modality_escalation"), + ("same_tier_repick_keeps_cause", "vision-cheap", "heuristic_scorer"), + ("keyword_tier_escalates", "vision-mid", "modality_escalation"), + ("no_ask_capable_default_kept", "vision-default", "default_fallback"), + ("no_ask_text_default_displaced", "vision-mid", "modality_escalation"), + ("custom_tiers_walk", "premium-model", "modality_escalation"), + ("pin_kept_bypasses", "text-cheap", "session_affinity_pin"), + ("pin_replacement_gated", "vision-big", "modality_escalation"), + ("adaptive_pick_rewritten", "vision-mid", "modality_escalation"), + ], + ) + async def test_placements_across_decision_paths(self, mock_router_instance, path, expected_model, expected_cause): + config = {"tiers": dict(self.BASE_TIERS), "modality_routing": True} + vision = dict(self.BASE_VISION) + request_kwargs = {} + messages = self.IMAGE_MESSAGE + if path == "same_tier_repick_keeps_cause": + config["tiers"]["SIMPLE"] = ["text-cheap", "vision-cheap"] + vision["vision-cheap"] = True + with patch( # test-quality-ok: the mixed-pool repick is unreachable deterministically without pinning the first random pick + "litellm.router_strategy.complexity_router.complexity_router.random.choice", + side_effect=lambda pool: sorted(pool)[0], + ): + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + assert result.routing_decision["signals"][-1] == "modality:image" + return + if path == "keyword_tier_escalates": + config["keyword_tier_rules"] = [{"keywords": ["quick lookup"], "tier": "SIMPLE"}] + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path == "no_ask_capable_default_kept": + config["default_model"] = "vision-default" + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "no_ask_text_default_displaced": + config["default_model"] = "text-default" + vision["text-default"] = False + messages = [{"role": "user", "content": [IMG_PART]}] + elif path == "custom_tiers_walk": + config = { + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-4o-mini"}, + "fallback_tier": "cheap", + "tier_definitions": [ + {"name": "cheap", "description": "trivial asks"}, + {"name": "premium", "description": "hard asks"}, + ], + "tiers": {"cheap": "cheap-model", "premium": "premium-model"}, + "keyword_tier_rules": [{"keywords": ["quick lookup"], "tier": "cheap"}], + "modality_routing": True, + } + vision = {"cheap-model": False, "premium-model": True} + messages = [ + {"role": "user", "content": [{"type": "text", "text": "quick lookup: what is this?"}, IMG_PART]} + ] + elif path in ("pin_kept_bypasses", "pin_replacement_gated"): + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model": "text-cheap", "tier": "SIMPLE"}) + mock_router_instance.cache = cache + config["session_affinity"] = True + request_kwargs = {"metadata": {"session_id": "s1"}} + if path == "pin_replacement_gated": + config["tiers"]["MEDIUM"] = "text-mid" + vision["text-mid"] = False + messages = [ + {"role": "user", "content": [{"type": "text", "text": "LITELLM ESCALATE describe this"}, IMG_PART]} + ] + elif path == "adaptive_pick_rewritten": + config["adaptive"] = True + mock_router_instance.model_list = [] + mock_router_instance.model_name_to_deployment_indices = {} + router = self._router(mock_router_instance, config, vision) + result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) + assert result.model == expected_model + assert result.routing_decision["cause"] == expected_cause + if path == "adaptive_pick_rewritten": + assert request_kwargs["metadata"]["adaptive_router_chosen_model"] == expected_model + + @pytest.mark.asyncio + async def test_plan_floored_decision_never_falls_to_default_model(self, mock_router_instance): + """An upward-only walk cannot undercut the floor; default_model must not either.""" + config = { + "tiers": {"SIMPLE": "vision-cheap", "MEDIUM": "text-mid"}, + "default_model": "vision-default", + "plan_mode_min_tier": "MEDIUM", + "modality_routing": True, + } + vision = {"vision-cheap": True, "text-mid": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + async def test_at_floor_plan_turn_never_falls_to_default_model(self, mock_router_instance): + """A sentinel turn whose classified tier already satisfies the floor keeps its ordinary + cause, so the record carries no floor marker; the default arm must still refuse it.""" + config = { + "tiers": {"SIMPLE": "text-a", "MEDIUM": "text-b"}, + "default_model": "vision-default", + "plan_mode_min_tier": "SIMPLE", + "modality_routing": True, + } + vision = {"text-a": False, "text-b": False, "vision-default": True} + router = self._router(mock_router_instance, config, vision) + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook( + model="m", + request_kwargs={"proxy_server_request": {"body": PLAN_BODY}}, + messages=[{"role": "user", "content": [{"type": "text", "text": "plan this"}, IMG_PART]}], + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "default_model, default_vision, expect_error", + [(None, None, True), ("text-default", False, True), ("vision-default", True, False)], + ids=["no_default", "text_only_default", "vision_default_serves"], + ) + async def test_no_capable_tier_above_uses_default_or_rejects( + self, mock_router_instance, default_model, default_vision, expect_error + ): + config = {"tiers": {"SIMPLE": "text-cheap", "COMPLEX": "text-big"}, "modality_routing": True} + vision = {"text-cheap": False, "text-big": False} + if default_model is not None: + config["default_model"] = default_model + vision[default_model] = default_vision + router = self._router(mock_router_instance, config, vision) + if expect_error: + with pytest.raises(litellm.BadRequestError, match="no model"): + await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + return + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-default" + assert result.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in result.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_mixed_deployment_group_is_treated_text_only(self, mock_router_instance): + def get_model_list(model_name=None): + declared = {"mixed-group": [True, False], "vision-big": [True]}.get(model_name) + if declared is None: + return [] + return [ + { + "model_name": model_name, + "litellm_params": {"model": f"openai/unmapped-{model_name}-{i}"}, + "model_info": {"supports_vision": accepts}, + } + for i, accepts in enumerate(declared) + ] + + mock_router_instance.get_model_list = get_model_list + router = ComplexityRouter( + model_name="modality-test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {"SIMPLE": "mixed-group", "COMPLEX": "vision-big"}, + "modality_routing": True, + }, + ) + result = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self.IMAGE_MESSAGE) + assert result.model == "vision-big" + assert result.routing_decision["cause"] == "modality_escalation" + + @pytest.mark.asyncio + async def test_continuation_turn_screenshot_escalates_past_the_held_model(self, mock_router_instance): + """classification_mode user_turn replays the held model on continuation turns; a + continuation carrying a screenshot must still be re-placed when that model is text-only.""" + mock_router_instance.cache = DualCache() + config = { + "tiers": dict(self.BASE_TIERS), + "classification_mode": "user_turn", + "modality_routing": True, + } + router = self._router(mock_router_instance, config, dict(self.BASE_VISION)) + first = await router.async_pre_routing_hook( + model="m", + request_kwargs={"metadata": {"session_id": "cont-1"}}, + messages=[{"role": "user", "content": "hi there"}], + ) + assert first.model == "text-cheap" + continuation = [ + {"role": "user", "content": "hi there"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "tu_1", "name": "screenshot", "input": {}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [{"type": "image", "source": {"type": "base64", "data": "aGk="}}], + } + ], + }, + ] + second = await router.async_pre_routing_hook( + model="m", request_kwargs={"metadata": {"session_id": "cont-1"}}, messages=continuation + ) + assert second.model == "vision-mid" + assert second.routing_decision["cause"] == "modality_escalation" + assert "modality_escalated_from:SIMPLE" in second.routing_decision["signals"] + + @pytest.mark.asyncio + async def test_rewrite_carries_the_context_escalation_record(self, mock_router_instance): + """A context-window escalation and a modality re-place are separate facts on one + record; rewriting for the image must not drop the sibling gate's fields.""" + from litellm.types.router import PreRoutingHookResponse + + router = self._router( + mock_router_instance, + {"tiers": dict(self.BASE_TIERS), "modality_routing": True}, + dict(self.BASE_VISION), + ) + decision = router._build_routing_decision( + routed_model="text-cheap", + cause="heuristic_scorer", + tier=ComplexityTier.SIMPLE, + context_escalation_original_tier=ComplexityTier.SIMPLE, + ) + response = PreRoutingHookResponse(model="text-cheap", messages=None, routing_decision=decision) + rewritten = await router._gate_response_modality(response, None, self.IMAGE_MESSAGE, {}) + assert rewritten.model == "vision-mid" + assert rewritten.routing_decision["cause"] == "modality_escalation" + assert rewritten.routing_decision["context_escalated"] is True + assert rewritten.routing_decision["context_escalation_original_tier"] == "SIMPLE" + + def test_modality_escalation_is_never_pinnable(self): + from litellm.router_strategy.complexity_router.complexity_router import _decision_is_pinnable + + assert _decision_is_pinnable({"cause": "modality_escalation"}) is False + assert _decision_is_pinnable({"cause": "heuristic_scorer"}) is True diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 826beb74a27..a96e8541e06 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -1,3 +1,4 @@ +import inspect import json from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -600,6 +601,72 @@ def test_reconnect_kwargs_in_cluster_kwargs(): assert "socket_keepalive" in kwargs +def test_retry_attempts_in_cluster_kwargs(): + """cluster_error_retry_attempts must survive the cluster kwarg allow-list so + operators can bound worst-case retry latency on a Redis Cluster: it was being + silently dropped because the allow-list was built from redis.RedisCluster's + decorated __init__ without unwrapping it, so getfullargspec saw an empty + (self, *args, **kwargs) wrapper signature.""" + kwargs = _get_redis_cluster_kwargs() + assert "cluster_error_retry_attempts" in kwargs + + +def test_async_only_kwargs_in_cluster_kwargs_when_async_client_requested(): + """decode_responses is on the async cluster client's constructor and not the sync + one, on every redis-py the matrix covers. Introspecting the sync class regardless + of which client is actually built silently drops it for every async cluster caller.""" + sync_kwargs = _get_redis_cluster_kwargs() + async_kwargs = _get_redis_cluster_kwargs(async_redis.RedisCluster) + + assert "decode_responses" not in sync_kwargs + assert "decode_responses" in async_kwargs + + +@patch( # test-quality-ok: redis-py >= 6 keeps no cluster_error_retry_attempts attribute on the built client, so the constructor call is the only place the value is observable + "litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class" +) +def test_async_cluster_forwards_retry_attempts(mock_get_cluster_class): + """Regression: cluster_error_retry_attempts must reach the constructed async + cluster client. Silently dropping it removes an operator's only lever for + bounding a stuck node's worst-case retry latency, and the client falls back + to redis-py's own default (3 retries) instead.""" + mock_cluster_cls = mock_get_cluster_class.return_value + get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + cluster_error_retry_attempts=2, + ) + + call_kwargs = mock_cluster_cls.call_args[1] + assert call_kwargs["cluster_error_retry_attempts"] == 2 + + +def test_async_cluster_passes_async_only_kwargs(): + """Regression: decode_responses is an async-cluster-only constructor arg. When + the allow-list came from the sync class it was filtered out and values came + back as bytes instead of str.""" + client = get_redis_async_client( + startup_nodes=[{"host": "cluster-node", "port": 6379}], + decode_responses=True, + ) + + assert client.connection_kwargs["decode_responses"] is True + + +@pytest.mark.parametrize("cluster_client", [redis.RedisCluster, async_redis.RedisCluster], ids=["sync", "async"]) +def test_cluster_kwargs_exclude_variadic_parameters(cluster_client): + """*args / **kwargs are signature placeholders, not connection settings, and + must never land in the allow-list regardless of which cluster client is + introspected.""" + variadic = { + name + for name, param in inspect.signature(cluster_client).parameters.items() + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD) + } + + leaked = variadic & set(_get_redis_cluster_kwargs(cluster_client)) + assert not leaked, f"variadic params leaked into the allow-list: {leaked}" + + @patch("litellm.caching.redis_cluster_node_isolation.get_litellm_async_redis_cluster_class") def test_async_cluster_sets_reconnect_defaults(mock_get_cluster_class): """ diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6524353aa48..627f341fe2f 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1416,6 +1416,26 @@ def test_get_provider_rerank_config(): assert isinstance(config, HostedVLLMRerankConfig) +def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts(): + """Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS + dropped response_format before the speech_to_completion bridge could honor it.""" + from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig + from litellm.utils import LlmProviders + + assert ( + ProviderConfigManager.get_provider_text_to_speech_config( + model="gemini-2.5-flash-preview-tts", provider=LlmProviders.VERTEX_AI + ) + is None + ) + assert isinstance( + ProviderConfigManager.get_provider_text_to_speech_config( + model="en-US-Studio-O", provider=LlmProviders.VERTEX_AI + ), + VertexAITextToSpeechConfig, + ) + + # Models that should be skipped during testing OLD_PROVIDERS = ["aleph_alpha", "palm"] SKIP_MODELS = [ @@ -5765,3 +5785,33 @@ class TestHuggingFaceConfigFetch: assert _get_max_position_embeddings("some-org/some-model") == 512 request_timeout = hf_config_route.calls.last.request.extensions["timeout"] assert request_timeout["read"] == HF_CONFIG_FETCH_TIMEOUT_SECONDS + + +class TestIsVisionExplicitlyDisabled: + """github_copilot and chatgpt run an OAuth device flow inside get_llm_provider; the + explicit-disable lookup must adopt the declared prefix instead of resolving it, exactly + as _supports_factory does, or a capability check on a copilot deployment blocks routing + on a device-code prompt.""" + + @pytest.mark.parametrize("model", ["github_copilot/gpt-4o", "chatgpt/gpt-5"]) + def test_never_resolves_an_authenticating_prefix(self, model, monkeypatch): + from litellm.utils import is_vision_explicitly_disabled + + lookups: list = [] + + def _record(*args, **kwargs): + lookups.append((args, kwargs)) + raise RuntimeError("provider resolution must not run for an authenticating provider") + + monkeypatch.setattr(litellm, "get_llm_provider", _record) + + assert is_vision_explicitly_disabled(model) is False + assert lookups == [] + + def test_explicit_false_detected_and_absent_reads_enabled(self): + from litellm.utils import is_vision_explicitly_disabled + + assert ( + is_vision_explicitly_disabled("fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731") is True + ) + assert is_vision_explicitly_disabled("anthropic/claude-sonnet-4-5") is False diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index 8b397f20552..64e03985f57 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -102,6 +102,7 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ job_id: "job-1", status: "running", router_name: "claude-auto", + router_names: ["claude-auto"], direction: "forward", baseline_model: null, judge_model: "anthropic/claude-sonnet-5", @@ -436,7 +437,7 @@ describe("ShadowEvalSection", () => { await user.click(within(keyList).getByText("prod-alpha")); await user.click(keyInput); await user.click(within(keyList).getByText("staging-beta")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); expect(screen.getByText("Start shadow eval")).toBeDisabled(); @@ -449,7 +450,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha", "hash-beta"], team_ids: [], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, duration_days: 7, @@ -469,7 +470,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search teams by alias")); const teamList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(teamList).getByText("engineering")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); @@ -479,7 +480,7 @@ describe("ShadowEvalSection", () => { api_key_ids: [], team_ids: ["team-eng"], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "forward", shadow_percentage: 10, duration_days: 7, @@ -501,7 +502,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByPlaceholderText("Search keys by alias")); const keyList = await screen.findByTestId("paginated-multi-select-list"); await user.click(within(keyList).getByText("prod-alpha")); - await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers")); await user.click(await screen.findByText("gpt-auto")); await user.click(screen.getByPlaceholderText("Select a judge model")); await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); @@ -517,7 +518,7 @@ describe("ShadowEvalSection", () => { api_key_ids: ["hash-alpha"], team_ids: [], user_ids: [], - router_name: "gpt-auto", + router_names: ["gpt-auto"], direction: "reverse", baseline_model: "prod-claude", shadow_percentage: 10, @@ -528,6 +529,119 @@ describe("ShadowEvalSection", () => { expect(start.mutate).toHaveBeenCalledWith(expectedBody); }); + it("submits every picked auto-router so one job compares them on the same traffic", async () => { + const user = userEvent.setup(); + const { start } = mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Search keys by alias")); + const keyList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(keyList).getByText("prod-alpha")); + const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers"); + await user.click(routerInput); + await user.click(await screen.findByText("gpt-auto")); + await user.click(routerInput); + await user.click(await screen.findByText("claude-auto")); + expect( + screen.getByText("Every router sees the same sampled requests, judged against the same live responses"), + ).toBeInTheDocument(); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByText("Start shadow eval")); + + const expectedBody = { + api_key_ids: ["hash-alpha"], + team_ids: [], + user_ids: [], + router_names: ["gpt-auto", "claude-auto"], + direction: "forward", + shadow_percentage: 10, + duration_days: 7, + max_budget: 10, + judge_model: "anthropic/claude-sonnet-5", + }; + expect(start.mutate).toHaveBeenCalledWith(expectedBody); + }); + + it("blocks starting a reverse job with more than one router and says why", async () => { + const user = userEvent.setup(); + mockHooks({}); + render(); + + await user.click(screen.getByPlaceholderText("Search keys by alias")); + const keyList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(keyList).getByText("prod-alpha")); + const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers"); + await user.click(routerInput); + await user.click(await screen.findByText("gpt-auto")); + await user.click(routerInput); + await user.click(await screen.findByText("claude-auto")); + await user.click(screen.getByText("Adoption check: key's traffic vs the router")); + await user.click(await screen.findByText("Regression check: router's picks vs a baseline")); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByPlaceholderText("Select a baseline model")); + await user.click(screen.getByRole("option", { name: /prod-claude/ })); + + expect(screen.getByText("A regression check compares one router to its baseline")).toBeInTheDocument(); + expect(screen.getByText("Start shadow eval")).toBeDisabled(); + }); + + it("renders a per-router comparison table only when the job ran several routers", () => { + const routerSlice = (group: string, wins: number) => ({ + group, + turn_count: 20, + real_win_rate_pct: 100 - wins - 10, + shadow_win_rate_pct: wins, + tie_rate_pct: 10, + avg_judge_confidence: 0.8, + real_spend: 0.4, + shadow_spend: 0.2, + cache_hit_turns: 0, + }); + const base = job(); + const multi = job({ + router_names: ["claude-auto", "gpt-auto"], + results: { ...base.results!, by_router: [routerSlice("claude-auto", 40), routerSlice("gpt-auto", 70)] }, + }); + mockHooks({ jobs: [multi], detailsById: { "job-1": multi } }); + render(); + + expect(screen.getByText("Router")).toBeInTheDocument(); + const rows = screen.getAllByRole("row").map((row) => row.textContent ?? ""); + expect(rows.some((text) => text.includes("claude-auto") && text.includes("40.0%"))).toBe(true); + expect(rows.some((text) => text.includes("gpt-auto") && text.includes("70.0%"))).toBe(true); + expect( + screen.getByText( + (_, element) => + element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto, gpt-auto" && + element.tagName === "P", + ), + ).toBeInTheDocument(); + }); + + it("renders a job from an older proxy that predates router_names", () => { + const legacy = { ...job(), router_names: undefined } as unknown as ShadowEvalJob; + mockHooks({ jobs: [legacy], detailsById: { "job-1": legacy } }); + render(); + + expect( + screen.getByText( + (_, element) => + element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto" && element.tagName === "P", + ), + ).toBeInTheDocument(); + }); + + it("keeps the per-router table hidden for a single-router job", () => { + const base = job(); + const single = job({ results: { ...base.results!, by_router: [] } }); + mockHooks({ jobs: [single], detailsById: { "job-1": single } }); + render(); + + expect(screen.queryByText("Router")).not.toBeInTheDocument(); + }); + it("flips the arm labels and headline for a reverse job's results", () => { const j = job({ direction: "reverse", baseline_model: "openai/gpt-4o" }); mockHooks({ jobs: [j], detailsById: { "job-1": j } }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index ec145bc617a..c66d74074c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -2,32 +2,21 @@ import React, { useMemo, useState } from "react"; -import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; -import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; -import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; -import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; -import TeamMultiSelect from "@/components/common_components/team_multi_select"; -import { userOptionLabel } from "@/components/common_components/UserDropdown"; -import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { CircleHelp } from "lucide-react"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card } from "@/components/ui/card"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { ApiError } from "@/lib/http/client"; import { usd } from "./costOptimizationUtils"; +import { StartForm } from "./ShadowEvalStartForm"; import { useShadowEvalJob, useShadowEvalJobs, - useStartShadowEval, useStopShadowEval, type ShadowEvalJob, type ShadowEvalJobTarget, @@ -96,17 +85,19 @@ const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string = return target.stopped_at != null ? "stopped" : "running"; }; +const jobRouters = (job: ShadowEvalJob): string => (job.router_names ?? [job.router_name]).join(", "); + const jobHeadline = (job: ShadowEvalJob): React.ReactNode => job.direction === "reverse" ? ( <> - Comparing {job.router_name} to{" "} + Comparing {jobRouters(job)} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} {shadowedTargetsLabel(job)} traffic ) : ( <> Shadowing {job.shadow_percentage}% of {shadowedTargetsLabel(job)}{" "} - traffic via {job.router_name} + traffic via {jobRouters(job)} ); @@ -352,6 +343,11 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ + {(results.by_router ?? []).length > 1 && ( +
+ +
+ )} {results.by_current_model.length > 0 && ( { - const { data: costMap } = useModelCostMap(); - return useMemo(() => { - if (!costMap) return []; - const chatModels = Object.entries(costMap as Record) - .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) - .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); - return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); - }, [costMap]); -}; - -const useJudgeModelOptions = (): SearchSelectOption[] => { - const chatModels = useChatModelNames(); - return useMemo(() => { - const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ - label: model, - value: model, - sublabel: "Recommended", - })); - const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); - const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); - return [...pinned, ...rest]; - }, [chatModels]); -}; - -const useBaselineModelOptions = (): SearchSelectOption[] => { - const configuredGroups = usePlainModelGroups(); - const chatModels = useChatModelNames(); - return useMemo(() => { - const configured = [...configuredGroups] - .toSorted((a, b) => a.localeCompare(b)) - .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); - const rest = chatModels - .filter((model) => !configuredGroups.has(model)) - .map((model) => ({ label: model, value: model })); - return [...configured, ...rest]; - }, [configuredGroups, chatModels]); -}; - -const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ - { value: "forward", label: "Adoption check: key's traffic vs the router" }, - { value: "reverse", label: "Regression check: router's picks vs a baseline" }, -] as const; - -const START_FORM_DESCRIPTION: Record = { - forward: - "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", - reverse: - "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", -}; - -const DURATION_OPTIONS = [ - { value: "1", label: "1 day" }, - { value: "3", label: "3 days" }, - { value: "7", label: "7 days" }, - { value: "14", label: "14 days" }, - { value: "30", label: "30 days" }, -] as const; - -const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({ - label, - htmlFor, - className, - children, -}) => ( -
- - {children} -
-); - -const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => { - const [search, setSearch] = useState(""); - const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, { - selectedKeyAlias: search || null, - }); - const options = useMemo( - () => - (data?.pages ?? []) - .flatMap((page) => page.keys) - .map((key) => ({ - label: key.key_alias || key.key_name || key.token, - value: key.token, - sublabel: key.token, - })), - [data], - ); - return ( - void fetchNextPage()} - hasNextPage={hasNextPage} - isFetchingNextPage={isFetchingNextPage} - isLoading={isPending} - placeholder="Search keys by alias" - emptyText="No matching keys" - errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined} - /> - ); -}; - -const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { - const [search, setSearch] = useState(""); - const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( - 50, - search || undefined, - ); - const options = useMemo( - () => - Array.from( - new Map( - (data?.pages ?? []) - .flatMap((page) => page.users) - .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), - ).values(), - ), - [data], - ); - return ( - void fetchNextPage()} - hasNextPage={hasNextPage} - isFetchingNextPage={isFetchingNextPage} - isLoading={isPending} - placeholder="Search users by email" - emptyText="No matching users" - errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} - /> - ); -}; - -const StartForm: React.FC = () => { - const { accessToken } = useAuthorized(); - const [apiKeyIds, setApiKeyIds] = useState([]); - const [teamIds, setTeamIds] = useState([]); - const [userIds, setUserIds] = useState([]); - const [routerName, setRouterName] = useState(""); - const [direction, setDirection] = useState("forward"); - const [baselineModel, setBaselineModel] = useState(""); - const [percentage, setPercentage] = useState("10"); - const [durationDays, setDurationDays] = useState("7"); - const [judgeModel, setJudgeModel] = useState(""); - const [maxBudget, setMaxBudget] = useState("10"); - const { data: autoRouters } = useAutoRouters(); - const judgeModelOptions = useJudgeModelOptions(); - const baselineModelOptions = useBaselineModelOptions(); - const start = useStartShadowEval(); - - const routerOptions = useMemo(() => { - const names = new Set( - (autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)), - ); - return [...names].toSorted().map((name) => ({ label: name, value: name })); - }, [autoRouters]); - - const parsedPct = Number.parseFloat(percentage); - const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; - const parsedMaxBudget = Number.parseFloat(maxBudget); - const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; - const baselinePicked = direction === "forward" || baselineModel !== ""; - const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0; - const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; - const boundsValid = percentageValid && maxBudgetValid; - const valid = Boolean(accessToken) && filled && boundsValid; - const handleStart = () => { - const startBody = { - api_key_ids: apiKeyIds, - team_ids: teamIds, - user_ids: userIds, - router_name: routerName, - direction, - ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), - shadow_percentage: parsedPct, - duration_days: Number.parseInt(durationDays, 10), - max_budget: parsedMaxBudget, - judge_model: judgeModel, - }; - start.mutate(startBody); - }; - - return ( - - - Start a shadow eval -

{START_FORM_DESCRIPTION[direction]}

- - -
- - - - - - - - - - - - - - - - -
- setPercentage(e.target.value)} - /> - % of traffic -
-
- {percentage.trim() !== "" && !percentageValid && ( -

Enter a value from 0.1 to 100

- )} -
- - - - - -
- $ - setMaxBudget(e.target.value)} - /> - max shadow + judge spend, per target -
- {maxBudget.trim() !== "" && !maxBudgetValid && ( -

Enter a value from 0.01 to 10000

- )} - - {direction === "reverse" && ( - - - - )} - - - -
- - - - ); -}; - const previousSummary = (job: ShadowEvalJob): string => { const results = job.results; if (results) return pct(routerMatchedOrBeatPct(job.direction, results)); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx new file mode 100644 index 00000000000..f96910a4ad6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx @@ -0,0 +1,432 @@ +"use client"; + +import React, { useMemo, useState } from "react"; + +import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; +import TeamMultiSelect from "@/components/common_components/team_multi_select"; +import { userOptionLabel } from "@/components/common_components/UserDropdown"; +import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; + +import { useStartShadowEval, type ShadowEvalJob } from "./useShadowEval"; + +type ShadowEvalDirection = ShadowEvalJob["direction"]; + +const MAX_ROUTERS = 4; + +const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const; + +interface CostMapEntry { + litellm_provider?: string; + mode?: string; +} + +const useChatModelNames = (): string[] => { + const { data: costMap } = useModelCostMap(); + return useMemo(() => { + if (!costMap) return []; + const chatModels = Object.entries(costMap as Record) + .filter(([, value]) => value?.mode === "chat" && value?.litellm_provider) + .map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`)); + return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b)); + }, [costMap]); +}; + +const useJudgeModelOptions = (): SearchSelectOption[] => { + const chatModels = useChatModelNames(); + return useMemo(() => { + const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({ + label: model, + value: model, + sublabel: "Recommended", + })); + const pinnedNames = new Set(RECOMMENDED_JUDGE_MODELS); + const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model })); + return [...pinned, ...rest]; + }, [chatModels]); +}; + +const useBaselineModelOptions = (): SearchSelectOption[] => { + const configuredGroups = usePlainModelGroups(); + const chatModels = useChatModelNames(); + return useMemo(() => { + const configured = [...configuredGroups] + .toSorted((a, b) => a.localeCompare(b)) + .map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" })); + const rest = chatModels + .filter((model) => !configuredGroups.has(model)) + .map((model) => ({ label: model, value: model })); + return [...configured, ...rest]; + }, [configuredGroups, chatModels]); +}; + +const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [ + { value: "forward", label: "Adoption check: key's traffic vs the router" }, + { value: "reverse", label: "Regression check: router's picks vs a baseline" }, +] as const; + +const START_FORM_DESCRIPTION: Record = { + forward: + "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", + reverse: + "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", +}; + +const DURATION_OPTIONS = [ + { value: "1", label: "1 day" }, + { value: "3", label: "3 days" }, + { value: "7", label: "7 days" }, + { value: "14", label: "14 days" }, + { value: "30", label: "30 days" }, +] as const; + +const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({ + label, + htmlFor, + className, + children, +}) => ( +
+ + {children} +
+); + +const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, { + selectedKeyAlias: search || null, + }); + const options = useMemo( + () => + (data?.pages ?? []) + .flatMap((page) => page.keys) + .map((key) => ({ + label: key.key_alias || key.key_name || key.token, + value: key.token, + sublabel: key.token, + })), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search keys by alias" + emptyText="No matching keys" + errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + +const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( + 50, + search || undefined, + ); + const options = useMemo( + () => + Array.from( + new Map( + (data?.pages ?? []) + .flatMap((page) => page.users) + .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), + ).values(), + ), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search users by email" + emptyText="No matching users" + errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + +const RouterField: React.FC<{ + options: SearchSelectOption[]; + routerNames: string[]; + onChange: (names: string[]) => void; + direction: ShadowEvalDirection; +}> = ({ options, routerNames, onChange, direction }) => ( + + + {routerNames.length > MAX_ROUTERS && ( +

Pick at most {MAX_ROUTERS} auto-routers

+ )} + {direction === "reverse" && routerNames.length > 1 && ( +

A regression check compares one router to its baseline

+ )} + {direction === "forward" && routerNames.length > 1 && ( +

+ Every router sees the same sampled requests, judged against the same live responses +

+ )} + +); + +interface StartFormValidityInputs { + accessToken: string | null | undefined; + apiKeyIds: string[]; + teamIds: string[]; + userIds: string[]; + routerNames: string[]; + direction: ShadowEvalDirection; + baselineModel: string; + judgeModel: string; + percentage: string; + maxBudget: string; +} + +const startFormValidity = (inputs: StartFormValidityInputs) => { + const parsedPct = Number.parseFloat(inputs.percentage); + const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; + const parsedMaxBudget = Number.parseFloat(inputs.maxBudget); + const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; + const baselinePicked = inputs.direction === "forward" || inputs.baselineModel !== ""; + const targetsPicked = inputs.apiKeyIds.length + inputs.teamIds.length + inputs.userIds.length > 0; + const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS; + const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1; + const routersValid = routerCountValid && routersMatchDirection; + const modelsPicked = routersValid && inputs.judgeModel !== "" && baselinePicked; + const filled = targetsPicked && modelsPicked; + const boundsValid = percentageValid && maxBudgetValid; + const valid = Boolean(inputs.accessToken) && filled && boundsValid; + return { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid }; +}; + +interface StartBodyInputs { + apiKeyIds: string[]; + teamIds: string[]; + userIds: string[]; + routerNames: string[]; + direction: ShadowEvalDirection; + baselineModel: string; + shadowPercentage: number; + durationDays: number; + maxBudget: number; + judgeModel: string; +} + +const buildStartBody = (inputs: StartBodyInputs) => ({ + api_key_ids: inputs.apiKeyIds, + team_ids: inputs.teamIds, + user_ids: inputs.userIds, + router_names: inputs.routerNames, + direction: inputs.direction, + ...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}), + shadow_percentage: inputs.shadowPercentage, + duration_days: inputs.durationDays, + max_budget: inputs.maxBudget, + judge_model: inputs.judgeModel, +}); + +export const StartForm: React.FC = () => { + const { accessToken } = useAuthorized(); + const [apiKeyIds, setApiKeyIds] = useState([]); + const [teamIds, setTeamIds] = useState([]); + const [userIds, setUserIds] = useState([]); + const [routerNames, setRouterNames] = useState([]); + const [direction, setDirection] = useState("forward"); + const [baselineModel, setBaselineModel] = useState(""); + const [percentage, setPercentage] = useState("10"); + const [durationDays, setDurationDays] = useState("7"); + const [judgeModel, setJudgeModel] = useState(""); + const [maxBudget, setMaxBudget] = useState("10"); + const { data: autoRouters } = useAutoRouters(); + const judgeModelOptions = useJudgeModelOptions(); + const baselineModelOptions = useBaselineModelOptions(); + const start = useStartShadowEval(); + + const routerOptions = useMemo(() => { + const names = new Set( + (autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)), + ); + return [...names].toSorted().map((name) => ({ label: name, value: name })); + }, [autoRouters]); + + const validityInputs: StartFormValidityInputs = { + accessToken, + apiKeyIds, + teamIds, + userIds, + routerNames, + direction, + baselineModel, + judgeModel, + percentage, + maxBudget, + }; + const { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid } = startFormValidity(validityInputs); + const handleStart = () => { + const bodyInputs: StartBodyInputs = { + apiKeyIds, + teamIds, + userIds, + routerNames, + direction, + baselineModel, + shadowPercentage: parsedPct, + durationDays: Number.parseInt(durationDays, 10), + maxBudget: parsedMaxBudget, + judgeModel, + }; + start.mutate(buildStartBody(bodyInputs)); + }; + + return ( + + + Start a shadow eval +

{START_FORM_DESCRIPTION[direction]}

+ + +
+ + + + + + + + + + + + + + +
+ setPercentage(e.target.value)} + /> + % of traffic +
+
+ {percentage.trim() !== "" && !percentageValid && ( +

Enter a value from 0.1 to 100

+ )} +
+ + + + + +
+ $ + setMaxBudget(e.target.value)} + /> + max shadow + judge spend, per target +
+ {maxBudget.trim() !== "" && !maxBudgetValid && ( +

Enter a value from 0.01 to 10000

+ )} + + {direction === "reverse" && ( + + + + )} + + + +
+ + + + ); +}; diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/ui/litellm-dashboard/src/autorouter_presets.json index 4cbb548a855..da35d6171dc 100644 --- a/ui/litellm-dashboard/src/autorouter_presets.json +++ b/ui/litellm-dashboard/src/autorouter_presets.json @@ -14,6 +14,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -30,6 +31,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -56,6 +58,7 @@ }, "classifier_context_window_size": 0, "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } @@ -72,6 +75,7 @@ }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], + "classification_mode": "every_request", "session_affinity": false, "deployment_affinity": true } diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 2947e29319b..96c93306611 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -15,9 +15,12 @@ import { RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { + ClassificationFrequency, ClassifierFallback, ClassifierType, ComplexityRouterConfigValue, + classificationFrequency, + withClassificationFrequency, DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS, MIN_QUOTED_CONTEXT_TURN_CHARS, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, @@ -111,7 +114,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = How Classification Works {scoringExplanation(value)} {scorerRuns && ranges && ( -
    +
    • {effectiveTierLabel("SIMPLE", value.tier_labels)}: Score < {ranges.simpleMedium}
      • @@ -211,6 +214,7 @@ const ClassificationMethodConfig: React.FC = ({ const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); const classifierType = effectiveClassifierType(value); + const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity"); const classifierModelMissing = showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); @@ -305,6 +309,10 @@ const ClassificationMethodConfig: React.FC = ({ onChange({ ...value, classifier_fallback: fallback }); }; + const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => { + onChange(withClassificationFrequency(value, frequency)); + }; + const handleClassifierContextWindowSizeChange = (windowSize: number) => { onChange({ ...value, @@ -367,6 +375,49 @@ const ClassificationMethodConfig: React.FC = ({ )} +
      + How often to classify + + handleClassificationFrequencyChange(frequency as ClassificationFrequency) + } + > +
      + + + +
      +       +

      + Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router + cannot match to a held decision, such as one with no session id or an expired one, is scored again +

      +
      + {usesLlmClassifier(classifierType) && (
      diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 33ce1169c46..bbee03135c7 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -80,6 +80,15 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); }); + it("leaves the score threshold list color to the theme instead of an inline style", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const list = screen.getByText(/Score < 0.15/).closest("ul"); + expect(list).toBeInTheDocument(); + expect(list).toHaveClass("text-muted-foreground"); + expect(list?.style.color).toBe(""); + }); + it("should default to heuristic and hide classifier model/timeout fields", () => { renderWithProviders(); expect(screen.getByText("Advanced: Classification Method")).toBeInTheDocument(); @@ -589,6 +598,82 @@ describe("ComplexityRouterConfig classifier fallback", () => { }); }); +describe("ComplexityRouterConfig classification frequency", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + + it("defaults to every request, matching both backend field defaults", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every request/ })).toBeChecked(); + expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + expect(screen.getByRole("radio", { name: /Once per session/ })).not.toBeChecked(); + }); + + it("writes both wire fields when the frequency moves to every new user message", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("radio", { name: /Every new user message/ })); + expect(onChange).toHaveBeenCalledWith({ + ...llmValue, + classification_mode: "user_turn", + session_affinity: false, + }); + }); + + it("writes session affinity, not a classification mode, when the frequency moves to once per session", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("radio", { name: /Once per session/ })); + expect(onChange).toHaveBeenCalledWith({ + ...llmValue, + classification_mode: "every_request", + session_affinity: true, + }); + }); + + it("shows a hand-authored config that sets both fields as once per session, matching the backend", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Once per session/ })).toBeChecked(); + expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + }); + + it("records a switch back to every request", () => { + const onChange = vi.fn(); + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeChecked(); + fireEvent.click(screen.getByRole("radio", { name: /Every request/ })); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classification_mode: "every_request" })); + }); + + it("offers the frequency on a heuristic router, where holding the tier still pins the model", () => { + // The backend pin is gated on the two fields alone, so a heuristic router that switches models + // mid tool loop is fixed by this control too. + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeInTheDocument(); + }); +}); + describe("ComplexityRouterConfig classifier rubric", () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -752,12 +837,12 @@ describe("ComplexityRouterConfig tier labels", () => { }); describe("ComplexityRouterConfig affinity panel", () => { - it("holds both affinity switches with their backend defaults", () => { + it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", () => { renderWithProviders(); fireEvent.click(screen.getByText("Advanced: Affinity")); expect(screen.getByRole("switch", { name: "Pin a session to one deployment per model group" })).toBeChecked(); - expect(screen.getByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + expect(screen.queryByRole("switch", { name: "Pin a session to its first model" })).not.toBeInTheDocument(); }); it("writes deployment_affinity through onChange without touching other keys", () => { @@ -1282,10 +1367,18 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); }); - it("disables session pinning and says why, rather than letting a stripped value look saved", () => { - renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); - expect(screen.getByLabelText("Pin a session to its first model")).toHaveAttribute("data-disabled"); + it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + const sessionOption = screen.getByRole("radio", { name: /Once per session/ }); + expect(sessionOption).toHaveAttribute("aria-disabled", "true"); + expect(sessionOption).not.toBeChecked(); expect( screen.getByText("Session pinning escalates along the built-in tier ladder", { exact: false }), ).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 153afa0b586..5fde87fd638 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -29,6 +29,7 @@ import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import { Restricted, restrictedBy } from "./TierRestrictions"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -55,6 +56,16 @@ export const MIN_QUOTED_CONTEXT_TURN_CHARS = 120; export const DEFAULT_SESSION_AFFINITY = false; export const DEFAULT_DEPLOYMENT_AFFINITY = true; +export type ClassificationMode = "every_request" | "user_turn"; + +export const DEFAULT_CLASSIFICATION_MODE: ClassificationMode = "every_request"; + +/** + * One operator-facing choice over the two wire fields that share the router's tier-pin machinery: + * session affinity pins every turn, user_turn pins every turn except a new human ask. + */ +export type ClassificationFrequency = ClassificationMode | "session"; + export type ComplexityTiers = { SIMPLE: string[]; MEDIUM: string[]; @@ -383,6 +394,7 @@ export interface ComplexityRouterConfigValue { classification_prompt?: string; /** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */ heuristic_first_max_tier?: string; + classification_mode?: ClassificationMode; session_affinity?: boolean; deployment_affinity?: boolean; /** Plan-mode floor as a tier ROW ID, unset meaning off. The wire carries the row's name. */ @@ -392,6 +404,13 @@ export interface ComplexityRouterConfigValue { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + /** + * Context-window escalation gate. Undefined means untouched, which keeps both keys out of the + * payload so the router tracks the backend defaults (enabled, 0.95 buffer); an explicit false + * is a real opt-out and must survive the edit round-trip. + */ + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; /** * Heuristic scorer knobs. Undefined means the operator never touched them, which keeps the key out of the * payload so the router tracks the backend defaults rather than freezing today's numbers. @@ -412,6 +431,21 @@ export interface ComplexityRouterConfigValue { tier_model_params?: TierModelParamsByTier; } +/** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */ +export const classificationFrequency = (value: ComplexityRouterConfigValue): ClassificationFrequency => { + if (!value.custom_tier_set && (value.session_affinity ?? DEFAULT_SESSION_AFFINITY)) return "session"; + return value.classification_mode === "user_turn" ? "user_turn" : "every_request"; +}; + +export const withClassificationFrequency = ( + value: ComplexityRouterConfigValue, + frequency: ClassificationFrequency, +): ComplexityRouterConfigValue => ({ + ...value, + classification_mode: frequency === "user_turn" ? "user_turn" : "every_request", + session_affinity: frequency === "session", +}); + interface ComplexityRouterConfigProps { modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; @@ -490,23 +524,10 @@ const AffinityControls: React.FC<{ /> Pin a session to one deployment per model group
      - + Keeps a session on the same deployment within a group, so provider prompt caches stay warm. Turn off to load-balance every turn. -
      - onChange({ ...value, session_affinity: sessionAffinity })} - aria-label="Pin a session to its first model" - /> - Pin a session to its first model -
      - - {restrictedBy(value, "sessionAffinity")?.reason ?? - "Keeps a session on its first turn's model instead of re-classifying each turn. Also pins the deployment."} - ); @@ -827,6 +848,11 @@ const ComplexityRouterConfig: React.FC = ({ ), }, + { + key: "context-window", + label: Advanced: Context Window Escalation, + children: , + }, { key: "response", label: Advanced: Response Format, diff --git a/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx new file mode 100644 index 00000000000..c0a65076d20 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ContextWindowEscalationConfig.tsx @@ -0,0 +1,60 @@ +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const ContextWindowEscalationConfig: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => { + const enabled = value.enable_context_window_escalation ?? true; + // A number input renders Number("0.") as "0", so a decimal cannot be typed without a local draft. + const [bufferDraft, setBufferDraft] = React.useState(null); + const commitBuffer = (raw: string) => { + setBufferDraft(null); + if (raw.trim() === "") { + onChange({ ...value, context_window_escalation_buffer: undefined }); + return; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed)) return; + onChange({ ...value, context_window_escalation_buffer: Math.min(1, Math.max(0.01, parsed)) }); + }; + return ( + <> +
      + onChange({ ...value, enable_context_window_escalation: next })} + aria-label="Escalate oversized prompts to a tier that fits" + /> + Escalate oversized prompts to a tier that fits +
      + + When a prompt provably cannot fit the decided tier's context windows, route it to the lowest tier whose + window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone. + + {enabled && ( +
      + + setBufferDraft(event.target.value)} + onBlur={(event) => commitBuffer(event.target.value)} + /> + + Fraction of a model's window the counted prompt must fit within, above 0 up to 1. Empty tracks the + backend default of 0.95. + +
      + )} + + ); +}; + +export default ContextWindowEscalationConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 01cb41bcb95..d8a955b719c 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -362,8 +362,8 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); expandDetailedConfiguration(); - await user.click(screen.getByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + await user.click(screen.getByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Once per session/ })).not.toBeChecked(); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -373,6 +373,71 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries a context-window escalation opt-out through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); + expect(toggle).toBeChecked(); + await user.click(toggle); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + enable_context_window_escalation: false, + }); + }); + + it("clamps the context-window buffer to 1 and keeps an untouched buffer out of the payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "1.5" } }); + fireEvent.blur(buffer, { target: { value: "1.5" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; + expect(config).toMatchObject({ context_window_escalation_buffer: 1 }); + expect(config).not.toHaveProperty("enable_context_window_escalation"); + }); + + it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Context Window Escalation")); + const buffer = await screen.findByLabelText("Window fit buffer"); + fireEvent.change(buffer, { target: { value: "0.8" } }); + fireEvent.blur(buffer, { target: { value: "0.8" } }); + fireEvent.change(buffer, { target: { value: "" } }); + fireEvent.blur(buffer, { target: { value: "" } }); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).not.toHaveProperty( + "context_window_escalation_buffer", + ); + }); + // The scalar floor is the one scorer knob with no group dict behind it, so its wiring into the create // payload is only proven end to end. 0 is the case a truthy check would silently drop. it("carries a reasoning override floor of 0 through to the create payload", async () => { @@ -403,8 +468,8 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); expandDetailedConfiguration(); - await user.click(screen.getByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Once per session/ })); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -414,6 +479,44 @@ describe("AddAutoRouterTab", () => { }); }); + it("carries every new user message through to the create payload", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "user-turn-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + classification_mode: "user_turn", + }); + }); + + it("writes every_request into the create payload when the default frequency stays selected", async () => { + const user = userEvent.setup(); + vi.mocked(getMissingTiersError).mockReturnValue(null); + + renderWithProviders(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "default-timing-router"); + expandDetailedConfiguration(); + await user.click(screen.getByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect( + vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config.classification_mode, + ).toBe("every_request"); + }); + it("defaults a new router to deployment affinity on, matching the backend field default", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4e5e5e8d460..c60ce5e4959 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -342,6 +342,7 @@ const AddAutoRouterTab: React.FC = ({ planModeMinTier: complexityRouterConfig.plan_mode_min_tier, classificationPrompt: complexityRouterConfig.classification_prompt, heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier, + classificationMode: complexityRouterConfig.classification_mode, tierLabels: complexityRouterConfig.tier_labels, classifierType: complexityRouterConfig.classifier_type, classifierLlmConfig: complexityRouterConfig.classifier_llm_config, @@ -367,6 +368,8 @@ const AddAutoRouterTab: React.FC = ({ tokenThresholds: complexityRouterConfig.token_thresholds, dimensionWeights: complexityRouterConfig.dimension_weights, reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, + enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, + contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, }; const submitRecommendedRouter = async (name: string) => { diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 84406a2093e..40addf086b6 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -52,6 +52,7 @@ describe("buildComplexityRouterConfig", () => { const expected = { tiers, classifier_type: "heuristic", + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, escalation_keywords: ["LITELLM ESCALATE"], @@ -59,6 +60,16 @@ describe("buildComplexityRouterConfig", () => { expect(config).toEqual(expected); }); + it("carries an explicit context-window escalation opt-out and buffer, false included", () => { + const config = buildComplexityRouterConfig({ + ...baseParams, + enableContextWindowEscalation: false, + contextWindowEscalationBuffer: 0.9, + }); + expect(config.enable_context_window_escalation).toBe(false); + expect(config.context_window_escalation_buffer).toBe(0.9); + }); + it("trims escalation keywords and drops blank entries", () => { const config = buildComplexityRouterConfig({ ...baseParams, @@ -780,6 +791,20 @@ describe("heuristic_first", () => { }); }); +describe("classification_mode", () => { + it("emits user_turn", () => { + const config = buildComplexityRouterConfig({ ...baseParams, classificationMode: "user_turn" }); + expect(config.classification_mode).toBe("user_turn"); + }); + + it("writes every_request explicitly, so a saved router never depends on the backend default", () => { + expect( + buildComplexityRouterConfig({ ...baseParams, classificationMode: "every_request" }).classification_mode, + ).toBe("every_request"); + expect(buildComplexityRouterConfig(baseParams).classification_mode).toBe("every_request"); + }); +}); + describe("buildComplexityRouterConfig with an edited tier set", () => { const customTierSet = { tiers: [ @@ -892,6 +917,10 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 }); }); + it("keeps classification_mode, which the backend accepts beside tier_definitions", () => { + expect(build({ classificationMode: "user_turn" }).classification_mode).toBe("user_turn"); + }); + it("carries the plan-mode floor as the row's name, not the row id the form holds", () => { expect(build({ planModeMinTier: "sec" }).plan_mode_min_tier).toBe("SECURITY_REVIEW"); }); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index ba500d116ce..6a087649c00 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -19,10 +19,12 @@ import { import { AdaptiveEligible, AdaptiveRouterWeights, + ClassificationMode, ClassifierFallback, ClassifierLLMConfig, ClassifierType, ComplexityTierLabels, + DEFAULT_CLASSIFICATION_MODE, ComplexityRouterConfigValue, ComplexityTiers, DimensionWeights, @@ -105,6 +107,7 @@ export interface BuildComplexityRouterConfigParams { classifierFallback: ClassifierFallback | undefined; classificationPrompt: string | undefined; heuristicFirstMaxTier: string | undefined; + classificationMode: ClassificationMode | undefined; sessionAffinity: boolean; deploymentAffinity: boolean; customTechnicalKeywords: string[]; @@ -123,6 +126,8 @@ export interface BuildComplexityRouterConfigParams { dimensionWeights?: DimensionWeights; reasoningOverrideMinScore?: number; tierModelParams?: TierModelParamsByTier; + enableContextWindowEscalation?: boolean; + contextWindowEscalationBuffer?: number; } /** @@ -157,6 +162,7 @@ export interface ComplexityRouterConfigPayload { classifier_fallback?: ClassifierFallback; classification_prompt?: string; heuristic_first_max_tier?: string; + classification_mode: ClassificationMode; session_affinity: boolean; deployment_affinity: boolean; custom_technical_keywords?: string[]; @@ -174,6 +180,8 @@ export interface ComplexityRouterConfigPayload { token_thresholds?: TokenThresholds; dimension_weights?: DimensionWeights; reasoning_override_min_score?: number; + enable_context_window_escalation?: boolean; + context_window_escalation_buffer?: number; tier_model_configs?: Record; } @@ -389,6 +397,7 @@ export const buildComplexityRouterConfig = ({ classifierFallback, classificationPrompt, heuristicFirstMaxTier, + classificationMode, sessionAffinity, deploymentAffinity, customTechnicalKeywords, @@ -407,6 +416,8 @@ export const buildComplexityRouterConfig = ({ dimensionWeights, reasoningOverrideMinScore, tierModelParams, + enableContextWindowEscalation, + contextWindowEscalationBuffer, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const serializedTierModelConfigs = customTierSet ? serializeTierModelConfigs( @@ -446,6 +457,7 @@ export const buildComplexityRouterConfig = ({ ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), classifier_type: classifierType, ...classifierWireFields(effectiveType, classifierInputs), + classification_mode: classificationMode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: sessionAffinity, deployment_affinity: deploymentAffinity, ...(customTechnicalKeywords.length > 0 && { custom_technical_keywords: customTechnicalKeywords }), @@ -463,6 +475,12 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), + ...(enableContextWindowEscalation !== undefined && { + enable_context_window_escalation: enableContextWindowEscalation, + }), + ...(contextWindowEscalationBuffer !== undefined && { + context_window_escalation_buffer: contextWindowEscalationBuffer, + }), ...scorerKnobs, }; if (!customTierSet) return payload; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index a7bd4b8eab4..3e63cbd3b32 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -257,6 +257,33 @@ describe("buildUpdatedComplexityRouterConfig session affinity", () => { }); }); +describe("buildUpdatedComplexityRouterConfig classification mode", () => { + it("round-trips a stored user_turn through hydrate then save", () => { + const stored = { ...STORED, classification_mode: "user_turn" }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(hydrated.classification_mode).toBe("user_turn"); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classification_mode).toBe("user_turn"); + }); + + it("round-trips an explicitly stored every_request, so an untouched save leaves it as written", () => { + const stored = { ...STORED, classification_mode: "every_request" }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(hydrated.classification_mode).toBe("every_request"); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classification_mode).toBe("every_request"); + }); + + it("rewrites a stored user_turn to every_request once the operator picks the default back", () => { + const stored = { ...STORED, classification_mode: "user_turn" }; + const result = buildUpdatedComplexityRouterConfig(stored, { + ...FORM_VALUE, + classification_mode: "every_request", + }); + expect(result.classification_mode).toBe("every_request"); + }); +}); + describe("buildUpdatedComplexityRouterConfig deployment affinity", () => { it("writes deployment_affinity=false when the toggle is off", () => { const result = buildUpdatedComplexityRouterConfig(STORED, { ...FORM_VALUE, deployment_affinity: false }); @@ -476,6 +503,7 @@ describe("managed keys survive an untouched open-and-save", () => { classifier_context_budget_chars: 4000, classifier_context_include_assistant_turns: true, classifier_fallback: "default_model", + classification_mode: "user_turn", session_affinity: true, deployment_affinity: false, adaptive: true, @@ -487,6 +515,8 @@ describe("managed keys survive an untouched open-and-save", () => { token_thresholds: { simple: 20, complex: 500 }, dimension_weights: { tokenCount: 0.1 }, reasoning_override_min_score: 0.3, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, }; // tier_definitions, fallback_tier and classification_prompt cannot sit beside heuristic_first, which diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 027e01a9351..d199af51b41 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -47,6 +47,7 @@ const expectedClassifiedTierConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, adaptive: true, @@ -68,6 +69,7 @@ const expectedAdaptiveDisabledConfig = { semantic_keyword_matching: true, embedding_model: "voyage-4-large", match_threshold: 0.65, + classification_mode: "every_request", session_affinity: false, deployment_affinity: true, }; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index a4921fcfcb5..96e8549eac8 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -364,7 +364,7 @@ describe("EditAutoRouterModal assistant turns", () => { }); }); -describe("EditAutoRouterModal session affinity", () => { +describe("EditAutoRouterModal classification frequency", () => { beforeEach(() => { modelPatchUpdateCall.mockClear(); }); @@ -381,15 +381,15 @@ describe("EditAutoRouterModal session affinity", () => { />, ); - // A stored config with no session_affinity key now runs with affinity OFF, because the backend - // field defaults to False. The toggle has to render what the router actually does, and an - // untouched save must not flip it. - it("shows a stored config with no session_affinity key as off", async () => { + // A stored config with neither key now runs with affinity OFF, because both backend fields + // default that way. The picker has to render what the router actually does, and an untouched + // save must not flip it. + it("shows a stored config with neither key as every request", async () => { const user = userEvent.setup(); renderWithStoredConfig(STORED_CONFIG); - await user.click(await screen.findByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).not.toBeChecked(); + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every request/ })).toBeChecked(); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -397,12 +397,12 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(false); }); - it("shows a stored session_affinity=true as on and preserves it through an untouched save", async () => { + it("shows a stored session_affinity=true as once per session and preserves it through an untouched save", async () => { const user = userEvent.setup(); renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); - await user.click(await screen.findByText("Advanced: Affinity")); - expect(await screen.findByRole("switch", { name: "Pin a session to its first model" })).toBeChecked(); + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Once per session/ })).toBeChecked(); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -410,12 +410,12 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(true); }); - it("persists turning session affinity on", async () => { + it("persists picking once per session", async () => { const user = userEvent.setup(); renderWithStoredConfig(STORED_CONFIG); - await user.click(await screen.findByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Once per session/ })); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -423,18 +423,71 @@ describe("EditAutoRouterModal session affinity", () => { expect(savedConfig().session_affinity).toBe(true); }); - it("persists turning session affinity back off", async () => { + it("persists picking every request back over a stored session pin", async () => { const user = userEvent.setup(); renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); - await user.click(await screen.findByText("Advanced: Affinity")); - await user.click(await screen.findByRole("switch", { name: "Pin a session to its first model" })); + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every request/ })); await user.click(screen.getByRole("button", { name: /save changes/i })); await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); expect(savedConfig().session_affinity).toBe(false); }); + + it("clears a stored session pin when the operator moves to every new user message", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, session_affinity: true }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().session_affinity).toBe(false); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("shows a stored user_turn as selected and preserves it through an untouched save", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(await screen.findByRole("radio", { name: /Every new user message/ })).toBeChecked(); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("persists switching a stored config to every new user message", async () => { + const user = userEvent.setup(); + renderWithStoredConfig(STORED_CONFIG); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every new user message/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("user_turn"); + }); + + it("rewrites the stored mode to every_request when the operator picks it back", async () => { + const user = userEvent.setup(); + renderWithStoredConfig({ ...STORED_CONFIG, classification_mode: "user_turn" }); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(await screen.findByRole("radio", { name: /Every request/ })); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classification_mode).toBe("every_request"); + }); }); describe("EditAutoRouterModal deployment affinity", () => { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 425d5d51f06..2aecf0b3483 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -96,6 +96,7 @@ export interface StoredComplexityRouterConfig { classifier_context_budget_chars?: unknown; classifier_context_include_assistant_turns?: unknown; classifier_fallback?: unknown; + classification_mode?: unknown; tier_boundaries?: unknown; token_thresholds?: unknown; dimension_weights?: unknown; @@ -107,6 +108,8 @@ export interface StoredComplexityRouterConfig { tier_distance_penalty?: number; adaptive_eligible?: AdaptiveEligible; return_raw_model_name?: boolean; + enable_context_window_escalation?: unknown; + context_window_escalation_buffer?: unknown; } /** @@ -163,6 +166,10 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== "" ? parsedConfig.heuristic_first_max_tier : undefined, + classification_mode: + parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request" + ? parsedConfig.classification_mode + : undefined, tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries), token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds), dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights), @@ -178,6 +185,14 @@ export const hydrateComplexityRouterConfig = ( tier_distance_penalty: parsedConfig.tier_distance_penalty, adaptive_eligible: parsedConfig.adaptive_eligible || "all", return_raw_model_name: parsedConfig.return_raw_model_name || false, + enable_context_window_escalation: + typeof parsedConfig.enable_context_window_escalation === "boolean" + ? parsedConfig.enable_context_window_escalation + : undefined, + context_window_escalation_buffer: + typeof parsedConfig.context_window_escalation_buffer === "number" + ? parsedConfig.context_window_escalation_buffer + : undefined, }; }; @@ -197,6 +212,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "classifier_fallback", "classification_prompt", "heuristic_first_max_tier", + "classification_mode", "session_affinity", "deployment_affinity", "adaptive", @@ -208,6 +224,8 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "token_thresholds", "dimension_weights", "reasoning_override_min_score", + "enable_context_window_escalation", + "context_window_escalation_buffer", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -282,6 +300,7 @@ export const buildUpdatedComplexityRouterConfig = ( planModeMinTier: value.plan_mode_min_tier, classificationPrompt: value.classification_prompt, heuristicFirstMaxTier: value.heuristic_first_max_tier, + classificationMode: value.classification_mode, tierLabels: value.tier_labels, classifierType: value.classifier_type, classifierLlmConfig: value.classifier_llm_config, @@ -307,6 +326,8 @@ export const buildUpdatedComplexityRouterConfig = ( dimensionWeights: value.dimension_weights, reasoningOverrideMinScore: value.reasoning_override_min_score, tierModelParams: value.tier_model_params, + enableContextWindowEscalation: value.enable_context_window_escalation, + contextWindowEscalationBuffer: value.context_window_escalation_buffer, }; const built = buildComplexityRouterConfig(builderParams); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index e084fdf37e6..bf6a20a4af8 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -186,6 +186,12 @@ describe("RoutingDecisionCard", () => { expect(screen.queryByText("housekeeping")).not.toBeInTheDocument(); }); + it("labels a modality escalation instead of showing the raw cause token", () => { + render(); + expect(screen.getByText("Escalated for image input")).toBeInTheDocument(); + expect(screen.queryByText("modality_escalation")).not.toBeInTheDocument(); + }); + it("shows the escalation keyword", () => { render( , diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index d2aa20901f5..aa1d45a859e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -90,6 +90,7 @@ const CONSTANT_CAUSE_LABELS: Record = { session_affinity_pin: "Pinned to session", session_affinity_escalation: "Escalated from session pin", user_turn_continuation: "Continuation turn, classifier skipped", + modality_escalation: "Escalated for image input", quality_tier: "Quality tier mapping", bandit: "Adaptive bandit", default_fallback: "Default model, no route matched", diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 2de1ac11db2..5632d6d947f 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -230,6 +230,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -273,6 +274,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -287,6 +289,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -320,6 +323,7 @@ describe("autorouter_presets", () => { const simpleTierConfig = (presetModel: string) => ({ tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }); @@ -563,6 +567,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, match_threshold: 0, @@ -573,11 +578,46 @@ describe("autorouter_presets", () => { expect(prefill.escalationKeywords).toEqual([]); }); + it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => { + const prefill = buildPresetPrefill( + { + tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic", + classification_mode: "every_request", + session_affinity: false, + deployment_affinity: true, + enable_context_window_escalation: false, + context_window_escalation_buffer: 0.9, + }, + groupsOnly(["gpt-5-nano"]), + ); + expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false); + expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); + }); + + it("carries a preset's classification_mode and defaults it when the preset omits one", () => { + const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }; + const base = { + tiers, + classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, + session_affinity: false, + deployment_affinity: true, + }; + const availability = groupsOnly(["gpt-5-nano"]); + expect( + buildPresetPrefill({ ...base, classification_mode: "user_turn" }, availability).complexityRouterConfig + .classification_mode, + ).toBe("user_turn"); + expect(buildPresetPrefill(base, availability).complexityRouterConfig.classification_mode).toBe("every_request"); + }); + it("falls back to the defaults when a preset omits match_threshold and escalation_keywords", () => { const prefill = buildPresetPrefill( { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }, @@ -594,6 +634,7 @@ describe("autorouter_presets", () => { const base = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -609,6 +650,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["claude-sonnet-4-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -623,6 +665,7 @@ describe("autorouter_presets", () => { REASONING: [{ model_name: "o3", litellm_params: { reasoning_effort: "high" } }], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -642,6 +685,7 @@ describe("autorouter_presets", () => { REASONING: [{ model_name: "claude-sonnet-4-5", litellm_params: { reasoning_effort: "high" } }], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; @@ -664,6 +708,9 @@ describe("autorouter_presets", () => { ], }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, + session_affinity: false, + deployment_affinity: true, }; const prefill = buildPresetPrefill(config, groupsOnly(["claude-sonnet-4.5"])); // temperature survives from the spelling that would otherwise have been overwritten; @@ -677,6 +724,7 @@ describe("autorouter_presets", () => { const config = { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic" as const, + classification_mode: "every_request" as const, session_affinity: false, deployment_affinity: true, }; diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index a35b868db5e..4c1f08b62e9 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -6,6 +6,7 @@ import { ComplexityRouterConfigValue, ClassifierType, ClassifierLLMConfig, + DEFAULT_CLASSIFICATION_MODE, DEFAULT_SESSION_AFFINITY, DEFAULT_DEPLOYMENT_AFFINITY, usesLlmClassifier, @@ -288,6 +289,7 @@ export const buildPresetPrefill = ( classifier_context_budget_chars: config.classifier_context_budget_chars, classifier_context_per_turn_chars: config.classifier_context_per_turn_chars, classifier_context_include_assistant_turns: config.classifier_context_include_assistant_turns, + classification_mode: config.classification_mode ?? DEFAULT_CLASSIFICATION_MODE, session_affinity: config.session_affinity ?? DEFAULT_SESSION_AFFINITY, deployment_affinity: config.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, adaptive: config.adaptive, @@ -295,6 +297,8 @@ export const buildPresetPrefill = ( tier_distance_penalty: config.tier_distance_penalty, adaptive_eligible: config.adaptive_eligible, return_raw_model_name: config.return_raw_model_name, + enable_context_window_escalation: config.enable_context_window_escalation, + context_window_escalation_buffer: config.context_window_escalation_buffer, }, customTechnicalKeywords: config.custom_technical_keywords ?? [], keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c20545f6fb2..e944062e15e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7710,6 +7710,10 @@ export interface paths { * - model_max_budget: dict - Per-model budgets, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}} * - model_max_budget_usage: dict | None - Current-window spend per model, present only when * the key has per-model budgets + * - budget_limits: list | None - Concurrent budget windows, exactly as stored + * - budget_limits_usage: dict | None - Current-window spend per budget window, e.g. + * {"1h": {"current_spend": 0.0009}}, present only when the key has budget windows + * (read from the same cross-pod spend counter the budget enforcement uses) * - models: list - Model_name's the key is allowed to call * - tpm_limit / rpm_limit: int | None - Tokens and requests per minute limits * - metadata: dict - Metadata for the key, e.g. {"team": "core-infra"} @@ -34463,6 +34467,12 @@ export interface components { * @default 0.5 */ match_threshold: number; + /** + * Modality Routing + * @description Route image-bearing requests only to models that can accept image input. The classifier reads text alone, so an image request whose text classifies cheap otherwise lands on a text-only model and fails with a provider 400. When enabled, a routed model explicitly declared supports_vision false (deployment model_info or the model cost map; unmapped names stay routable) is replaced by the nearest HIGHER tier holding a capable model, then default_model, else a clear 400. A kept session-affinity pin still wins even when an image arrives. + * @default false + */ + modality_routing: boolean; /** * Plan Mode Min Tier * @description When set, requests carrying a coding-agent plan-mode sentinel (Claude Code plan mode, VS Code Copilot Plan mode, Copilot CLI's exit_plan_mode tool) are routed to at least this tier: the classified tier still wins when it is higher, and the floor also overrides a session-affinity pin to a lower tier for exactly the turns carrying the sentinel, without rewriting the pin -- the first turn after plan mode exits routes as if plan mode had never happened. Names a built-in tier, or with tier_definitions set, one of the defined tier names (list order is ascending severity, same as keyword_tier_rules). Unset disables detection entirely. The sentinels ride in client-injected prompt text, so a caller who pastes one can spend up to this tier's models -- never down, and never outside the configured pools. @@ -35321,8 +35331,17 @@ export interface components { last_error?: string | null; /** @description Stratified verdicts; detail endpoint only */ results?: components["schemas"]["ShadowEvalResult"] | null; - /** Router Name */ - router_name: string; + /** + * Router Name + * @description The first router, kept for callers that predate router_names; derived so the + * two fields can never disagree. + */ + readonly router_name: string; + /** + * Router Names + * @description Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of traffic and judge every arm against the same real responses + */ + router_names: string[]; /** Shadow Percentage */ shadow_percentage: number; /** @@ -35410,6 +35429,12 @@ export interface components { * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; + /** + * By Router + * @description One slice per router arm, grouped on the router name. Every arm of a multi-router job is judged against the same real responses over the same sampled requests, so these slices compare routers head-to-head: like-for-like win rates and spends on identical traffic. Verdicts from before arm stamping existed count toward the job's own router + * @default [] + */ + by_router: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** @@ -35423,13 +35448,13 @@ export interface components { overall_tie_rate_pct: number; /** * Sampled Real Spend - * @description USD the real arm billed across all judged turns, cache-served turns excluded + * @description USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn is one (request, router arm) verdict, so a multi-router job counts the real response once per arm it was judged against; per-router comparisons read by_router * @default 0 */ sampled_real_spend: number; /** * Sampled Shadow Spend - * @description USD the shadow arm billed across the same turns, judge excluded, like for like + * @description USD the shadow arms billed across the same turns, judge excluded, like for like * @default 0 */ sampled_shadow_spend: number; @@ -35620,7 +35645,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ @@ -35723,15 +35748,21 @@ export interface components { judge_model: string; /** * Max Budget - * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window + * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window. Every router arm draws from the same per-target budget, so a multi-router job reaches it proportionally sooner * @default 10 */ max_budget: number; /** * Router Name - * @description The auto-router under evaluation, in either direction + * @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields */ - router_name: string; + router_name?: string | null; + /** + * Router Names + * @description The auto-routers under evaluation, at most 4. Every sampled request runs through every router listed and each arm is judged independently against the same real response, so routers compare head-to-head on identical traffic. More than one router requires direction 'forward'. After validation this field always carries the full deduplicated set, whichever spelling the caller used + * @default [] + */ + router_names: string[]; /** * Shadow Percentage * @description Percentage of each target's requests to duplicate through the router