From b9fcfb26d01aaba21316ff9cb2ab1f4b2abf0fe4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:13:37 -0700 Subject: [PATCH 01/83] test(e2e): run the memory cell alone on the shared stack (#42518) * test(e2e): run the memory cell alone on the shared stack * test(e2e): hold the stack lock for every collected test, marker or not * test(e2e): prove the stack lock's reader sharing and writer preference across processes --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- tests/e2e/conftest.py | 20 ++- tests/e2e/pytest.ini | 1 + .../e2e/router/test_reliability_memory_e2e.py | 2 +- tests/e2e/stack_lock.py | 45 +++++++ tests/e2e/test_stack_lock.py | 117 ++++++++++++++++++ 6 files changed, 179 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/stack_lock.py create mode 100644 tests/e2e/test_stack_lock.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 6212a172fe3..d948c6fd1d9 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -97,7 +97,7 @@ Each suite provides its own `client` fixture (see `llm_translation/passthrough_c Request and response bodies are typed pydantic models in `models.py`; only the fields a test reads are modelled, and nothing passes raw dicts. Outcomes come back as a `Result[R]` tagged union (`Success`, `NetworkError`, `UnauthorizedError`, `RateLimitedError`, `ValidationError`, `UnknownApiError`). Handle them with `match`, or call `unwrap(...)` when a non-success should fail the test. The harness hard-fails and never skips: a test marked `e2e` fails when no proxy answers its liveness probe, and once a request reaches the proxy any wrong behavior is likewise a hard failure, so a missing proxy turns the run red instead of being mistaken for a pass -Mark live tests with `@pytest.mark.e2e` (on the class or the module). Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +Mark live tests with `@pytest.mark.e2e` (on the class or the module). Add `@pytest.mark.quiet_stack` to a test that measures the proxy itself (RSS, latency): the shared stack lock in `stack_lock.py` then runs it while no other test on the host is hitting the stack, marked or not, so the reading depends only on the test's own traffic. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache ## Record and replay fixtures diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index f1f94f2d2b3..f4de4ac01ba 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -51,6 +51,7 @@ from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client +from stack_lock import stack_lock _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() @@ -148,6 +149,11 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) + config.addinivalue_line( + "markers", + "quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; " + "every other test waits for it to finish", + ) config.addinivalue_line( "markers", "mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless " @@ -172,9 +178,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: """Abort before collection when E2E_FIXTURE_MODE can never work: an unknown mode value, or replay against a missing, unreadable, or stale bundle (the stale message names the bundle's age). Live and record modes pass through.""" - reason = fixture_mode_collection_error( - FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc) - ) + reason = fixture_mode_collection_error(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)) if reason is not None: raise pytest.UsageError(reason) @@ -267,6 +271,12 @@ def _proxy_fail_reason() -> str | None: return None +@pytest.hookimpl(wrapper=True) +def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None) -> Generator[None, object, object]: + with stack_lock(exclusive=item.get_closest_marker("quiet_stack") is not None): + return (yield) + + @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. @@ -326,9 +336,7 @@ def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: LIVE_PROVIDER_REQUIRED.set(False) if not item.stash.get(_CALL_PASSED, False): return result - reason = replay_leftover_error( - mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid - ) + reason = replay_leftover_error(mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid) if reason is not None: pytest.fail(reason) return result diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index a77459d3683..4866511af3f 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -12,6 +12,7 @@ markers = prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 2568434a08f..77d3a68cae5 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -86,7 +86,7 @@ from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] DEPLOYMENTS_PER_GROUP: Final = 2 RSS_SAMPLE_CAP: Final = 4 * MEMORY_RSS_SETTLE_SAMPLES diff --git a/tests/e2e/stack_lock.py b/tests/e2e/stack_lock.py new file mode 100644 index 00000000000..06df7a20b6a --- /dev/null +++ b/tests/e2e/stack_lock.py @@ -0,0 +1,45 @@ +"""Cross-process reader/writer lock over the proxy stack every xdist worker shares. +Every collected test holds it shared, marker or not, since the Claude Code cells and +other unmarked suites drive the same stack; a `quiet_stack` test holds it exclusive, +and the `gate` file makes a waiting exclusive holder win over readers that arrive +after it.""" + +from __future__ import annotations + +import fcntl +import hashlib +import tempfile +from collections.abc import Generator +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Final + +from e2e_config import PROXY_BASE_URL + +STACK_DIGEST: Final = hashlib.sha256(PROXY_BASE_URL.encode()).hexdigest()[:12] +LOCK_DIR: Final = Path(tempfile.gettempdir()) / f"litellm-e2e-stack-{STACK_DIGEST}" +GATE_FILE: Final = LOCK_DIR / "gate" +STACK_FILE: Final = LOCK_DIR / "stack" + + +@contextmanager +def _flock(path: Path, operation: int) -> Generator[None]: + with path.open("a") as handle: + fcntl.flock(handle, operation) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +@contextmanager +def stack_lock(exclusive: bool) -> Generator[None]: + LOCK_DIR.mkdir(parents=True, exist_ok=True) + if exclusive: + with _flock(GATE_FILE, fcntl.LOCK_EX), _flock(STACK_FILE, fcntl.LOCK_EX): + yield + return + with ExitStack() as held: + with _flock(GATE_FILE, fcntl.LOCK_SH): + held.enter_context(_flock(STACK_FILE, fcntl.LOCK_SH)) + yield diff --git a/tests/e2e/test_stack_lock.py b/tests/e2e/test_stack_lock.py new file mode 100644 index 00000000000..af071d2cc18 --- /dev/null +++ b/tests/e2e/test_stack_lock.py @@ -0,0 +1,117 @@ +"""Cross-process behavior of the stack lock: readers share it, an exclusive holder waits for +every reader and keeps them out, and a reader arriving behind a waiting exclusive holder +queues behind it instead of starving it.""" + +from __future__ import annotations + +import fcntl +import os +import subprocess +import sys +import time +from contextlib import ExitStack +from pathlib import Path +from typing import Final + +import pytest + +from stack_lock import STACK_DIGEST + +HARNESS_DIR: Final = Path(__file__).resolve().parent +DEADLINE_SECONDS: Final = 30.0 +SETTLE_SECONDS: Final = 0.5 +HOLDER_SCRIPT: Final = """ +import sys, time +from pathlib import Path +from stack_lock import stack_lock +name, mode, release_path, log_path = sys.argv[1:] + + +def record(event): + with Path(log_path).open("a") as log: + log.write(f"{name} {event}\\n") + + +record("waiting") +with stack_lock(exclusive=mode == "exclusive"): + record("enter") + while not Path(release_path).exists(): + time.sleep(0.02) + record("exit") +""" + + +def _events(log_path: Path) -> tuple[str, ...]: + return tuple(log_path.read_text().splitlines()) if log_path.exists() else () + + +def _wait_for_event(log_path: Path, event: str) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + while event not in _events(log_path): + if time.monotonic() > deadline: + pytest.fail(f"{event!r} never appeared; events so far: {_events(log_path)}") + time.sleep(0.02) + + +def _wait_until_gate_is_held_exclusively(gate_path: Path) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + with gate_path.open("a") as handle: + while True: + try: + fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError: + return + fcntl.flock(handle, fcntl.LOCK_UN) + if time.monotonic() > deadline: + pytest.fail("no exclusive holder ever took the gate") + time.sleep(0.02) + + +def _start_holder(held: ExitStack, tmp_path: Path, name: str, mode: str) -> subprocess.Popen[bytes]: + holder: Final = held.enter_context( + subprocess.Popen( + ( + sys.executable, + "-P", + "-c", + HOLDER_SCRIPT, + name, + mode, + str(tmp_path / f"release-{name}"), + str(tmp_path / "events"), + ), + cwd=HARNESS_DIR, + env={**os.environ, "TMPDIR": str(tmp_path), "PYTHONPATH": str(HARNESS_DIR)}, + ) + ) + held.callback(holder.kill) + return holder + + +def test_readers_share_exclusive_waits_and_a_waiting_exclusive_beats_later_readers(tmp_path: Path) -> None: + lock_dir: Final = tmp_path / f"litellm-e2e-stack-{STACK_DIGEST}" + lock_dir.mkdir() + log_path: Final = tmp_path / "events" + with ExitStack() as held: + first_reader: Final = _start_holder(held, tmp_path, "A", "shared") + _wait_for_event(log_path, "A enter") + second_reader: Final = _start_holder(held, tmp_path, "R", "shared") + _wait_for_event(log_path, "R enter") + (tmp_path / "release-R").touch() + _wait_for_event(log_path, "R exit") + writer: Final = _start_holder(held, tmp_path, "W", "exclusive") + _wait_until_gate_is_held_exclusively(lock_dir / "gate") + late_reader: Final = _start_holder(held, tmp_path, "B", "shared") + _wait_for_event(log_path, "B waiting") + time.sleep(SETTLE_SECONDS) + (tmp_path / "release-A").touch() + _wait_for_event(log_path, "W enter") + (tmp_path / "release-W").touch() + _wait_for_event(log_path, "B enter") + (tmp_path / "release-B").touch() + for holder in (first_reader, second_reader, writer, late_reader): + assert holder.wait(timeout=DEADLINE_SECONDS) == 0 + events: Final = _events(log_path) + assert events.index("R enter") < events.index("A exit") + assert events.index("W enter") > events.index("A exit") + assert events.index("B enter") > events.index("W exit") From a173657dfb2df85511d07febd1f15adb4e576c6a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:14:44 -0500 Subject: [PATCH 02/83] fix(caching): keep embedding cache hits aligned with request inputs (#42571) * fix(caching): keep embedding cache hits aligned with request inputs Partial hits now send only the uncached inputs to the provider and merge fresh vectors back into their original positions. Responses whose item count differs from the input count (one input scoring many documents) are no longer written to the per-input cache, since a later hit would return a single item. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): drop mutable collection builds flagged by the type discipline gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): bypass embedding cache entries written before the per input cardinality check Embedding cache entries now carry format_version and readers treat entries without it as misses, so entries that only hold the first row of a multi row response are refetched instead of served until their TTL expires. The provider call also receives a copy of the request kwargs with the uncached inputs rather than mutating the caller's mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): assert a partial embedding cache hit becomes a full hit on repeat Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(caching): await pending embedding cache writes before asserting on cache hits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): validate cached embeddings without mutating responses or request kwargs Validate cache rows through a frozen pydantic model so import does not depend on TypeAdapter support for ReadOnly TypedDicts, accept string embeddings, build the merged partial hit response instead of mutating the cached one, and hand the provider request mapping to post call hooks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(caching): keep cache_hit and response_ms on merged partial embedding hits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 55 ++++----- litellm/caching/caching_handler.py | 89 +++++++++----- litellm/types/caching.py | 18 +-- litellm/utils.py | 16 ++- tests/test_litellm/caching/test_caching.py | 113 +++++++++++++++++- .../caching/test_caching_handler.py | 45 ++++++- 6 files changed, 264 insertions(+), 72 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 6f99eb616b0..a31cad4af29 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -781,35 +781,23 @@ class Cache: Convert any embedding response into the standardized CachedEmbedding TypedDict format. """ try: - if isinstance(embedding_response, dict): - return { - "embedding": embedding_response.get("embedding"), - "index": embedding_response.get("index"), - "object": embedding_response.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - elif hasattr(embedding_response, "model_dump"): - data = embedding_response.model_dump() - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - else: - data = vars(embedding_response) - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } + data: Final = ( + embedding_response + if isinstance(embedding_response, dict) + else embedding_response.model_dump() + if hasattr(embedding_response, "model_dump") + else vars(embedding_response) + ) + cached: Final[CachedEmbedding] = { + "embedding": data.get("embedding"), + "index": data.get("index"), + "object": data.get("object"), + "model": model, + "prompt_tokens": prompt_tokens, + "prompt_tokens_details": prompt_tokens_details, + "format_version": EMBEDDING_CACHE_FORMAT_VERSION, + } + return cached except KeyError as e: raise ValueError(f"Missing expected key in embedding response: {e}") @@ -925,6 +913,15 @@ class Cache: if self.should_use_cache(**kwargs) is not True: return + input_count: Final = len(kwargs["input"]) if isinstance(kwargs["input"], list) else 1 + if len(result.data) != input_count: + verbose_logger.debug( + "LiteLLM Cache: skipping embedding cache write, %d inputs but %d embeddings in the response", + input_count, + len(result.data), + ) + return + # set default ttl if not set if self.ttl is not None: kwargs["ttl"] = self.ttl diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 36c3b744a06..4afd0e7caaa 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -21,7 +21,7 @@ import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm._logging import print_verbose, verbose_logger @@ -34,7 +34,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( from litellm.litellm_core_utils.logging_utils import ( _assemble_complete_response_from_streaming_chunks, ) -from litellm.types.caching import CachedEmbedding +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.rerank import RerankResponse @@ -77,6 +77,7 @@ class CachingHandlerResponse(BaseModel): cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call + embedding_uncached_input: list[str | list[int]] | None = None in_memory_cache_obj: Final = InMemoryCache() @@ -168,6 +169,37 @@ def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None: return request_kwargs.get("cache_key", None) +class _CachedEmbeddingRecord(BaseModel): + model_config = ConfigDict(frozen=True) + + embedding: list[float] | str | None + index: int | None + object: str | None + model: str | None + prompt_tokens: int | None + prompt_tokens_details: dict | None + format_version: int + + +def _current_format_embedding_entry(entry: object) -> CachedEmbedding | None: + try: + record: Final = _CachedEmbeddingRecord.model_validate(entry) + except ValidationError: + return None + if record.format_version != EMBEDDING_CACHE_FORMAT_VERSION: + return None + cached: Final[CachedEmbedding] = { + "embedding": record.embedding, + "index": record.index, + "object": record.object, + "model": record.model, + "prompt_tokens": record.prompt_tokens, + "prompt_tokens_details": record.prompt_tokens_details, + "format_version": record.format_version, + } + return cached + + class LLMCachingHandler: def __init__( self, @@ -320,6 +352,7 @@ class LLMCachingHandler: return CachingHandlerResponse( final_embedding_cached_response=final_embedding_cached_response, embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, + embedding_uncached_input=self.handle_kwargs_input_list_or_str(kwargs), ) verbose_logger.debug("CACHE RESULT: %s", cached_result) @@ -657,32 +690,30 @@ class LLMCachingHandler: if _caching_handler_response.final_embedding_cached_response is None: return embedding_response - idx = 0 - final_data_list: Final = [] - for item in _caching_handler_response.final_embedding_cached_response.data: - if item is None and embedding_response.data is not None: - final_data_list.append(embedding_response.data[idx]) - idx += 1 - else: - final_data_list.append(item) - - _caching_handler_response.final_embedding_cached_response.data = final_data_list - _caching_handler_response.final_embedding_cached_response._hidden_params["cache_hit"] = True - _caching_handler_response.final_embedding_cached_response._response_ms = ( - end_time - start_time - ).total_seconds() * 1000 - - ## USAGE - if ( - _caching_handler_response.final_embedding_cached_response.usage is not None - and embedding_response.usage is not None - ): - _caching_handler_response.final_embedding_cached_response.usage = self.combine_usage( - usage1=_caching_handler_response.final_embedding_cached_response.usage, - usage2=embedding_response.usage, - ) - - return _caching_handler_response.final_embedding_cached_response + cached: Final = _caching_handler_response.final_embedding_cached_response + fresh_items: Final = iter(embedding_response.data or ()) + merged_usage: Final = ( + self.combine_usage(usage1=cached.usage, usage2=embedding_response.usage) + if cached.usage is not None and embedding_response.usage is not None + else cached.usage + ) + merged: Final = EmbeddingResponse( + model=cached.model, + data=[ # mutable-ok: EmbeddingResponse.data is a pydantic list field + item + if item is not None + else Embedding(embedding=next(fresh_items)["embedding"], index=position, object="embedding") + for position, item in enumerate(cached.data) + ], + usage=merged_usage, + hidden_params={ # mutable-ok: EmbeddingResponse._hidden_params is a mutable dict field + **cached._hidden_params, + "cache_hit": True, + }, + _response_headers=cached._response_headers, + ) + merged._response_ms = (end_time - start_time).total_seconds() * 1000 + return merged def _async_log_cache_hit_on_callbacks( self, @@ -770,7 +801,7 @@ class LLMCachingHandler: dynamic_cache_object=self.dual_cache, ) ) - cached_result = await asyncio.gather(*tasks) + cached_result = [_current_format_embedding_entry(entry) for entry in await asyncio.gather(*tasks)] ## check if cached result is None ## if cached_result is not None and isinstance(cached_result, list): # set cached_result to None if all elements are None diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 747f202d35e..6d42556a9c4 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Final, Literal, Optional, Union from pydantic import BaseModel -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class LiteLLMCacheType(str, Enum): @@ -137,12 +137,16 @@ class HealthCheckCacheParams(BaseModel): redis_version: str | int | float | None = None +EMBEDDING_CACHE_FORMAT_VERSION: Final = 2 + + class CachedEmbedding(TypedDict): """Type definition for cached embedding objects""" - embedding: list[float] | None - index: int | None - object: str | None - model: str | None - prompt_tokens: int | None - prompt_tokens_details: dict | None + embedding: ReadOnly[list[float] | str | None] + index: ReadOnly[int | None] + object: ReadOnly[str | None] + model: ReadOnly[str | None] + prompt_tokens: ReadOnly[int | None] + prompt_tokens_details: ReadOnly[dict | None] + format_version: ReadOnly[int] diff --git a/litellm/utils.py b/litellm/utils.py index b9d56b25653..b5d396030f7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2015,13 +2015,19 @@ def client(original_function): print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL + call_kwargs: Final = ( + {**kwargs, "input": _caching_handler_response.embedding_uncached_input} + if _caching_handler_response is not None + and _caching_handler_response.embedding_uncached_input is not None + else kwargs + ) try: - result = await original_function(*args, **kwargs) + result = await original_function(*args, **call_kwargs) except Exception as deployment_error: _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with try: await async_post_call_failure_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, exception=deployment_error, call_type=call_type, ) @@ -2062,7 +2068,7 @@ def client(original_function): post_call_processing( original_response=result, model=model, - optional_params=kwargs, + optional_params=call_kwargs, original_function=original_function, rules_obj=rules_obj, ) @@ -2070,7 +2076,7 @@ def client(original_function): _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: result = await async_post_call_success_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, response=result, call_type=_call_type_enum, ) @@ -2079,7 +2085,7 @@ def client(original_function): await _llm_caching_handler.async_set_cache( result=result, original_function=original_function, - kwargs=kwargs, + kwargs=call_kwargs, args=args, ) diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index c7ec8abc31e..2e4122530d8 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -1,3 +1,4 @@ +import asyncio import logging import re from unittest.mock import MagicMock @@ -6,8 +7,9 @@ import pytest import litellm.caching.redis_cache as redis_cache_module from litellm.caching.caching import Cache +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.caching.redis_cache import RedisCache, _RedisTimeoutLogThrottle -from litellm.types.caching import LiteLLMCacheType, SemanticCacheScope +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, LiteLLMCacheType, SemanticCacheScope from litellm.types.utils import Embedding, EmbeddingResponse, Usage @@ -278,3 +280,112 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key( model="claude-sonnet-4-5", messages=messages, **anthropic_param ) + + +@pytest.mark.asyncio +async def test_embedding_cache_skips_write_when_one_input_yields_many_embeddings(monkeypatch): + """A cross-encoder behind /embeddings returns one score per document for a single + input string; caching data[0] per input would make the second call return 1 score.""" + import litellm + from litellm import CustomLLM + + class ScoreEveryDocument(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding=[float(i)], index=i, object="embedding") for i in range(5)], + ) + + scorer = ScoreEveryDocument() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "score-every-doc", "custom_handler": scorer}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "score-every-doc"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "score-every-doc"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + batch = '{"query": "q", "documents": ["a", "b", "c", "d", "e"]}' + first = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + + assert scorer.provider_calls == 2 + assert [len(first.data), len(second.data)] == [5, 5] + + +@pytest.mark.asyncio +async def test_embedding_cache_refetches_entries_written_without_format_version(monkeypatch): + import litellm + from litellm import CustomLLM + + class EmbedLength(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = EmbedLength() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-length", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-length"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-length"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + await litellm.aembedding(model="embed-length/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + store = litellm.cache.cache.cache_dict + stored = [entry["response"] for entry in store.values()] + assert [entry["format_version"] for entry in stored] == [EMBEDDING_CACHE_FORMAT_VERSION], stored + legacy_store = { + key: { + **entry, + "response": { + field: value + for field, value in {**entry["response"], "embedding": [-1.0]}.items() + if field != "format_version" + }, + } + for key, entry in store.items() + } + monkeypatch.setattr(litellm.cache.cache, "cache_dict", legacy_store) + + refetched = await litellm.aembedding(model="embed-length/m", input=["abcd"]) + + assert embedder.provider_calls == 2, "an entry written without format_version must be a cache miss" + assert [item["embedding"] for item in refetched.data] == [[4.0]] + + +@pytest.mark.asyncio +async def test_embedding_cache_serves_base64_string_embeddings_on_repeat(monkeypatch): + import litellm + from litellm import CustomLLM + + class Base64Embedder(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding="AACAPwAAAEA=", index=idx, object="embedding") for idx, _ in enumerate(input)], + ) + + embedder = Base64Embedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-b64", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-b64"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-b64"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + first = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + + assert embedder.provider_calls == 1, "a string embedding written to the cache must be served on repeat" + assert [item["embedding"] for item in second.data] == [item["embedding"] for item in first.data] == ["AACAPwAAAEA="] diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 39018dca41d..6956a6932d5 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -11,7 +11,7 @@ from fastapi.testclient import TestClient from datetime import datetime from unittest.mock import AsyncMock -from litellm.caching.caching_handler import LLMCachingHandler +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES, LLMCachingHandler @pytest.mark.asyncio @@ -780,3 +780,46 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl assert hit.cached_result.choices[0].message.content == "done" logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once() assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True + + +@pytest.mark.asyncio +async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_order(monkeypatch): + import litellm + from litellm import CustomLLM + from litellm.caching.caching import Cache + from litellm.types.utils import Embedding, EmbeddingResponse + + class RecordingEmbedder(CustomLLM): + provider_inputs: tuple[tuple[str, ...], ...] = () + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_inputs = (*self.provider_inputs, tuple(input)) + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = RecordingEmbedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "recording-embedder", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "recording-embedder"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "recording-embedder"]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + + await litellm.aembedding(model="recording-embedder/m", input=["aa", "bbbb"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + mixed_input = ["c", "aa", "ddd", "bbbb", "eeeee"] + response = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert embedder.provider_inputs == (("aa", "bbbb"), ("c", "ddd", "eeeee")), embedder.provider_inputs + assert [item["index"] for item in response.data] == [0, 1, 2, 3, 4] + assert [item["embedding"] for item in response.data] == [[float(len(text))] for text in mixed_input] + assert response._hidden_params["cache_hit"] is True, "a partial hit must still be reported as a cache hit" + + repeat = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + + assert len(embedder.provider_inputs) == 2, embedder.provider_inputs + assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input] From 95d4613bd3f64cbbdb752f77c935015166718409 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:14:48 -0700 Subject: [PATCH 03/83] chore(prices): sync xAI prices: 3 models, 3 new [3 with gaps] (#42591) * chore(prices): sync xAI prices: 3 models, 3 new [3 with gaps] xai/grok-code-fast: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema xai/grok-code-fast-1: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema xai/grok-code-fast-1-0825: supports_vision, supports_prompt_caching, input_cost_per_token, output_cost_per_token, input_cost_per_image_token, cache_read_input_token_cost, input_cost_per_token_above_200k_tokens, output_cost_per_token_above_200k_tokens, cache_read_input_token_cost_above_200k_tokens, supports_function_calling, supports_tool_choice, supports_response_schema * fix(prices): add context limits and reasoning flag to xai grok-code-fast aliases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prices): keep the xai sync diff limited to the alias fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 63 +++++++++++++++++++ model_prices_and_context_window.json | 63 +++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3f1d26e9adc..224129fbd25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72335,5 +72335,68 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3f1d26e9adc..224129fbd25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72335,5 +72335,68 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 238f4341532ae6dd150de55e9302c08a8e031c66 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:16:46 -0700 Subject: [PATCH 04/83] fix(otel): record the GenAI exception event through the Logs API on both OpenTelemetry lines (#42431) * fix(otel): record the GenAI exception event without the removed Events API OpenTelemetry removed opentelemetry._events in 1.44.0, so the three imports of it broke 7 modules under litellm.integrations.otel, including the entry point. Two things then failed quietly: with LITELLM_OTEL_V2 set the otel callback resolved to None and nothing was exported, and with it unset the newrelic callback was dropped as well, because that branch imports the v2 logger ungated Build and emit the event through the Logs API, which both lines carry. The event name keeps riding the event.name attribute: the event_name log record field that replaces it only exists from 1.44.0, and this package pins 1.28.0, so the attribute is the only form both can write. It is also what the Events API wrote, so exported events keep their shape Emitting a plain record drops the default the Events SDK applied, so the timestamp now falls back to time_ns() here * style(otel): trim the event name key and regression test prose Keep only the constraint a reader cannot infer from the code, that the event_name record field does not exist on the pinned OpenTelemetry line * fix(otel): export the GenAI exception event on both OpenTelemetry 1.28 and 1.44 lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the record selection comment Co-authored-by: Pawan-Shahane Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): collapse the record selection conditional for ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): build the record fields with a dict literal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): set the native event_name on the 1.44 log record Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): spell out the record kwargs so the type gate sees each call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): suppress the version-gated kwargs for the type gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(otel): drop the version-window prose and correct the event_name suppression reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(otel): wrap the compat test docstring to the line limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Pawan-Shahane Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/logger.py | 2 +- litellm/integrations/otel/model/semconv.py | 1 + litellm/integrations/otel/plumbing/events.py | 54 +++++--- .../integrations/otel/plumbing/providers.py | 8 +- .../otel/test_otel_v2_components.py | 125 ++++++++++++++++++ 5 files changed, 168 insertions(+), 22 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6b673967427..20cdf9662e1 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -240,7 +240,7 @@ class OpenTelemetryV2(CustomLogger): provider: Final = resolve_logger_provider(self.config, logger_provider) if provider is None: return None - return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) # ====================================================================== # # Proxy global registration diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index d3628005bac..f552ba37655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -245,6 +245,7 @@ class GenAIEvent: details, unlike the deprecated ``error.message`` span attribute. """ + NAME_KEY: Final = "event.name" OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py index e7b8e22ddcd..e95d31f886f 100644 --- a/litellm/integrations/otel/plumbing/events.py +++ b/litellm/integrations/otel/plumbing/events.py @@ -9,18 +9,28 @@ emitting that event; the exporter pipeline it rides is built in """ from dataclasses import dataclass +from time import time_ns from typing import Final -from opentelemetry._events import Event, EventLogger +from opentelemetry._logs import Logger, LogRecord from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk.resources import Resource from opentelemetry.trace import SpanContext from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent +try: + from opentelemetry.sdk._logs import LogRecord as _SDKLogRecord +except ImportError: + _SDKLogRecord = None + +SDK_LOG_RECORD: Final[type[LogRecord] | None] = _SDKLogRecord + @dataclass(frozen=True, slots=True) class GenAIEventRecorder: - event_logger: EventLogger + event_logger: Logger + resource: Resource | None = None def record_operation_exception( self, @@ -30,24 +40,36 @@ class GenAIEventRecorder: stack_trace: str | None, timestamp_ns: int | None, ) -> None: - # ``exception.type`` and ``exception.message`` are the semconv-required - # pair and always ride the event; only the recommended stacktrace is - # conditional on the payload carrying one. stacktrace: Final = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () - self.event_logger.emit( - Event( - name=GenAIEvent.OPERATION_EXCEPTION, - timestamp=timestamp_ns, + attributes: Final = dict( + ( + (GenAIEvent.NAME_KEY, GenAIEvent.OPERATION_EXCEPTION), + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ) + record: Final[LogRecord] = ( + SDK_LOG_RECORD( + timestamp=timestamp_ns or time_ns(), trace_id=span_context.trace_id, span_id=span_context.span_id, trace_flags=span_context.trace_flags, severity_number=SeverityNumber.WARN, - attributes=dict( - ( - (ExceptionEvent.TYPE, error_type), - (ExceptionEvent.MESSAGE, message), - *stacktrace, - ) - ), + body=message, + attributes=attributes, + resource=self.resource, # pyright: ignore[reportCallIssue] # SDK-only kwarg absent from the API LogRecord signature on the pin + ) + if SDK_LOG_RECORD is not None + else LogRecord( + timestamp=timestamp_ns or time_ns(), + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + body=message, + attributes=attributes, + event_name=GenAIEvent.OPERATION_EXCEPTION, # pyright: ignore[reportCallIssue] # kwarg exists only on OTel 1.38+, absent from the pinned API signature ) ) + self.event_logger.emit(record) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index d52736a1303..e3474edaf14 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -9,11 +9,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics, trace -from opentelemetry._events import EventLogger -from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider +from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.sdk._events import EventLoggerProvider from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider from opentelemetry.sdk._logs.export import ( BatchLogRecordProcessor, @@ -1042,8 +1040,8 @@ def resolve_logger_provider( return provider -def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: - return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> Logger: + return provider.get_logger(name, litellm_version) def build_meter_provider( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 79747ac9956..07705e17d9a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -1290,6 +1290,50 @@ def test_operation_exception_log_event_always_carries_required_pair(): assert ExceptionEvent.STACKTRACE not in attributes +def test_operation_exception_log_event_records_without_the_events_api(): + """Recording must not import the Events API modules (removed upstream in 1.44.0); + the SDK record path still exports.""" + import importlib + import sys + from unittest.mock import patch + + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + plumbing = ("litellm.integrations.otel.plumbing.events", "litellm.integrations.otel.plumbing.providers") + without_events_api = { + **{name: module for name, module in sys.modules.items() if name not in plumbing}, + "opentelemetry._events": None, + "opentelemetry.sdk._events": None, + } + with patch.dict(sys.modules, without_events_api, clear=True): + events_mod = importlib.import_module(plumbing[0]) + providers_mod = importlib.import_module(plumbing[1]) + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers_mod.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = events_mod.GenAIEventRecorder(providers_mod.get_event_logger(logger_provider)) + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.severity_number == SeverityNumber.WARN + assert record.timestamp is not None + + def test_operation_exception_log_event_not_emitted_on_success(): engine, span_exporter, log_exporter = _engine_with_event_recorder() engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) @@ -1425,6 +1469,87 @@ def test_genai_mapper_guardrail_cost_in_spend_attr(): assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed)) +def _sampled_span_context(): + from opentelemetry.trace import SpanContext, TraceFlags, TraceState + + return SpanContext( + trace_id=0x0AF7651916CD43DD8448EB211C80319C, + span_id=0x00F067AA0BA902B7, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + ) + + +def test_operation_exception_log_event_exports_through_console_exporter(): + """The emitted record serializes through a real SDK exporter: the console + exporter only handles SDK-shaped records (``to_json`` plus a resource), so + an API-shaped record crashed the export under the repo's pinned OTel.""" + import io + import json as json_mod + + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import ConsoleLogExporter, SimpleLogRecordProcessor + from opentelemetry.sdk.resources import Resource + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + out = io.StringIO() + logger_provider = LoggerProvider(resource=Resource.create({"service.name": "otel-event-test"})) + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(ConsoleLogExporter(out=out))) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + exported = json_mod.loads(out.getvalue()) + assert exported["attributes"][GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert exported["attributes"][ExceptionEvent.TYPE] == "RateLimitError" + assert exported["attributes"][ExceptionEvent.MESSAGE] == "rate limited" + assert exported["body"] == "rate limited" + assert exported["resource"]["attributes"]["service.name"] == "otel-event-test" + + +def test_operation_exception_log_event_encodes_for_otlp(): + """The OTLP log encoder reads ``log_record.resource`` and rejects a None + body on the pinned OTel line, so the event must encode into a real + ExportLogsServiceRequest, not only land in an in-memory exporter.""" + from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.model.semconv import GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + request = encode_logs(log_exporter.get_finished_logs()) + (resource_logs,) = request.resource_logs + (scope_logs,) = resource_logs.scope_logs + (encoded,) = scope_logs.log_records + encoded_attrs = {a.key: a.value.string_value for a in encoded.attributes} + assert encoded_attrs[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert encoded.body.string_value == "rate limited" + resource_attrs = {a.key: a.value.string_value for a in resource_logs.resource.attributes} + assert resource_attrs["service.name"] == logger_provider.resource.attributes["service.name"] + + + + def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: for key in ( "SSL_VERIFY", From 3db94b932ec792c101149acafc3a1ff2e3004456 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:44:00 -0700 Subject: [PATCH 05/83] fix(spend): return 400 from /spend/calculate for a model with no pricing row (#42497) * fix(spend): return 400 from /spend/calculate for a model with no pricing row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): assert error type and param for unpriced /spend/calculate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): move the repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: alias ModelNotMappedError re-export to satisfy F401 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(utils): raise ModelNotMappedError only when the pricing row is missing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/exceptions.py | 4 ++++ .../spend_management_endpoints.py | 7 +++++++ litellm/utils.py | 18 +++++++++++------ tests/integration/contracts.json | 3 +++ .../integration/spend/test_spend_calculate.py | 20 +++++++++++++++++++ .../test_spend_management_endpoints.py | 17 ++++++++++++++++ tests/test_litellm/test_utils.py | 7 +++++++ 8 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 tests/integration/spend/test_spend_calculate.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 471b273f00d..c8df4394a06 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1402,6 +1402,7 @@ from .exceptions import ( JSONSchemaValidationError, LITELLM_EXCEPTION_TYPES, MockException, + ModelNotMappedError as ModelNotMappedError, ) from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server diff --git a/litellm/exceptions.py b/litellm/exceptions.py index c8de2ab12ed..3bae8a95ef6 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -991,6 +991,10 @@ LITELLM_EXCEPTION_TYPES: Final = [ ] +class ModelNotMappedError(Exception): + pass + + class BudgetExceededError(Exception): def __init__( self, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index b5980f9b224..822b827f985 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2323,6 +2323,13 @@ async def calculate_spend(request: SpendCalculateRequest): param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), ) + if isinstance(e, litellm.exceptions.ModelNotMappedError): + raise ProxyException( + message=str(e), + type="invalid_request_error", + param="model", + code=status.HTTP_400_BAD_REQUEST, + ) error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), diff --git a/litellm/utils.py b/litellm/utils.py index b5d396030f7..01f6fee3594 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -472,6 +472,7 @@ from .exceptions import ( BudgetExceededError, ContentPolicyViolationError, ContextWindowExceededError, + ModelNotMappedError, NotFoundError, OpenAIError, PermissionDeniedError, @@ -5830,6 +5831,13 @@ def _is_potential_model_name_in_model_cost( _ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN +def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str: + return ( + f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. " + "Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." + ) + + def _get_model_info_helper( model: str, custom_llm_provider: str | None = None, @@ -6012,9 +6020,7 @@ def _get_model_info_helper( key, _model_info = generalization if _model_info is None or key is None: - raise ValueError( - "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" - ) + raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") if _input_cost_per_token is None: # default value to 0, be noisy about this @@ -6249,11 +6255,11 @@ def _get_model_info_helper( if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: returned_model_info[cost_key] = cost_value return returned_model_info + except ModelNotMappedError: + raise except Exception as e: verbose_logger.debug("Error getting model info: %s", e) - raise Exception( - f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json." - ) + raise Exception(_model_not_mapped_message(model, custom_llm_provider)) def _build_model_info( diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index cdf534bb5a1..1ae760467f1 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -275,6 +275,9 @@ "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" ], + "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ + "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" + ], "tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [ "mgmt.key.update.project_detach_denied_to_restricted_actor" ], diff --git a/tests/integration/spend/test_spend_calculate.py b/tests/integration/spend/test_spend_calculate.py new file mode 100644 index 00000000000..855dd2b21f3 --- /dev/null +++ b/tests/integration/spend/test_spend_calculate.py @@ -0,0 +1,20 @@ +import uuid +from typing import Final + +import pytest +from integration._support.client import JSON_OBJECT, Gateway, object_value, string_value + + +@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.rejects_unpriced_model") +def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> None: + model: Final = f"openrouter/integration-unpriced-{uuid.uuid4().hex}" + response: Final = gateway.request( + "POST", + "/spend/calculate", + {"model": model, "messages": [{"role": "user", "content": "price this request"}]}, + ) + assert response.status_code == 400, response.text + error: Final = object_value(JSON_OBJECT.validate_json(response.text)["error"]) + assert error["type"] == "invalid_request_error", response.text + assert error["param"] == "model", response.text + assert model in string_value(error["message"]), response.text diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index e41c027d962..c6a4173b583 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -257,6 +257,8 @@ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.proxy._types import ( LitellmUserRoles, Member, + ProxyException, + SpendCalculateRequest, SpendLogsPayload, UserAPIKeyAuth, ) @@ -7835,3 +7837,18 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo assert custom_logger.requested_ids == [] finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_calculate_spend_unpriced_model_returns_400(): + model = "openrouter/unit-test-unpriced-model" + with patch("litellm.proxy.proxy_server.llm_router", None): + with pytest.raises(ProxyException) as exc_info: + await spend_management_endpoints.calculate_spend( + SpendCalculateRequest(model=model, messages=[{"role": "user", "content": "hi"}]) + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "model" + assert model in exc_info.value.message diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 07982f51153..79462d16a8c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -230,6 +230,13 @@ def test_get_model_info_prefers_exact_dated_key_over_stripped( assert info["key"] == expected_key +def test_get_model_info_internal_failure_is_not_reported_as_unmapped() -> None: + with patch("litellm.utils._get_potential_model_names", side_effect=RuntimeError("malformed metadata")): + with pytest.raises(Exception, match="This model isn't mapped yet") as exc_info: + litellm.utils._get_model_info_helper(model="gpt-4o", custom_llm_provider="openai") + assert not isinstance(exc_info.value, litellm.ModelNotMappedError) + + def test_check_provider_match_azure_ai_allows_openai_and_azure(): """ Test that azure_ai provider can match openai and azure models. From 58a05a9eae5b31d2a71d1766aa5b5a153d8c2263 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:44:30 -0700 Subject: [PATCH 06/83] fix(anthropic): return 400 instead of 500 when a content list holds a bare string (#42420) * fix(anthropic): skip non-dict content items in beta-header and file-id helpers so malformed content lists return 400 instead of 500 Fixes #42094 Supersedes #42101 Co-authored-by: Pawan-Shahane Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): spawn the DB-less regression proxy with -P so the cwd cannot shadow the pinned checkout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): launch the DB-less proxy via -I -c with an explicit sys.path so python 3.10 works, drop DIRECT_URL, remove restating docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): gate the self-booted DB-less proxy behind the owned_gateway opt-in the Buildkite container cannot satisfy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): move the bare string content item repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): wrap the anthropic bare string wire test to the 120 column limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): wrap anthropic common_utils test literals to the 120 column limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style: fix ruff findings in touched test files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Pawan-Shahane Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 2 +- litellm/llms/anthropic/common_utils.py | 4 +- tests/integration/contracts.json | 6 ++ .../providers/test_anthropic_wire.py | 24 ++++++- ...ore_utils_prompt_templates_common_utils.py | 15 +++- .../anthropic/test_anthropic_common_utils.py | 70 +++++++++++++++++++ 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 1e96e20a03b..6c45622649f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1657,7 +1657,7 @@ def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: if isinstance(content, str): continue for c in content: - if c["type"] == "file": + if isinstance(c, dict) and c["type"] == "file": file_object = cast(ChatCompletionFileObject, c) file_object_file_field = file_object.get("file") if not isinstance(file_object_file_field, dict): diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0c3c8996789..c0e6006633e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -314,7 +314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): _message_content = message.get("content") if _message_content is not None and isinstance(_message_content, list): for content in _message_content: - if "cache_control" in content: + if isinstance(content, dict) and "cache_control" in content: return True return False @@ -359,7 +359,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): for message in messages: if "content" in message and message["content"] is not None and isinstance(message["content"], list): for content in message["content"]: - if "type" in content and content["type"] != "text": + if isinstance(content, dict) and "type" in content and content["type"] != "text": return True return False diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 1ae760467f1..8c4ce4debbc 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -159,6 +159,12 @@ "tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": [ "other.routing.redis.owned_outage_recovers_serving_and_response_cache" ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], + "tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": [ + "other.provider_wire.anthropic.bare_string_content_item_is_client_error" + ], "tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": [ "other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields", "quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates" diff --git a/tests/integration/providers/test_anthropic_wire.py b/tests/integration/providers/test_anthropic_wire.py index 64160fa85aa..27895440712 100644 --- a/tests/integration/providers/test_anthropic_wire.py +++ b/tests/integration/providers/test_anthropic_wire.py @@ -3,7 +3,6 @@ import uuid from typing import Final import pytest - from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server @@ -59,3 +58,26 @@ def test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contra parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) assert parsed["cost_breakdown"]["input_cost"] == pytest.approx(0.0245) assert parsed["cost_breakdown"]["output_cost"] == pytest.approx(0.008) + + +@pytest.mark.covers("other.provider_wire.anthropic.bare_string_content_item_is_client_error") +@pytest.mark.parametrize( + "text", [pytest.param("what type of file is this?", id="type_word"), pytest.param("hello", id="plain")] +) +def test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire( + gateway: Gateway, text: str +) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"upstream must not be reached: {request.target}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-4-5-20250929", api_base=wire.url, api_key="synthetic-anthropic-key" + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 16, "timeout": 5, "messages": [{"role": "system", "content": [text]}]}, + ) + assert response.status_code == 400, response.text + assert wire.drain() == () diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 62cf5680266..79c50bf2369 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -4,7 +4,6 @@ import json import os import sys from typing import Final -from unittest.mock import MagicMock, patch import pytest @@ -356,6 +355,20 @@ def test_get_file_ids_from_messages_file_field_not_dict(): assert get_file_ids_from_messages(messages) == [] +def test_get_file_ids_from_messages_skips_bare_string_content_items(): + messages = [ + { + "role": "user", + "content": [ + "what type of file is this?", + {"type": "file", "file": {"file_id": "file-abc"}}, + ], + } + ] + + assert get_file_ids_from_messages(messages) == ["file-abc"] + + def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): """`update_messages_with_model_file_ids` is also called on user content before provider dispatch. It must tolerate non-OpenAI file blocks the same diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 945033c5cac..1a21b6d4394 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -14,6 +14,7 @@ import json import os import sys from types import SimpleNamespace +from typing import Final from unittest.mock import patch import pytest @@ -2275,3 +2276,72 @@ def test_create_anthropic_model_list_response_lists_ids_as_told(): assert (gpt["id"], gpt["display_name"], gpt["max_input_tokens"]) == ("claude-router-gpt-4o[1m]", "GPT 4o", 1000000) assert (haiku["id"], haiku["display_name"]) == ("claude-haiku-4-5", "claude-haiku-4-5") assert (response["first_id"], response["last_id"]) == ("claude-router-gpt-4o[1m]", "claude-haiku-4-5") + + +class TestMalformedContentListItems: + @pytest.mark.parametrize( + "content", + [ + pytest.param(["what type of file is this?"], id="string_containing_type"), + pytest.param(["how do I set cache_control?"], id="string_containing_cache_control"), + pytest.param([None], id="none_item"), + pytest.param([5], id="int_item"), + pytest.param([["nested"]], id="list_item"), + ], + ) + def test_beta_headers_resolve_for_non_dict_content_items(self, content: list[object]) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + messages: Final = [{"role": "user", "content": content}] + + headers: Final = config.validate_environment( + headers={}, + model="claude-sonnet-4-5", + messages=messages, + optional_params={}, + litellm_params={}, + api_key=FAKE_REGULAR_KEY, + ) + + assert headers["x-api-key"] == FAKE_REGULAR_KEY + assert config.is_cache_control_set(messages) is False + assert config.is_pdf_used(messages) is False + + def test_real_content_parts_still_set_their_beta_headers(self) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + + assert config.is_pdf_used([{"role": "user", "content": [{"type": "image", "source": {}}]}]) is True + assert config.is_pdf_used([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]) is False + assert ( + config.is_cache_control_set( + [ + { + "role": "user", + "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}], + } + ] + ) + is True + ) + + def test_mixed_list_keeps_detecting_the_valid_part(self) -> None: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config: Final = AnthropicModelInfo() + messages: Final = [{"role": "user", "content": ["what type of file is this?", {"type": "image", "source": {}}]}] + + assert config.is_pdf_used(messages) is True + + def test_litellm_completion_rejects_bare_string_content_item_as_bad_request(self) -> None: + import litellm + + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model="anthropic/claude-haiku-4-5-20251001", + messages=[{"role": "user", "content": ["what type of file is this?"]}], + api_key=FAKE_REGULAR_KEY, + max_tokens=5, + ) From 7177d3b6d11dc208e2531f05df4df7a1f33b228a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:50:28 +0000 Subject: [PATCH 07/83] feat(rust): add standalone cost calculator (#42604) * feat: add standalone Rust text pricing crate * feat(rust): harden standalone cost calculator --------- Co-authored-by: Yujong Lee --- litellm-rust/Cargo.lock | 8 + litellm-rust/crates/cost/Cargo.toml | 14 + litellm-rust/crates/cost/README.md | 13 + litellm-rust/crates/cost/benches/calculate.rs | 66 +++ litellm-rust/crates/cost/examples/charge.rs | 40 ++ litellm-rust/crates/cost/src/lib.rs | 405 ++++++++++++++++ litellm-rust/crates/cost/tests/calculation.rs | 458 ++++++++++++++++++ .../cost/tests/generate_python_reference.py | 96 ++++ .../crates/cost/tests/python_reference.tsv | 9 + 9 files changed, 1109 insertions(+) create mode 100644 litellm-rust/crates/cost/Cargo.toml create mode 100644 litellm-rust/crates/cost/README.md create mode 100644 litellm-rust/crates/cost/benches/calculate.rs create mode 100644 litellm-rust/crates/cost/examples/charge.rs create mode 100644 litellm-rust/crates/cost/src/lib.rs create mode 100644 litellm-rust/crates/cost/tests/calculation.rs create mode 100644 litellm-rust/crates/cost/tests/generate_python_reference.py create mode 100644 litellm-rust/crates/cost/tests/python_reference.tsv diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index d9425fc6bd7..f3510000c44 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2956,6 +2956,14 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-cost" +version = "0.1.0" +dependencies = [ + "criterion", + "proptest", +] + [[package]] name = "litellm-framing" version = "0.1.0" diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml new file mode 100644 index 00000000000..b85f8a8f876 --- /dev/null +++ b/litellm-rust/crates/cost/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-cost" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dev-dependencies] +criterion.workspace = true +proptest.workspace = true + +[[bench]] +name = "calculate" +harness = false diff --git a/litellm-rust/crates/cost/README.md b/litellm-rust/crates/cost/README.md new file mode 100644 index 00000000000..ee4722b6438 --- /dev/null +++ b/litellm-rust/crates/cost/README.md @@ -0,0 +1,13 @@ +# litellm-cost + +This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration + +Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values + +The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates + +`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract + +The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes + +This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage diff --git a/litellm-rust/crates/cost/benches/calculate.rs b/litellm-rust/crates/cost/benches/calculate.rs new file mode 100644 index 00000000000..2561b606e6c --- /dev/null +++ b/litellm-rust/crates/cost/benches/calculate.rs @@ -0,0 +1,66 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use litellm_cost::{ + Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates, + Usage, calculate, compile, +}; +use std::hint::black_box; + +fn bench(c: &mut Criterion) { + let pricing = Pricing { + standard: Rates { + input: Rate::Value(0.000002), + output: Rate::Value(0.000008), + cache_read: Rate::Value(0.0000005), + cache_write: Rate::Missing, + cache_write_1h: Rate::Missing, + }, + tiers: &[], + thresholds: &[], + off_peak: None, + }; + let request = Request { + usage: Usage { + prompt_tokens: 1000, + completion_tokens: 200, + cache_read_tokens: 250, + cache_write_tokens: 0, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + }; + let plan = compile(&pricing).unwrap(); + c.bench_function("native_compiled_calculation", |b| { + b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap())) + }); + c.bench_function("native_full_wrapper", |b| { + b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap())) + }); + c.bench_function("native_rate_compilation", |b| { + b.iter(|| black_box(compile(black_box(&pricing)).unwrap())) + }); + let threshold = ThresholdRates { + above_prompt_tokens: 1000, + standard: Rates { + input: Rate::Value(0.000004), + output: Rate::Value(0.000016), + ..Rates::EMPTY + }, + tiers: &[], + }; + let threshold_pricing = Pricing { + thresholds: &[threshold], + ..pricing + }; + let threshold_plan = compile(&threshold_pricing).unwrap(); + c.bench_function("native_threshold_boundary", |b| { + b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap())) + }); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/litellm-rust/crates/cost/examples/charge.rs b/litellm-rust/crates/cost/examples/charge.rs new file mode 100644 index 00000000000..a3a83febd74 --- /dev/null +++ b/litellm-rust/crates/cost/examples/charge.rs @@ -0,0 +1,40 @@ +use litellm_cost::{ + Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile, +}; + +fn main() { + let pricing = Pricing { + standard: Rates { + input: Rate::Value(2.0), + output: Rate::Value(4.0), + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + cache_write_1h: Rate::Missing, + }, + tiers: &[], + thresholds: &[], + off_peak: None, + }; + let request = Request { + usage: Usage { + prompt_tokens: 100, + completion_tokens: 20, + cache_read_tokens: 25, + cache_write_tokens: 10, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + }; + let cost = compile(&pricing).unwrap().calculate(&request).unwrap(); + println!( + "input={} output={} total={}", + cost.input(), + cost.output(), + cost.total() + ); +} diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs new file mode 100644 index 00000000000..32cc755ce15 --- /dev/null +++ b/litellm-rust/crates/cost/src/lib.rs @@ -0,0 +1,405 @@ +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Rate { + Missing, + Null, + Value(f64), +} + +impl Rate { + fn value(self) -> Option { + match self { + Self::Value(value) => Some(value), + Self::Missing | Self::Null => None, + } + } + + fn or(self, fallback: Self) -> Self { + if self.value().is_some() { + self + } else { + fallback + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Rates { + pub input: Rate, + pub output: Rate, + pub cache_read: Rate, + pub cache_write: Rate, + pub cache_write_1h: Rate, +} + +impl Rates { + pub const EMPTY: Self = Self { + input: Rate::Missing, + output: Rate::Missing, + cache_read: Rate::Missing, + cache_write: Rate::Missing, + cache_write_1h: Rate::Missing, + }; + + fn overlay(self, base: Self) -> Self { + Self { + input: self.input.or(base.input), + output: self.output.or(base.output), + cache_read: self.cache_read.or(base.cache_read), + cache_write: self.cache_write.or(base.cache_write), + cache_write_1h: self.cache_write_1h.or(base.cache_write_1h), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ServiceTier { + Standard, + Flex, + Priority, + Fast, + Ultrafast, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ThresholdPolicy { + Exclusive, + Inclusive, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PromptConvention { + IncludesCache, + ExcludesCache, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Usage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub cache_write_5m_tokens: Option, + pub cache_write_1h_tokens: Option, + pub prompt_convention: PromptConvention, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TierRates { + pub tier: ServiceTier, + pub rates: Rates, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ThresholdRates<'a> { + pub above_prompt_tokens: u64, + pub standard: Rates, + pub tiers: &'a [TierRates], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct OffPeakRates { + pub start_utc_minute: u16, + pub end_utc_minute: u16, + pub rates: Rates, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pricing<'a> { + pub standard: Rates, + pub tiers: &'a [TierRates], + pub thresholds: &'a [ThresholdRates<'a>], + pub off_peak: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Request { + pub usage: Usage, + pub service_tier: ServiceTier, + pub threshold_policy: ThresholdPolicy, + pub region_multiplier: Option, + pub billed_at_utc_minute: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Cost { + pub uncached_input: f64, + pub cache_read: f64, + pub cache_write_5m: f64, + pub cache_write_1h: f64, + pub output: f64, + pub multiplier: f64, + pub rates: EffectiveRates, +} + +impl Cost { + pub fn input(self) -> f64 { + (self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h) + * self.multiplier + } + + pub fn output(self) -> f64 { + self.output * self.multiplier + } + + pub fn total(self) -> f64 { + self.input() + self.output() + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EffectiveRates { + pub input: f64, + pub output: f64, + pub cache_read: f64, + pub cache_write_5m: f64, + pub cache_write_1h: f64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PricingError { + MissingInputRate, + MissingOutputRate, + InvalidRate, + InvalidRegionMultiplier, + InvalidBillingTime, + InvalidOffPeakWindow, + CacheExceedsPrompt, + InvalidCacheWriteDetails, + TokenCountOverflow, + DuplicateTier, + DuplicateThreshold, + DuplicateThresholdTier, +} + +fn selected_tier(tier: ServiceTier) -> ServiceTier { + if tier == ServiceTier::Fast { + ServiceTier::Priority + } else { + tier + } +} + +#[derive(Clone, Debug)] +struct CompiledThreshold { + above_prompt_tokens: u64, + standard: Rates, + tiers: Vec, +} + +#[derive(Clone, Debug)] +pub struct PricingPlan { + standard: Rates, + tiers: Vec, + thresholds: Vec, + off_peak: Option, +} + +fn valid_rates(rates: Rates) -> bool { + [ + rates.input, + rates.output, + rates.cache_read, + rates.cache_write, + rates.cache_write_1h, + ] + .into_iter() + .all(|rate| { + rate.value() + .is_none_or(|value| value.is_finite() && value >= 0.0) + }) +} + +fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> { + if tiers.iter().any(|entry| !valid_rates(entry.rates)) { + return Err(PricingError::InvalidRate); + } + if tiers.iter().enumerate().any(|(index, entry)| { + matches!( + entry.tier, + ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast + ) || tiers[..index] + .iter() + .any(|previous| previous.tier == entry.tier) + }) { + return Err(duplicate); + } + Ok(()) +} + +pub fn compile(pricing: &Pricing<'_>) -> Result { + if !valid_rates(pricing.standard) { + return Err(PricingError::InvalidRate); + } + validate_tiers(pricing.tiers, PricingError::DuplicateTier)?; + if let Some(window) = pricing.off_peak { + if window.start_utc_minute >= 1440 + || window.end_utc_minute > 1440 + || window.start_utc_minute >= window.end_utc_minute + { + return Err(PricingError::InvalidOffPeakWindow); + } + if !valid_rates(window.rates) { + return Err(PricingError::InvalidRate); + } + } + let mut thresholds: Vec<_> = pricing + .thresholds + .iter() + .map(|entry| { + if !valid_rates(entry.standard) { + return Err(PricingError::InvalidRate); + } + validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?; + Ok(CompiledThreshold { + above_prompt_tokens: entry.above_prompt_tokens, + standard: entry.standard, + tiers: entry.tiers.to_vec(), + }) + }) + .collect::>()?; + thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens); + if thresholds + .windows(2) + .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens) + { + return Err(PricingError::DuplicateThreshold); + } + Ok(PricingPlan { + standard: pricing.standard, + tiers: pricing.tiers.to_vec(), + thresholds, + off_peak: pricing.off_peak, + }) +} + +impl PricingPlan { + fn resolve_rates( + &self, + request: &Request, + threshold_tokens: u64, + ) -> Result { + let tier = selected_tier(request.service_tier); + let base = self + .tiers + .iter() + .find(|entry| tier != ServiceTier::Standard && entry.tier == tier) + .map_or(self.standard, |entry| entry.rates.overlay(self.standard)); + let threshold = self.thresholds.iter().rev().find(|entry| { + threshold_tokens > entry.above_prompt_tokens + || (request.threshold_policy == ThresholdPolicy::Inclusive + && threshold_tokens == entry.above_prompt_tokens) + }); + let selected = threshold.map_or(base, |entry| { + let standard = entry.standard.overlay(base); + entry + .tiers + .iter() + .find(|specific| tier != ServiceTier::Standard && specific.tier == tier) + .map_or(standard, |specific| specific.rates.overlay(standard)) + }); + match self.off_peak { + None => Ok(selected), + Some(window) => { + if window.start_utc_minute >= 1440 + || window.end_utc_minute > 1440 + || window.start_utc_minute >= window.end_utc_minute + { + return Err(PricingError::InvalidOffPeakWindow); + } + let minute = request + .billed_at_utc_minute + .ok_or(PricingError::InvalidBillingTime)?; + if minute >= 1440 { + return Err(PricingError::InvalidBillingTime); + } + if (window.start_utc_minute..window.end_utc_minute).contains(&minute) { + Ok(window.rates.overlay(selected)) + } else { + Ok(selected) + } + } + } + } + + fn checked_rate(rate: Rate, missing: PricingError) -> Result { + let value = rate.value().ok_or(missing)?; + if !value.is_finite() || value < 0.0 { + return Err(PricingError::InvalidRate); + } + Ok(value) + } + + pub fn calculate(&self, request: &Request) -> Result { + let usage = request.usage; + let cached = usage + .cache_read_tokens + .checked_add(usage.cache_write_tokens) + .ok_or(PricingError::TokenCountOverflow)?; + let (regular, threshold_tokens) = match usage.prompt_convention { + PromptConvention::IncludesCache => ( + usage + .prompt_tokens + .checked_sub(cached) + .ok_or(PricingError::CacheExceedsPrompt)?, + usage.prompt_tokens, + ), + PromptConvention::ExcludesCache => ( + usage.prompt_tokens, + usage + .prompt_tokens + .checked_add(cached) + .ok_or(PricingError::TokenCountOverflow)?, + ), + }; + let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) { + (None, None) => (usage.cache_write_tokens, 0), + (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => { + (five, one) + } + _ => return Err(PricingError::InvalidCacheWriteDetails), + }; + let rates = self.resolve_rates(request, threshold_tokens)?; + let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?; + let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?; + let read = Self::checked_rate( + rates.cache_read.or(rates.input), + PricingError::MissingInputRate, + )?; + let write = Self::checked_rate( + rates.cache_write.or(rates.input), + PricingError::MissingInputRate, + )?; + let write_1h = Self::checked_rate( + rates.cache_write_1h.or(rates.cache_write).or(rates.input), + PricingError::MissingInputRate, + )?; + let multiplier = request.region_multiplier.unwrap_or(1.0); + if !multiplier.is_finite() || multiplier <= 0.0 { + return Err(PricingError::InvalidRegionMultiplier); + } + let cost = Cost { + uncached_input: regular as f64 * input, + cache_read: usage.cache_read_tokens as f64 * read, + cache_write_5m: writes.0 as f64 * write, + cache_write_1h: writes.1 as f64 * write_1h, + output: usage.completion_tokens as f64 * output, + multiplier, + rates: EffectiveRates { + input, + output, + cache_read: read, + cache_write_5m: write, + cache_write_1h: write_1h, + }, + }; + if !cost.total().is_finite() { + return Err(PricingError::TokenCountOverflow); + } + Ok(cost) + } +} + +pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { + compile(pricing)?.calculate(request) +} diff --git a/litellm-rust/crates/cost/tests/calculation.rs b/litellm-rust/crates/cost/tests/calculation.rs new file mode 100644 index 00000000000..8cd06e74ec8 --- /dev/null +++ b/litellm-rust/crates/cost/tests/calculation.rs @@ -0,0 +1,458 @@ +use litellm_cost::{ + OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier, + ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile, +}; + +fn rates(input: Rate, output: Rate) -> Rates { + Rates { + input, + output, + ..Rates::EMPTY + } +} + +fn request() -> Request { + Request { + usage: Usage { + prompt_tokens: 100, + completion_tokens: 20, + cache_read_tokens: 25, + cache_write_tokens: 10, + cache_write_5m_tokens: None, + cache_write_1h_tokens: None, + prompt_convention: PromptConvention::IncludesCache, + }, + service_tier: ServiceTier::Standard, + threshold_policy: ThresholdPolicy::Exclusive, + region_multiplier: None, + billed_at_utc_minute: None, + } +} + +fn pricing(standard: Rates) -> Pricing<'static> { + Pricing { + standard, + tiers: &[], + thresholds: &[], + off_peak: None, + } +} + +#[test] +fn breakdown_and_total_agree() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let result = calculate(&pricing(standard), &request()).unwrap(); + assert_eq!(result.uncached_input, 65.0 * 2.0); + assert_eq!(result.cache_read, 25.0 * 0.5); + assert_eq!(result.cache_write_5m, 10.0 * 3.0); + assert_eq!(result.output(), 20.0 * 4.0); + assert_eq!(result.total(), result.input() + result.output()); + assert_eq!(result.rates.cache_read, 0.5); +} + +#[test] +fn absent_null_and_zero_cache_rates_are_distinct() { + let base = rates(Rate::Value(2.0), Rate::Value(4.0)); + for read in [Rate::Missing, Rate::Null] { + let standard = Rates { + cache_read: read, + ..base + }; + assert_eq!( + calculate(&pricing(standard), &request()).unwrap().input(), + 200.0 + ); + } + let standard = Rates { + cache_read: Rate::Value(0.0), + cache_write: Rate::Value(0.0), + ..base + }; + assert_eq!( + calculate(&pricing(standard), &request()).unwrap().input(), + 130.0 + ); +} + +#[test] +fn equivalent_prompt_conventions_select_the_same_threshold() { + let threshold = ThresholdRates { + above_prompt_tokens: 90, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[threshold], + off_peak: None, + }; + let included = request(); + let excluded = Request { + usage: Usage { + prompt_tokens: 65, + prompt_convention: PromptConvention::ExcludesCache, + ..included.usage + }, + ..included + }; + let plan = compile(&specification).unwrap(); + assert_eq!(plan.calculate(&included), plan.calculate(&excluded)); + assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0); +} + +#[test] +fn split_writes_and_invalid_accounting() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + cache_write_1h: Rate::Value(5.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let base = request(); + let split = Request { + usage: Usage { + cache_write_5m_tokens: Some(4), + cache_write_1h_tokens: Some(6), + ..base.usage + }, + ..base + }; + let result = calculate(&pricing(standard), &split).unwrap(); + assert_eq!(result.cache_write_5m, 12.0); + assert_eq!(result.cache_write_1h, 30.0); + let overlapping = Request { + usage: Usage { + prompt_tokens: 30, + ..split.usage + }, + ..split + }; + assert_eq!( + calculate(&pricing(standard), &overlapping), + Err(PricingError::CacheExceedsPrompt) + ); + let incomplete = Request { + usage: Usage { + cache_write_1h_tokens: None, + ..split.usage + }, + ..split + }; + assert_eq!( + calculate(&pricing(standard), &incomplete), + Err(PricingError::InvalidCacheWriteDetails) + ); +} + +#[test] +fn threshold_tiers_and_boundaries() { + let priority = TierRates { + tier: ServiceTier::Priority, + rates: rates(Rate::Value(3.0), Rate::Missing), + }; + let threshold = ThresholdRates { + above_prompt_tokens: 100, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[ + TierRates { + tier: ServiceTier::Priority, + rates: rates(Rate::Value(7.0), Rate::Missing), + }, + TierRates { + tier: ServiceTier::Flex, + rates: rates(Rate::Value(6.0), Rate::Missing), + }, + ], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[priority], + thresholds: &[threshold], + off_peak: None, + }; + let base = request(); + let no_cache = Request { + usage: Usage { + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + let fast = Request { + service_tier: ServiceTier::Fast, + ..no_cache + }; + let inclusive = Request { + threshold_policy: ThresholdPolicy::Inclusive, + ..fast + }; + let flex = Request { + service_tier: ServiceTier::Flex, + ..inclusive + }; + assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0); + assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0); + assert_eq!( + calculate(&specification, &inclusive).unwrap().input(), + 700.0 + ); + assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0); +} + +#[test] +fn compile_rejects_ambiguous_rates() { + let duplicate = ThresholdRates { + above_prompt_tokens: 100, + standard: Rates::EMPTY, + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(1.0), Rate::Value(1.0)), + tiers: &[], + thresholds: &[duplicate, duplicate], + off_peak: None, + }; + assert_eq!( + compile(&specification).err(), + Some(PricingError::DuplicateThreshold) + ); + let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0))); + assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate)); +} + +#[test] +fn off_peak_is_one_non_wrapping_utc_window() { + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[], + off_peak: Some(OffPeakRates { + start_utc_minute: 60, + end_utc_minute: 120, + rates: rates(Rate::Value(1.0), Rate::Value(2.0)), + }), + }; + let base = request(); + let start = Request { + billed_at_utc_minute: Some(60), + ..base + }; + let end = Request { + billed_at_utc_minute: Some(120), + ..base + }; + assert_eq!( + calculate(&specification, &base), + Err(PricingError::InvalidBillingTime) + ); + assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0); + assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0); +} + +#[test] +fn missing_rates_and_free_rates_remain_distinct() { + let base = request(); + let empty = Request { + usage: Usage { + prompt_tokens: 0, + completion_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + assert_eq!( + calculate(&pricing(Rates::EMPTY), &empty), + Err(PricingError::MissingInputRate) + ); + assert_eq!( + calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty), + Err(PricingError::MissingOutputRate) + ); + assert_eq!( + calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty) + .unwrap() + .total(), + 0.0 + ); +} + +#[test] +fn matches_executed_python_reference_cases() { + for row in include_str!("python_reference.tsv") + .lines() + .filter(|line| !line.starts_with('#')) + { + let fields: Vec<_> = row.split('\t').collect(); + let count = |index: usize| fields[index].parse::().unwrap(); + let number = |index: usize| fields[index].parse::().unwrap(); + let optional_rate = |index: usize| { + if fields[index].is_empty() { + Rate::Missing + } else { + Rate::Value(number(index)) + } + }; + let threshold = ThresholdRates { + above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) }, + standard: rates(optional_rate(10), optional_rate(11)), + tiers: &[], + }; + let thresholds = if fields[9].is_empty() { + &[][..] + } else { + std::slice::from_ref(&threshold) + }; + let specification = Pricing { + standard: Rates { + cache_read: optional_rate(7), + cache_write: optional_rate(8), + ..rates(Rate::Value(number(5)), Rate::Value(number(6))) + }, + tiers: &[], + thresholds, + off_peak: None, + }; + let base = request(); + let input = Request { + usage: Usage { + prompt_tokens: count(1), + completion_tokens: count(2), + cache_read_tokens: count(3), + cache_write_tokens: count(4), + ..base.usage + }, + ..base + }; + let actual = calculate(&specification, &input).unwrap(); + assert_eq!(actual.input(), number(12), "{}", fields[0]); + assert_eq!(actual.output(), number(13), "{}", fields[0]); + } +} + +proptest::proptest! { + #[test] + fn equivalent_usage_conventions_and_breakdown_agree( + regular in 0_u64..1000, + read in 0_u64..1000, + write in 0_u64..1000, + output in 0_u64..1000, + ) { + let threshold = ThresholdRates { + above_prompt_tokens: 1000, + standard: rates(Rate::Value(5.0), Rate::Value(8.0)), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[threshold], + off_peak: None, + }; + let base = request(); + let included = Request { + usage: Usage { + prompt_tokens: regular + read + write, + completion_tokens: output, + cache_read_tokens: read, + cache_write_tokens: write, + ..base.usage + }, + ..base + }; + let excluded = Request { + usage: Usage { + prompt_tokens: regular, + prompt_convention: PromptConvention::ExcludesCache, + ..included.usage + }, + ..included + }; + let plan = compile(&specification).unwrap(); + let left = plan.calculate(&included).unwrap(); + let right = plan.calculate(&excluded).unwrap(); + proptest::prop_assert_eq!(left, right); + proptest::prop_assert_eq!(left.total(), left.input() + left.output()); + } +} + +#[test] +fn regional_multiplier_applies_after_input_components_are_summed() { + let standard = Rates { + cache_read: Rate::Value(0.5), + cache_write: Rate::Value(3.0), + ..rates(Rate::Value(2.0), Rate::Value(4.0)) + }; + let base = request(); + let regional = Request { + region_multiplier: Some(1.1), + ..base + }; + let result = calculate(&pricing(standard), ®ional).unwrap(); + assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1); + assert_eq!(result.output(), 20.0 * 4.0 * 1.1); + let invalid = Request { + region_multiplier: Some(f64::NAN), + ..base + }; + assert_eq!( + calculate(&pricing(standard), &invalid), + Err(PricingError::InvalidRegionMultiplier) + ); +} + +#[test] +fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() { + let high = ThresholdRates { + above_prompt_tokens: 200, + standard: rates(Rate::Value(7.0), Rate::Missing), + tiers: &[], + }; + let low = ThresholdRates { + above_prompt_tokens: 100, + standard: rates(Rate::Value(5.0), Rate::Missing), + tiers: &[], + }; + let specification = Pricing { + standard: rates(Rate::Value(2.0), Rate::Value(4.0)), + tiers: &[], + thresholds: &[high, low], + off_peak: None, + }; + let base = request(); + let above_both = Request { + usage: Usage { + prompt_tokens: 201, + cache_read_tokens: 0, + cache_write_tokens: 0, + ..base.usage + }, + ..base + }; + assert_eq!( + compile(&specification) + .unwrap() + .calculate(&above_both) + .unwrap() + .rates + .input, + 7.0 + ); + let duplicate = TierRates { + tier: ServiceTier::Flex, + rates: Rates::EMPTY, + }; + let invalid = Pricing { + tiers: &[duplicate, duplicate], + thresholds: &[], + ..specification + }; + assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier)); +} diff --git a/litellm-rust/crates/cost/tests/generate_python_reference.py b/litellm-rust/crates/cost/tests/generate_python_reference.py new file mode 100644 index 00000000000..4a179555650 --- /dev/null +++ b/litellm-rust/crates/cost/tests/generate_python_reference.py @@ -0,0 +1,96 @@ +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import Usage + + +@dataclass(frozen=True, slots=True) +class Case: + name: str + prompt: int + completion: int + cache_read: int + cache_write: int + input_rate: float + output_rate: float + cache_read_rate: float | None = None + cache_write_rate: float | None = None + threshold: int | None = None + threshold_input_rate: float | None = None + threshold_output_rate: float | None = None + + +CASES = ( + Case("ordinary", 100, 20, 0, 0, 2.0, 4.0), + Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0), + Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0), + Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0), + Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0), + Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0), + Case( + "threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0 + ), + Case( + "cache_threshold_above", + 101, + 20, + 25, + 10, + 2.0, + 4.0, + threshold=100, + threshold_input_rate=5.0, + threshold_output_rate=8.0, + ), +) + + +def reference(case: Case) -> tuple[float, float]: + info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate} + if case.cache_read_rate is not None: + info["cache_read_input_token_cost"] = case.cache_read_rate + if case.cache_write_rate is not None: + info["cache_creation_input_token_cost"] = case.cache_write_rate + if case.threshold is not None: + info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate + info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate + details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write} + usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details) + return generic_cost_per_token( + model="synthetic", + usage=usage, + custom_llm_provider="openai", + model_info=info, + ) + + +def main() -> None: + revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip() + rows = ("# Python reference commit: " + revision,) + tuple( + "\t".join( + str(value) if value is not None else "" + for value in ( + case.name, + case.prompt, + case.completion, + case.cache_read, + case.cache_write, + case.input_rate, + case.output_rate, + case.cache_read_rate, + case.cache_write_rate, + case.threshold, + case.threshold_input_rate, + case.threshold_output_rate, + *reference(case), + ) + ) + for case in CASES + ) + Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n") + + +if __name__ == "__main__": + main() diff --git a/litellm-rust/crates/cost/tests/python_reference.tsv b/litellm-rust/crates/cost/tests/python_reference.tsv new file mode 100644 index 00000000000..5abb1329a61 --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_reference.tsv @@ -0,0 +1,9 @@ +# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627 +ordinary 100 20 0 0 2.0 4.0 200.0 80.0 +cache_fallback 100 20 25 10 2.0 4.0 200.0 80.0 +cache_specific 100 20 25 10 2.0 4.0 0.5 3.0 172.5 80.0 +free_cache 100 20 25 10 2.0 4.0 0.0 0.0 130.0 80.0 +threshold_below 99 20 0 0 2.0 4.0 100 5.0 8.0 198.0 80.0 +threshold_at 100 20 0 0 2.0 4.0 100 5.0 8.0 200.0 80.0 +threshold_above 101 20 0 0 2.0 4.0 100 5.0 8.0 505.0 160.0 +cache_threshold_above 101 20 25 10 2.0 4.0 100 5.0 8.0 505.0 160.0 From 392e80717200b36b083ce859c866af75d7721569 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:56:50 -0700 Subject: [PATCH 08/83] feat(logging): add normalized_error cluster key to error_information (#41715) * feat(logging): add normalized_error cluster key to error_information Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): stop classifying parameter length errors as context window errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): assert failure spend rows share normalized_error across provider wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): map agent model access denials and ignore non-string proxy error types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(logging): cover budget exceeded errors with custom wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster router no-healthy and provider-budget wording correctly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): let the exception class win over router fallback wording Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster peer closed connection errors as provider connection errors Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): cluster tag routing denials as 403_MODEL_ACCESS_DENIED Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- .../litellm_core_utils/error_normalization.py | 195 +++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/types/utils.py | 1 + .../coverage_registry/quota_management.yaml | 1 + tests/e2e/models.py | 8 + .../spend_tracking/test_spend_tracking_e2e.py | 41 ++++ .../test_error_normalization.py | 231 ++++++++++++++++++ 7 files changed, 479 insertions(+) create mode 100644 litellm/litellm_core_utils/error_normalization.py create mode 100644 tests/test_litellm/litellm_core_utils/test_error_normalization.py diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py new file mode 100644 index 00000000000..2a9ff748899 --- /dev/null +++ b/litellm/litellm_core_utils/error_normalization.py @@ -0,0 +1,195 @@ +""" +Map any exception litellm logs to one stable ``normalized_error`` code so dashboards can cluster +failures without parsing free-text messages that embed team names, token counts, model names, etc. +""" + +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +from litellm.exceptions import ( + APIConnectionError, + AuthenticationError, + BadGatewayError, + BadRequestError, + BlockedPiiEntityError, + BudgetExceededError, + ContentPolicyViolationError, + ContextWindowExceededError, + GuardrailRaisedException, + InternalServerError, + MidStreamFallbackError, + NotFoundError, + PermissionDeniedError, + RateLimitError, + RateLimitType, + ServiceUnavailableError, + Timeout, + UnprocessableEntityError, + UnsupportedParamsError, +) + +RATE_LIMIT_EXCEEDED: Final = "429_RATE_LIMIT_EXCEEDED" +BUDGET_EXCEEDED: Final = "429_BUDGET_EXCEEDED" +NO_HEALTHY_DEPLOYMENTS: Final = "429_NO_HEALTHY_DEPLOYMENTS" +AUTHENTICATION_FAILED: Final = "401_AUTHENTICATION_FAILED" +MODEL_ACCESS_DENIED: Final = "403_MODEL_ACCESS_DENIED" +PERMISSION_DENIED: Final = "403_PERMISSION_DENIED" +MISSING_REQUIRED_PARAMETER: Final = "400_MISSING_REQUIRED_PARAMETER" +INVALID_PARAMETER_VALUE: Final = "400_INVALID_PARAMETER_VALUE" +CONTEXT_WINDOW_EXCEEDED: Final = "400_CONTEXT_WINDOW_EXCEEDED" +CONTENT_POLICY_VIOLATION: Final = "400_CONTENT_POLICY_VIOLATION" +INVALID_REQUEST: Final = "400_INVALID_REQUEST" +RESOURCE_NOT_FOUND: Final = "404_RESOURCE_NOT_FOUND" +UPSTREAM_TIMEOUT: Final = "408_UPSTREAM_TIMEOUT" +PROVIDER_CONNECTION_ERROR: Final = "500_PROVIDER_CONNECTION_ERROR" +PROVIDER_OVERLOADED: Final = "503_PROVIDER_OVERLOADED" +PROVIDER_INTERNAL_ERROR: Final = "500_PROVIDER_INTERNAL_ERROR" +ROUTER_NO_FALLBACK: Final = "500_ROUTER_NO_FALLBACK" +ROUTER_FALLBACK_FAILURE: Final = "500_ROUTER_FALLBACK_FAILURE" +UPSTREAM_PASSTHROUGH: Final = "500_UPSTREAM_PASSTHROUGH" +UNSUPPORTED_OPERATION: Final = "500_UNSUPPORTED_OPERATION" +INTERNAL_STATE_ERROR: Final = "500_INTERNAL_STATE_ERROR" +UNCLASSIFIED: Final = "UNCLASSIFIED" + + +@runtime_checkable +class _HasProxyErrorType(Protocol): + type: str + + +_MESSAGE_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + ( + re.compile(r"budget has been exceeded|max budget|exceeded.*budget|crossed budget", re.IGNORECASE), + BUDGET_EXCEEDED, + ), + (re.compile(r"no healthy deployments?|no deployments available", re.IGNORECASE), NO_HEALTHY_DEPLOYMENTS), + (re.compile(r"not allowed to access model due to tags configuration", re.IGNORECASE), MODEL_ACCESS_DENIED), + (re.compile(r"upstream passthrough request failed", re.IGNORECASE), UPSTREAM_PASSTHROUGH), + (re.compile(r"is not supported for provider|not implemented", re.IGNORECASE), UNSUPPORTED_OPERATION), + ( + re.compile(r"context window|context length|(prompt|input) is too long|tokens? ?> ?\d+ ?maximum", re.IGNORECASE), + CONTEXT_WINDOW_EXCEEDED, + ), + (re.compile(r"missing required parameter|field required", re.IGNORECASE), MISSING_REQUIRED_PARAMETER), + (re.compile(r"overloaded|unable to process your request", re.IGNORECASE), PROVIDER_OVERLOADED), + ( + re.compile( + r"connection error|APIConnectionError|TransferEncodingError|payload is not completed|connection reset" + r"|peer closed connection|incomplete chunked read", + re.IGNORECASE, + ), + PROVIDER_CONNECTION_ERROR, + ), + (re.compile(r"timed? ?out", re.IGNORECASE), UPSTREAM_TIMEOUT), +) + +_ROUTER_WRAPPER_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = ( + (re.compile(r"no fallback model group found", re.IGNORECASE), ROUTER_NO_FALLBACK), + (re.compile(r"error doing the fallback|MidStreamFallbackError", re.IGNORECASE), ROUTER_FALLBACK_FAILURE), +) + +_PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "budget_exceeded": BUDGET_EXCEEDED, + "auth_error": AUTHENTICATION_FAILED, + "expired_key": AUTHENTICATION_FAILED, + "token_not_found_in_db": AUTHENTICATION_FAILED, + "auth_provider_unavailable": AUTHENTICATION_FAILED, + "key_model_access_denied": MODEL_ACCESS_DENIED, + "team_model_access_denied": MODEL_ACCESS_DENIED, + "user_model_access_denied": MODEL_ACCESS_DENIED, + "org_model_access_denied": MODEL_ACCESS_DENIED, + "project_model_access_denied": MODEL_ACCESS_DENIED, + "agent_model_access_denied": MODEL_ACCESS_DENIED, + "key_vector_store_access_denied": PERMISSION_DENIED, + "team_vector_store_access_denied": PERMISSION_DENIED, + "org_vector_store_access_denied": PERMISSION_DENIED, + "tool_access_denied": PERMISSION_DENIED, + "team_member_permission_error": PERMISSION_DENIED, + "not_found_error": RESOURCE_NOT_FOUND, + } +) + +_STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "400": INVALID_REQUEST, + "401": AUTHENTICATION_FAILED, + "403": PERMISSION_DENIED, + "404": RESOURCE_NOT_FOUND, + "408": UPSTREAM_TIMEOUT, + "422": INVALID_PARAMETER_VALUE, + "429": RATE_LIMIT_EXCEEDED, + "500": PROVIDER_INTERNAL_ERROR, + "502": PROVIDER_INTERNAL_ERROR, + "503": PROVIDER_OVERLOADED, + "504": UPSTREAM_TIMEOUT, + } +) + +_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( + TypeError, + KeyError, + AttributeError, + IndexError, + RuntimeError, + AssertionError, + ZeroDivisionError, +) + +_CLASS_CODE_TABLE: Final[tuple[tuple[tuple[type[BaseException], ...], str], ...]] = ( + ((AuthenticationError,), AUTHENTICATION_FAILED), + ((PermissionDeniedError,), PERMISSION_DENIED), + ((ContextWindowExceededError,), CONTEXT_WINDOW_EXCEEDED), + ((ContentPolicyViolationError, GuardrailRaisedException, BlockedPiiEntityError), CONTENT_POLICY_VIOLATION), + ((UnsupportedParamsError,), INVALID_PARAMETER_VALUE), + ((NotFoundError,), RESOURCE_NOT_FOUND), + ((Timeout,), UPSTREAM_TIMEOUT), + ((MidStreamFallbackError,), ROUTER_FALLBACK_FAILURE), + ((APIConnectionError,), PROVIDER_CONNECTION_ERROR), + ((ServiceUnavailableError,), PROVIDER_OVERLOADED), + ((InternalServerError, BadGatewayError), PROVIDER_INTERNAL_ERROR), + ((BadRequestError, UnprocessableEntityError), INVALID_REQUEST), + ((NotImplementedError,), UNSUPPORTED_OPERATION), +) + + +def _classify_by_message(message: str, patterns: tuple[tuple[re.Pattern[str], str], ...]) -> str | None: + return next((code for pattern, code in patterns if pattern.search(message)), None) + + +def _classify_by_class(exc: Exception) -> str | None: + if isinstance(exc, BudgetExceededError): + return BUDGET_EXCEEDED + if isinstance(exc, RateLimitError): + return BUDGET_EXCEEDED if exc.rate_limit_type == RateLimitType.BUDGET.value else RATE_LIMIT_EXCEEDED + for exc_types, code in _CLASS_CODE_TABLE: + if isinstance(exc, exc_types): + return code + if isinstance(exc, _INTERNAL_STATE_EXCEPTIONS): + return INTERNAL_STATE_ERROR + return None + + +def normalize_error(exc: Exception | None, status_code: str, message: str) -> str | None: + """ + Return a stable cluster key for ``exc``. ``status_code`` and ``message`` are the values + ``get_error_information`` already extracted, so the same exception always yields the same code. + """ + if exc is None: + return None + proxy_type: Final = exc.type if isinstance(exc, _HasProxyErrorType) else None + by_proxy_type: Final = _PROXY_ERROR_TYPE_MAP.get(proxy_type) if isinstance(proxy_type, str) else None + if by_proxy_type is not None: + return by_proxy_type + by_message: Final = _classify_by_message(message, _MESSAGE_PATTERNS) + if by_message is not None: + return by_message + by_class: Final = _classify_by_class(exc) + if by_class is not None: + return by_class + by_router_wrapper: Final = _classify_by_message(message, _ROUTER_WRAPPER_PATTERNS) + if by_router_wrapper is not None: + return by_router_wrapper + return _STATUS_CODE_MAP.get(status_code, UNCLASSIFIED) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6e5a37a7226..8e28a0d543d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.litellm_core_utils.core_helpers import ( reconstruct_model_name, set_response_cost_in_hidden_params, ) +from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -6100,6 +6101,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + normalized_error=normalize_error(original_exception, error_status, error_message), ) @staticmethod diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f2c8f0e7044..dfc98a9d89d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3256,6 +3256,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + normalized_error: ReadOnly[str | None] class GuardrailMode(TypedDict, total=False): diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 5ea48fdcd9d..5740a878608 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -51,6 +51,7 @@ - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} +- {id: quota_management.spend_tracking.failure.writes_normalized_error, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_normalized_error], exercised_on: [chat_completions], source: "litellm_core_utils/error_normalization.py", rationale: "Failure rows carry a stable metadata.error_information.normalized_error key next to the unchanged error_message, so two upstream auth failures with different provider wording share one cluster key a dashboard can group by"} - {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index bc81d1015ba..bd7f5171172 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -937,10 +937,18 @@ class GuardrailRunRecord(BaseModel): guardrail_response: object | None = None +class SpendLogErrorInformation(BaseModel): + error_code: str | None = None + error_class: str | None = None + error_message: str | None = None + normalized_error: str | None = None + + class SpendLogMetadata(BaseModel): user_api_key_alias: str | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None + error_information: SpendLogErrorInformation | None = None class SpendLogRow(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 6ca6f8cee55..286421e2e3f 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -499,6 +499,47 @@ def test_failure_call_writes_failure_status_row( assert (failure_row.spend or 0) == 0.0, "failed call must not be charged" +@pytest.mark.covers("quota_management.spend_tracking.failure.writes_normalized_error") +def test_failure_rows_share_normalized_error_across_provider_wording( + client: SpendClient, resources: ResourceManager, scoped_key: str +) -> None: + """Two upstream auth failures with different provider wording land as failure rows + whose metadata.error_information keeps each provider's own error_message and + carries the same stable normalized_error cluster key.""" + marker = unique_marker() + deployments: Final = ( + (f"e2e-norm-openai-{marker}", "openai/gpt-5.5"), + (f"e2e-norm-anthropic-{marker}", "anthropic/claude-haiku-4-5"), + ) + for name, provider_model in deployments: + model_id = client.proxy.create_model( + name, LiteLLMParamsBody(model=provider_model, api_key=f"sk-invalid-{marker}") + ) + resources.defer(lambda model_id=model_id: client.proxy.delete_model(model_id)) + result = client.chat(scoped_key, name, f"normalize failure {marker}", max_tokens=1) + assert not is_ok(result), f"{name}: invalid upstream key must fail the call, got {result}" + + rows = client.poll_logs_for_key( + scoped_key, + min_rows=2, + predicate=lambda rs: sum(1 for r in rs if r.status == "failure") >= 2, + ) + failure_rows = [r for r in rows if r.status == "failure"] + assert len(failure_rows) == 2, f"expected one failure row per deployment: {_summarize(rows)}" + + infos = [r.metadata.error_information if r.metadata else None for r in failure_rows] + assert all(info is not None for info in infos), ( + f"failure rows must carry metadata.error_information: {[r.model_dump() for r in failure_rows]}" + ) + messages = {info.error_message for info in infos if info is not None} + assert len(messages) == 2, f"provider wording must stay distinct in error_message: {messages}" + normalized = {info.normalized_error for info in infos if info is not None} + assert normalized == {"401_AUTHENTICATION_FAILED"}, ( + f"both auth failures must share one normalized_error cluster key; saw {normalized} " + f"for messages {messages}" + ) + + @pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") def test_pre_call_rejection_row_attributes_provider_and_model_id( client: SpendClient, resources: ResourceManager diff --git a/tests/test_litellm/litellm_core_utils/test_error_normalization.py b/tests/test_litellm/litellm_core_utils/test_error_normalization.py new file mode 100644 index 00000000000..9d5469ddb4c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_error_normalization.py @@ -0,0 +1,231 @@ +import httpx +import pytest + +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.error_normalization import normalize_error +from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.types.router import RouterErrors + +_RESPONSE = httpx.Response(status_code=500, request=httpx.Request("POST", "https://example.invalid")) + + +def _proxy_exc(message: str, error_type: str, code: int) -> ProxyException: + return ProxyException(message=message, type=error_type, param=None, code=code) + + +@pytest.mark.parametrize( + ("messages", "expected"), + [ + ( + ( + _proxy_exc("Rate limit exceeded for team X. Reset at 10:01", "rate_limit_error", 429), + _proxy_exc("Rate limit exceeded for team Y. Reset at 10:02", "rate_limit_error", 429), + ), + "429_RATE_LIMIT_EXCEEDED", + ), + ( + ( + litellm.BudgetExceededError(current_cost=3501.85, max_budget=3500), + _proxy_exc( + "User=abc, Current cost=1000.03, Max budget=1000", ProxyErrorTypes.budget_exceeded.value, 400 + ), + litellm.RateLimitError( + "budget", + llm_provider="openai", + model="gpt", + rate_limit_type=litellm.exceptions.RateLimitType.BUDGET, + ), + ), + "429_BUDGET_EXCEEDED", + ), + ( + ( + _proxy_exc("Token Expired", ProxyErrorTypes.expired_key.value, 401), + _proxy_exc("Malformed API Key", ProxyErrorTypes.auth_error.value, 401), + litellm.AuthenticationError("Signature verification failed", llm_provider="azure", model="gpt"), + ), + "401_AUTHENTICATION_FAILED", + ), + ( + ( + _proxy_exc("No team has access to gpt-5.5-mini", ProxyErrorTypes.team_model_access_denied.value, 401), + _proxy_exc("key not allowed to access claude", ProxyErrorTypes.key_model_access_denied.value, 401), + ValueError( + "Not allowed to access model due to tags configuration. Passed model=gpt-5.5 and tags=['team-a']" + ), + ), + "403_MODEL_ACCESS_DENIED", + ), + ( + ( + _proxy_exc("Missing required parameter: messages", ProxyErrorTypes.bad_request_error.value, 400), + litellm.BadRequestError("Missing required parameter: input", llm_provider="openai", model="gpt"), + ), + "400_MISSING_REQUIRED_PARAMETER", + ), + ( + ( + litellm.ContextWindowExceededError( + "1002823 tokens > 1000000 maximum", model="g", llm_provider="vertex" + ), + litellm.BadRequestError("Input is too long for requested model", llm_provider="anthropic", model="c"), + ), + "400_CONTEXT_WINDOW_EXCEEDED", + ), + ( + ( + litellm.NotFoundError("Response id xxx not found", llm_provider="openai", model="gpt"), + _proxy_exc("No vector store found with id abc", ProxyErrorTypes.not_found_error.value, 404), + ), + "404_RESOURCE_NOT_FOUND", + ), + ( + ( + litellm.APIConnectionError("Connection error", llm_provider="openai", model="gpt"), + litellm.InternalServerError("TransferEncodingError", llm_provider="openai", model="gpt"), + litellm.APIError(500, "Response payload is not completed", llm_provider="openai", model="gpt"), + httpx.RemoteProtocolError( + "peer closed connection without sending complete message body (incomplete chunked read)" + ), + ), + "500_PROVIDER_CONNECTION_ERROR", + ), + ( + ( + litellm.ServiceUnavailableError("server_is_overloaded", llm_provider="anthropic", model="c"), + litellm.InternalServerError( + "Bedrock is unable to process your request", llm_provider="bedrock", model="c" + ), + litellm.APIError(529, "Overloaded", llm_provider="anthropic", model="c"), + ), + "503_PROVIDER_OVERLOADED", + ), + ( + ( + litellm.InternalServerError( + "The server had an error while processing your request", llm_provider="openai", model="gpt" + ), + litellm.APIError(500, "server_error", llm_provider="openai", model="gpt"), + ), + "500_PROVIDER_INTERNAL_ERROR", + ), + ( + ( + _proxy_exc("No fallback model group found for gpt-5.6", "internal_server_error", 500), + _proxy_exc("No fallback model group found for claude-46-sonnet", "internal_server_error", 500), + ), + "500_ROUTER_NO_FALLBACK", + ), + ( + ( + _proxy_exc("Error doing the fallback: RateLimitError", "internal_server_error", 500), + MidStreamFallbackError( + "stream died", model="gpt", llm_provider="openai", original_exception=ValueError("boom") + ), + ), + "500_ROUTER_FALLBACK_FAILURE", + ), + ( + ( + TypeError("cannot pickle '_thread.RLock' object"), + RuntimeError("dictionary changed size during iteration"), + TypeError("'NoneType' object is not iterable"), + ), + "500_INTERNAL_STATE_ERROR", + ), + ( + ( + litellm.Timeout("Timeout on reading data from socket", model="gpt", llm_provider="openai"), + litellm.APIError(504, "Request timed out", llm_provider="openai", model="gpt"), + ), + "408_UPSTREAM_TIMEOUT", + ), + ( + ( + _proxy_exc("500: Upstream passthrough request failed", "internal_server_error", 500), + _proxy_exc("503: Upstream passthrough request failed", "internal_server_error", 503), + ), + "500_UPSTREAM_PASSTHROUGH", + ), + ( + ( + _proxy_exc("OCR is not supported for provider openai", "internal_server_error", 500), + NotImplementedError("rerank"), + ), + "500_UNSUPPORTED_OPERATION", + ), + ], +) +def test_variants_of_one_failure_share_a_normalized_error(messages: tuple[Exception, ...], expected: str) -> None: + normalized = {StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] for exc in messages} + assert normalized == {expected} + + +def test_router_no_healthy_deployment_wording_clusters_as_no_healthy_deployments() -> None: + for message in (RouterErrors.no_healthy_deployments.value, "No healthy deployments found."): + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_NO_HEALTHY_DEPLOYMENTS", message + + +def test_provider_budget_routing_wording_clusters_as_budget_exceeded() -> None: + message = RouterErrors.no_deployments_with_provider_budget_routing.value + exc = litellm.BadRequestError(message, llm_provider="openai", model="gpt-4o") + assert normalize_error(exc, "400", message) == "429_BUDGET_EXCEEDED" + + +def test_router_fallback_wording_does_not_hide_the_wrapped_exception_class() -> None: + provider_message = "litellm.AuthenticationError: OpenAIException - Incorrect API key provided" + exc = litellm.AuthenticationError( + provider_message + "\nNo fallback model group found for lookup_groups=['x']", + llm_provider="openai", + model="gpt", + ) + assert normalize_error(exc, "401", str(exc)) == "401_AUTHENTICATION_FAILED" + wrapped = litellm.AuthenticationError( + "Error doing the fallback: " + provider_message, llm_provider="openai", model="gpt" + ) + assert normalize_error(wrapped, "401", str(wrapped)) == "401_AUTHENTICATION_FAILED" + + +def test_parameter_length_error_is_not_a_context_window_error() -> None: + exc = litellm.BadRequestError("string too long: 'user' max 64 chars", llm_provider="openai", model="gpt") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "400_INVALID_REQUEST" + + +def test_no_exception_has_no_normalized_error() -> None: + assert StandardLoggingPayloadSetup.get_error_information(None)["normalized_error"] is None + + +def test_unknown_exception_falls_back_to_status_then_unclassified() -> None: + assert normalize_error(Exception("x"), "429", "x") == "429_RATE_LIMIT_EXCEEDED" + assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED" + + +def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None: + exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key") + assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED" + + +def test_every_model_access_denied_proxy_type_shares_one_cluster() -> None: + access_denied_types = tuple(t for t in ProxyErrorTypes if t.value.endswith("_model_access_denied")) + assert len(access_denied_types) >= 6, access_denied_types + codes = {normalize_error(_proxy_exc("denied", t.value, 403), "403", "denied") for t in access_denied_types} + assert codes == {"403_MODEL_ACCESS_DENIED"}, codes + + +def test_non_string_type_attribute_falls_through_to_status() -> None: + class _OddType(Exception): + type = {"kind": "odd"} + + assert normalize_error(_OddType("odd"), "500", "odd") == "500_PROVIDER_INTERNAL_ERROR" + + +def test_normalized_error_never_embeds_dynamic_parts() -> None: + exc = _proxy_exc( + "No team has access to anthropic.claude-sonnet-4-5", ProxyErrorTypes.team_model_access_denied.value, 401 + ) + info = StandardLoggingPayloadSetup.get_error_information(exc) + assert info["error_message"] == "No team has access to anthropic.claude-sonnet-4-5" + assert "claude" not in (info["normalized_error"] or "") From 3c3803f37afa488a5fad08b2fc2f39d1c36f740c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:04:44 -0700 Subject: [PATCH 09/83] test(bedrock): point unit tests at model ids still in the cost map (#42606) #42521 retired the cohere.command-r ids from the cost map, and a bare Bedrock id resolves its provider through that map, so test_model_group_info and the cohere cases in test_bedrock_dynamic_auth_params_unit_tests failed with LLM Provider NOT provided. The completion tests keep the cohere invoke request and URL assertions through the bedrock/ prefix, the bare parametrize entry moves to amazon.nova-2-lite-v1:0 with the mock response shape picked by the real Bedrock route, and the router test builds its group info from bedrock/amazon.nova-2-lite-v1:0. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../test_bedrock_dynamic_auth_params_unit_tests.py | 11 ++++++----- tests/local_testing/test_router.py | 9 ++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py index 2d1d2815026..184a0ae0749 100644 --- a/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py +++ b/tests/llm_translation/test_bedrock_dynamic_auth_params_unit_tests.py @@ -9,6 +9,7 @@ import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler from unittest.mock import Mock from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockModelInfo @@ -42,7 +43,7 @@ def test_bedrock_completion_with_region_name(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_region_name="us-west-12", client=client, @@ -98,7 +99,7 @@ def test_bedrock_completion_with_dynamic_authentication_params(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_access_key_id="dynamically_generated_access_key_id", aws_secret_access_key="dynamically_generated_secret_access_key", @@ -146,7 +147,7 @@ def test_bedrock_completion_with_dynamic_bedrock_runtime_endpoint(): # Pass the client so that the HTTP call will be intercepted. response = litellm.completion( - model="cohere.command-r-v1:0", + model="bedrock/cohere.command-r-v1:0", messages=[{"role": "user", "content": "Hello, world!"}], aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com", client=client, @@ -179,7 +180,7 @@ class DummyCredentials: "model", [ "bedrock/converse/cohere.command-r-v1:0", - "cohere.command-r-v1:0", + "amazon.nova-2-lite-v1:0", "bedrock/cohere.command-r-v1:0", "bedrock/invoke/cohere.command-r-v1:0", ], @@ -250,7 +251,7 @@ def test_dynamic_aws_params_propagation(model, param_name, param_value, expected "finish_reason": "COMPLETE", } ) - if "converse" in model: + if BedrockModelInfo.get_bedrock_route(model) == "converse": mock_response.text = json.dumps( { "output": { diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index bd0a9bf8df4..4c62c28530d 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1284,15 +1284,18 @@ def test_model_group_info(): router = Router( model_list=[ { - "model_name": "command-r-plus", - "litellm_params": {"model": "cohere.command-r-plus-v1:0"}, + "model_name": "nova-2-lite", + "litellm_params": {"model": "bedrock/amazon.nova-2-lite-v1:0"}, } ] ) - response = router.get_model_group_info(model_group="command-r-plus") + response = router.get_model_group_info(model_group="nova-2-lite") assert response is not None + assert response.model_group == "nova-2-lite" + assert response.providers == ["bedrock"] + assert response.max_input_tokens is not None def test_consistent_model_id(): From dd327156c80865c58f0eff5599c987020ed545f7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:07:02 +0000 Subject: [PATCH 10/83] feat(rust): add immutable model catalog crate (#42605) * feat(rust): add immutable model catalog crate * feat(rust): add typed model info mirror, error module, and schema feature Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(rust): split catalog module, rstest tests, and repo data parity tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 38 + litellm-rust/crates/model-catalog/Cargo.toml | 25 + litellm-rust/crates/model-catalog/README.md | 25 + .../crates/model-catalog/benches/catalog.rs | 21 + .../crates/model-catalog/src/catalog.rs | 241 +++++++ .../crates/model-catalog/src/error.rs | 28 + litellm-rust/crates/model-catalog/src/lib.rs | 16 + .../crates/model-catalog/src/model_info.rs | 665 ++++++++++++++++++ .../crates/model-catalog/src/schema.rs | 7 + .../crates/model-catalog/tests/catalog.rs | 253 +++++++ .../crates/model-catalog/tests/spec_parity.rs | 121 ++++ 11 files changed, 1440 insertions(+) create mode 100644 litellm-rust/crates/model-catalog/Cargo.toml create mode 100644 litellm-rust/crates/model-catalog/README.md create mode 100644 litellm-rust/crates/model-catalog/benches/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/src/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/src/error.rs create mode 100644 litellm-rust/crates/model-catalog/src/lib.rs create mode 100644 litellm-rust/crates/model-catalog/src/model_info.rs create mode 100644 litellm-rust/crates/model-catalog/src/schema.rs create mode 100644 litellm-rust/crates/model-catalog/tests/catalog.rs create mode 100644 litellm-rust/crates/model-catalog/tests/spec_parity.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f3510000c44..4427057a3b9 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3054,6 +3054,20 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-model-catalog" +version = "0.1.0" +dependencies = [ + "criterion", + "indexmap 2.14.0", + "litellm-model-catalog", + "rstest", + "schemars 1.2.2", + "serde", + "serde_json", + "thiserror 2.0.19", +] + [[package]] name = "litellm-python-bridge" version = "0.1.0" @@ -4756,10 +4770,23 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "dyn-clone", "ref-cast", + "schemars_derive", "serde", "serde_json", ] +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4848,6 +4875,17 @@ dependencies = [ "syn 3.0.0", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.0", +] + [[package]] name = "serde_json" version = "1.0.150" diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml new file mode 100644 index 00000000000..ea75c6386d8 --- /dev/null +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "litellm-model-catalog" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +schema = ["dep:schemars"] + +[dependencies] +indexmap = { version = "2.14.0", features = ["serde"] } +schemars = { version = "1.0", optional = true } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +criterion.workspace = true +rstest.workspace = true +litellm-model-catalog = { path = ".", features = ["schema"] } + +[[bench]] +name = "catalog" +harness = false diff --git a/litellm-rust/crates/model-catalog/README.md b/litellm-rust/crates/model-catalog/README.md new file mode 100644 index 00000000000..973f7190614 --- /dev/null +++ b/litellm-rust/crates/model-catalog/README.md @@ -0,0 +1,25 @@ +# Model catalog + +`litellm-model-catalog` builds an immutable snapshot from caller supplied JSON bytes. It has no network, Python, registration, or refresh behavior. The caller supplies optional source, revision, and ETag provenance. Parse and validation are separate so small synthetic catalogs can use explicit integrity limits + +The parser treats `sample_spec` and `fallback_generalizations` as reserved top level metadata. `fallback_rules()` exposes the typed rule array when present; this crate does not execute regex generalizations. Model entries retain all JSON fields except `aliases`, including unknown fields. `field()` returns `None` for an absent key and a JSON null, false, or zero value for a present key. The returned values are borrowed, so callers cannot mutate the snapshot + +Each entry also deserializes into `ModelInfo`, a typed mirror of `model_prices_and_context_window.schema.json`'s `modelEntry` definition, reachable via `ModelEntry::info()`. All schema fields are optional on `ModelInfo`, including `litellm_provider` which the schema marks required, so small synthetic catalogs still parse. Unknown fields are not part of `ModelInfo`; they remain on `fields()`. Building with the `schema` feature adds `schemars` derives and exposes `model_entry_json_schema()` for emitting the entry's JSON Schema. Parse and validation failures are reported by the `Error` enum in `error.rs`, while catalog logic lives in `catalog.rs` + +The integration tests read the repository's catalog and schema files at test time, assert every entry round-trips through `ModelInfo`, and verify that the generated schema's properties match the repository schema + +Aliases point to their canonical entries. An alias that exactly matches any canonical key is skipped; the first canonical entry claiming an alias wins. Invalid alias lists and nonstring names are skipped and reported by `alias_issues()`. Exact lookup wins. For a case insensitive miss, the last key with the same lowercase spelling wins, following Python's lowercase map built after aliases are appended. This uses Rust Unicode lowercasing, which can differ from Python for unusual Unicode model IDs + +`validate()` counts canonical entries before alias expansion and excludes both reserved keys. It enforces an explicit minimum and backup shrink ratio, with Python defaults of 50 models and 0.5. Parsing rejects nonobject model entries and known fields with the wrong JSON type, but ignores unknown fields. It does not enforce every constraint in the JSON schema, calculate prices, resolve providers, or check provenance authenticity. The caller decides how to handle validation failures + +This snapshot does not represent Python's live mutable `litellm.model_cost`, nested dict and list mutation, or mutation of dicts previously returned by Python APIs. It has no bridge or runtime integration + +## Benchmarks + +`cargo bench -p litellm-model-catalog --bench catalog` measures parsing plus alias indexing and exact lookup. For a local Python baseline on the same fixture, use: + +```sh +python3 -m timeit -s 'import json, pathlib; body = pathlib.Path("../model_prices_and_context_window.json").read_bytes()' 'json.loads(body)' +``` + +Run these commands from `litellm-rust`. Python's command measures JSON loading only, without alias expansion or snapshot construction. The Rust benchmark does not include future Python object materialization, so these numbers are not an end to end runtime comparison diff --git a/litellm-rust/crates/model-catalog/benches/catalog.rs b/litellm-rust/crates/model-catalog/benches/catalog.rs new file mode 100644 index 00000000000..d1f51507c2b --- /dev/null +++ b/litellm-rust/crates/model-catalog/benches/catalog.rs @@ -0,0 +1,21 @@ +use criterion::{Criterion, criterion_group, criterion_main}; +use litellm_model_catalog::{Catalog, Provenance}; +use std::hint::black_box; + +fn benchmarks(c: &mut Criterion) { + let body = include_bytes!("../../../../model_prices_and_context_window.json"); + c.bench_function("parse_current_catalog", |b| { + b.iter(|| Catalog::parse(black_box(body), Provenance::default()).unwrap()) + }); + let catalog = Catalog::parse(body, Provenance::default()).unwrap(); + let key = catalog + .model_names() + .next() + .expect("catalog must have a benchmark key"); + c.bench_function("lookup_catalog_key", |b| { + b.iter(|| black_box(&catalog).lookup(black_box(key))) + }); +} + +criterion_group!(benches, benchmarks); +criterion_main!(benches); diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs new file mode 100644 index 00000000000..dc7564f9bee --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -0,0 +1,241 @@ +use crate::error::Error; +use crate::model_info::{FallbackGeneralizations, FallbackRule, ModelInfo}; +use indexmap::IndexMap; +use serde::Deserialize; +use serde_json::{Map, Value}; +use std::collections::HashMap; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Provenance { + pub source: Option, + pub revision: Option, + pub etag: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct IntegrityLimits { + pub backup_model_count: usize, + pub min_model_count: usize, + pub min_backup_ratio: f64, +} + +impl IntegrityLimits { + pub fn python_defaults(backup_model_count: usize) -> Self { + Self { + backup_model_count, + min_model_count: 50, + min_backup_ratio: 0.5, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AliasIssue { + InvalidList { model: String }, + InvalidName { model: String }, + CanonicalCollision { model: String, alias: String }, + AliasCollision { model: String, alias: String }, +} + +#[derive(Clone, Debug)] +pub struct ModelEntry { + fields: Map, + info: ModelInfo, +} + +impl ModelEntry { + pub fn field(&self, name: &str) -> Option<&Value> { + self.fields.get(name) + } + pub fn fields(&self) -> &Map { + &self.fields + } + /// The entry deserialized into the typed mirror of the catalog schema. + pub fn info(&self) -> &ModelInfo { + &self.info + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ModelMatch<'a> { + pub matched_key: &'a str, + pub canonical_key: &'a str, + pub entry: &'a ModelEntry, +} + +#[derive(Debug)] +pub struct Catalog { + entries: IndexMap, + aliases: IndexMap, + lowercase_keys: HashMap, + sample_spec: Option, + fallback_generalizations: Option, + provenance: Provenance, + alias_issues: Vec, +} + +impl Catalog { + pub fn parse(body: &[u8], provenance: Provenance) -> Result { + let root: IndexMap = serde_json::from_slice(body)?; + if root.is_empty() { + return Err(Error::Empty); + } + + let mut entries = IndexMap::with_capacity(root.len()); + let mut alias_lists = Vec::new(); + let mut alias_issues = Vec::new(); + let mut sample_spec = None; + let mut fallback_generalizations = None; + for (name, value) in root { + match name.as_str() { + "sample_spec" => { + sample_spec = Some(value); + continue; + } + "fallback_generalizations" => { + fallback_generalizations = + Some(serde_json::from_value::(value)?); + continue; + } + _ => {} + } + let Value::Object(ref object) = value else { + return Err(Error::EntryNotObject { model: name }); + }; + let info = ModelInfo::deserialize(object)?; + let Value::Object(mut fields) = value else { + unreachable!("value checked is_object above") + }; + if let Some(aliases) = fields.remove("aliases") + && !aliases.is_null() + { + match aliases { + Value::Array(names) => alias_lists.push((name.clone(), names)), + _ => alias_issues.push(AliasIssue::InvalidList { + model: name.clone(), + }), + } + } + entries.insert(name, ModelEntry { fields, info }); + } + + let mut aliases = IndexMap::new(); + for (model, names) in alias_lists { + for name in names { + let Value::String(alias) = name else { + alias_issues.push(AliasIssue::InvalidName { + model: model.clone(), + }); + continue; + }; + if entries.contains_key(&alias) { + alias_issues.push(AliasIssue::CanonicalCollision { + model: model.clone(), + alias, + }); + } else if aliases.contains_key(&alias) { + alias_issues.push(AliasIssue::AliasCollision { + model: model.clone(), + alias, + }); + } else { + aliases.insert(alias, model.clone()); + } + } + } + + let lowercase_keys = entries + .keys() + .chain(aliases.keys()) + .map(|key| (key.to_lowercase(), key.clone())) + .collect(); + Ok(Self { + entries, + aliases, + lowercase_keys, + sample_spec, + fallback_generalizations, + provenance, + alias_issues, + }) + } + + pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> { + if !limits.min_backup_ratio.is_finite() || !(0.0..=1.0).contains(&limits.min_backup_ratio) { + return Err(Error::InvalidRatio); + } + let actual = self.entries.len(); + if actual < limits.min_model_count { + return Err(Error::BelowMinimum { + actual, + minimum: limits.min_model_count, + }); + } + if limits.backup_model_count > 0 + && (actual as f64) < (limits.backup_model_count as f64) * limits.min_backup_ratio + { + return Err(Error::Shrunk { + actual, + backup: limits.backup_model_count, + ratio: limits.min_backup_ratio, + }); + } + Ok(()) + } + + pub fn lookup(&self, key: &str) -> Option> { + let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) { + key + } else { + self.lowercase_keys.get(&key.to_lowercase())?.as_str() + }; + let canonical_key = self + .aliases + .get(matched_key) + .map(String::as_str) + .unwrap_or(matched_key); + let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?; + let matched_key = self + .entries + .get_key_value(matched_key) + .map(|(key, _)| key.as_str()) + .or_else(|| { + self.aliases + .get_key_value(matched_key) + .map(|(key, _)| key.as_str()) + })?; + Some(ModelMatch { + matched_key, + canonical_key, + entry, + }) + } + + pub fn model_count(&self) -> usize { + self.entries.len() + } + pub fn model_names(&self) -> impl Iterator { + self.entries.keys().map(String::as_str) + } + pub fn alias_count(&self) -> usize { + self.aliases.len() + } + pub fn aliases(&self) -> &IndexMap { + &self.aliases + } + pub fn alias_issues(&self) -> &[AliasIssue] { + &self.alias_issues + } + pub fn sample_spec(&self) -> Option<&Value> { + self.sample_spec.as_ref() + } + pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> { + self.fallback_generalizations.as_ref() + } + pub fn fallback_rules(&self) -> Option<&[FallbackRule]> { + Some(self.fallback_generalizations.as_ref()?.rules.as_slice()) + } + pub fn provenance(&self) -> &Provenance { + &self.provenance + } +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs new file mode 100644 index 00000000000..83617312fff --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -0,0 +1,28 @@ +use thiserror::Error; + +/// Failures from parsing or validating a catalog snapshot. +#[derive(Debug, Error)] +pub enum Error { + /// The body is not valid JSON, or a model entry fails typed deserialization. + #[error("invalid JSON: {0}")] + Json(#[from] serde_json::Error), + /// The catalog has no entries at all. + #[error("catalog is empty")] + Empty, + /// A non-reserved top level value is not a JSON object. + #[error("model {model:?} must be an object")] + EntryNotObject { model: String }, + /// Canonical entry count is under the configured minimum. + #[error("catalog has {actual} models, below minimum {minimum}")] + BelowMinimum { actual: usize, minimum: usize }, + /// Canonical entry count is under the configured backup shrink ratio. + #[error("catalog has {actual} models, below {ratio} of backup count {backup}")] + Shrunk { + actual: usize, + backup: usize, + ratio: f64, + }, + /// The configured minimum backup ratio is not finite or outside `[0, 1]`. + #[error("minimum backup ratio must be finite and between zero and one")] + InvalidRatio, +} diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs new file mode 100644 index 00000000000..9c942a5521c --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -0,0 +1,16 @@ +mod catalog; +mod error; +mod model_info; +#[cfg(feature = "schema")] +mod schema; + +pub use catalog::{AliasIssue, Catalog, IntegrityLimits, ModelEntry, ModelMatch, Provenance}; +pub use error::Error; +pub use model_info::{ + AudioFormat, FallbackGeneralizations, FallbackRule, InputModality, Mode, ModelInfo, + OffPeakPricing, OffPeakWindow, OutputModality, ReasoningEffort, SearchContextCostPerQuery, + TieredRate, UtcHours, VertexAiAudioApi, WebSearchBillingUnit, Weekday, +}; + +#[cfg(feature = "schema")] +pub use schema::model_entry_json_schema; diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs new file mode 100644 index 00000000000..77a8b768e38 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -0,0 +1,665 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Primary API surface / task type of the model. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum Mode { + AudioSpeech, + AudioTranscription, + Chat, + Completion, + Embedding, + Evaluation, + Guardrail, + ImageEdit, + ImageGeneration, + Moderation, + Ocr, + Realtime, + Rerank, + Responses, + Search, + VectorStore, + VideoGeneration, +} + +/// Reasoning effort level accepted or applied by the model. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, + Xhigh, + Max, +} + +/// Gemini audio generation API the model is served through. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum VertexAiAudioApi { + LyriaPredict, + LyriaInteractions, +} + +/// Whether web search is billed per query or per prompt. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum WebSearchBillingUnit { + PerQuery, + PerPrompt, +} + +/// Audio container format the model can return. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + Mp3, + Wav, +} + +/// Input modality the model accepts. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum InputModality { + Text, + Image, + Audio, + Video, +} + +/// Output modality the model can produce. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum OutputModality { + Text, + Image, + Audio, + Video, + Code, +} + +/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum UtcHours { + Single(String), + Multiple(Vec), +} + +/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum Weekday { + Number(u8), + Name(String), +} + +/// One off-peak window entry inside `windows`. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct OffPeakWindow { + pub hours_utc: UtcHours, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub weekdays: Option>, +} + +/// Rates that replace the same-named base fields inside the stated UTC windows. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct OffPeakPricing { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hours_utc: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub windows: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub weekday_timezone: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, +} + +/// USD cost per web search query, keyed by search context size. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct SearchContextCostPerQuery { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_low: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_medium: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_size_high: Option, +} + +/// One tier of a context-length or result-count tiered rate. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct TieredRate { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range: Option<[f64; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_results_range: Option<[f64; 2]>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_query: Option, +} + +/// One regex rule generalizing unknown model ids to known families. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct FallbackRule { + pub name: String, + pub pattern: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(flatten)] + pub extra: BTreeMap, +} + +/// Regex rules that generalize unknown model ids to known families; not a model entry. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields)] +pub struct FallbackGeneralizations { + pub rules: Vec, +} + +/// Typed mirror of one catalog model entry. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ModelInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub annotation_cost_per_page: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub annotation_cost_per_page_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_transcription_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bedrock_converse_supports_strict_tools: Option, + /// Highest reasoning effort the Bedrock output_config accepts for this model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bedrock_output_config_effort_ceiling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_audio_token_cost: Option, + /// USD per token written to the provider's prompt cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_1hr: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_200k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_above_272k_tokens_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_token_cost_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_audio_token_cost: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_image_token_cost: Option, + /// USD per prompt token served from the provider's prompt cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_above_512k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub citation_cost_per_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_interpreter_cost_per_session: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub comment: Option, + /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_reasoning_effort: Option, + /// Date the provider deprecates the model, YYYY-MM-DD. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deprecation_date: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gemini_audio_only_live: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gemini_native_audio: Option, + /// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub google_maps_grounding_cost_per_query: Option, + /// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub guardrail_cost_per_unit: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_per_second: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_per_second_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token_batches: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_audio_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_character: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_character_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_image_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_pixel: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_query: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_request: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_second: Option, + /// USD per prompt token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_above_512k_tokens: Option, + /// USD per prompt token via the provider's batch API. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_cache_hit: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_15s_interval: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_per_second_above_8s_interval: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_video_token_batches: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_dbu_cost_per_token: Option, + /// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub litellm_provider: Option, + /// Maximum prompt/context tokens the model accepts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_input_tokens: Option, + /// Maximum tokens the model can generate in one response. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + /// Legacy field: max output tokens if the provider specifies it, else max input tokens. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + /// Free-form notes about the entry (e.g. pricing derivation). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + /// Primary API surface / task type of the model. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_credit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_page: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocr_cost_per_page_batches: Option, + /// Rates that replace the same-named base fields while the request falls inside the stated UTC windows. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub off_peak_pricing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_audio_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_character: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_character_above_128k_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_1024: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_1536: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_512: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_image_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_pixel: Option, + /// USD per reasoning/thinking token, when billed separately. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_reasoning_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_1080p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_2k: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_480p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_4k: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_720p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_second_768p: Option, + /// USD per generated token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_128k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_200k_tokens: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_200k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_256k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_272k_tokens_priority: Option, + /// Rate applied once the prompt exceeds the token threshold in the field name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_above_512k_tokens: Option, + /// USD per generated token via the provider's batch API. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_batches: Option, + /// Flex service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_flex: Option, + /// Priority service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_priority: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_video_per_second: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_video_token: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_dbu_cost_per_token: Option, + /// Embedding dimension for embedding models. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_vector_size: Option, + /// Smallest prefix the provider will actually cache; absent means the provider default applies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_cache_min_tokens: Option, + /// Provider-internal routing hints (e.g. bedrock_invocation_schema). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_entry: Option>, + /// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_effort_levels: Option>, + /// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_endpoint_uplift_multiplier: Option, + /// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_processing_uplift_multiplier_eu: Option, + /// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub regional_processing_uplift_multiplier_us: Option, + /// Provider default requests-per-minute limit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rpm: Option, + /// USD cost per web search query, keyed by search context size. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub search_context_cost_per_query: Option, + /// URL of the provider pricing/model page this entry was taken from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Audio container formats the model can return. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_audio_formats: Option>, + /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_endpoints: Option>, + /// Input modalities the model accepts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_modalities: Option>, + /// Output modalities the model can produce. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_output_modalities: Option>, + /// Cloud regions the model is available in ('global' or region ids). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supported_regions: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_adaptive_thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_anthropic_compaction: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_anthropic_thinking_payload: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_assistant_prefill: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_audio_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_audio_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_computer_use: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_embedding_image_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_fast_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_forced_tool_use: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_function_calling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_image_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_image_size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_legacy_thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_low_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_max_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_mid_conversation_system: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_minimal_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_multimodal: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_native_streaming: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_native_structured_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_none_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_nova_canvas_image_edit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_output_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_parallel_function_calling: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_parallel_tool_use_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_pdf_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_prompt_cache_breakpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_prompt_caching: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_reasoning: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_response_schema: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_sampling_params: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_speed: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_system_messages: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_thinking_cache_preservation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tool_choice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tool_search: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_url_context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_video_input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_vision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_web_search: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_xhigh_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_always_on: Option, + /// Context-length or result-count tiered rates; each tier's costs apply within its range. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tiered_pricing: Option>, + /// Provider default tokens-per-minute limit. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tpm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub use_openai_responses_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub uses_embed_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vertex_ai_audio_api: Option, + /// Whether web search is billed per query or per prompt. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub web_search_billing_unit: Option, +} diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs new file mode 100644 index 00000000000..82cfd6c0352 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -0,0 +1,7 @@ +use crate::model_info::ModelInfo; + +/// JSON Schema for one catalog model entry, mirroring +/// `model_prices_and_context_window.schema.json`'s `modelEntry` definition. +pub fn model_entry_json_schema() -> schemars::Schema { + schemars::schema_for!(ModelInfo) +} diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs new file mode 100644 index 00000000000..bcadd38e908 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -0,0 +1,253 @@ +use std::path::{Path, PathBuf}; + +use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance}; +use rstest::{fixture, rstest}; +use serde_json::json; + +const ALPHA_FIXTURE: &[u8] = br#"{ + "sample_spec":{"explanation":"example"}, + "fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}, + "Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false, + "optional":null,"unknown":{"nested":[1,{"x":true}]}} +}"#; + +#[fixture] +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +#[fixture] +fn current_catalog(repo_root: PathBuf) -> Catalog { + let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); + Catalog::parse(&body, Provenance::default()).unwrap() +} + +#[fixture] +fn backup_catalog(repo_root: PathBuf) -> Catalog { + let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json")) + .unwrap(); + Catalog::parse(&body, Provenance::default()).unwrap() +} + +#[fixture] +fn fixture_catalog() -> Catalog { + Catalog::parse( + ALPHA_FIXTURE, + Provenance { + source: Some("fixture".into()), + revision: Some("rev".into()), + etag: None, + }, + ) + .unwrap() +} + +#[rstest] +fn preserves_fields_and_metadata(fixture_catalog: Catalog) { + let catalog = fixture_catalog; + let entry = catalog.lookup("SHORT").unwrap(); + assert_eq!(entry.canonical_key, "Alpha"); + assert_eq!(entry.matched_key, "short"); + assert_eq!(entry.entry.field("price"), Some(&json!(0))); + assert_eq!(entry.entry.field("enabled"), Some(&json!(false))); + assert_eq!(entry.entry.field("optional"), Some(&json!(null))); + assert_eq!(entry.entry.field("missing"), None); + assert_eq!( + entry.entry.field("unknown"), + Some(&json!({"nested":[1,{"x":true}]})) + ); + assert_eq!(entry.entry.field("aliases"), None); + assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test")); + assert_eq!( + catalog.sample_spec(), + Some(&json!({"explanation":"example"})) + ); + assert_eq!(catalog.fallback_rules().unwrap().len(), 1); + assert_eq!(catalog.provenance().revision.as_deref(), Some("rev")); + assert_eq!(catalog.model_count(), 1); +} + +#[rstest] +fn snapshot_does_not_borrow_source() { + let mut source = ALPHA_FIXTURE.to_vec(); + let catalog = Catalog::parse(&source, Provenance::default()).unwrap(); + source.fill(b' '); + + let entry = catalog.lookup("short").unwrap(); + assert_eq!(entry.canonical_key, "Alpha"); + assert_eq!(entry.entry.field("price"), Some(&json!(0))); +} + +#[rstest] +#[case("Shared", "First")] +#[case("Second", "Second")] +#[case("shared", "Second")] +#[case("FIRST", "First")] +#[case("sHaReD", "Second")] +fn alias_collisions_and_case_fallback_follow_python_order( + #[case] lookup: &str, + #[case] expected: &str, +) { + let catalog = Catalog::parse( + br#"{ + "First":{"aliases":["Shared","Second","first"],"value":1}, + "Second":{"aliases":["Shared","sHaReD"],"value":2}, + "SHARED":{"value":3} + }"#, + Provenance::default(), + ) + .unwrap(); + assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected); + assert_eq!(catalog.alias_count(), 3); + assert!( + catalog + .alias_issues() + .contains(&AliasIssue::CanonicalCollision { + model: "First".into(), + alias: "Second".into(), + }) + ); + assert!( + catalog + .alias_issues() + .contains(&AliasIssue::AliasCollision { + model: "Second".into(), + alias: "Shared".into(), + }) + ); +} + +#[derive(Debug)] +enum ValidationOutcome { + Ok, + Shrunk, + BelowMinimum, + InvalidRatio, +} + +#[rstest] +#[case( + IntegrityLimits { + backup_model_count: 2, + min_model_count: 1, + min_backup_ratio: 0.5, + }, + ValidationOutcome::Ok +)] +#[case( + IntegrityLimits { + backup_model_count: 3, + min_model_count: 1, + min_backup_ratio: 0.5, + }, + ValidationOutcome::Shrunk +)] +#[case( + IntegrityLimits { + backup_model_count: 0, + min_model_count: 2, + min_backup_ratio: 0.5, + }, + ValidationOutcome::BelowMinimum +)] +#[case( + IntegrityLimits { + backup_model_count: 0, + min_model_count: 0, + min_backup_ratio: f64::NAN, + }, + ValidationOutcome::InvalidRatio +)] +fn integrity_uses_canonical_count_and_strict_shrink_boundary( + #[case] limits: IntegrityLimits, + #[case] expected: ValidationOutcome, +) { + let catalog = Catalog::parse( + br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#, + Provenance::default(), + ) + .unwrap(); + let actual = catalog.validate(limits); + match expected { + ValidationOutcome::Ok => assert!(actual.is_ok()), + ValidationOutcome::Shrunk => { + assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. }))) + } + ValidationOutcome::BelowMinimum => { + assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. }))) + } + ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))), + } +} + +#[derive(Debug)] +enum MalformedOutcome { + Empty, + Json, + EntryNotObject, +} + +#[rstest] +#[case::empty(b"{}", MalformedOutcome::Empty)] +#[case::invalid_json(b"{", MalformedOutcome::Json)] +#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)] +#[case::fallback_rules_missing( + br#"{"fallback_generalizations":{},"a":{}}"#, + MalformedOutcome::Json +)] +fn malformed_input_and_aliases_have_typed_outcomes( + #[case] body: &[u8], + #[case] expected: MalformedOutcome, +) { + let actual = Catalog::parse(body, Provenance::default()); + match expected { + MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))), + MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))), + MalformedOutcome::EntryNotObject => { + assert!(matches!(actual, Err(Error::EntryNotObject { .. }))) + } + } +} + +#[rstest] +fn invalid_aliases_are_reported_not_fatal() { + let catalog = Catalog::parse( + br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#, + Provenance::default(), + ) + .unwrap(); + assert_eq!( + catalog.alias_issues(), + &[ + AliasIssue::InvalidList { model: "a".into() }, + AliasIssue::InvalidName { model: "b".into() }, + ] + ); + assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b"); + assert!(catalog.lookup("missing").is_none()); +} + +#[rstest] +fn parses_current_and_packaged_catalogs_without_pinning_counts( + current_catalog: Catalog, + backup_catalog: Catalog, +) { + assert!(current_catalog.model_count() > 0); + assert!(backup_catalog.model_count() > 0); + assert!(current_catalog.sample_spec().is_some()); + assert!(backup_catalog.sample_spec().is_some()); + assert!( + current_catalog + .validate(IntegrityLimits::python_defaults( + backup_catalog.model_count() + )) + .is_ok() + ); + for name in current_catalog.model_names() { + let entry = current_catalog.lookup(name).unwrap().entry; + assert_eq!( + entry.info().litellm_provider.is_some(), + entry.field("litellm_provider").is_some() + ); + } +} diff --git a/litellm-rust/crates/model-catalog/tests/spec_parity.rs b/litellm-rust/crates/model-catalog/tests/spec_parity.rs new file mode 100644 index 00000000000..7296d96f798 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/spec_parity.rs @@ -0,0 +1,121 @@ +use std::collections::{BTreeSet, HashSet}; +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use litellm_model_catalog::{ + Catalog, FallbackGeneralizations, ModelInfo, Provenance, model_entry_json_schema, +}; +use rstest::{fixture, rstest}; +use serde_json::{Map, Value}; + +#[fixture] +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn json_eq(left: &Value, right: &Value) -> bool { + match (left, right) { + (Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(), + (Value::Array(left), Value::Array(right)) => { + left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b)) + } + (Value::Object(left), Value::Object(right)) => { + left.len() == right.len() + && left + .iter() + .all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other))) + } + _ => left == right, + } +} + +fn keys(value: &Map) -> BTreeSet { + value.keys().cloned().collect() +} + +fn symmetric_difference(left: &BTreeSet, right: &BTreeSet) -> BTreeSet { + left.symmetric_difference(right).cloned().collect() +} + +#[rstest] +#[case("model_prices_and_context_window.json")] +#[case("litellm/model_prices_and_context_window_backup.json")] +fn every_entry_round_trips_through_model_info(repo_root: PathBuf, #[case] filename: &str) { + let body = std::fs::read(repo_root.join(filename)).unwrap(); + let document: IndexMap = serde_json::from_slice(&body).unwrap(); + for (model_name, value) in document { + if matches!( + model_name.as_str(), + "sample_spec" | "fallback_generalizations" + ) { + continue; + } + let object = value + .as_object() + .unwrap_or_else(|| panic!("{model_name} is not an object")); + let info: ModelInfo = serde_json::from_value(value.clone()) + .unwrap_or_else(|error| panic!("{model_name} does not deserialize: {error}")); + let serialized = serde_json::to_value(info).unwrap(); + let serialized_object = serialized + .as_object() + .unwrap_or_else(|| panic!("{model_name} did not serialize as an object")); + let mut expected = object.clone(); + expected.remove("aliases"); + let expected_keys = keys(&expected); + let serialized_keys = keys(serialized_object); + assert_eq!( + expected_keys, + serialized_keys, + "{model_name} key difference: {:?}", + symmetric_difference(&expected_keys, &serialized_keys) + ); + assert!( + json_eq(&Value::Object(expected), &serialized), + "{model_name} changed during ModelInfo round-trip" + ); + } +} + +#[rstest] +fn fallback_generalizations_are_typed(repo_root: PathBuf) { + let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap(); + let document: Map = serde_json::from_slice(&body).unwrap(); + let Some(raw_rules) = document.get("fallback_generalizations") else { + return; + }; + let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap(); + let catalog = Catalog::parse(&body, Provenance::default()).unwrap(); + assert!( + catalog + .fallback_rules() + .is_some_and(|rules| !rules.is_empty()) + ); +} + +#[rstest] +fn generated_schema_properties_match_repo_schema(repo_root: PathBuf) { + let body = + std::fs::read(repo_root.join("model_prices_and_context_window.schema.json")).unwrap(); + let document: Value = serde_json::from_slice(&body).unwrap(); + let repo_entry_properties = document["$defs"]["modelEntry"]["properties"] + .as_object() + .unwrap(); + let generated = serde_json::to_value(model_entry_json_schema()).unwrap(); + let generated_properties = generated["properties"].as_object().unwrap(); + let expected = keys(repo_entry_properties); + let actual = keys(generated_properties); + assert_eq!( + expected, + actual, + "modelEntry property difference: {:?}", + symmetric_difference(&expected, &actual) + ); + + let repo_root_properties = document["properties"].as_object().unwrap(); + let actual_root: HashSet = repo_root_properties.keys().cloned().collect(); + let expected_root: HashSet = ["sample_spec", "fallback_generalizations"] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!(actual_root, expected_root); +} From da82ea8e94b737b7f5873609edf1e842699e5aa2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:30:29 -0700 Subject: [PATCH 11/83] fix(ui): let the Create Key user picker find users by user_id, not just email (#41687) * feat(ui): search users by id or email when assigning a key owner Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): label users without an email by user id in key owner picker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): freeze merged user-filter where, format create key test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): suppress module-global patch findings in ui_view_users search test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep merged user-filter where as a plain dict for prisma serialization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): forward search param from userFilterUICall to /user/filter/ui Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): mention user ID in the Create Key user picker helper text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: jesus Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../internal_user_endpoints.py | 14 +++- .../test_internal_user_endpoints.py | 66 +++++++++++++++++++ .../src/components/networking.test.ts | 23 +++++++ .../src/components/networking.tsx | 1 + .../create_key_button.integration.test.tsx | 53 ++++++++++----- .../organisms/create_key_button.tsx | 10 +-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 7 files changed, 148 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 133181e9203..587ae416096 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2741,6 +2741,10 @@ async def _resolve_team_org_filter( async def ui_view_users( user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"), user_email: str | None = fastapi.Query(default=None, description="User email in the request parameters"), + search: str | None = fastapi.Query( + default=None, + description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).", + ), team_id: str | None = fastapi.Query( default=None, description="Team ID — used when a team admin searches for users to add to their team", @@ -2750,7 +2754,7 @@ async def ui_view_users( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Filter users based on partial match of user_id or email with pagination. + Filter users based on partial match of user_id or email, or combined ``search``, with pagination. Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag (stored in the ``litellm_uisettings`` table): @@ -2802,9 +2806,15 @@ async def ui_view_users( if org_filter_ids is not None: where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}} + where: Final[Mapping[str, object]] = { # mutable-ok: prisma serializes `where`, keep it a plain dict + key: value + for key, value in (*where_conditions.items(), *_user_search_where(search).items()) + if value is not None + } + # Query users with pagination and filters users: Final = await _user_table(prisma_client).find_many( - where=where_conditions, + where=where, skip=skip, take=page_size, order={"created_at": "desc"}, diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 2d1049b143e..0465572235b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -2,6 +2,7 @@ import asyncio import hashlib import json import logging +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from types import SimpleNamespace from typing import Final @@ -37,6 +38,7 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( ui_view_users, ) from litellm.proxy.proxy_server import app +from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains from tests.test_litellm.proxy.management_endpoints.jwt_key_mapping_doubles import ( CascadingJWTMappingTable, JWTMappingRow, @@ -119,6 +121,70 @@ async def test_ui_view_users_proxy_admin_no_org_filter(mocker): ) +UserWhereCondition = InsensitiveContains | Sequence[Mapping[str, InsensitiveContains]] + + +def _matches_user_where(row: LiteLLM_UserTableFiltered, where: Mapping[str, UserWhereCondition]) -> bool: + def matches(field: str, condition: UserWhereCondition) -> bool: + if not isinstance(condition, Mapping): + return any(_matches_user_where(row, branch) for branch in condition) + value: Final = {"user_id": row.user_id, "user_email": row.user_email}[field] + return value is not None and condition["contains"].lower() in value.lower() + + return all(matches(field, condition) for field, condition in where.items()) + + +@pytest.mark.parametrize( + "params, expected_user_ids", + [ + ({"search": "SVC"}, ["svc-bot"]), + ({"search": "ali"}, ["alice-admin"]), + ({"search": "example.com"}, ["alice-admin"]), + ({"search": "admin"}, ["alice-admin"]), + ({"user_email": "svc"}, []), + ({"user_id": "svc"}, ["svc-bot"]), + ({"search": "ali", "user_id": "svc"}, []), + ], +) +def test_ui_view_users_search_matches_user_id_or_email( + mocker: MockerFixture, params: Mapping[str, str], expected_user_ids: list[str] +): + """ + search= returns users whose user_id or user_email contains the value (case-insensitive), + including users with no email; user_id=/user_email= keep filtering a single field and AND with search. + """ + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + users = ( + LiteLLM_UserTableFiltered(user_id="alice-admin", user_email="alice@example.com"), + LiteLLM_UserTableFiltered(user_id="svc-bot", user_email=None), + LiteLLM_UserTableFiltered(user_id="bob", user_email="bob@corp.io"), + ) + + async def mock_find_many(*, where: Mapping[str, UserWhereCondition], **_: object): + return [user for user in users if _matches_user_where(user, where)] + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_many = mock_find_many + mocker.patch( # test-quality-ok: endpoint reads settings via module global; same seam as sibling tests + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={}, + ) + mocker.patch( # test-quality-ok: endpoint reads prisma_client via module global; same seam as sibling tests + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + response = client.get("/user/filter/ui", params=params) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert [user["user_id"] for user in response.json()] == expected_user_ids + + @pytest.mark.asyncio async def test_ui_view_users_org_admin_filtered_by_org(mocker): """ diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 3b2a17101ee..e14f1939ee1 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -913,3 +913,26 @@ describe("fetchMemoryList search serialization", () => { expect(lastParams(mockFetch).has("search")).toBe(false); }); }); + +describe("userFilterUICall", () => { + let currentFetch: typeof global.fetch; + + beforeEach(() => { + currentFetch = global.fetch; + }); + + afterEach(() => { + global.fetch = currentFetch; + }); + + it("forwards the search param to /user/filter/ui", async () => { + const mockFetch = vi.fn().mockResolvedValue({ ok: true, text: async () => "[]" } as any); + global.fetch = mockFetch as any; + + await Networking.userFilterUICall("sk-test", new URLSearchParams({ search: "svc" })); + + const parsed = new URL(mockFetch.mock.calls[0][0] as string, "http://localhost"); + expect(parsed.pathname).toContain("/user/filter/ui"); + expect(parsed.searchParams.get("search")).toBe("svc"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d358d23408c..76c6a3cb935 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1983,6 +1983,7 @@ export const userFilterUICall = async (accessToken: string, params: URLSearchPar user_email: params.get("user_email") || undefined, user_id: params.get("user_id") || undefined, team_id: params.get("team_id") || undefined, + search: params.get("search") || undefined, }, }); } catch (error) { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index a6b37ac26ff..0bf3268379c 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -204,7 +204,8 @@ const openModal = async (props: Partial> return view; }; -const userSearchInput = (): Promise => screen.findByPlaceholderText("Type email to search for users"); +const userSearchInput = (): Promise => + screen.findByPlaceholderText("Type email or user ID to search for users"); const openSection = async (name: RegExp) => { await userEvent.click(await screen.findByRole("button", { name })); @@ -619,7 +620,7 @@ describe("CreateKey", () => { it("mounts the user search control only once Another User is chosen", async () => { await openModal(); - expect(screen.queryByPlaceholderText("Type email to search for users")).not.toBeInTheDocument(); + expect(screen.queryByPlaceholderText("Type email or user ID to search for users")).not.toBeInTheDocument(); await userEvent.click(screen.getByRole("radio", { name: "Another User" })); @@ -946,18 +947,18 @@ describe("CreateKey", () => { expect(vi.mocked(userFilterUICall)).toHaveBeenCalledTimes(1); const params = vi.mocked(userFilterUICall).mock.calls[0][1] as URLSearchParams; - expect(params.get("user_email")).toBe("alice"); + expect(params.get("search")).toBe("alice"); } finally { vi.useRealTimers(); } }); it("keeps the current search's users when an abandoned search answers last", async () => { - const answers = new Map void>(); + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -984,12 +985,36 @@ describe("CreateKey", () => { expect(screen.getByRole("option", { name: "alice.smith@example.com (u-smith)" })).toBeInTheDocument(); }); - it("stops searching once the box is cleared and the abandoned search answers", async () => { - const answers = new Map void>(); + it("labels a user with no email by their user id", async () => { + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); + }) as never, + ); + + const user = userEvent.setup(); + renderCreateKey({ autoOpenCreate: true, prefillData: { owned_by: "another_user" } }); + const search = await userSearchInput(); + + await user.type(search, "svc"); + await waitFor(() => expect(answers.has("svc")).toBe(true), { timeout: 3000 }); + + await act(async () => { + answers.get("svc")?.([{ user_id: "svc-bot", user_email: null }]); + }); + + expect(await screen.findByRole("option", { name: "svc-bot" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /null/ })).not.toBeInTheDocument(); + }); + + it("stops searching once the box is cleared and the abandoned search answers", async () => { + const answers = new Map void>(); + vi.mocked(userFilterUICall).mockImplementation( + (_accessToken, params) => + new Promise((resolve) => { + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -1013,11 +1038,11 @@ describe("CreateKey", () => { }); it("keeps searching while a newer search is still in flight", async () => { - const answers = new Map void>(); + const answers = new Map void>(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve) => { - answers.set(params.get("user_email") ?? "", resolve); + answers.set(params.get("search") ?? "", resolve); }) as never, ); @@ -1047,12 +1072,12 @@ describe("CreateKey", () => { it("only warns about a failed search when it is the one the box is waiting on", async () => { const answers = new Map< string, - { resolve: (users: { user_id: string; user_email: string }[]) => void; reject: (error: Error) => void } + { resolve: (users: { user_id: string; user_email: string | null }[]) => void; reject: (error: Error) => void } >(); vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => new Promise((resolve, reject) => { - answers.set(params.get("user_email") ?? "", { resolve, reject }); + answers.set(params.get("search") ?? "", { resolve, reject }); }) as never, ); @@ -1099,9 +1124,7 @@ describe("CreateKey", () => { ]; vi.mocked(userFilterUICall).mockImplementation( (_accessToken, params) => - Promise.resolve( - directory.filter((entry) => entry.user_email.includes(params.get("user_email") ?? "")), - ) as never, + Promise.resolve(directory.filter((entry) => entry.user_email.includes(params.get("search") ?? ""))) as never, ); const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 3127eab249e..25b986e4c9e 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -161,7 +161,7 @@ interface CreateKeyProps { interface User { user_id: string; - user_email: string; + user_email: string | null; role?: string; } @@ -570,7 +570,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp setUserSearchLoading(true); try { const params = new URLSearchParams(); - params.append("user_email", searchText); // Always search by email + params.append("search", searchText); if (accessToken == null) { return; } @@ -579,7 +579,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp const data: User[] = response; const options: SearchSelectOption[] = data.map((user) => ({ - label: `${user.user_email} (${user.user_id})`, + label: user.user_email ? `${user.user_email} (${user.user_id})` : user.user_id, value: user.user_id, })); @@ -729,7 +729,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp onValueChange={control.onChange} onSearchChange={fetchUsers} isLoading={userSearchLoading} - placeholder="Type email to search for users" + placeholder="Type email or user ID to search for users" emptyText="No users found" loadingText="Searching..." inputId={control.id} @@ -741,7 +741,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp Create User -
Search by email to find users
+
Search by email or user ID to find users
)} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index be1e000bd66..df294a20e19 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17325,7 +17325,7 @@ export interface paths { }; /** * Ui View Users - * @description Filter users based on partial match of user_id or email with pagination. + * @description Filter users based on partial match of user_id or email, or combined ``search``, with pagination. * * Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag * (stored in the ``litellm_uisettings`` table): @@ -64338,6 +64338,8 @@ export interface operations { user_id?: string | null; /** @description User email in the request parameters */ user_email?: string | null; + /** @description Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive). */ + search?: string | null; /** @description Team ID — used when a team admin searches for users to add to their team */ team_id?: string | null; /** @description Page number for pagination */ From 13691426c4610f10da9627388a53419196a989ee Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:52:05 -0700 Subject: [PATCH 12/83] test(cost_calculator): point image-generation deployment price test at a live gemini row (#42615) The test priced gemini/gemini-3.1-flash-image-preview, which #42435 removed from the cost map as deprecated, so the calculator had no per-token rates to keep and the hardcoded expected value no longer matched. Price the live gemini/gemini-3.1-flash-image row instead and derive the expected cost from that row in litellm.model_cost, so a rate change on it cannot break the test while a calculator that drops the map's token rates still fails it. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/test_litellm/test_cost_calculator.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index e7ce9a74797..c6b57fa7604 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -595,6 +595,8 @@ def test_completion_cost_image_generation_registered_deployment_price_keeps_map_ deployment_id, {"mode": "image_generation", "litellm_provider": "gemini", "output_cost_per_image": 0.1}, ) + map_model: Final = "gemini/gemini-3.1-flash-image" + row: Final = litellm.model_cost[map_model] usage: Final = ImageUsage( input_tokens=10, input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), @@ -604,7 +606,7 @@ def test_completion_cost_image_generation_registered_deployment_price_keeps_map_ cost = completion_cost( completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")], usage=usage), - model="gemini/gemini-3.1-flash-image-preview", + model=map_model, custom_llm_provider="gemini", call_type="image_generation", custom_pricing=True, @@ -612,7 +614,10 @@ def test_completion_cost_image_generation_registered_deployment_price_keeps_map_ litellm_logging_obj=SimpleNamespace(litellm_params={"metadata": {"model_info": {"id": deployment_id}}}), ) - assert cost == pytest.approx(10 * 5e-07 + 1290 * 6e-05) + expected: Final = ( + usage.input_tokens * row["input_cost_per_token"] + usage.output_tokens * row["output_cost_per_image_token"] + ) + assert cost == pytest.approx(expected) def test_completion_cost_image_generation_ignores_deployment_model_info_without_custom_pricing( From cf08cb89e81d4e744644428a09241da08b2996ce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:04:13 -0700 Subject: [PATCH 13/83] test(utils): accept the per-size image cost keys in the price-map schema check (#42612) The cost map's fal_ai/fal-ai/trellis-2 entry prices its output by resolution with output_cost_per_image_512, output_cost_per_image_1024, and output_cost_per_image_1536, which litellm/types/utils.py types and the fal_ai cost calculator reads, but INTENDED_SCHEMA in test_aaamodel_prices_and_context_window_json_is_valid never allowed them, so the test fails on main with "Additional properties are not allowed". Add the three keys next to output_cost_per_image in the schema and in the cost-under-1 field list so a per-size image price is validated like the per-size video ones Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 79462d16a8c..283cff97ce0 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -631,6 +631,9 @@ def validate_model_cost_values(model_data, exceptions=None): "output_cost_per_character", "input_cost_per_image", "output_cost_per_image", + "output_cost_per_image_512", + "output_cost_per_image_1024", + "output_cost_per_image_1536", "input_cost_per_pixel", "output_cost_per_pixel", "input_cost_per_second", @@ -858,6 +861,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "output_cost_per_character": {"type": "number"}, "output_cost_per_character_above_128k_tokens": {"type": "number"}, "output_cost_per_image": {"type": "number"}, + "output_cost_per_image_512": {"type": "number"}, + "output_cost_per_image_1024": {"type": "number"}, + "output_cost_per_image_1536": {"type": "number"}, "output_cost_per_image_token": {"type": "number"}, "output_cost_per_video_token": {"type": "number"}, "output_cost_per_pixel": {"type": "number"}, From 9bad2c35e51f34abd0557877fe03f565f9657067 Mon Sep 17 00:00:00 2001 From: Pawan Shahane <110886433+Pawan-Shahane@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:39:39 +0530 Subject: [PATCH 14/83] fix(ollama): send PNG and JPEG images without requiring Pillow. (#41979) * fix(ollama): send PNG and JPEG images without requiring Pillow The ollama/ completion transport imported Pillow before it looked at the image, so every image request failed with a 500 on installs without Pillow. That includes the Docker image, where Pillow is only a CI dependency Detect PNG and JPEG from their leading bytes and pass them through untouched. Pillow is now imported only when another format has to be re-encoded as JPEG, and that case still raises the same install hint * fix(ollama): address Greptile findings on image conversion Catch all exceptions on Pillow import, not just ImportError, so the helpful install hint always appears. Break a line that exceeded 120 characters --- litellm/llms/ollama/common_utils.py | 36 ++++----- .../test_ollama_completion_transformation.py | 74 +++++++++++++++++++ 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index ed4bab22a84..9f46cbc5cd5 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -1,3 +1,5 @@ +import base64 +import io from typing import Any, Final import httpx @@ -11,37 +13,35 @@ class OllamaError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) -def _convert_image(image): - """ - Convert image to base64 encoded image if not already in base64 format +_JPEG_AND_PNG_SIGNATURES: Final = (b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n") - If image is already in base64 format AND is a jpeg/png, return it - - If image is not JPEG/PNG, convert it to JPEG base64 format - """ - import base64 - import io +def _reencode_as_jpeg(raw_image: bytes, original: str) -> str: try: from PIL import Image except Exception: raise Exception("ollama image conversion failed please run `pip install Pillow`") - orig: Final = image - if image.startswith("data:"): - image = image.split(",")[-1] try: - image_data: Final = Image.open(io.BytesIO(base64.b64decode(image))) - if image_data.format in ["JPEG", "PNG"]: - return image + picture: Final = Image.open(io.BytesIO(raw_image)) except Exception: - return orig + return original jpeg_image: Final = io.BytesIO() - image_data.convert("RGB").save(jpeg_image, "JPEG") - jpeg_image.seek(0) + picture.convert("RGB").save(jpeg_image, "JPEG") return base64.b64encode(jpeg_image.getvalue()).decode("utf-8") +def _convert_image(image: str) -> str: + payload: Final = image.split(",")[-1] if image.startswith("data:") else image + try: + raw_image: Final = base64.b64decode(payload) + except ValueError: + return image + if raw_image.startswith(_JPEG_AND_PNG_SIGNATURES): + return payload + return _reencode_as_jpeg(raw_image, original=image) + + from litellm.llms.base_llm.base_utils import BaseLLMModelInfo diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index b2071155f3f..28e86e40944 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -1,4 +1,7 @@ +import base64 +import io import json +import sys from litellm._uuid import uuid from unittest.mock import MagicMock, patch @@ -544,3 +547,74 @@ async def test_ollama_async_completion_inlines_remote_images_off_the_event_loop( assert response.choices[0].message.content == "Green" assert async_only_image_fetch.fetched == [image_url] assert captured["body"]["images"] == [async_only_image_fetch.base64_png] + + +def _image_base64(image_format: str) -> str: + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", (4, 4), "green").save(buffer, image_format) + return base64.b64encode(buffer.getvalue()).decode("utf-8") + + +def _transform_image_request(image_base64: str, mime_subtype: str) -> dict: + return OllamaConfig().transform_request( + model="llava", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What colour is this?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/{mime_subtype};base64,{image_base64}"}, + }, + ], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("image_format", ["PNG", "JPEG"]) +def test_transform_request_sends_png_and_jpeg_images_without_pillow( + image_format: str, monkeypatch: pytest.MonkeyPatch +) -> None: + image_base64 = _image_base64(image_format) + monkeypatch.setitem(sys.modules, "PIL", None) + + data = _transform_image_request(image_base64, image_format.lower()) + + assert data["images"] == [image_base64] + + +def test_transform_request_without_pillow_says_how_to_convert_other_image_formats( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gif_base64 = _image_base64("GIF") + monkeypatch.setitem(sys.modules, "PIL", None) + + with pytest.raises(Exception, match="pip install Pillow"): + _transform_image_request(gif_base64, "gif") + + +def test_transform_request_reencodes_other_image_formats_as_jpeg() -> None: + from PIL import Image + + data = _transform_image_request(_image_base64("GIF"), "gif") + + (encoded,) = data["images"] + assert Image.open(io.BytesIO(base64.b64decode(encoded))).format == "JPEG" + + +@pytest.mark.parametrize( + "payload", + [base64.b64encode(b"not an image").decode("utf-8"), "abc"], + ids=["decodable_but_not_an_image", "invalid_base64"], +) +def test_transform_request_leaves_unreadable_images_untouched(payload: str) -> None: + data = _transform_image_request(payload, "png") + + assert data["images"] == [payload] From ca95fc2bd4185483e91cdd62093b6fdb339f82f3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:23:44 -0700 Subject: [PATCH 15/83] fix: answer get_api_base for github_copilot and chatgpt without running the login flow (#42602) * fix: answer get_api_base for github_copilot and chatgpt without running the login flow * refactor(get_api_base): dispatch the provider helpers with if-chains --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llm_response_utils/get_api_base.py | 39 +++++--- litellm/llms/chatgpt/chat/transformation.py | 5 +- .../github_copilot/chat/transformation.py | 15 +-- .../llm_response_utils/test_get_api_base.py | 93 +++++++++++++++++++ tests/test_litellm/rerank_api/test_main.py | 4 +- 5 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/llm_response_utils/test_get_api_base.py diff --git a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py index 26e79fa0ea8..3815ea91b51 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py @@ -3,10 +3,30 @@ from typing import Final import litellm from litellm import verbose_logger -from ...litellm_core_utils.get_llm_provider_logic import get_llm_provider +from ...litellm_core_utils.get_llm_provider_logic import ( + declared_authenticating_provider, + get_llm_provider, +) from ...types.router import LiteLLM_Params +def _api_base_without_login(provider: str) -> str | None: + if provider == "github_copilot": + return litellm.GithubCopilotConfig().api_base_without_login() + if provider == "chatgpt": + return litellm.ChatGPTConfig().api_base_without_login() + return None + + +def _provider_default_api_base(model: str, custom_llm_provider: str | None, stream: bool) -> str | None: + if custom_llm_provider == "gemini": + action: Final = "streamGenerateContent" if stream else "generateContent" + return f"https://generativelanguage.googleapis.com/v1beta/models/{model}:{action}" + if custom_llm_provider == "openai": + return "https://api.openai.com" + return None + + def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | None: """ Returns the api base used for calling the model. @@ -42,6 +62,9 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No if litellm.model_alias_map and model in litellm.model_alias_map: model = litellm.model_alias_map[model] + declared: Final = declared_authenticating_provider(model, _optional_params.custom_llm_provider) + if declared is not None: + return _api_base_without_login(declared) try: ( model, @@ -83,16 +106,4 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No _api_base = f"{_optional_params.vertex_location}-aiplatform.googleapis.com/v1/projects/{_optional_params.vertex_project}/locations/{_optional_params.vertex_location}/publishers/google/models/{model}:generateContent" return _api_base - if custom_llm_provider is None: - return None - - if custom_llm_provider == "gemini": - if stream: - _api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:streamGenerateContent" - else: - _api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent" - return _api_base - elif custom_llm_provider == "openai": - _api_base = "https://api.openai.com" - return _api_base - return None + return _provider_default_api_base(model, custom_llm_provider, stream) diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index e35408b0829..1b110704c8b 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -23,6 +23,9 @@ class ChatGPTConfig(OpenAIConfig): super().__init__() self.authenticator = Authenticator() + def api_base_without_login(self) -> str: + return self.authenticator.get_api_base() + def _get_openai_compatible_provider_info( self, model: str, @@ -30,7 +33,7 @@ class ChatGPTConfig(OpenAIConfig): api_key: str | None, custom_llm_provider: str, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = self.authenticator.get_api_base() + dynamic_api_base: Final = self.api_base_without_login() try: dynamic_api_key: Final = self.authenticator.get_access_token() except GetAccessTokenError as e: diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 8634b374f1b..169b9a037a5 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -31,6 +31,14 @@ class GithubCopilotConfig(OpenAIConfig): super().__init__() self.authenticator = Authenticator() + def api_base_without_login(self, api_base: str | None = None) -> str: + return ( + api_base + or self.authenticator.get_api_base() + or os.getenv("GITHUB_COPILOT_API_BASE") + or DEFAULT_GITHUB_COPILOT_API_BASE + ) + def _get_openai_compatible_provider_info( self, model: str, @@ -38,12 +46,7 @@ class GithubCopilotConfig(OpenAIConfig): api_key: str | None, custom_llm_provider: str, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = ( - api_base - or self.authenticator.get_api_base() - or os.getenv("GITHUB_COPILOT_API_BASE") - or DEFAULT_GITHUB_COPILOT_API_BASE - ) + dynamic_api_base: Final = self.api_base_without_login(api_base) try: dynamic_api_key: Final = self.authenticator.get_api_key() except GetAPIKeyError as e: diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_get_api_base.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_get_api_base.py new file mode 100644 index 00000000000..63977c30270 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_get_api_base.py @@ -0,0 +1,93 @@ +import json + +import pytest + +import litellm +from litellm.litellm_core_utils.llm_response_utils import get_api_base as get_api_base_module +from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE +from litellm.llms.github_copilot.common_utils import DEFAULT_GITHUB_COPILOT_API_BASE + + +@pytest.fixture +def isolated_token_dirs(tmp_path, monkeypatch): + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "github_copilot")) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt")) + monkeypatch.delenv("GITHUB_COPILOT_API_BASE", raising=False) + monkeypatch.delenv("CHATGPT_API_BASE", raising=False) + monkeypatch.delenv("OPENAI_CHATGPT_API_BASE", raising=False) + return tmp_path + + +@pytest.fixture +def resolution_lookups(monkeypatch): + lookups: list = [] + + def _record(*args, **kwargs): + lookups.append((args, kwargs)) + raise RuntimeError("provider resolution must not run for an authenticating provider") + + monkeypatch.setattr(get_api_base_module, "get_llm_provider", _record) + return lookups + + +class TestDeclaredAuthenticatingProvider: + """get_llm_provider runs the OAuth device flow for github_copilot and chatgpt, and get_api_base + runs on every response's hidden params and on every mapped exception, so it must answer from + the declaration without resolving. The recorder appends before raising, and get_api_base + swallows resolver errors, so an empty list proves the lookup never ran.""" + + @pytest.mark.parametrize( + "model, custom_llm_provider, expected", + [ + ("github_copilot/gpt-4o", None, DEFAULT_GITHUB_COPILOT_API_BASE), + ("gpt-4o", "github_copilot", DEFAULT_GITHUB_COPILOT_API_BASE), + ("chatgpt/gpt-5", None, CHATGPT_API_BASE), + ("gpt-5", "chatgpt", CHATGPT_API_BASE), + ], + ) + def test_answers_without_resolving( + self, model, custom_llm_provider, expected, isolated_token_dirs, resolution_lookups + ): + api_base = litellm.get_api_base(model=model, optional_params={"custom_llm_provider": custom_llm_provider}) + + assert resolution_lookups == [] + assert api_base == expected + + def test_copilot_keeps_the_enterprise_endpoint_from_disk(self, isolated_token_dirs, resolution_lookups): + token_dir = isolated_token_dirs / "github_copilot" + token_dir.mkdir() + (token_dir / "api-key.json").write_text( + json.dumps({"endpoints": {"api": "https://api.enterprise.githubcopilot.com"}}) + ) + + api_base = litellm.get_api_base(model="github_copilot/gpt-4o", optional_params={}) + + assert resolution_lookups == [] + assert api_base == "https://api.enterprise.githubcopilot.com" + + def test_explicit_api_base_still_wins(self, isolated_token_dirs, resolution_lookups): + api_base = litellm.get_api_base( + model="github_copilot/gpt-4o", optional_params={"api_base": "https://copilot.example/v1"} + ) + + assert resolution_lookups == [] + assert api_base == "https://copilot.example/v1" + + def test_other_providers_still_resolve(self, isolated_token_dirs, resolution_lookups): + litellm.get_api_base(model="openai/gpt-4o", optional_params={}) + + assert len(resolution_lookups) == 1 + + +@pytest.mark.parametrize( + "model, expected", + [ + ("gemini/gemini-2.5-pro", "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent"), + ("openai/gpt-4o", "https://api.openai.com"), + ], +) +def test_providers_with_a_fixed_base_still_get_it(model, expected, monkeypatch): + for env in ("GEMINI_API_BASE", "OPENAI_API_BASE", "OPENAI_BASE_URL"): + monkeypatch.delenv(env, raising=False) + + assert litellm.get_api_base(model=model, optional_params={}) == expected diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py index aca0c970dd5..f56673fec30 100644 --- a/tests/test_litellm/rerank_api/test_main.py +++ b/tests/test_litellm/rerank_api/test_main.py @@ -239,7 +239,6 @@ async def test_arerank_error_is_mapped_to_litellm_exception(respx_mock: respx.Mo @pytest.mark.asyncio -@pytest.mark.timeout(300) async def test_arerank_declared_authenticating_provider_skips_resolution(monkeypatch): """Regression for the event-loop hazard in arerank's provider pre-resolution: get_llm_provider runs the blocking OAuth device flow for github_copilot/chatgpt, @@ -257,6 +256,9 @@ async def test_arerank_declared_authenticating_provider_skips_resolution(monkeyp raise BaseLLMException(status_code=401, message='{"error":"bad key"}') monkeypatch.setattr(litellm, "get_llm_provider", record_resolution) + monkeypatch.setattr( + "litellm.litellm_core_utils.llm_response_utils.get_api_base.get_llm_provider", record_resolution + ) monkeypatch.setattr("litellm.rerank_api.main.rerank", rerank_raises_provider_error) with pytest.raises(litellm.AuthenticationError) as exc_info: From 4f93e2c3da75289393af1b9e2ddb28935b4e11da Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 22 Sep 2026 17:28:34 -0700 Subject: [PATCH 16/83] test: point CircleCI-only suites at models still in the cost map (#42617) * test: point CircleCI-only suites at models still in the cost map #42435 removed cost map entries past their deprecation date and #42437 added litellm_uisettings to the config-synced tables, but both only updated tests/test_litellm. The CircleCI-only suites (local_testing, llm_translation, logging_callback_tests, litellm_utils_tests, unit) kept using the removed models or the old table list and went red on main. Each test keeps its assertions and swaps the removed model for a current one with the same provider and capabilities. The fireworks tests pick a vision model from the cost map because #34941 set supports_vision false on minimax-m3, and the vertex image provider test injects the image model set because #42435 removed every vertex_ai-image-models entry. * test(vertex_ai): register the image model through add_known_models in the provider test --- tests/litellm_utils_tests/test_utils.py | 2 +- .../test_fireworks_ai_translation.py | 14 ++++-- .../test_gemini_image_usage.py | 2 +- tests/llm_translation/test_groq.py | 2 +- tests/llm_translation/test_optional_params.py | 8 +-- tests/llm_translation/test_xai.py | 2 +- .../test_amazing_vertex_completion.py | 6 +-- tests/local_testing/test_completion_cost.py | 50 +++++++++---------- tests/local_testing/test_exceptions.py | 2 +- .../test_function_call_parsing.py | 2 +- tests/local_testing/test_get_llm_provider.py | 18 +++++-- tests/local_testing/test_get_model_info.py | 4 +- .../local_testing/test_lowest_cost_routing.py | 2 +- .../test_openai_moderations_hook.py | 6 +-- tests/local_testing/test_router_utils.py | 24 ++++----- .../test_spend_calculate_endpoint.py | 4 +- .../completion_with_vertex_call.json | 10 ++-- tests/logging_callback_tests/test_alerting.py | 6 +-- .../test_langfuse_e2e_test.py | 4 +- tests/unit/repositories/test_repositories.py | 1 + 20 files changed, 93 insertions(+), 76 deletions(-) diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index f7575b969c4..fb20cdf7e0e 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -263,7 +263,7 @@ def test_trimming_should_not_change_original_messages(): assert messages == messages_copy -@pytest.mark.parametrize("model", ["gpt-4-0125-preview", "claude-sonnet-4-6"]) +@pytest.mark.parametrize("model", ["gpt-5.4-mini", "claude-sonnet-4-6"]) def test_trimming_with_model_cost_max_input_tokens(model): messages = [ {"role": "system", "content": "This is a normal system message"}, diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index e20134fc1bf..a7dc913c388 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -9,6 +9,12 @@ from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig fireworks = FireworksAIConfig() +VISION_MODEL = next( + key.removeprefix("fireworks_ai/") + for key, info in litellm.model_cost.items() + if key.startswith("fireworks_ai/accounts/fireworks/models/") and info.get("supports_vision") is True +) + def test_map_openai_params_tool_choice(): # Test case 1: tool_choice is "required" @@ -97,7 +103,7 @@ def test_document_inlining_example(disable_add_transform_inline_image_block): with patch.object(client, "post") as mock_post: try: completion( - model="fireworks_ai/accounts/fireworks/models/minimax-m3", + model=f"fireworks_ai/{VISION_MODEL}", messages=[ { "role": "user", @@ -157,7 +163,7 @@ def test_transform_inline_no_longer_added(content, expected_url): result = litellm.FireworksAIConfig()._transform_messages_helper( messages=messages, - model="accounts/fireworks/models/minimax-m3", + model=VISION_MODEL, litellm_params={}, ) result_image_block = result[0]["content"][0] @@ -182,7 +188,7 @@ def test_global_disable_flag_no_longer_adds_transform_inline(is_disabled): ] result = litellm.FireworksAIConfig()._transform_messages_helper( messages=messages, - model="accounts/fireworks/models/minimax-m3", + model=VISION_MODEL, litellm_params={}, ) assert result[0]["content"][0]["image_url"] == url @@ -204,7 +210,7 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch): ) as mock_post: try: completion( - model="fireworks_ai/accounts/fireworks/models/minimax-m3", + model=f"fireworks_ai/{VISION_MODEL}", messages=[ { "role": "user", diff --git a/tests/llm_translation/test_gemini_image_usage.py b/tests/llm_translation/test_gemini_image_usage.py index 096f9c4796c..0be8b6c23e1 100644 --- a/tests/llm_translation/test_gemini_image_usage.py +++ b/tests/llm_translation/test_gemini_image_usage.py @@ -238,7 +238,7 @@ def test_gemini_image_generation_accumulates_multiple_image_prompt_token_details os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" + model = "gemini/gemini-3-pro-image" config = GoogleImageGenConfig() usage_metadata = { diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py index c720f818eaf..fbecbeab08b 100644 --- a/tests/llm_translation/test_groq.py +++ b/tests/llm_translation/test_groq.py @@ -32,7 +32,7 @@ class TestGroq(BaseLLMChatTest): @pytest.mark.parametrize( "model", - ["groq/qwen/qwen3-32b", "groq/openai/gpt-oss-20b", "groq/openai/gpt-oss-120b"], + ["groq/qwen/qwen3.8-27b", "groq/openai/gpt-oss-20b", "groq/openai/gpt-oss-120b"], ) def test_reasoning_effort_in_supported_params(self, model): """Test that reasoning_effort is in the list of supported parameters for Groq""" diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 997f5b3b73f..58446014bdf 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -537,7 +537,7 @@ def test_dynamic_drop_params_e2e(): ) as mock_response: try: response = litellm.completion( - model="command-r", + model="command-r-08-2024", messages=[{"role": "user", "content": "Hey, how's it going?"}], response_format={"key": "value"}, drop_params=True, @@ -556,7 +556,7 @@ def test_dynamic_pass_additional_params(): ) as mock_response: try: response = litellm.completion( - model="command-r", + model="command-r-08-2024", messages=[{"role": "user", "content": "Hey, how's it going?"}], custom_param="test", api_key="my-custom-key", @@ -606,7 +606,7 @@ def test_dynamic_drop_params_parallel_tool_calls(): ) as mock_response: try: response = litellm.completion( - model="command-r", + model="command-r-08-2024", messages=[{"role": "user", "content": "Hey, how's it going?"}], parallel_tool_calls=True, drop_params=True, @@ -663,7 +663,7 @@ def test_dynamic_drop_additional_params_e2e(): ) as mock_response: try: response = litellm.completion( - model="command-r", + model="command-r-08-2024", messages=[{"role": "user", "content": "Hey, how's it going?"}], response_format={"key": "value"}, additional_drop_params=["response_format"], diff --git a/tests/llm_translation/test_xai.py b/tests/llm_translation/test_xai.py index 7a121afc3fa..d6d42ed215e 100644 --- a/tests/llm_translation/test_xai.py +++ b/tests/llm_translation/test_xai.py @@ -164,7 +164,7 @@ def test_xai_message_name_filtering(): class TestXAIReasoningEffort(BaseReasoningLLMTests): def get_base_completion_call_args(self): return { - "model": "xai/grok-3-mini-beta", + "model": "xai/grok-4.7", "messages": [{"role": "user", "content": "Hello"}], } diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 3d66064f5c0..8b45ae08813 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -2863,7 +2863,7 @@ def test_gemini_function_call_parameter_in_messages(): mock_client.return_value = mock_response try: completion( - model="vertex_ai/gemini-2.0-flash", + model="vertex_ai/gemini-2.5-flash-preview-09-2025", messages=messages, tools=tools, tool_choice="auto", @@ -3263,7 +3263,7 @@ def test_vertex_anthropic_completion(): client, "post", side_effect=vertex_ai_anthropic_thinking_mock_response ): response = completion( - model="vertex_ai/claude-3-7-sonnet@20250219", + model="vertex_ai/claude-sonnet-4-6@default", messages=[{"role": "user", "content": "Hello, world!"}], vertex_ai_location="us-east5", vertex_ai_project="test-project", @@ -3271,7 +3271,7 @@ def test_vertex_anthropic_completion(): client=client, ) print(response) - assert response.model == "claude-3-7-sonnet@20250219" + assert response.model == "claude-sonnet-4-6@default" assert response._hidden_params["response_cost"] is not None assert response._hidden_params["response_cost"] > 0 diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index f40818b9bf1..3ce99f893d8 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -445,7 +445,7 @@ def test_groq_response_cost_tracking(is_streaming): response_cost = litellm.response_cost_calculator( response_object=response, - model="groq/llama-3.3-70b-versatile", + model="groq/openai/gpt-oss-120b", custom_llm_provider="groq", call_type=CallTypes.acompletion.value, optional_params={}, @@ -515,7 +515,7 @@ def test_gemini_completion_cost(provider): """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - model_name = "gemini-2.0-flash" + model_name = "gemini-3.8-flash" prompt_tokens = 128.0 output_tokens = 228.0 ## GET MODEL FROM LITELLM.MODEL_INFO @@ -543,7 +543,7 @@ def test_vertex_ai_completion_cost(): prompt_tokens = 100 - model_info = litellm.get_model_info(model="gemini-2.0-flash") + model_info = litellm.get_model_info(model="gemini-3.8-flash") print("\nExpected model info:\n{}\n\n".format(model_info)) @@ -551,7 +551,7 @@ def test_vertex_ai_completion_cost(): ## CALCULATED COST calculated_input_cost, calculated_output_cost = cost_per_token( - model="gemini-2.0-flash", + model="gemini-3.8-flash", custom_llm_provider="vertex_ai", prompt_tokens=prompt_tokens, completion_tokens=0, @@ -676,7 +676,7 @@ async def test_completion_cost_hidden_params(sync_mode): def test_vertex_ai_gemini_predict_cost(): - model = "gemini-2.0-flash" + model = "gemini-3.8-flash" messages = [{"role": "user", "content": "Hey, hows it going???"}] predictive_cost = completion_cost(model=model, messages=messages) @@ -757,24 +757,24 @@ def test_completion_cost_tts(model): def test_completion_cost_anthropic(): """ - model_name: claude-3-haiku-20240307 + model_name: claude-haiku-4-5 litellm_params: - model: anthropic/claude-3-haiku-20240307 + model: anthropic/claude-haiku-4-5 max_tokens: 4096 """ router = litellm.Router( model_list=[ { - "model_name": "claude-3-haiku-20240307", + "model_name": "claude-haiku-4-5", "litellm_params": { - "model": "anthropic/claude-3-haiku-20240307", + "model": "anthropic/claude-haiku-4-5", "max_tokens": 4096, }, } ] ) data = { - "model": "claude-3-haiku-20240307", + "model": "claude-haiku-4-5", "prompt_tokens": 21, "completion_tokens": 20, "response_time_ms": 871.7040000000001, @@ -2068,14 +2068,14 @@ def test_completion_cost_params(): """ litellm.set_verbose = True resp1_prompt_cost, resp1_completion_cost = cost_per_token( - model="gemini-2.0-flash", + model="gemini-3.8-flash", prompt_tokens=1000, completion_tokens=1000, custom_llm_provider="vertex_ai_beta", ) resp2_prompt_cost, resp2_completion_cost = cost_per_token( - model="gemini-2.0-flash", prompt_tokens=1000, completion_tokens=1000 + model="gemini-3.8-flash", prompt_tokens=1000, completion_tokens=1000 ) assert resp2_prompt_cost > 0 @@ -2084,7 +2084,7 @@ def test_completion_cost_params(): assert resp1_completion_cost == resp2_completion_cost resp3_prompt_cost, resp3_completion_cost = cost_per_token( - model="vertex_ai/gemini-2.0-flash", prompt_tokens=1000, completion_tokens=1000 + model="vertex_ai/gemini-3.8-flash", prompt_tokens=1000, completion_tokens=1000 ) assert resp3_prompt_cost > 0 @@ -2102,14 +2102,14 @@ def test_completion_cost_params_2(): prompt_tokens = 1000 completion_tokens = 1000 resp1_prompt_cost, resp1_completion_cost = cost_per_token( - model="gemini-2.0-flash", + model="gemini-3.8-flash", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, ) print(resp1_prompt_cost, resp1_completion_cost) - model_info = litellm.get_model_info("gemini-2.0-flash") + model_info = litellm.get_model_info("gemini-3.8-flash") input_cost_per_token = model_info["input_cost_per_token"] output_cost_per_token = model_info["output_cost_per_token"] @@ -2148,7 +2148,7 @@ def test_completion_cost_params_gemini_3(): ) ], created=1728529259, - model="gemini-2.0-flash", + model="gemini-3.8-flash", object="chat.completion", system_fingerprint=None, usage=usage, @@ -2172,7 +2172,7 @@ def test_completion_cost_params_gemini_3(): pc, cc = cost_per_character( **{ - "model": "gemini-2.0-flash", + "model": "gemini-3.8-flash", "custom_llm_provider": "vertex_ai", "prompt_characters": None, "completion_characters": 3, @@ -2180,9 +2180,9 @@ def test_completion_cost_params_gemini_3(): } ) - model_info = litellm.get_model_info("gemini-2.0-flash") + model_info = litellm.get_model_info("gemini-3.8-flash") - # gemini-2.0-flash has no per-character pricing, so cost_per_character + # gemini-3.8-flash has no per-character pricing, so cost_per_character # falls back to per-token pricing using usage.prompt_tokens / usage.completion_tokens assert round(pc, 10) == round(3771 * model_info["input_cost_per_token"], 10) assert round(cc, 10) == round( @@ -2239,16 +2239,16 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): ) ], created=1729282652, - model="gpt-4o-audio-preview", + model="gpt-audio-1.5", object="chat.completion", system_fingerprint="fp_4eafc16e9d", usage=usage_object, service_tier=None, ) - cost = completion_cost(completion, model="gpt-4o-audio-preview") + cost = completion_cost(completion, model="gpt-audio-1.5") - model_info = litellm.get_model_info("gpt-4o-audio-preview") + model_info = litellm.get_model_info("gpt-audio-1.5") print(f"model_info: {model_info}") ## input cost @@ -2517,7 +2517,7 @@ def test_cost_calculator_with_base_model(): resp = litellm.completion( model="bedrock/random-model", messages=[{"role": "user", "content": "Hello, how are you?"}], - base_model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + base_model="bedrock/anthropic.claude-sonnet-5", mock_response="Hello, how are you?", ) assert resp.model == "random-model" @@ -2551,10 +2551,10 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg): if base_model_arg == "litellm_param": model_item["litellm_params"][ "base_model" - ] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0" + ] = "bedrock/anthropic.claude-sonnet-5" elif base_model_arg == "model_info": model_item["model_info"] = { - "base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + "base_model": "bedrock/anthropic.claude-sonnet-5", } router = Router(model_list=[model_item]) diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index e6392cda406..813146f8ace 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -1148,7 +1148,7 @@ def test_openai_gateway_timeout_error(): @pytest.mark.parametrize( "provider, model, call_type", [ - ("anthropic", "claude-3-haiku-20240307", "chat_completion"), + ("anthropic", "claude-haiku-4-5-20251001", "chat_completion"), ], ) @pytest.mark.asyncio diff --git a/tests/local_testing/test_function_call_parsing.py b/tests/local_testing/test_function_call_parsing.py index c98f170a98f..ebb13e0018d 100644 --- a/tests/local_testing/test_function_call_parsing.py +++ b/tests/local_testing/test_function_call_parsing.py @@ -136,7 +136,7 @@ def trade(model_name: str) -> List[Trade]: # type: ignore @pytest.mark.parametrize( - "model", ["claude-haiku-4-5-20251001", "anthropic.claude-3-haiku-20240307-v1:0"] + "model", ["claude-haiku-4-5-20251001", "us.anthropic.claude-haiku-4-5-20251001-v1:0"] ) @pytest.mark.flaky(retries=6, delay=10) def test_function_call_parsing(model): diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index ebad0fbafc5..4ac7cecb97a 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -67,7 +67,17 @@ def test_get_llm_provider_deepseek_custom_api_base(): os.environ.pop("DEEPSEEK_API_BASE") -def test_get_llm_provider_vertex_ai_image_models(): +def test_get_llm_provider_vertex_ai_image_models(monkeypatch): + monkeypatch.setattr(litellm, "vertex_ai_image_models", set()) + monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) + litellm.add_known_models( + model_cost_map={ + "vertex_ai/imagegeneration@006": { + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + } + } + ) model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( model="imagegeneration@006", custom_llm_provider=None ) @@ -101,17 +111,17 @@ def test_get_llm_provider_ai21_chat_test2(): def test_get_llm_provider_cohere_chat_test2(): """ - if user prefix with cohere/ but calls command-r-plus then it should be cohere_chat provider + if user prefix with cohere/ but calls command-r-plus-08-2024 then it should be cohere_chat provider """ model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( - model="cohere/command-r-plus", + model="cohere/command-r-plus-08-2024", ) print("model=", model) print("custom_llm_provider=", custom_llm_provider) print("api_base=", api_base) assert custom_llm_provider == "cohere_chat" - assert model == "command-r-plus" + assert model == "command-r-plus-08-2024" def test_get_llm_provider_azure_o1(): diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 37f4ece611d..1e46a1bf853 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -16,7 +16,7 @@ def test_get_model_info_simple_model_name(): """ tests if model name given, and model exists in model info - the object is returned """ - model = "claude-3-opus-20240229" + model = "claude-opus-5-5" litellm.get_model_info(model) @@ -24,7 +24,7 @@ def test_get_model_info_custom_llm_with_model_name(): """ Tests if {custom_llm_provider}/{model_name} name given, and model exists in model info, the object is returned """ - model = "anthropic/claude-3-opus-20240229" + model = "anthropic/claude-opus-5-5" litellm.get_model_info(model) diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 5bf3a3ee98b..631271ca710 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -28,7 +28,7 @@ async def test_get_available_deployments(): }, { "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "groq/llama-3.1-8b-instant"}, + "litellm_params": {"model": "groq/openai/gpt-oss-20b"}, "model_info": {"id": "groq-llama"}, }, ] diff --git a/tests/local_testing/test_openai_moderations_hook.py b/tests/local_testing/test_openai_moderations_hook.py index 530ab714eae..7ce4bc2e4bf 100644 --- a/tests/local_testing/test_openai_moderations_hook.py +++ b/tests/local_testing/test_openai_moderations_hook.py @@ -31,7 +31,7 @@ async def test_openai_moderation_error_raising(monkeypatch): from unittest.mock import AsyncMock, MagicMock from litellm.types.llms.openai import OpenAIModerationResponse - litellm.openai_moderations_model_name = "text-moderation-latest" + litellm.openai_moderations_model_name = "omni-moderation-latest" openai_mod = _ENTERPRISE_OpenAI_Moderation() _api_key = "sk-12345" _api_key = hash_token("sk-12345") @@ -41,9 +41,9 @@ async def test_openai_moderation_error_raising(monkeypatch): llm_router = litellm.Router( model_list=[ { - "model_name": "text-moderation-latest", + "model_name": "omni-moderation-latest", "litellm_params": { - "model": "text-moderation-latest", + "model": "omni-moderation-latest", "api_key": os.environ.get("OPENAI_API_KEY", "fake-key"), }, } diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index 1b3e361bb1f..635bda55144 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -188,7 +188,7 @@ def test_router_get_model_info_wildcard_routes(): ] ) model_info = router.get_router_model_info( - deployment=None, received_model_name="gemini/gemini-1.5-flash", id="1" + deployment=None, received_model_name="gemini/gemini-2.5-flash", id="1" ) print(model_info) assert model_info is not None @@ -212,7 +212,7 @@ async def test_router_get_model_group_usage_wildcard_routes(): ) resp = await router.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="Hello, I'm good.", ) @@ -220,7 +220,7 @@ async def test_router_get_model_group_usage_wildcard_routes(): await asyncio.sleep(2) - tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-1.5-flash") + tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-2.5-flash") assert tpm is not None, "tpm is None" assert rpm is not None, "rpm is None" @@ -242,7 +242,7 @@ async def test_call_router_callbacks_on_success(): router.cache, "async_increment_cache_pipeline", new=AsyncMock() ) as mock_callback: await router.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="Hello, I'm good.", ) @@ -255,12 +255,12 @@ async def test_call_router_callbacks_on_success(): for increment in increment_list: if "tpm" in increment["key"]: assert increment["key"].startswith( - "global_router:1:gemini/gemini-1.5-flash:tpm" + "global_router:1:gemini/gemini-2.5-flash:tpm" ) assert increment["increment_value"] == 30 elif "rpm" in increment["key"]: assert increment["key"].startswith( - "global_router:1:gemini/gemini-1.5-flash:rpm" + "global_router:1:gemini/gemini-2.5-flash:rpm" ) assert increment["increment_value"] == 1 @@ -283,7 +283,7 @@ async def test_call_router_callbacks_on_failure(): ) as mock_callback: with pytest.raises(litellm.RateLimitError): await router.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="litellm.RateLimitError", num_retries=0, @@ -295,7 +295,7 @@ async def test_call_router_callbacks_on_failure(): assert ( mock_callback.call_args_list[0] .kwargs["key"] - .startswith("global_router:1:gemini/gemini-1.5-flash:rpm") + .startswith("global_router:1:gemini/gemini-2.5-flash:rpm") ) @@ -317,7 +317,7 @@ async def test_router_model_group_headers(): for _ in range(2): resp = await router.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="Hello, I'm good.", ) @@ -325,7 +325,7 @@ async def test_router_model_group_headers(): assert ( resp._hidden_params["additional_headers"]["x-litellm-model-group"] - == "gemini/gemini-1.5-flash" + == "gemini/gemini-2.5-flash" ) assert "x-ratelimit-remaining-requests" in resp._hidden_params["additional_headers"] @@ -349,7 +349,7 @@ async def test_get_remaining_model_group_usage(): ) for _ in range(2): resp = await router.acompletion( - model="gemini/gemini-1.5-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="Hello, I'm good.", ) @@ -363,7 +363,7 @@ async def test_get_remaining_model_group_usage(): await asyncio.sleep(1) remaining_usage = await router.get_remaining_model_group_usage( - model_group="gemini/gemini-1.5-flash" + model_group="gemini/gemini-2.5-flash" ) assert remaining_usage is not None assert "x-ratelimit-remaining-requests" in remaining_usage diff --git a/tests/local_testing/test_spend_calculate_endpoint.py b/tests/local_testing/test_spend_calculate_endpoint.py index 3bedab794e2..054dc398039 100644 --- a/tests/local_testing/test_spend_calculate_endpoint.py +++ b/tests/local_testing/test_spend_calculate_endpoint.py @@ -38,7 +38,7 @@ async def test_spend_calc_model_on_router_messages(): { "model_name": "special-llama-model", "litellm_params": { - "model": "groq/llama-3.1-8b-instant", + "model": "groq/openai/gpt-oss-20b", }, } ] @@ -81,7 +81,7 @@ async def test_spend_calc_using_response(): } ], "created": "1677652288", - "model": "groq/llama-3.1-8b-instant", + "model": "groq/openai/gpt-oss-20b", "object": "chat.completion", "system_fingerprint": "fp_873a560973", "usage": { diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json index b6c11f96953..5998c52659c 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json @@ -31,14 +31,14 @@ "model_id": null, "cache_key": null, "api_base": null, - "response_cost": 7.5e-06, + "response_cost": 3.5e-05, "additional_headers": {}, "litellm_overhead_time_ms": null, "batch_models": null, - "litellm_model_name": "vertex_ai/gemini-2.0-flash-001", + "litellm_model_name": "vertex_ai/gemini-3-flash-preview", "usage_object": null }, - "litellm_response_cost": 7.5e-06, + "litellm_response_cost": 3.5e-05, "cache_hit": false, "requester_metadata": {} }, @@ -54,13 +54,13 @@ "id": "time-14-15-40-349639_chatcmpl-59a988d0-7ef1-4dc4-bc18-d2e78961817f", "endTime": "2025-05-26T14:15:40.607266-07:00", "completionStartTime": "2025-05-26T14:15:40.607266-07:00", - "model": "gemini-2.0-flash-001", + "model": "gemini-3-flash-preview", "modelParameters": {}, "usage": { "input": 10, "output": 10, "unit": "TOKENS", - "totalCost": 7.5e-06 + "totalCost": 3.5e-05 }, "usageDetails": { "input": 10, diff --git a/tests/logging_callback_tests/test_alerting.py b/tests/logging_callback_tests/test_alerting.py index 3074e973a8e..0a3e1a0e982 100644 --- a/tests/logging_callback_tests/test_alerting.py +++ b/tests/logging_callback_tests/test_alerting.py @@ -582,7 +582,7 @@ async def test_webhook_alerting(alerting_type): None, None, ), - ("gemini-2.0-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"), + ("gemini-3.8-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"), ], ) @pytest.mark.parametrize("error_code", [500, 408, 400]) @@ -688,7 +688,7 @@ async def test_outage_alerting_called( None, None, ), - ("gemini-2.0-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"), + ("gemini-3.8-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"), ], ) @pytest.mark.parametrize("error_code", [500, 408, 400]) @@ -775,7 +775,7 @@ async def test_region_outage_alerting_called( await slack_alerting.region_outage_alerts( exception=error_to_raise, deployment_id=deployment_id # type: ignore ) - if model == "gemini-2.0-flash" and (error_code == 500 or error_code == 408): + if model == "gemini-3.8-flash" and (error_code == 500 or error_code == 408): mock_send_alert.assert_called_once() else: mock_send_alert.assert_not_called() diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index 5682d3720d8..76ebd2b9a28 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -481,12 +481,12 @@ class TestLangfuseLogging: completion_tokens=10, total_tokens=20, ), - model="vertex/gemini-2.0-flash-001", + model="vertex/gemini-3-flash-preview", object="chat.completion", created=1723081200, ).model_dump() await litellm.acompletion( - model="vertex_ai/gemini-2.0-flash-001", + model="vertex_ai/gemini-3-flash-preview", messages=[{"role": "user", "content": "Hello!"}], mock_response=mock_response, metadata={"trace_id": setup["trace_id"]}, diff --git a/tests/unit/repositories/test_repositories.py b/tests/unit/repositories/test_repositories.py index 87cf2fc4268..e185d95ffb8 100644 --- a/tests/unit/repositories/test_repositories.py +++ b/tests/unit/repositories/test_repositories.py @@ -2120,6 +2120,7 @@ class TestPrismaTableRepository: "litellm_prompttable", "litellm_searchtoolstable", "litellm_ssoconfig", + "litellm_uisettings", } ) From 7688f56256484f866687e147d0a994fb7649dd6d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:28:48 -0700 Subject: [PATCH 17/83] fix(pricing): drop the duplicate cache_read_input_token_cost_batches key from 23 entries (#42623) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 69 +++++++------------ model_prices_and_context_window.json | 69 +++++++------------ 2 files changed, 46 insertions(+), 92 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 224129fbd25..ad66cb76f8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25314,8 +25314,7 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query", - "cache_read_input_token_cost_batches": 1e-07 + "web_search_billing_unit": "per_query" }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -25401,8 +25400,7 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query", - "cache_read_input_token_cost_batches": 2.5e-08 + "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -25482,8 +25480,7 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_vision": true }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -25593,8 +25590,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07, - "cache_read_input_token_cost_batches": 1.25e-08 + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -25652,8 +25648,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 1.5e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -25689,8 +25684,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true, - "cache_read_input_token_cost_batches": 1e-07 + "supports_web_search": true }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -26321,8 +26315,7 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08, - "cache_read_input_token_cost_batches": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -26380,8 +26373,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -26440,8 +26432,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -26500,8 +26491,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -28250,8 +28240,7 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08, - "cache_read_input_token_cost_batches": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -28309,8 +28298,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -28369,8 +28357,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -28429,8 +28416,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -33110,8 +33096,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 6.25e-08 + "supports_minimal_reasoning_effort": true }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -33292,8 +33277,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -33392,8 +33376,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 2.5e-09 + "supports_minimal_reasoning_effort": true }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -47531,8 +47514,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 1e-07 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -47570,8 +47552,7 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 2.5e-08 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -47627,8 +47608,7 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_vision": true }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -47739,8 +47719,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07, - "cache_read_input_token_cost_batches": 1.25e-08 + "input_cost_per_audio_token_batches": 2.5e-07 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -47799,8 +47778,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 1.5e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -47817,8 +47795,7 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 1e-07 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 224129fbd25..ad66cb76f8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25314,8 +25314,7 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query", - "cache_read_input_token_cost_batches": 1e-07 + "web_search_billing_unit": "per_query" }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -25401,8 +25400,7 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query", - "cache_read_input_token_cost_batches": 2.5e-08 + "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -25482,8 +25480,7 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_vision": true }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -25593,8 +25590,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07, - "cache_read_input_token_cost_batches": 1.25e-08 + "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -25652,8 +25648,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 1.5e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -25689,8 +25684,7 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true, - "cache_read_input_token_cost_batches": 1e-07 + "supports_web_search": true }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -26321,8 +26315,7 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08, - "cache_read_input_token_cost_batches": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -26380,8 +26373,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -26440,8 +26432,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -26500,8 +26491,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -28250,8 +28240,7 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08, - "cache_read_input_token_cost_batches": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -28309,8 +28298,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, @@ -28369,8 +28357,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, @@ -28429,8 +28416,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 3.75e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -33110,8 +33096,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 6.25e-08 + "supports_minimal_reasoning_effort": true }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -33292,8 +33277,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_minimal_reasoning_effort": true }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, @@ -33392,8 +33376,7 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, - "cache_read_input_token_cost_batches": 2.5e-09 + "supports_minimal_reasoning_effort": true }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -47531,8 +47514,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 1e-07 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -47570,8 +47552,7 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 2.5e-08 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -47627,8 +47608,7 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true, - "cache_read_input_token_cost_batches": 1.25e-08 + "supports_vision": true }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -47739,8 +47719,7 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07, - "cache_read_input_token_cost_batches": 1.25e-08 + "input_cost_per_audio_token_batches": 2.5e-07 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", @@ -47799,8 +47778,7 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014, - "cache_read_input_token_cost_batches": 1.5e-08 + "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, @@ -47817,8 +47795,7 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "cache_read_input_token_cost_batches": 1e-07 + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, From b4ccb5b7474270bd69f1b491a3febf0bd2c44210 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 22 Sep 2026 20:33:44 -0400 Subject: [PATCH 18/83] fix(s3): replace colons in generated log filenames (#40452) * fix(s3): replace colons in generated log filenames Bedrock and Vertex AI batch file uploads use s3:// and gs:// URIs as response ids. The shared filename sanitizer replaced slashes but kept the scheme colon, producing log object keys that Hadoop-style consumers reject as a relative path in an absolute URI. Fixes #40234 * test(s3): drop docstrings flagged by review --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/s3.py | 2 +- tests/test_litellm/integrations/test_s3_v2.py | 28 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index 796784fb993..54ec876fd0d 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -277,7 +277,7 @@ def get_s3_object_key( start_time: datetime, s3_file_name: str, ) -> str: - sanitized_s3_file_name: Final = s3_file_name.replace("/", "_") + sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_") configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix date_segment: Final = start_time.strftime("%Y-%m-%d") + "/" # we need the s3 key to include the time, so we log cache hits too diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 52fbbe40b0e..a9d13038180 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -1273,9 +1273,7 @@ async def test_combined_prefix_reflects_in_s3_object_key(): assert "myteam/apikey/" in key, f"Expected both prefixes in key: {key}" -def test_s3_object_key_sanitizes_slashes_in_file_name(): - """Response ids containing slashes (e.g. bedrock batch job ARNs) must not - create nested S3 folders; only path/prefix/date slashes are separators.""" +def test_s3_object_key_sanitizes_slashes_and_colons_in_file_name(): from litellm.integrations.s3 import get_s3_object_key start_time = datetime(2026, 2, 11, 0, 35, 18, 391582) @@ -1290,10 +1288,32 @@ def test_s3_object_key_sanitizes_slashes_in_file_name(): assert key == ( "LiteLLMAPPLogs/myteam/2026-02-11/" - "time-00-35-18-391582_arn:aws:bedrock:us-east-1:123456789012:model-invocation-job_gl18r6skk9yy.json" + "time-00-35-18-391582_arn_aws_bedrock_us-east-1_123456789012_model-invocation-job_gl18r6skk9yy.json" ) +@pytest.mark.parametrize( + "response_id", + [ + "s3://example-batch-bucket/litellm-bedrock-files/input.jsonl", + "gs://example-batch-bucket/litellm-vertex-files/input.jsonl", + ], +) +def test_s3_object_key_has_no_colon_for_cloud_uri_file_ids(response_id: str): + from litellm.integrations.s3 import get_s3_object_key + + key = get_s3_object_key( + s3_path="", + prefix="", + start_time=datetime(2026, 9, 7, 4, 51, 6, 685889), + s3_file_name=f"time-04-51-06-685889_{response_id}", + ) + + filename = key.rsplit("/", 1)[-1] + assert ":" not in filename + assert filename.endswith("_input.jsonl.json") + + def test_create_s3_batch_logging_element_flat_key_for_arn_response_id(): """End-to-end through the s3_v2 element builder: an ARN response id must yield a flat file directly under the date segment.""" From 38f0eb876bf2f05b4850adb8e80bcb48885e557e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:42:12 -0700 Subject: [PATCH 19/83] test(realtime): drop legacy InvalidStatusCode tests and pin websockets imports (#42624) The two redaction tests raised the deprecated InvalidStatusCode, which the websockets 15 asyncio client never raises, and asserted the raw 403 close code that the handshake refusal path replaced with 1008. The refusal path builds its close reason from the status code alone, so there is no secret to redact there, and the handshake refusal tests already cover the error event and the 1008 close. Those refusal tests only passed when run after a sibling test had imported websockets.asyncio.client, since websockets lazy-loads its exceptions submodule. Importing InvalidStatus, Response, and Headers from their own submodules makes them pass in any order. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/azure/realtime/test_handler.py | 8 +-- .../realtime/test_openai_realtime_handler.py | 8 +-- .../test_redact_string_in_error_paths.py | 56 +------------------ 3 files changed, 9 insertions(+), 63 deletions(-) diff --git a/tests/test_litellm/llms/azure/realtime/test_handler.py b/tests/test_litellm/llms/azure/realtime/test_handler.py index edf1b8b290f..e9d24b459d8 100644 --- a/tests/test_litellm/llms/azure/realtime/test_handler.py +++ b/tests/test_litellm/llms/azure/realtime/test_handler.py @@ -21,7 +21,9 @@ class _RecordingClientWebSocket: @pytest.mark.asyncio async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): - import websockets + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatus + from websockets.http11 import Response from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime from litellm.types.realtime import RealtimeErrorEvent @@ -32,9 +34,7 @@ async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_ dummy_websocket = _RecordingClientWebSocket() dummy_logging_obj = MagicMock() - refused = websockets.exceptions.InvalidStatus( - websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) - ) + refused = InvalidStatus(Response(401, "Unauthorized", Headers())) with patch("websockets.connect", side_effect=refused): await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is a Protocol here but the mock connect type is incomplete diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 7cd2b9e259c..f7a88b5ba63 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -422,7 +422,9 @@ async def test_async_realtime_ws_url_has_no_ssl(): async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): from typing import cast - import websockets + from websockets.datastructures import Headers + from websockets.exceptions import InvalidStatus + from websockets.http11 import Response from litellm.llms.openai.realtime.handler import OpenAIRealtime from litellm.types.realtime import RealtimeErrorEvent @@ -445,9 +447,7 @@ async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_ dummy_websocket = RecordingClientWebSocket() dummy_logging_obj = MagicMock() - refused = websockets.exceptions.InvalidStatus( - websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) - ) + refused = InvalidStatus(Response(401, "Unauthorized", Headers())) with patch("websockets.connect", side_effect=refused): await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is Any diff --git a/tests/test_litellm/test_redact_string_in_error_paths.py b/tests/test_litellm/test_redact_string_in_error_paths.py index 07d1ec5f523..a5128a87b0d 100644 --- a/tests/test_litellm/test_redact_string_in_error_paths.py +++ b/tests/test_litellm/test_redact_string_in_error_paths.py @@ -2,7 +2,7 @@ Tests for _redact_string usage in error/logging paths. Covers actual execution of redaction in: -- WebSocket close reasons in realtime handlers (openai, azure, bedrock) +- WebSocket close reasons in realtime handlers (openai, bedrock) - Gemini RAG ingestion x-goog-api-key header usage - Traceback redaction pattern used in proxy streaming - Router fallback-failure traceback redaction @@ -72,25 +72,6 @@ class TestOpenAIRealtimeRedaction: api_key="test-key", ) - @pytest.mark.asyncio - async def test_invalid_status_code_redacts_reason(self): - import websockets.exceptions - - from litellm.llms.openai.realtime.handler import OpenAIRealtime - - handler = OpenAIRealtime() - exc = websockets.exceptions.InvalidStatusCode(403, None) - exc.status_code = 403 - - kwargs = self._call_kwargs() - mock_ws = kwargs["websocket"] - p1, p2, p3 = self._make_patches(handler) - with p1, p2, p3, patch("websockets.connect", side_effect=exc): - await handler.async_realtime(**kwargs) - - mock_ws.close.assert_called_once() - assert mock_ws.close.call_args[1]["code"] == 403 - @pytest.mark.asyncio async def test_generic_exception_redacts_reason(self): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -111,41 +92,6 @@ class TestOpenAIRealtimeRedaction: assert "sk-1234567890abcdefghij" not in mock_ws.close.call_args[1]["reason"] -class TestAzureRealtimeRedaction: - """Test that Azure realtime handler redacts secrets in websocket close reasons.""" - - @pytest.mark.asyncio - async def test_invalid_status_code_redacts_reason(self): - import websockets.exceptions - - from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime - - handler = AzureOpenAIRealtime() - mock_ws = AsyncMock() - exc = websockets.exceptions.InvalidStatusCode(403, None) - exc.status_code = 403 - - with ( - patch.object( - handler, - "_construct_url", - return_value="wss://test.openai.azure.com/openai/realtime", - ), - patch("websockets.connect", side_effect=exc), - ): - await handler.async_realtime( - model="gpt-4", - websocket=mock_ws, - logging_obj=MagicMock(), - api_base="https://test.openai.azure.com/", - api_key="test-key", - api_version="2024-10-01-preview", - ) - - mock_ws.close.assert_called_once() - assert mock_ws.close.call_args[1]["code"] == 403 - - class TestBedrockRealtimeRedaction: """Test that _redact_string produces safe close reasons for Bedrock-style errors.""" From 8ee6bab52931798d9aec00003797c25d0190e144 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:55:22 -0700 Subject: [PATCH 20/83] fix(bedrock): treat blank AWS_S3_* env vars as unset for batch jobs (#42528) * test(e2e): pin bedrock batch create with blank S3 env vars Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): treat blank S3 env vars as unset for batch jobs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): trim blank S3 env gateway config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): register blank_s3_env capability and clean gateway tempdir Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(e2e): move blank S3 env batch test to its own module Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/common_utils.py | 2 +- tests/e2e/batches/COVERAGE.md | 1 + tests/e2e/batches/bedrock_env_gateway.py | 145 ++++++++++++++++++ .../batches/test_bedrock_blank_s3_env_e2e.py | 109 +++++++++++++ .../llm_nonconversational.yaml | 1 + tests/e2e/coverage_registry/schema.py | 1 + .../bedrock/batches/test_transformation.py | 41 ++++- 7 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/batches/bedrock_env_gateway.py create mode 100644 tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d9fc813a594..2e20aafffcb 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1593,7 +1593,7 @@ def _resolve_s3_setting( source.get(param_name) for source in (litellm_params, optional_params) if source is not None ) explicit: Final = next((value for value in candidates if isinstance(value, str) and value), None) - return explicit or get_secret_str(env_var) + return explicit or get_secret_str(env_var) or None class CommonBatchFilesUtils: diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index 1731b4c620d..862eef5c0f4 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -22,6 +22,7 @@ failures are hard test failures (see `tests/e2e/AGENTS.md`). | Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | | Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) | | Bedrock split S3 identity | no | no | no | no | yes (file upload, content, delete) | S3 signed with `s3_access_key_id` / `s3_secret_access_key` (`AWS_S3_ONLY_ACCESS_KEY_ID` / `AWS_S3_ONLY_SECRET_ACCESS_KEY`, object rights on `AWS_BATCH_S3_BUCKET` only) while `aws_*` is `AWS_BEDROCK_ONLY_ACCESS_KEY_ID` / `AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY`, an identity with no S3 rights on that bucket | +| Bedrock blank S3 env | yes (unified only, on an owned gateway exporting `AWS_S3_ENCRYPTION_KEY_ID` / `AWS_S3_BUCKET_OWNER` as empty strings) | no | no | no | no | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` in the gateway config); blank env vars must be treated as unset, not serialized | Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`). diff --git a/tests/e2e/batches/bedrock_env_gateway.py b/tests/e2e/batches/bedrock_env_gateway.py new file mode 100644 index 00000000000..fb3ec60c87c --- /dev/null +++ b/tests/e2e/batches/bedrock_env_gateway.py @@ -0,0 +1,145 @@ +"""An owned, source-built proxy whose process env exports AWS_S3_* vars blank. + +The shared fixture proxy inherits the harness env, which cannot reproduce a user +shell that exports AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER as empty +strings. This gateway boots a second proxy with both vars present but blank, so +a batch create through it proves blank means unset, not an empty string. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final + +from e2e_config import unique_marker +from e2e_http import NoBody +from idp import stop_process_group +from proxy_client import ProxyClient, build_proxy_client +from pydantic import TypeAdapter + +STARTUP_TIMEOUT_SECONDS: Final = 240 +LOG_TAIL_BYTES: Final = 4000 +REPO_ROOT: Final = Path(__file__).resolve().parents[3] + +_CONFIG_YAML: Final = """model_list: + - model_name: bedrock-blank-s3-batch + litellm_params: + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION + s3_region_name: os.environ/AWS_REGION + s3_bucket_name: os.environ/AWS_BATCH_S3_BUCKET + s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID + s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL +""" + + +def available_port() -> int: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())[1] + + +@dataclass(slots=True) +class BedrockEnvGateway: + base_url: str + master_key: str + proxy: ProxyClient + _environment: Mapping[str, str] = field(repr=False) + _command: tuple[str, ...] = field(repr=False) + _log_path: Path + _child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False) + + @classmethod + def start(cls) -> BedrockEnvGateway: + assert os.environ.get("DATABASE_URL"), "DATABASE_URL is required for the blank-S3-env gateway" + port: Final = available_port() + base_url: Final = f"http://127.0.0.1:{port}" + master_key: Final = f"sk-e2e-blank-s3-{unique_marker()}" + directory: Final = Path(tempfile.mkdtemp(prefix="litellm-e2e-blank-s3-")) + config: Final = directory / "blank-s3-gateway.yaml" + config.write_text(_CONFIG_YAML) + environment: Final = { + **{key: value for key, value in os.environ.items() if not key.startswith("REDIS_")}, + "DATABASE_URL": os.environ["DATABASE_URL"], + "LITELLM_MASTER_KEY": master_key, + "STORE_MODEL_IN_DB": "False", + "PYTHONPATH": str(REPO_ROOT), + "AWS_S3_ENCRYPTION_KEY_ID": "", + "AWS_S3_BUCKET_OWNER": "", + } + gateway: Final = cls( + base_url=base_url, + master_key=master_key, + proxy=build_proxy_client( + base_url=base_url, + control_plane_base_url=base_url, + replica_urls=(base_url,), + master_key=master_key, + ), + _environment=environment, + _command=( + sys.executable, + "-m", + "litellm.proxy.proxy_cli", + "--config", + str(config), + "--port", + str(port), + "--host", + "127.0.0.1", + ), + _log_path=directory / "blank-s3-gateway.log", + ) + with gateway._log_path.open("ab") as log: + gateway._child = subprocess.Popen( + gateway._command, + env=dict(gateway._environment), + stdout=log, + stderr=log, + start_new_session=True, + cwd=REPO_ROOT, + ) + deadline: Final = time.monotonic() + STARTUP_TIMEOUT_SECONDS + while time.monotonic() < deadline: + assert gateway._child.poll() is None, ( + f"blank-S3-env gateway exited early; log tail:\n{gateway.log_tail()}" + ) + result = gateway.proxy.transport.probe("/health/liveliness", params=NoBody()) + if result.status_code == 200: + return gateway + time.sleep(0.5) + tail: Final = gateway.log_tail() + gateway.stop() + raise AssertionError( + f"blank-S3-env gateway did not become ready in {STARTUP_TIMEOUT_SECONDS}s; log tail:\n{tail}" + ) + + def log_tail(self) -> str: + if not self._log_path.exists(): + return "" + with self._log_path.open("rb") as log: + log.seek(0, 2) + size: Final = log.tell() + log.seek(max(0, size - LOG_TAIL_BYTES)) + return log.read().decode("utf-8", errors="replace") + + def stop(self) -> None: + if self._child is not None: + stop_process_group(self._child) + shutil.rmtree(self._log_path.parent, ignore_errors=True) diff --git a/tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py b/tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py new file mode 100644 index 00000000000..77eb8427e59 --- /dev/null +++ b/tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py @@ -0,0 +1,109 @@ +"""Live e2e pin for Bedrock batch create with blank AWS_S3_* env vars. + +Owns its own file (not test_batches_e2e.py) so the PR changed-file e2e gate +stays a single tiny file: this class boots its own gateway with +AWS_S3_ENCRYPTION_KEY_ID and AWS_S3_BUCKET_OWNER exported empty, then runs the +unified target_model_names upload + batch create lifecycle against real Bedrock. +""" + +from __future__ import annotations + +import json +from typing import Final + +import pytest +from batch_cleanup import cleanup_batch, cleanup_file +from batch_client import BatchClient, BatchCreateBody, BatchObject, FileObject +from bedrock_env_gateway import BedrockEnvGateway +from capabilities import is_managed_id +from e2e_http import FileUploadForm, require_successful_call, unwrap +from lifecycle import ResourceManager +from models import KeyGenerateBody + +pytestmark = pytest.mark.e2e + +CREATED_BATCH_STATUSES = {"validating", "in_progress", "finalizing"} +BLANK_S3_RAW_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +def render_jsonl(model: str) -> bytes: + line = { + "custom_id": "req-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 8, + }, + } + return (json.dumps(line) + "\n").encode() + + +def assert_file_object(file: FileObject, *, provider: str) -> None: + assert file.object == "file", f"file.object={file.object!r}" + assert file.purpose == "batch", f"file.purpose={file.purpose!r}" + assert file.bytes is not None, f"file.bytes={file.bytes!r}" + if provider != "bedrock": + assert file.bytes > 0, f"file.bytes={file.bytes!r}" + assert file.status, "file.status missing" + assert file.created_at is not None and file.created_at > 0, "file.created_at missing" + + +def assert_batch_object(batch: BatchObject) -> None: + assert batch.object == "batch", f"batch.object={batch.object!r}" + if batch.endpoint: + assert batch.endpoint == "/v1/chat/completions", f"batch.endpoint={batch.endpoint!r}" + assert batch.completion_window == "24h", f"window={batch.completion_window!r}" + assert batch.input_file_id, "batch.input_file_id missing" + assert batch.created_at is not None and batch.created_at > 0, "batch.created_at missing" + + +class TestBedrockBatchBlankS3EnvVars: + """Bedrock batch create with AWS_S3_* env vars exported but blank. + + Regression: a blank AWS_S3_ENCRYPTION_KEY_ID or AWS_S3_BUCKET_OWNER env var + resolved to "" and was serialized into the create-job request, which Bedrock + rejects. The owned gateway exports both vars empty, so the unified lifecycle + only passes when blank is treated as unset. + """ + + @pytest.mark.covers( + "llm.batches.bedrock.blank_s3_env.nonstream.works", + "llm.files.bedrock.upload.nonstream.works", + exercised_on=["batches", "files"], + ) + def test_unified_batch_create_ignores_blank_s3_env_vars(self, resources: ResourceManager) -> None: + gateway: Final = BedrockEnvGateway.start() + resources.defer(gateway.stop) + client: Final = BatchClient(proxy=gateway.proxy) + + key: Final = client.proxy.generate_key(KeyGenerateBody(models=[], user_id="e2e-test-user")) + resources.defer(lambda: client.proxy.delete_key(key)) + + file: Final = unwrap( + client.upload_file( + content=render_jsonl(BLANK_S3_RAW_MODEL), + form=FileUploadForm(purpose="batch", target_model_names="bedrock-blank-s3-batch"), + key=key, + ) + ) + resources.defer(lambda: cleanup_file(client, file.id, key=key)) + assert_file_object(file, provider="bedrock") + + created: Final = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key) + assert created.status_code < 400, ( + f"blank AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER must be treated as " + f"unset; Bedrock rejected the job: {created.body[:400]}" + ) + require_successful_call(created) + batch: Final = BatchObject.model_validate_json(created.body) + resources.defer(lambda: cleanup_batch(client, batch.id, key=key)) + + assert is_managed_id(batch.id), ( + f"blank-S3-env create via target_model_names must return a managed batch id, got {batch.id!r}" + ) + assert batch.status in CREATED_BATCH_STATUSES, ( + f"blank-S3-env batch has non-transitional status {batch.status!r}" + ) + assert_batch_object(batch) diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index c58c8af44ff..3e389acc2a9 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -24,6 +24,7 @@ - {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"} - {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"} - {id: llm.batches.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create in the us-gov-west-1 partition"} +- {id: llm.batches.bedrock.blank_s3_env.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: blank_s3_env, streaming: nonstream, assertions: [works], source: "test_bedrock_blank_s3_env_e2e.py", rationale: "Bedrock batch create treats blank AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER env vars as unset instead of serializing empty strings"} - {id: llm.batches.bedrock.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch cancel (StopModelInvocationJob) returns the same id with a cancelling/cancelled status"} - {id: llm.batches.bedrock.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "A Bedrock managed batch is present in the GET /v1/batches list envelope"} - {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index f3ac1ef8a83..e009b02b69c 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -65,6 +65,7 @@ LlmCapability = Literal[ "assume_role", "basic", "batch_deployment", + "blank_s3_env", "count_tokens", "govcloud_partition", "split_s3_credentials", diff --git a/tests/test_litellm/llms/bedrock/batches/test_transformation.py b/tests/test_litellm/llms/bedrock/batches/test_transformation.py index eb08c19cbdf..347c459a369 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_transformation.py +++ b/tests/test_litellm/llms/bedrock/batches/test_transformation.py @@ -19,9 +19,8 @@ from unittest.mock import MagicMock, patch import httpx import pytest - from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig -from litellm.types.utils import LiteLLMBatch, LlmProviders +from litellm.types.utils import LlmProviders # AWS JobStatus -> OpenAI BatchJobStatus, exactly as encoded in transformation.py # (both transform_create_batch_response and transform_retrieve_batch_response). @@ -270,6 +269,44 @@ def test_create_request_keeps_kms_key_alongside_s3_bucket_owner(config, monkeypa } +def test_create_request_omits_kms_key_when_env_var_is_blank(config, monkeypatch): + monkeypatch.setenv("AWS_S3_ENCRYPTION_KEY_ID", "") + monkeypatch.delenv("AWS_S3_BUCKET_OWNER", raising=False) + + bedrock_request = _signed_batch_request(config, {}, {}) + + assert bedrock_request["outputDataConfig"] == { + "s3OutputDataConfig": {"s3Uri": "s3://in-bucket/litellm-batch-outputs/litellm-batch-1/"} + } + + +def test_create_request_omits_s3_bucket_owner_when_env_var_is_blank(config, monkeypatch): + monkeypatch.setenv("AWS_S3_BUCKET_OWNER", "") + monkeypatch.delenv("AWS_S3_ENCRYPTION_KEY_ID", raising=False) + + bedrock_request = _signed_batch_request(config, {}, {}) + + assert bedrock_request["inputDataConfig"] == {"s3InputDataConfig": {"s3Uri": "s3://in-bucket/in.jsonl"}} + assert bedrock_request["outputDataConfig"] == { + "s3OutputDataConfig": {"s3Uri": "s3://in-bucket/litellm-batch-outputs/litellm-batch-1/"} + } + + +def test_create_request_emits_real_values_alongside_blank_sibling_env_var(config, monkeypatch): + monkeypatch.setenv("AWS_S3_ENCRYPTION_KEY_ID", "kms-key-123") + monkeypatch.setenv("AWS_S3_BUCKET_OWNER", "") + + bedrock_request = _signed_batch_request(config, {}, {}) + + assert bedrock_request["inputDataConfig"] == {"s3InputDataConfig": {"s3Uri": "s3://in-bucket/in.jsonl"}} + assert bedrock_request["outputDataConfig"] == { + "s3OutputDataConfig": { + "s3Uri": "s3://in-bucket/litellm-batch-outputs/litellm-batch-1/", + "s3EncryptionKeyId": "kms-key-123", + } + } + + def test_create_request_missing_input_file_id_raises(config): with pytest.raises(ValueError, match="input_file_id is required"): config.transform_create_batch_request( From 21a2d828dffd70b0c868d2dba487d3c372239475 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:56:32 +0000 Subject: [PATCH 21/83] ci(test-unit): drop dead misc shard paths and skip missing paths with a warning (#42603) Eight directories the misc shard named moved to tests/unit on 2026-09-20, and one missing path makes pytest-xdist collect [0 items] for the whole shard, which the exit-5 tolerance turned into a green required check running nothing. The shared Run tests step now drops a path that does not exist with a ::warning:: and runs pytest over the rest, keeping option tokens verbatim. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/_test-unit-base.yml | 32 +++++--- .github/workflows/test-unit.yml | 8 -- .../test_unit_shard_missing_paths.py | 74 +++++++++++++++++++ 3 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 tests/test_litellm/test_unit_shard_missing_paths.py diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index f4d4fb54ba6..8faddd11df8 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -4,7 +4,13 @@ on: workflow_call: inputs: test-path: - description: "Pytest path(s) to run" + description: >- + Space-separated pytest paths to run. A path that no longer exists is + dropped with a warning instead of being passed to pytest, because one + missing path makes pytest-xdist collect nothing and report exit 5, which + the step treats as a drained shard. Options are passed through as + written, so use the `--flag=value` form: a bare `--ignore path` would + have its path existence-checked like any other token. required: true type: string workers: @@ -165,14 +171,22 @@ jobs: DIST: ${{ inputs.dist }} COVERAGE_CORE: sysmon run: | - found_path=false - for path in ${TEST_PATH}; do - if [ -e "${path%%::*}" ]; then - found_path=true - break - fi + pytest_args=() + existing_paths=0 + for token in ${TEST_PATH:?}; do + case "${token}" in + -*) pytest_args+=("${token}") ;; + *) + if [ -e "${token%%::*}" ]; then + pytest_args+=("${token}") + existing_paths=$((existing_paths + 1)) + else + echo "::warning::${token} does not exist; drop it from this shard's test-path" + fi + ;; + esac done - if [ "$found_path" = false ]; then + if [ "${existing_paths}" -eq 0 ]; then echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run" exit 0 fi @@ -181,7 +195,7 @@ jobs: xdist_args=(-n "${WORKERS}" --dist="${DIST}") fi set +e - uv run --no-sync pytest ${TEST_PATH:?} \ + uv run --no-sync pytest "${pytest_args[@]}" \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ "${xdist_args[@]}" \ diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 49e6d7040d4..ac60a9b5f05 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -107,26 +107,18 @@ jobs: tests/test_litellm/batches tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol - tests/test_litellm/anthropic_interface tests/test_litellm/chat_completions tests/test_litellm/completion_extras - tests/test_litellm/compression tests/test_litellm/containers tests/test_litellm/endpoints - tests/test_litellm/models - tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/messages tests/test_litellm/ocr tests/test_litellm/passthrough tests/test_litellm/rag - tests/test_litellm/realtime_api tests/test_litellm/rerank_api tests/test_litellm/rust_bridge - tests/test_litellm/sandbox - tests/test_litellm/skills - tests/test_litellm/test_router tests/test_litellm/vector_stores tests/test_litellm/videos tests/test_litellm/test_*.py diff --git a/tests/test_litellm/test_unit_shard_missing_paths.py b/tests/test_litellm/test_unit_shard_missing_paths.py new file mode 100644 index 00000000000..b91c2cff764 --- /dev/null +++ b/tests/test_litellm/test_unit_shard_missing_paths.py @@ -0,0 +1,74 @@ +import os +import subprocess +import sys +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_BASE_WORKFLOW: Final = _REPO_ROOT / ".github" / "workflows" / "_test-unit-base.yml" +_SHARD_ENV: Final = MappingProxyType( + {"MAX_FAILURES": "10", "RERUNS": "0", "DIST": "loadscope", "TEST_TIMEOUT_SECONDS": "60", "COVERAGE_CORE": "sysmon"} +) +_UV_SHIM: Final = f'#!/usr/bin/env bash\nshift 2\nexec "{sys.executable}" -m "$@"\n' +_PASSING_TEST: Final = "def test_passes():\n assert True\n" +_FAILING_TEST: Final = "def test_fails():\n assert False\n" + + +def _run_tests_script() -> str: + workflow: Final = yaml.safe_load(_BASE_WORKFLOW.read_text()) + return next(step["run"] for step in workflow["jobs"]["run"]["steps"] if step.get("name") == "Run tests") + + +def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.CompletedProcess[str]: + shim_dir: Final = tmp_path / "bin" + shim_dir.mkdir() + (shim_dir / "uv").write_text(_UV_SHIM) + (shim_dir / "uv").chmod(0o755) + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n") + return subprocess.run( + ("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()), + cwd=tmp_path, + env={ + **os.environ, + **_SHARD_ENV, + "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}", + "TEST_PATH": test_path, + "WORKERS": workers, + }, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + +def _write_passing_test(tmp_path: Path) -> Path: + present: Final = tmp_path / "tests" / "present" + present.mkdir(parents=True) + (present / "test_present.py").write_text(_PASSING_TEST) + return present + + +@pytest.mark.parametrize("workers", ("0", "2"), ids=("serial", "xdist")) +def test_a_missing_path_is_dropped_and_the_existing_paths_still_run(tmp_path: Path, workers: str) -> None: + _write_passing_test(tmp_path) + + result: Final = _run_shard(tmp_path, "tests/gone tests/present", workers) + + assert result.returncode == 0, result.stdout + result.stderr + assert "1 passed" in result.stdout, result.stdout + assert "::warning::tests/gone does not exist" in result.stdout + + +def test_ignore_flags_survive_the_path_filter(tmp_path: Path) -> None: + present: Final = _write_passing_test(tmp_path) + (present / "test_ignored.py").write_text(_FAILING_TEST) + + result: Final = _run_shard(tmp_path, "tests/present --ignore=tests/present/test_ignored.py", "0") + + assert result.returncode == 0, result.stdout + result.stderr + assert "1 passed" in result.stdout, result.stdout From a80379baf8910f5aa72990cd1bdbd8088d5e6b2c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:58:08 -0700 Subject: [PATCH 22/83] fix(proxy): keep the in-flight daily spend batch when shutdown cancels the flush (#42593) * fix(proxy): keep the in-flight daily spend batch when shutdown cancels the flush A daily spend batch drained from the in-memory queue was dropped for good when the scheduler tick was cancelled by shutdown, because asyncio.CancelledError bypasses the except Exception requeue. The flush now requeues the drained rows on cancellation and re-raises, and each daily batch upsert runs in an interactive transaction so a statement that already reached Postgres is rolled back with the cancel instead of committing behind the requeue Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): requeue the cancelled daily spend batch before its rollback returns Behind a lock the rollback of the cancelled interactive transaction only returns once the blocked statement does, which is after the shutdown flush has already run. The commit now runs as a shielded task so the cancelled tick requeues the batch at once and lets the rollback finish in the background. The final flush then finds the rows and writes them exactly once Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): give the recording db a transaction seam for the bulk upsert tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): route the mocked daily tag spend upsert through the transaction seam Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): restore the drained Redis tag batch when shutdown cancels its commit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 25 +- tests/integration/_support/process.py | 37 ++- tests/integration/contracts.json | 6 + .../integration/spend/test_shutdown_flush.py | 189 ++++++++++++ .../test_update_daily_tag_spend.py | 1 + .../proxy/db/test_daily_spend_bulk_upsert.py | 9 + .../proxy/db/test_db_spend_update_writer.py | 270 +++++++++++++++++- 7 files changed, 520 insertions(+), 17 deletions(-) create mode 100644 tests/integration/spend/test_shutdown_flush.py diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index d9c8b271646..e38214c98a6 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1606,13 +1606,21 @@ class DBSpendUpdateWriter: proxy_logging_obj: ProxyLogging, ) -> None: transactions: Final = await queue.flush_and_get_aggregated_daily_spend_update_transactions() - try: - await commit( + commit_task: Final = asyncio.ensure_future( + commit( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=cast(dict[str, _DailySpendTransactionT], transactions), ) + ) + try: + await asyncio.shield(commit_task) + except asyncio.CancelledError: + commit_task.cancel() + if transactions: + await queue.add_update(transactions) + raise except Exception as e: # noqa: BLE001 # whatever failed here, the other tables must still flush if not transactions: return @@ -1839,14 +1847,18 @@ class DBSpendUpdateWriter: if not daily_tag_spend_update_transactions: return - try: - await DBSpendUpdateWriter.update_daily_tag_spend( + commit_task: Final = asyncio.ensure_future( + DBSpendUpdateWriter.update_daily_tag_spend( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, daily_spend_transactions=daily_tag_spend_update_transactions, ) - except Exception: + ) + try: + await asyncio.shield(commit_task) + except BaseException: # noqa: BLE001 # a cancel must restore the drained rows before its rollback returns + commit_task.cancel() await self.redis_update_buffer.restore_transactions_to_redis( daily_tag_spend_update_transactions=daily_tag_spend_update_transactions, ) @@ -2368,7 +2380,8 @@ class DBSpendUpdateWriter: table=table, transactions=tuple(transactions_to_process.values()) ) sql, params = build_bulk_upsert(table=table, batch=merged_batch) - await prisma_client.db.execute_raw(sql, *params) + async with _spend_update_tx(prisma_client) as transaction: + await transaction.execute_raw(sql, *params) except Exception as batch_error: if _spend_commit_failure_is_requeue_safe(batch_error): spend_log_error( diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index e0923c9d055..0798927c6d1 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -7,6 +7,7 @@ import time import uuid from collections.abc import Iterator, Mapping from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Final @@ -45,8 +46,37 @@ def stop_root_process(process: subprocess.Popen[bytes]) -> bool: return True +@dataclass(frozen=True, slots=True) +class OwnedProxy: + gateway: Gateway + process: subprocess.Popen[bytes] + log: Path + + @contextmanager -def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], *, config: Path | None = None, remove_environment: tuple[str, ...] = ()) -> Iterator[Gateway]: +def owned_proxy( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), +) -> Iterator[Gateway]: + with owned_proxy_process( + gateway, directory, overrides, config=config, remove_environment=remove_environment + ) as owned: + yield owned.gateway + + +@contextmanager +def owned_proxy_process( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), +) -> Iterator[OwnedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] @@ -60,7 +90,8 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], } output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory))) output.mkdir(parents=True, exist_ok=True) - with (output / f"owned-proxy-{uuid.uuid4().hex}.log").open("w") as log: + log_path: Final = output / f"owned-proxy-{uuid.uuid4().hex}.log" + with log_path.open("w") as log: process: Final = subprocess.Popen( [ sys.executable, @@ -95,7 +126,7 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], pass assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" time.sleep(0.1) - yield Gateway(client, gateway.key, gateway.upstream_url) + yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, log_path) finally: root_stopped: Final = stop_root_process(process) residual: Final = group_members(process.pid) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 8c4ce4debbc..cfe31e2d8ec 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -281,6 +281,12 @@ "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" ], + "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": [ + "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" + ], + "tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": [ + "quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch" + ], "tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [ "quota_management.spend_tracking.spend_calculate.rejects_unpriced_model" ], diff --git a/tests/integration/spend/test_shutdown_flush.py b/tests/integration/spend/test_shutdown_flush.py new file mode 100644 index 00000000000..d27275f5213 --- /dev/null +++ b/tests/integration/spend/test_shutdown_flush.py @@ -0,0 +1,189 @@ +import json +import os +import signal +import threading +import uuid +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psycopg +import pytest +import yaml + +from integration._support.client import Gateway, delete_key_if_present, eventually, string_value +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, wire_server + +REQUESTS_WHILE_BLOCKED: Final = 6 +CANCEL_LOG_LINE: Final = "in-flight scheduled job(s) for shutdown" +BATCH_DRAINED_LOG_LINE: Final = f"flushed {REQUESTS_WHILE_BLOCKED} daily spend update items from in-memory queue" +MODEL_INSERT_ARRIVED_LOG_LINE: Final = "path=/model/new" + + +def _api_requests(table: str, column: str, identity: str) -> int: + rows: Final = read_rows( + f'SELECT coalesce(sum(api_requests), 0)::int AS total FROM "{table}" WHERE {column}=%s', (identity,) + ) + total: Final = rows[0]["total"] + assert isinstance(total, int) + return total + + +def _waiting_on(table: str) -> int: + rows: Final = read_rows( + "SELECT count(*)::int AS waiting FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE %s", + (f'%"{table}"%',), + ) + waiting: Final = rows[0]["waiting"] + assert isinstance(waiting, int) + return waiting + + +def _provider(request: Request) -> Reply: + if request.method != "POST": + return Reply(status=404, body=b'{"error":"not scripted"}') + assert request.target == "/v1/chat/completions" + return Reply( + body=json.dumps( + { + "id": "chatcmpl-" + uuid.uuid4().hex, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + + +@dataclass(frozen=True, slots=True) +class _Shutdown: + owner: str + team: str + owned: OwnedProxy + key: str + model: str + + def chat(self) -> None: + body: Final = {"model": self.model, "messages": [{"role": "user", "content": f"spend {uuid.uuid4().hex}"}]} + assert self.owned.gateway.request("POST", "/v1/chat/completions", body, key=self.key).status_code == 200 + + def daily_user_requests(self) -> int: + return _api_requests("LiteLLM_DailyUserSpend", "user_id", self.owner) + + def logged(self, line: str, times: int = 1) -> bool: + return self.owned.log.read_text(errors="replace").count(line) >= times + + def chat_while_spend_update_is_blocked(self, blocker: psycopg.Connection, table: str) -> None: + blocker.execute(f'LOCK TABLE "{table}" IN EXCLUSIVE MODE') + for _ in range(REQUESTS_WHILE_BLOCKED): + self.chat() + eventually(lambda: _waiting_on(table), lambda waiting: waiting == 1, seconds=30) + + def start_blocked_model_insert(self) -> threading.Thread: + body: Final = { + "model_name": f"integration-blocked-{uuid.uuid4().hex}", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "integration-provider-key"}, + "model_info": {}, + } + + def insert() -> None: + try: + self.owned.gateway.request("POST", "/model/new", body) + except httpx.TransportError: + pass + + thread: Final = threading.Thread(target=insert, daemon=True) + thread.start() + return thread + + def terminate_once(self, blocked: Callable[[], bool], release: Callable[[], None]) -> None: + eventually(blocked, lambda state: state, seconds=60) + self.owned.process.send_signal(signal.SIGTERM) + eventually(lambda: self.logged(CANCEL_LOG_LINE), lambda seen: seen, seconds=60) + release() + self.owned.process.wait(timeout=120) + + +def _config_with_pool_limit(tmp_path: Path, pool_limit: int) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"]["database_connection_pool_limit"] = pool_limit + config["general_settings"]["database_connection_pool_timeout"] = 60 + path: Final = tmp_path / f"pool-{pool_limit}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _proxy_with_one_seeded_row(gateway: Gateway, tmp_path: Path, pool_limit: int) -> Iterator[_Shutdown]: + owner: Final = f"integration-owner-{uuid.uuid4().hex}" + with gateway.scenario() as scenario, wire_server(_provider) as wire: + model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) + team: Final = scenario.team(models=[model]) + with owned_proxy_process( + gateway, + tmp_path, + { + "LITELLM_LOG": "DEBUG", + "GRACEFUL_SHUTDOWN_TIMEOUT": "1", + "SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS": "1", + "SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS": "5", + }, + config=_config_with_pool_limit(tmp_path, pool_limit), + ) as owned: + key: Final = string_value( + owned.gateway.post("/key/generate", {"user_id": owner, "team_id": team, "models": [model]})["key"] + ) + scenario.cleanups.callback(delete_key_if_present, gateway, key) + shutdown: Final = _Shutdown(owner, team, owned, key, model) + shutdown.chat() + eventually(shutdown.daily_user_requests, lambda total: total == 1, seconds=60) + yield shutdown + assert _api_requests("LiteLLM_DailyUserSpend", "user_id", owner) == 1 + REQUESTS_WHILE_BLOCKED + assert _api_requests("LiteLLM_DailyTeamSpend", "team_id", team) == 1 + REQUESTS_WHILE_BLOCKED + + +@pytest.mark.covers("quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch") +def test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush( + gateway: Gateway, tmp_path: Path +) -> None: + with ( + _proxy_with_one_seeded_row(gateway, tmp_path, pool_limit=2) as shutdown, + psycopg.connect(os.environ["DATABASE_URL"]) as models, + psycopg.connect(os.environ["DATABASE_URL"]) as memberships, + ): + models.execute('LOCK TABLE "LiteLLM_ProxyModelTable" IN EXCLUSIVE MODE') + first: Final = shutdown.start_blocked_model_insert() + eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 1, seconds=30) + shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership") + second: Final = shutdown.start_blocked_model_insert() + eventually(lambda: shutdown.logged(MODEL_INSERT_ARRIVED_LOG_LINE, times=2), lambda seen: seen, seconds=30) + memberships.rollback() + eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 2, seconds=30) + shutdown.terminate_once(lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE), models.rollback) + first.join(timeout=30) + second.join(timeout=30) + + +@pytest.mark.covers("quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch") +def test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once( + gateway: Gateway, tmp_path: Path +) -> None: + with ( + _proxy_with_one_seeded_row(gateway, tmp_path, pool_limit=10) as shutdown, + psycopg.connect(os.environ["DATABASE_URL"]) as holder, + psycopg.connect(os.environ["DATABASE_URL"]) as memberships, + ): + holder.execute('SELECT 1 FROM "LiteLLM_DailyUserSpend" WHERE user_id=%s FOR UPDATE', (shutdown.owner,)) + shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership") + memberships.rollback() + shutdown.terminate_once( + lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE) and _waiting_on("LiteLLM_DailyUserSpend") == 1, + holder.rollback, + ) diff --git a/tests/proxy_unit_tests/test_update_daily_tag_spend.py b/tests/proxy_unit_tests/test_update_daily_tag_spend.py index 35e9c6796eb..530c0d16767 100644 --- a/tests/proxy_unit_tests/test_update_daily_tag_spend.py +++ b/tests/proxy_unit_tests/test_update_daily_tag_spend.py @@ -100,6 +100,7 @@ async def test_daily_tag_spend_retries_then_succeeds(): 1, ] ) + prisma_client.db.tx.return_value.__aenter__.return_value.execute_raw = prisma_client.db.execute_raw daily_spend_transactions: Dict[str, DailyTagSpendTransaction] = { "k": { diff --git a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py index 510f77cecec..7893fb82281 100644 --- a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py @@ -1,6 +1,8 @@ """Tests for the single-statement daily spend upsert (LIT-5291).""" import re +from collections.abc import AsyncIterator +from contextlib import AbstractAsyncContextManager, asynccontextmanager import pytest @@ -149,6 +151,13 @@ class _RecordingDb: self.statements.append((query, args)) return len(args) + @asynccontextmanager + async def _tx(self) -> AsyncIterator["_RecordingDb"]: + yield self + + def tx(self, timeout: object = None) -> AbstractAsyncContextManager["_RecordingDb"]: + return self._tx() + class _RecordingPrismaClient: def __init__(self) -> None: diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index a997939ed34..daa07c8224a 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -5,11 +5,11 @@ import logging import re -from collections.abc import Callable -from contextlib import asynccontextmanager -from datetime import datetime, timezone +from collections.abc import AsyncIterator, Callable +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from datetime import datetime, timedelta, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, call, patch import httpx @@ -20,7 +20,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import Litellm_EntityType, SpendUpdateQueueItem +from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, _TEAM_MEMBER_SPEND_SQL, @@ -28,6 +28,8 @@ from litellm.proxy.db.db_spend_update_writer import ( _SpendTableName, _spend_tables_left_to_send, ) +from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue +from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, @@ -303,6 +305,13 @@ class _RecordingDb: return self._execute_raw() return len(args) + @asynccontextmanager + async def _tx(self) -> AsyncIterator["_RecordingDb"]: + yield self + + def tx(self, timeout: timedelta | None = None) -> AbstractAsyncContextManager["_RecordingDb"]: + return self._tx() + class _RecordingPrisma: def __init__(self, execute_raw: Callable[[], int] | None = None) -> None: @@ -3770,8 +3779,15 @@ async def test_commit_spend_updates_does_not_retry_non_deadlock_data_error(monke @pytest.mark.asyncio async def test_update_daily_spend_retries_deadlock(monkeypatch): """The daily-spend upsert path retries a deadlock on the bulk upsert and then drains successfully.""" - mock_prisma_client = MagicMock() - mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[_deadlock_error(), None]) + outcomes = iter([_deadlock_error(), None]) + + def first_attempt_deadlocks(): + outcome = next(outcomes) + if outcome is not None: + raise outcome + return 1 + + mock_prisma_client = _RecordingPrisma(execute_raw=first_attempt_deadlocks) proxy_logging = MagicMock() proxy_logging.failure_handler = AsyncMock() @@ -3786,7 +3802,7 @@ async def test_update_daily_spend_retries_deadlock(monkeypatch): entity_id_field="user_id", ) - assert mock_prisma_client.db.execute_raw.call_count == 2 + assert len(mock_prisma_client.db.statements) == 2 assert daily_spend_transactions == {} proxy_logging.failure_handler.assert_not_called() @@ -4247,3 +4263,241 @@ async def test_daily_transaction_attributes_caching_savings_only_with_an_injecti assert transaction["cache_creation_input_tokens"] == 1111 assert transaction["prompt_caching_savings_spend"] != 0.0 assert transaction["gateway_injected_caching_savings_spend"] == 0.0 + + +class _StallingDailySpendFakeDB(_DailySpendFakeDB): + """Holds the daily upsert aimed at one table until it is cancelled, like a starved pool does. + + The rollback of that transaction waits for ``rollback_release``: the query engine only + rolls back once the statement it is running has returned, which behind a lock takes + as long as the lock is held.""" + + def __init__(self, stalled_table: str) -> None: + super().__init__(failing_table=None) + self.stalled_table = stalled_table + self.stalled = asyncio.Event() + self.rollback_release = asyncio.Event() + self.rolled_back = asyncio.Event() + self.transaction_outcomes: list[str] = [] + + async def execute_raw(self, query: str, *args: object) -> int: + if self.stalled_table in query: + self.stalled.set() + await asyncio.Event().wait() + return await super().execute_raw(query, *args) + + @asynccontextmanager + async def _tx(self) -> AsyncIterator["_StallingDailySpendFakeDB"]: + try: + yield self + except BaseException: + await self.rollback_release.wait() + self.transaction_outcomes.append("rollback") + self.rolled_back.set() + raise + self.transaction_outcomes.append("commit") + + +def _daily_entity_txn(entity_id_field: str) -> dict: + return {key: value for key, value in _daily_txn().items() if key != "user_id"} | {entity_id_field: "entity-1"} + + +_DAILY_SPEND_ENTITIES: Final = [ + pytest.param("daily_spend_update_queue", "user", "user_id", "LiteLLM_DailyUserSpend", id="user"), + pytest.param("daily_team_spend_update_queue", "team", "team_id", "LiteLLM_DailyTeamSpend", id="team"), + pytest.param("daily_org_spend_update_queue", "org", "organization_id", "LiteLLM_DailyOrganizationSpend", id="org"), + pytest.param("daily_tag_spend_update_queue", "tag", "tag", "LiteLLM_DailyTagSpend", id="tag"), + pytest.param( + "daily_end_user_spend_update_queue", "end_user", "end_user_id", "LiteLLM_DailyEndUserSpend", id="end_user" + ), + pytest.param("daily_agent_spend_update_queue", "agent", "agent_id", "LiteLLM_DailyAgentSpend", id="agent"), +] + +_DAILY_SPEND_QUEUES: Final[dict[str, Callable[[DBSpendUpdateWriter], DailySpendUpdateQueue]]] = { + "daily_spend_update_queue": lambda writer: writer.daily_spend_update_queue, + "daily_team_spend_update_queue": lambda writer: writer.daily_team_spend_update_queue, + "daily_org_spend_update_queue": lambda writer: writer.daily_org_spend_update_queue, + "daily_tag_spend_update_queue": lambda writer: writer.daily_tag_spend_update_queue, + "daily_end_user_spend_update_queue": lambda writer: writer.daily_end_user_spend_update_queue, + "daily_agent_spend_update_queue": lambda writer: writer.daily_agent_spend_update_queue, +} + +_DAILY_SPEND_COMMITS: Final = { + "user": DBSpendUpdateWriter.update_daily_user_spend, + "team": DBSpendUpdateWriter.update_daily_team_spend, + "org": DBSpendUpdateWriter.update_daily_org_spend, + "tag": DBSpendUpdateWriter.update_daily_tag_spend, + "end_user": DBSpendUpdateWriter.update_daily_end_user_spend, + "agent": DBSpendUpdateWriter.update_daily_agent_spend, +} + + +@pytest.mark.parametrize(("queue_name", "entity_type", "entity_id_field", "table"), _DAILY_SPEND_ENTITIES) +@pytest.mark.asyncio +async def test_daily_spend_batch_cancelled_mid_flight_is_rolled_back_requeued_and_written_once_by_the_next_flush( + queue_name: str, entity_type: str, entity_id_field: str, table: str +): + """Shutdown cancels the scheduler tick while a drained batch waits on the database. The + batch has left the queue, so unless the cancellation puts it back, the final flush finds + nothing and the spend is gone (F2). The upsert runs in an interactive transaction so a + statement that did reach Postgres is rolled back with the cancel and the requeued rows + land exactly once.""" + db_writer = DBSpendUpdateWriter() + queue = _DAILY_SPEND_QUEUES[queue_name](db_writer) + await queue.add_update({"key-a": _daily_entity_txn(entity_id_field)}) + await queue.add_update({"key-a": _daily_entity_txn(entity_id_field)}) + db = _StallingDailySpendFakeDB(stalled_table=table) + proxy_logging_obj = MagicMock() + proxy_logging_obj.failure_handler = AsyncMock() + + def flush(prisma_db: _DailySpendFakeDB): + return db_writer._flush_daily_spend_queue( + queue=queue, + entity_type=entity_type, + commit=_DAILY_SPEND_COMMITS[entity_type], + n_retry_times=0, + prisma_client=_WindowSpendFakePrisma(prisma_db), + proxy_logging_obj=proxy_logging_obj, + ) + + tick = asyncio.ensure_future(flush(db)) + await asyncio.wait_for(db.stalled.wait(), timeout=5) + tick.cancel() + finished, _ = await asyncio.wait({tick}, timeout=1) + assert finished == {tick}, "the cancelled tick must return before the rolled-back statement unwinds" + with pytest.raises(asyncio.CancelledError): + tick.result() + + assert not queue.update_queue.empty(), "the cancelled batch must go back on the queue before the rollback lands" + assert db.transaction_outcomes == [] + db.rollback_release.set() + await asyncio.wait_for(db.rolled_back.wait(), timeout=5) + assert db.transaction_outcomes == ["rollback"] + assert _daily_upserts(db, table) == [] + + final_db = _DailySpendFakeDB(failing_table=None) + await flush(final_db) + + (upsert,) = _daily_upserts(final_db, table) + assert _row_values(upsert, entity_id_field) == ["entity-1"] + assert _row_values(upsert, "spend") == [pytest.approx(0.2)] + assert _row_values(upsert, "api_requests") == [2] + assert queue.update_queue.empty() + + +@pytest.mark.asyncio +async def test_cancelled_flush_of_an_empty_daily_queue_requeues_nothing(): + """A cancel that lands with nothing drained must not push an empty batch onto the queue.""" + db_writer = DBSpendUpdateWriter() + db = _StallingDailySpendFakeDB(stalled_table="LiteLLM_DailyUserSpend") + + class _CancellingQueue(type(db_writer.daily_spend_update_queue)): + async def flush_and_get_aggregated_daily_spend_update_transactions(self): + drained = await super().flush_and_get_aggregated_daily_spend_update_transactions() + asyncio.current_task().cancel() + await asyncio.sleep(0) + return drained + + queue = _CancellingQueue() + with pytest.raises(asyncio.CancelledError): + await db_writer._flush_daily_spend_queue( + queue=queue, + entity_type="user", + commit=DBSpendUpdateWriter.update_daily_user_spend, + n_retry_times=0, + prisma_client=_WindowSpendFakePrisma(db), + proxy_logging_obj=MagicMock(), + ) + + assert queue.update_queue.empty() + + +class _AnnouncingDailySpendFakeDB(_DailySpendFakeDB): + """Signals ``written`` the moment the daily upsert has been committed.""" + + def __init__(self) -> None: + super().__init__(failing_table=None) + self.written = asyncio.Event() + + async def execute_raw(self, query: str, *args: object) -> int: + rows = await super().execute_raw(query, *args) + self.written.set() + return rows + + +@pytest.mark.asyncio +async def test_cancel_that_lands_after_the_daily_batch_committed_does_not_requeue_it(): + """The commit has returned but the tick has not resumed yet when the cancel arrives. + Putting the batch back now would write the same spend twice on the final flush.""" + db_writer = DBSpendUpdateWriter() + queue = db_writer.daily_spend_update_queue + await queue.add_update({"key-a": _daily_txn()}) + db = _AnnouncingDailySpendFakeDB() + + tick = asyncio.ensure_future( + db_writer._flush_daily_spend_queue( + queue=queue, + entity_type="user", + commit=DBSpendUpdateWriter.update_daily_user_spend, + n_retry_times=0, + prisma_client=_WindowSpendFakePrisma(db), + proxy_logging_obj=MagicMock(), + ) + ) + await db.written.wait() + tick.cancel() + with pytest.raises(asyncio.CancelledError): + await tick + + assert len(_daily_upserts(db, "LiteLLM_DailyUserSpend")) == 1 + assert queue.update_queue.empty(), "a batch that already committed must not be requeued" + + +class _DrainedTagRedisBuffer: + """Hands out one drained tag batch and records whatever is restored.""" + + def __init__(self, drained: dict[str, DailyTagSpendTransaction]) -> None: + self.drained = drained + self.restored: list[dict[str, DailyTagSpendTransaction]] = [] + + async def get_all_daily_tag_spend_update_transactions_from_redis_buffer( + self, + ) -> dict[str, DailyTagSpendTransaction]: + return self.drained + + async def restore_transactions_to_redis( + self, daily_tag_spend_update_transactions: dict[str, DailyTagSpendTransaction] + ) -> None: + self.restored.append(daily_tag_spend_update_transactions) + + +@pytest.mark.asyncio +async def test_tag_batch_drained_from_redis_and_cancelled_mid_flight_is_restored_before_its_rollback_returns(): + """The Redis tag drain is destructive. A shutdown cancel used to leave the batch nowhere: + Redis no longer had it and the interactive transaction rolled the statement back.""" + db_writer = DBSpendUpdateWriter() + drained = {"key-a": cast(DailyTagSpendTransaction, _daily_entity_txn("tag"))} + redis_buffer = _DrainedTagRedisBuffer(drained) + db_writer.redis_update_buffer = cast(RedisUpdateBuffer, redis_buffer) + db = _StallingDailySpendFakeDB(stalled_table="LiteLLM_DailyTagSpend") + + tick = asyncio.ensure_future( + db_writer._drain_and_commit_daily_tag_spend_from_redis( + prisma_client=_WindowSpendFakePrisma(db), + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + ) + await asyncio.wait_for(db.stalled.wait(), timeout=5) + tick.cancel() + finished, _ = await asyncio.wait({tick}, timeout=1) + assert finished == {tick}, "the cancelled drain must return before the rolled-back statement unwinds" + with pytest.raises(asyncio.CancelledError): + tick.result() + + assert redis_buffer.restored == [drained], "the drained tag batch must be back in Redis before the rollback lands" + assert db.transaction_outcomes == [] + db.rollback_release.set() + await asyncio.wait_for(db.rolled_back.wait(), timeout=5) + assert db.transaction_outcomes == ["rollback"] + assert _daily_upserts(db, "LiteLLM_DailyTagSpend") == [] From 944f44d82be735616448f92b534dee1f063008b7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:59:50 -0700 Subject: [PATCH 23/83] fix(utils): isolate callback errors in async_post_call_success_deployment_hook (#42535) * fix(utils): isolate callback errors in async_post_call_success_deployment_hook A callback that raises inside async_post_call_success_deployment_hook no longer fails the completed request. The exception is logged with the callback class and call_type, the response stays as it was, and later callbacks still run. Guardrail callbacks are exempt because raising is how a post-call guardrail blocks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(utils): drop unrelated ruff autofixes from test_utils Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(utils): drop fastapi import from guardrail propagation regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(utils): cover every success deployment hook call type with a raising hook Parametrize the unit regression over video, embedding, responses, image, rerank, transcription, chat and anthropic messages responses and assert the failure log names the callback and call type. Run the integration test through a real proxy for /v1/chat/completions, /v1/embeddings, /v1/responses and /v1/videos Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): move raising success hook cases into the existing callback delivery file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 17 +- tests/integration/contracts.json | 12 ++ .../observability/test_callback_delivery.py | 166 +++++++++++++++++- tests/test_litellm/test_utils.py | 113 ++++++++++++ 4 files changed, 303 insertions(+), 5 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 01f6fee3594..dd35c17809f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1440,11 +1440,22 @@ async def async_post_call_success_deployment_hook( modified_response = response CustomLogger: Final = _get_cached_custom_logger() + CustomGuardrail: Final = _get_cached_custom_guardrail() for callback in litellm.callbacks: if isinstance(callback, CustomLogger): - result = await callback.async_post_call_success_deployment_hook( - request_data, cast(LLMResponseTypes, modified_response), typed_call_type - ) + try: + result = await callback.async_post_call_success_deployment_hook( + request_data, cast(LLMResponseTypes, modified_response), typed_call_type + ) + except Exception: # noqa: BLE001 # a broken callback must not fail a completed request + if isinstance(callback, CustomGuardrail): + raise + verbose_logger.exception( + "async_post_call_success_deployment_hook error in %s for call_type=%s", + type(callback).__name__, + typed_call_type, + ) + continue if result is not None: modified_response = result diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index cfe31e2d8ec..ee9b81f63dc 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1782,6 +1782,18 @@ ], "tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": [ "other.mcp.permissions.same_url_servers_enforce_discovery_and_execution" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" + ], + "tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": [ + "other.observability.callbacks.raising_success_deployment_hook_keeps_response" ] }, "browser": { diff --git a/tests/integration/observability/test_callback_delivery.py b/tests/integration/observability/test_callback_delivery.py index c44c1f30b80..7a5f420d2cc 100644 --- a/tests/integration/observability/test_callback_delivery.py +++ b/tests/integration/observability/test_callback_delivery.py @@ -1,13 +1,15 @@ +import base64 import json import uuid +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Final import pytest import yaml - -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, JsonValue, eventually, object_value, string_value from integration._support.database import read_rows from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server @@ -152,3 +154,163 @@ def test_concurrent_success_and_failure_join_callbacks_and_rows_without_credenti assert rows[0]["prompt_tokens"] == event["prompt_tokens"] else: assert event["prompt_tokens"] == event["completion_tokens"] == rows[0]["completion_tokens"] == 0 + + +_RAISING_HOOK: Final = """ +from litellm.integrations.custom_logger import CustomLogger + + +class RaisingHook(CustomLogger): + async def async_post_call_success_deployment_hook(self, request_data, response, call_type): + raise RuntimeError(f"hook rejected {type(response).__name__} for {call_type}") + + +instance = RaisingHook() +""" + +_VIDEO_JOB: Final = { + "id": "video_hook_isolation", + "object": "video", + "status": "queued", + "model": "sora-2", + "seconds": "4", + "size": "720x1280", +} + +_UPSTREAM_REPLIES: Final[Mapping[str, Mapping[str, JsonValue]]] = { + "/v1/chat/completions": { + "id": "chatcmpl_hook_isolation", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + "/v1/embeddings": { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + "/v1/responses": { + "id": "resp_hook_isolation", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_hook_isolation", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + "/v1/videos": _VIDEO_JOB, +} + + +def _item(value: JsonValue, index: int) -> JsonValue: + assert isinstance(value, list), f"Expected a list, received {type(value).__name__}" + return value[index] + + +def _chat_text(body: dict[str, JsonValue]) -> str: + return string_value(object_value(object_value(_item(body["choices"], 0))["message"])["content"]) + + +def _embedding_vector(body: dict[str, JsonValue]) -> JsonValue: + return object_value(_item(body["data"], 0))["embedding"] + + +def _responses_text(body: dict[str, JsonValue]) -> str: + return string_value(object_value(_item(object_value(_item(body["output"], 0))["content"], 0))["text"]) + + +def _video_job(body: dict[str, JsonValue]) -> tuple[str, str]: + encoded_id: Final = string_value(body["id"]).removeprefix("video_") + decoded: Final = base64.b64decode(encoded_id).decode() + return decoded.rsplit("video_id:", 1)[-1], string_value(body["status"]) + + +@dataclass(frozen=True, slots=True) +class _Surface: + route: str + upstream_model: str + body: Callable[[str], dict[str, JsonValue]] + observed: Callable[[dict[str, JsonValue]], JsonValue | tuple[str, str]] + expected: JsonValue | tuple[str, str] + + +_SURFACES: Final = ( + pytest.param( + _Surface( + "/v1/chat/completions", + "openai/gpt-5.6", + lambda model: {"model": model, "messages": [{"role": "user", "content": "hook isolation"}]}, + _chat_text, + "hi", + ), + id="chat", + ), + pytest.param( + _Surface( + "/v1/embeddings", + "openai/text-embedding-3-small", + lambda model: {"model": model, "input": "hook isolation"}, + _embedding_vector, + [0.1, 0.2], + ), + id="embeddings", + ), + pytest.param( + _Surface( + "/v1/responses", + "openai/gpt-5.6", + lambda model: {"model": model, "input": "hook isolation"}, + _responses_text, + "hi", + ), + id="responses", + ), + pytest.param( + _Surface( + "/v1/videos", + "openai/sora-2", + lambda model: {"model": model, "prompt": "a cat"}, + _video_job, + (_VIDEO_JOB["id"], _VIDEO_JOB["status"]), + ), + id="videos", + ), +) + + +@pytest.mark.covers("other.observability.callbacks.raising_success_deployment_hook_keeps_response") +@pytest.mark.parametrize("surface", _SURFACES) +def test_response_survives_raising_success_deployment_hook(gateway: Gateway, tmp_path: Path, surface: _Surface) -> None: + def upstream(request: Request) -> Reply: + assert request.target == surface.route, request.target + assert b"hook isolation" in request.body or b"a cat" in request.body, request.body[:300] + return Reply(body=json.dumps(_UPSTREAM_REPLIES[surface.route]).encode()) + + (tmp_path / "raising_hook.py").write_text(_RAISING_HOOK) + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"].update({"callbacks": ["raising_hook.instance"]}) + path: Final = tmp_path / "hook.yaml" + path.write_text(yaml.safe_dump(config)) + with ( + wire_server(upstream) as provider, + owned_proxy(gateway, tmp_path, {}, config=path) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model=surface.upstream_model, api_base=provider.url + "/v1") + response: Final = candidate.request("POST", surface.route, surface.body(model)) + assert response.status_code == 200, response.text + assert surface.observed(object_value(response.json())) == surface.expected, response.text diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 283cff97ce0..89472cbd20f 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -32,27 +32,35 @@ from litellm._logging import ( from litellm.caching.caching import Cache from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.proxy.utils import is_valid_api_key from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, CallTypes, Choices, Delta, + EmbeddingResponse, + ImageResponse, LlmProviders, + LLMResponseTypes, ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, + RerankResponse, StreamingChoices, + TranscriptionResponse, Usage, all_litellm_params, bedrock_batch_litellm_params, ) +from litellm.types.videos.main import VideoObject from litellm.utils import ( CustomStreamWrapper, ProviderConfigManager, @@ -4412,6 +4420,111 @@ async def test_converted_chat_stream_hook_skips_unhandled_wrappers( assert wrapper.completion_stream is completion_stream +class _ChatShapedSuccessDeploymentHook(CustomLogger): + async def async_post_call_success_deployment_hook( + self, request_data: dict[str, object], response: object, call_type: CallTypes | None + ) -> None: + raise AttributeError(f"{type(response).__name__!r} object has no attribute 'choices'") + + +class _RecordingSuccessDeploymentHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_responses: tuple[object, ...] = () + + async def async_post_call_success_deployment_hook( + self, request_data: dict[str, object], response: object, call_type: CallTypes | None + ) -> None: + self.seen_responses = (*self.seen_responses, response) + + +_SUCCESS_RESPONSES_BY_CALL_TYPE: Final = ( + pytest.param( + VideoObject(id="video_abc", object="video", status="queued", model="sora-2", seconds="4", size="720x1280"), + CallTypes.avideo_generation, + id="video", + ), + pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"), + pytest.param( + ResponsesAPIResponse( + id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6" + ), + CallTypes.aresponses, + id="responses", + ), + pytest.param(ImageResponse(), CallTypes.aimage_generation, id="image"), + pytest.param(RerankResponse(id="rerank_abc"), CallTypes.arerank, id="rerank"), + pytest.param(TranscriptionResponse(text="hi"), CallTypes.atranscription, id="transcription"), + pytest.param(ModelResponse(model="gpt-5.6"), CallTypes.acompletion, id="chat"), + pytest.param(ModelResponse(model="claude-sonnet-4-5"), CallTypes.aanthropic_messages, id="anthropic_messages"), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("response", "call_type"), _SUCCESS_RESPONSES_BY_CALL_TYPE) +async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hooks( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, response: object, call_type: CallTypes +) -> None: + second_hook: Final = _RecordingSuccessDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [_ChatShapedSuccessDeploymentHook(), second_hook]) + + with caplog.at_level(logging.ERROR, logger=verbose_logger.name): + result: Final = await async_post_call_success_deployment_hook( + request_data={"model": "m"}, response=response, call_type=call_type + ) + + assert result is response + assert second_hook.seen_responses == (response,) + failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message) + assert len(failure_logs) == 1 + assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message + assert str(call_type) in failure_logs[0].message + assert failure_logs[0].exc_info is not None + + +@pytest.mark.asyncio +async def test_success_deployment_hook_raising_keeps_earlier_hook_rewrite(monkeypatch: pytest.MonkeyPatch) -> None: + rewriter: Final = _RewritingSuccessDeploymentHook() + trailing_hook: Final = _RecordingSuccessDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [rewriter, _ChatShapedSuccessDeploymentHook(), trailing_hook]) + original: Final = ModelResponse(model="gpt-5.6") + + result: Final = await async_post_call_success_deployment_hook( + request_data={"model": "gpt-5.6"}, response=original, call_type=CallTypes.acompletion + ) + + assert isinstance(result, ModelResponse) + assert result is not original + assert result.choices[0].message.content == "rewritten by deployment hook" + assert trailing_hook.seen_responses == (result,) + + +class _GuardrailBlocked(Exception): + pass + + +class _BlockingSuccessDeploymentGuardrail(CustomGuardrail): + async def async_post_call_success_deployment_hook( + self, request_data: dict, response: LLMResponseTypes, call_type: CallTypes | None + ) -> LLMResponseTypes | None: + raise _GuardrailBlocked("Violated moderation policy") + + +@pytest.mark.asyncio +async def test_success_deployment_hook_still_propagates_guardrail_block(monkeypatch: pytest.MonkeyPatch) -> None: + later_hook: Final = _RewritingSuccessDeploymentHook() + monkeypatch.setattr( + litellm, "callbacks", [_BlockingSuccessDeploymentGuardrail(guardrail_name="blocking"), later_hook] + ) + + with pytest.raises(_GuardrailBlocked): + await async_post_call_success_deployment_hook( + request_data={"model": "gpt-5.6"}, response=ModelResponse(model="gpt-5.6"), call_type=CallTypes.acompletion + ) + + assert later_hook.seen_responses == () + + @pytest.mark.asyncio @respx.mock async def test_wrapper_async_leaves_success_deployment_hook_off_requested_fake_stream( From 630c4624f6027c4c075aa91439702878690ece78 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Tue, 22 Sep 2026 18:02:32 -0700 Subject: [PATCH 24/83] test(e2e): add secret manager lanes for HashiCorp Vault and CyberArk Conjur (#42503) * test(e2e): add a HashiCorp Vault secret manager lane key_management_system had no end-to-end coverage: the Rust crates and the Python unit tests all run against mocked managers. This adds a secret_manager suite that drives a proxy configured with hashicorp_vault against a real Vault. The tests seed a fresh secret name per test with the runner's OPENAI_API_KEY and register a deployment pointing at os.environ/. The proxy's env never holds that name, so get_secret's os.environ fallback cannot mask a broken manager, and a bogus value in Vault must come back as the provider's 401. Virtual keys are checked written to and removed from Vault under prefix_for_stored_virtual_keys. The setting is global to the proxy, so the lane has its own config and the secret_manager_vault opt-in marker, and stays out of the per-PR selector. Co-Authored-By: Claude Opus 5 * test(e2e): make the secret manager suite backend-agnostic One marker and opt-in (secret_manager / E2E_SECRET_MANAGER=) pick the backend from secret_backends.BACKENDS. The tests reach the manager through a SecretStore protocol, and each backend contributes a secret_store_.py module, a registry entry, and gateway/secret_manager__ci_config.yml. requires_capability deselects tests a backend cannot support (CyberArk does not delete), and test_secret_backends.py checks every lane config against its backend without a live stack. Co-Authored-By: Claude Opus 5 * test(e2e): add a CyberArk Conjur secret manager lane Adds cyberark as the second secret_manager backend: a Conjur store over its REST API (policy-declared variables, raw-text values, policy-patch teardown), its lane config, and a registry entry without deletes_stored_keys, since the proxy's CyberArk delete answers not_supported and Conjur keeps the key. secret_manager/backend.sh up|down boots any backend in Docker and writes proxy.env and tests.env, so every lane runs the same way; the registry test checks the script boots exactly the registered backends. e2e_http gains send_text_external for APIs that speak raw text rather than JSON. Co-Authored-By: Claude Opus 5 * fix(e2e): give the secret manager suite a client with .proxy and address review The shared resources fixture reads client.proxy, so a bare ProxyClient errored every live test at setup. backend.sh now writes its env under a per-user directory with umask 077, the markerless unit tests are gone per tests/e2e/AGENTS.md, and routine comments are trimmed. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 --- .github/e2e-stack/select_tests.py | 1 + tests/e2e/AGENTS.md | 1 + tests/e2e/CONTRIBUTING.md | 27 +++- tests/e2e/conftest.py | 7 + tests/e2e/coverage_registry/other.yaml | 5 +- tests/e2e/e2e_config.py | 1 + tests/e2e/e2e_http.py | 30 +++- .../secret_manager_cyberark_ci_config.yml | 8 ++ ...cret_manager_hashicorp_vault_ci_config.yml | 8 ++ tests/e2e/pytest.ini | 1 + tests/e2e/secret_manager/backend.sh | 76 +++++++++++ tests/e2e/secret_manager/conftest.py | 57 ++++++++ tests/e2e/secret_manager/secret_backends.py | 25 ++++ tests/e2e/secret_manager/secret_store.py | 29 ++++ .../secret_manager/secret_store_cyberark.py | 118 ++++++++++++++++ .../secret_store_hashicorp_vault.py | 113 ++++++++++++++++ .../secret_manager/test_secret_manager_e2e.py | 128 ++++++++++++++++++ 17 files changed, 630 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/gateway/secret_manager_cyberark_ci_config.yml create mode 100644 tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml create mode 100755 tests/e2e/secret_manager/backend.sh create mode 100644 tests/e2e/secret_manager/conftest.py create mode 100644 tests/e2e/secret_manager/secret_backends.py create mode 100644 tests/e2e/secret_manager/secret_store.py create mode 100644 tests/e2e/secret_manager/secret_store_cyberark.py create mode 100644 tests/e2e/secret_manager/secret_store_hashicorp_vault.py create mode 100644 tests/e2e/secret_manager/test_secret_manager_e2e.py diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index 492c52233cd..e425c313d6a 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$" + r"|^tests/e2e/secret_manager/" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index d948c6fd1d9..94035cfe849 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -46,6 +46,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite +- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 15c2d6763d6..7e1f516422e 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, `load/`, and `secret_manager/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set. The `secret_manager/` lanes each need a proxy configured against their own secret manager (see Secret manager lanes below) Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch @@ -117,6 +117,31 @@ Fetched values of eight characters or more are masked before use, while shorter To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use +### Secret manager lanes + +`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager__ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally: + +```bash +bash tests/e2e/secret_manager/backend.sh up cyberark +(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \ + LITELLM_MASTER_KEY=sk-1234 DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000) +(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \ + uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v) +bash tests/e2e/secret_manager/backend.sh down cyberark +``` + +`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors + +A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane + +To add a backend, leave the tests and markers alone and add: + +1. `secret_manager/secret_store_.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E__*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports +2. its entry in `secret_backends.BACKENDS` +3. `gateway/secret_manager__ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed +4. an `up_` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env` +5. a CI step that runs `backend.sh up ` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env` + ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index f4de4ac01ba..603591006d9 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -36,6 +36,7 @@ from e2e_config import ( PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, REDIS_CHAOS_OPT_IN_ENV, + SECRET_MANAGER_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV, unique_marker, ) @@ -70,6 +71,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, "otel_v2": OTEL_V2_OPT_IN_ENV, "otel_tls": OTEL_TLS_OPT_IN_ENV, + "secret_manager": SECRET_MANAGER_OPT_IN_ENV, } ) @@ -172,6 +174,11 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set", ) + config.addinivalue_line( + "markers", + "secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live " + "secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 1706b2f8f25..f695af3cb11 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -40,7 +40,10 @@ - {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"} - {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"} - {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"} -- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "proxy_server.py:3984-4010", rationale: "Resolves secrets from Vault/KMS at startup"} +- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "A deployment whose api_key is os.environ/ gets its key from the configured secret manager when that name exists only in the manager"} +- {id: other.config.secret_resolution.manager_value_used, module: other, tier: P1, area: config, assertions: [manager_value_used], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "The value the manager holds is what reaches the provider: a bogus key in the manager is rejected by the provider with 401, so a passing resolution test cannot be an env fallback"} +- {id: other.config.secret_manager.virtual_key_stored, module: other, tier: P1, area: config, assertions: [virtual_key_stored], source: "key_management_event_hooks.py _store_virtual_key_in_secret_manager", rationale: "With store_virtual_keys, /key/generate writes the new key under prefix_for_stored_virtual_keys + key_alias in the manager"} +- {id: other.config.secret_manager.virtual_key_deleted, module: other, tier: P1, area: config, assertions: [virtual_key_deleted], source: "key_management_event_hooks.py _delete_virtual_keys_from_secret_manager", rationale: "/key/delete removes the stored key from the manager, so a revoked key does not linger there"} - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"} - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index ac26b2a3875..ca3b74281ae 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -150,6 +150,7 @@ MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" +SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 1b62a1dbd8c..24bd76d4429 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -129,9 +129,9 @@ class ProbeResult(BaseModel): class ExternalWrite(BaseModel): - """Outcome of a write to a non-proxy API (an identity provider's admin API) - that answers with a status and, on create, a Location header naming the new - resource rather than a JSON body.""" + """Outcome of a call to a non-proxy API (an identity provider's admin API, a + secret manager) that answers with a status, on create a Location header naming + the new resource, and a body kept as text rather than parsed as JSON.""" status_code: int location: str = "" @@ -491,6 +491,30 @@ def post_json_external( ) +def send_text_external( + method: Literal["GET", "POST", "PATCH"], + url: str, + *, + headers: BaseModel, + content: str | None = None, + timeout: float = 30.0, +) -> ExternalWrite: + """Send an absolute URL outside the proxy a raw text body (or none) and keep the + answer as text, for an API that takes and returns neither JSON nor forms: CyberArk + Conjur takes a secret value or a YAML policy and returns a secret as its raw value.""" + try: + resp = requests.request( + method, + url, + headers=_headers(headers), + data=content.encode() if content is not None else None, + timeout=timeout, + ) + except requests.RequestException as exc: + return ExternalWrite(status_code=-1, body=str(exc)) + return ExternalWrite(status_code=resp.status_code, body=resp.text) + + def delete_external(url: str, *, headers: BaseModel, timeout: float = 30.0) -> ExternalWrite: try: resp = requests.delete(url, headers=_headers(headers), timeout=timeout) diff --git a/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml b/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml new file mode 100644 index 00000000000..32602783b02 --- /dev/null +++ b/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml @@ -0,0 +1,8 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + key_management_system: cyberark + key_management_settings: + access_mode: read_and_write + store_virtual_keys: true + prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml b/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml new file mode 100644 index 00000000000..8e6b7e74f8e --- /dev/null +++ b/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml @@ -0,0 +1,8 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + key_management_system: hashicorp_vault + key_management_settings: + access_mode: read_and_write + store_virtual_keys: true + prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 4866511af3f..d01caeff3ea 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -17,3 +17,4 @@ markers = provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set + secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py) diff --git a/tests/e2e/secret_manager/backend.sh b/tests/e2e/secret_manager/backend.sh new file mode 100755 index 00000000000..70272c237c5 --- /dev/null +++ b/tests/e2e/secret_manager/backend.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +usage() { + local systems + systems=$(declare -F | sed -n 's/^declare -f up_//p' | paste -sd '|' -) + echo "usage: $0 up|down $systems" >&2 + exit 2 +} + +action=${1:-} +system=${2:-} +dir=${E2E_SECRET_MANAGER_DIR:-$HOME/.cache/litellm-e2e-secret-manager}/$system +name=litellm-e2e-$system + +wait_for() { + local url=$1 + for _ in $(seq 1 90); do + if curl -sf -o /dev/null "$url"; then + return 0 + fi + sleep 2 + done + echo "$system did not answer at $url" >&2 + return 1 +} + +down() { + docker rm -f "$name" "$name-db" >/dev/null 2>&1 || true + docker network rm "$name" >/dev/null 2>&1 || true + rm -rf "$dir" +} + +up_hashicorp_vault() { + local port=${E2E_SECRET_MANAGER_PORT:-8200} + local token + token=e2e-$(openssl rand -hex 16) + docker run -d --name "$name" -p "127.0.0.1:$port:8200" --cap-add IPC_LOCK \ + -e VAULT_DEV_ROOT_TOKEN_ID="$token" hashicorp/vault:1.20 >/dev/null + wait_for "http://127.0.0.1:$port/v1/sys/health" + printf 'HCP_VAULT_ADDR=http://127.0.0.1:%s\nHCP_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/proxy.env" + printf 'E2E_VAULT_ADDR=http://127.0.0.1:%s\nE2E_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/tests.env" +} + +up_cyberark() { + local port=${E2E_SECRET_MANAGER_PORT:-8080} + local data_key api_key + docker network create "$name" >/dev/null + docker run -d --name "$name-db" --network "$name" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:15 >/dev/null + data_key=$(docker run --rm cyberark/conjur:1.24 data-key generate) + docker run -d --name "$name" --network "$name" -p "127.0.0.1:$port:80" \ + -e DATABASE_URL="postgres://postgres@$name-db/postgres" -e CONJUR_DATA_KEY="$data_key" \ + -e CONJUR_AUTHENTICATORS=authn cyberark/conjur:1.24 server >/dev/null + wait_for "http://127.0.0.1:$port/" + docker exec "$name" conjurctl account create --name default >/dev/null + api_key=$(docker exec "$name" conjurctl role retrieve-key default:user:admin | tr -d '\r\n') + printf 'CYBERARK_API_BASE=http://127.0.0.1:%s\nCYBERARK_ACCOUNT=default\nCYBERARK_USERNAME=admin\nCYBERARK_API_KEY=%s\n' \ + "$port" "$api_key" >"$dir/proxy.env" + printf 'E2E_CYBERARK_API_BASE=http://127.0.0.1:%s\nE2E_CYBERARK_ACCOUNT=default\nE2E_CYBERARK_USERNAME=admin\nE2E_CYBERARK_API_KEY=%s\n' \ + "$port" "$api_key" >"$dir/tests.env" +} + +[[ $# -eq 2 && -n $system ]] && declare -F "up_$system" >/dev/null || usage + +case $action in + up) + down + mkdir -p "$dir" + "up_$system" + echo "E2E_SECRET_MANAGER=$system" >>"$dir/tests.env" + echo "$system is up; env in $dir/proxy.env (proxy) and $dir/tests.env (pytest)" + ;; + down) down ;; + *) usage ;; +esac diff --git a/tests/e2e/secret_manager/conftest.py b/tests/e2e/secret_manager/conftest.py new file mode 100644 index 00000000000..46ef598d711 --- /dev/null +++ b/tests/e2e/secret_manager/conftest.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Final + +import pytest + +from e2e_config import SECRET_MANAGER_OPT_IN_ENV +from proxy_client import ProxyClient +from secret_backends import BACKENDS, selected_backend +from secret_store import SecretBackend, SecretStore + +REQUIRES_CAPABILITY: Final = "requires_capability" + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + f"{REQUIRES_CAPABILITY}(capability): secret_manager test deselected when the backend " + f"{SECRET_MANAGER_OPT_IN_ENV} names lacks the capability (secret_store.Capability)", + ) + + +def _lacks_capability(item: pytest.Item, backend: SecretBackend) -> bool: + marker: Final = item.get_closest_marker(REQUIRES_CAPABILITY) + return marker is not None and marker.args[0] not in backend.capabilities + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + backend: Final = BACKENDS.get(os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip()) + if backend is None: + return + deselected: Final = [item for item in items if _lacks_capability(item, backend)] + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = [item for item in items if not _lacks_capability(item, backend)] + + +@dataclass(frozen=True, slots=True) +class SecretManagerClient: + proxy: ProxyClient + + +@pytest.fixture(scope="session") +def client(proxy: ProxyClient) -> SecretManagerClient: + return SecretManagerClient(proxy) + + +@pytest.fixture(scope="session") +def backend() -> SecretBackend: + return selected_backend() + + +@pytest.fixture(scope="session") +def store(backend: SecretBackend) -> SecretStore: + return backend.from_env() diff --git a/tests/e2e/secret_manager/secret_backends.py b/tests/e2e/secret_manager/secret_backends.py new file mode 100644 index 00000000000..578b56970a3 --- /dev/null +++ b/tests/e2e/secret_manager/secret_backends.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import os +from types import MappingProxyType +from typing import Final + +import pytest + +from e2e_config import SECRET_MANAGER_OPT_IN_ENV +from secret_store import SecretBackend +from secret_store_cyberark import CYBERARK +from secret_store_hashicorp_vault import HASHICORP_VAULT + +BACKENDS: Final = MappingProxyType({backend.system: backend for backend in (HASHICORP_VAULT, CYBERARK)}) + + +def selected_backend() -> SecretBackend: + system: Final = os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip() + backend: Final = BACKENDS.get(system) + if backend is None: + pytest.fail( + f"{SECRET_MANAGER_OPT_IN_ENV}={system!r} names no secret manager backend; " + f"set it to one of {sorted(BACKENDS)}" + ) + return backend diff --git a/tests/e2e/secret_manager/secret_store.py b/tests/e2e/secret_manager/secret_store.py new file mode 100644 index 00000000000..cff5005b73a --- /dev/null +++ b/tests/e2e/secret_manager/secret_store.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final, Literal, Protocol + +SECRET_MANAGER_CONFIG_DIR: Final = "gateway" + + +class SecretStore(Protocol): + def write(self, name: str, value: str) -> None: ... + + def read(self, name: str) -> str | None: ... + + def destroy(self, name: str) -> None: ... + + +Capability = Literal["deletes_stored_keys"] + + +@dataclass(frozen=True, slots=True) +class SecretBackend: + system: str + from_env: Callable[[], SecretStore] + capabilities: frozenset[Capability] + + @property + def proxy_config(self) -> str: + return f"{SECRET_MANAGER_CONFIG_DIR}/secret_manager_{self.system}_ci_config.yml" diff --git a/tests/e2e/secret_manager/secret_store_cyberark.py b/tests/e2e/secret_manager/secret_store_cyberark.py new file mode 100644 index 00000000000..87bcd2ffb1c --- /dev/null +++ b/tests/e2e/secret_manager/secret_store_cyberark.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import base64 +import os +from dataclasses import dataclass, field +from typing import Final, Literal +from urllib.parse import quote + +import pytest +import yaml +from e2e_http import ExternalWrite, Headers, send_text_external +from pydantic import Field + +from secret_store import SecretBackend + +CYBERARK_API_BASE_ENV: Final = "E2E_CYBERARK_API_BASE" +CYBERARK_ACCOUNT_ENV: Final = "E2E_CYBERARK_ACCOUNT" +CYBERARK_USERNAME_ENV: Final = "E2E_CYBERARK_USERNAME" +CYBERARK_API_KEY_ENV: Final = "E2E_CYBERARK_API_KEY" + +# The same defaults CyberArkSecretManager falls back to for CYBERARK_*. +DEFAULT_API_BASE: Final = "http://127.0.0.1:8080" +DEFAULT_ACCOUNT: Final = "default" +DEFAULT_USERNAME: Final = "admin" + +SYSTEM: Final = "cyberark" + +_START_HINT: Final = ( + f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " + f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" +) + + +class ConjurHeaders(Headers): + authorization: str = Field(repr=False) + content_type: str | None = Field(default=None, serialization_alias="Content-Type") + + +def _policy_scalar(name: str) -> str: + # Quoted the way CyberArkSecretManager._ensure_variable_exists quotes it. + return yaml.safe_dump(name, default_style='"').strip() + + +@dataclass(frozen=True, slots=True) +class Conjur: + base_url: str + account: str + username: str + api_key: str = field(repr=False) + + def _fail_unless_reached(self, result: ExternalWrite, action: str) -> None: + if result.status_code == -1: + pytest.fail(f"No live Conjur at {self.base_url}: {result.body}. {_START_HINT}") + if result.status_code == 401: + pytest.fail(f"Conjur rejected {self.username}'s credentials while trying to {action}. {_START_HINT}") + + def _headers(self, content_type: str | None = None) -> ConjurHeaders: + # Tokens last about eight minutes, so each call authenticates afresh rather than + # letting a long session outlive a cached one. + auth: Final = send_text_external( + "POST", + f"{self.base_url}/authn/{self.account}/{quote(self.username, safe='')}/authenticate", + headers=Headers(), + content=self.api_key, + ) + self._fail_unless_reached(auth, "authenticate") + if not auth.ok: + pytest.fail(f"Conjur refused to authenticate {self.username}: HTTP {auth.status_code} {auth.body[:300]}") + token: Final = base64.b64encode(auth.body.encode()).decode() + return ConjurHeaders(authorization=f'Token token="{token}"', content_type=content_type) + + def _secret_url(self, name: str) -> str: + return f"{self.base_url}/secrets/{self.account}/variable/{quote(name, safe='')}" + + def _update_root_policy(self, method: Literal["POST", "PATCH"], policy: str, action: str) -> None: + result: Final = send_text_external( + method, + f"{self.base_url}/policies/{self.account}/policy/root", + headers=self._headers(content_type="application/x-yaml"), + content=policy, + ) + self._fail_unless_reached(result, action) + if not result.ok: + pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}") + + def write(self, name: str, value: str) -> None: + self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}") + result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value) + self._fail_unless_reached(result, f"write {name}") + if not result.ok: + pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}") + + def read(self, name: str) -> str | None: + result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers()) + self._fail_unless_reached(result, f"read {name}") + if result.status_code == 404: + return None + if not result.ok: + pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}") + return result.body + + def destroy(self, name: str) -> None: + self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}") + + +def conjur_from_env() -> Conjur: + api_key: Final = os.environ.get(CYBERARK_API_KEY_ENV, "").strip() + if not api_key: + pytest.fail(f"The {SYSTEM} lane needs {CYBERARK_API_KEY_ENV} to reach its Conjur. {_START_HINT}") + return Conjur( + base_url=os.environ.get(CYBERARK_API_BASE_ENV, "").strip().rstrip("/") or DEFAULT_API_BASE, + account=os.environ.get(CYBERARK_ACCOUNT_ENV, "").strip() or DEFAULT_ACCOUNT, + username=os.environ.get(CYBERARK_USERNAME_ENV, "").strip() or DEFAULT_USERNAME, + api_key=api_key, + ) + + +CYBERARK: Final = SecretBackend(system=SYSTEM, from_env=conjur_from_env, capabilities=frozenset()) diff --git a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py new file mode 100644 index 00000000000..ccf8cefe716 --- /dev/null +++ b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Final + +import pytest +from e2e_http import ( + Headers, + NetworkError, + Success, + UnknownApiError, + delete_external, + get_external, + post_json_external, +) +from pydantic import BaseModel, Field + +from secret_store import SecretBackend + +VAULT_ADDR_ENV: Final = "E2E_VAULT_ADDR" +VAULT_TOKEN_ENV: Final = "E2E_VAULT_TOKEN" +VAULT_MOUNT_ENV: Final = "E2E_VAULT_MOUNT_NAME" + +DEFAULT_VAULT_ADDR: Final = "http://127.0.0.1:8200" +DEFAULT_MOUNT: Final = "secret" + +SYSTEM: Final = "hashicorp_vault" + +_START_HINT: Final = ( + f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " + f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" +) + + +class VaultHeaders(Headers): + x_vault_token: str = Field(serialization_alias="X-Vault-Token", repr=False) + + +class KvData(BaseModel): + key: str = Field(repr=False) + + +class KvWriteBody(BaseModel): + data: KvData + + +class KvReadData(BaseModel): + data: KvData + + +class KvReadResponse(BaseModel): + data: KvReadData + + +@dataclass(frozen=True, slots=True) +class Vault: + base_url: str + token: str = field(repr=False) + mount: str = DEFAULT_MOUNT + + def _headers(self) -> VaultHeaders: + return VaultHeaders(x_vault_token=self.token) + + def _data_url(self, name: str) -> str: + return f"{self.base_url}/v1/{self.mount}/data/{name}" + + def _metadata_url(self, name: str) -> str: + return f"{self.base_url}/v1/{self.mount}/metadata/{name}" + + def write(self, name: str, value: str) -> None: + write: Final = post_json_external( + self._data_url(name), headers=self._headers(), json=KvWriteBody(data=KvData(key=value)) + ) + if write.status_code == -1: + pytest.fail(f"No live Vault at {self.base_url}: {write.body}. {_START_HINT}") + if not write.ok: + pytest.fail(f"Vault refused to write {name}: HTTP {write.status_code} {write.body[:300]}") + + def read(self, name: str) -> str | None: + result: Final = get_external(self._data_url(name), headers=self._headers(), response_type=KvReadResponse) + match result: + case Success(data=body): + return body.data.data.key + case UnknownApiError(status_code=404): + return None + case NetworkError(message=message): + return pytest.fail(f"No live Vault at {self.base_url}: {message}. {_START_HINT}") + case _: + return pytest.fail(f"Vault refused to read {name}: {result}") + + def destroy(self, name: str) -> None: + write: Final = delete_external(self._metadata_url(name), headers=self._headers()) + if not write.ok and write.status_code != 404: + pytest.fail(f"Vault refused to destroy {name}: HTTP {write.status_code} {write.body[:300]}") + + +def vault_from_env() -> Vault: + token: Final = os.environ.get(VAULT_TOKEN_ENV, "").strip() + if not token: + pytest.fail(f"The hashicorp_vault lane needs {VAULT_TOKEN_ENV} to reach its Vault. {_START_HINT}") + return Vault( + base_url=os.environ.get(VAULT_ADDR_ENV, DEFAULT_VAULT_ADDR).rstrip("/"), + token=token, + mount=os.environ.get(VAULT_MOUNT_ENV, "").strip() or DEFAULT_MOUNT, + ) + + +HASHICORP_VAULT: Final = SecretBackend( + system=SYSTEM, + from_env=vault_from_env, + capabilities=frozenset({"deletes_stored_keys"}), +) diff --git a/tests/e2e/secret_manager/test_secret_manager_e2e.py b/tests/e2e/secret_manager/test_secret_manager_e2e.py new file mode 100644 index 00000000000..a9c9024718d --- /dev/null +++ b/tests/e2e/secret_manager/test_secret_manager_e2e.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import os +import time +from collections.abc import Callable +from typing import Final + +import pytest + +from e2e_config import unique_marker +from e2e_http import Result, Success, UnauthorizedError, unwrap +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody +from proxy_client import ProxyClient +from secret_store import SecretStore + +pytestmark = [pytest.mark.e2e, pytest.mark.secret_manager] + +BACKEND_MODEL: Final = "openai/gpt-4o-mini" +VIRTUAL_KEY_PREFIX: Final = "litellm-e2e/virtual-keys/" +PROVIDER_KEY_ENV: Final = "OPENAI_API_KEY" + + +# The proxy's env never holds OPENAI_API_KEY and each test seeds it under a fresh name, so a passing +# call proves the key came from the manager and not get_secret's os.environ fallback. +def _provider_key() -> str: + key: Final = os.environ.get(PROVIDER_KEY_ENV, "").strip() + if not key: + pytest.fail(f"The secret manager suite seeds the manager with the runner's {PROVIDER_KEY_ENV}, which is unset") + return key + + +def _seed(store: SecretStore, resources: ResourceManager, value: str) -> str: + name: Final = f"litellm-e2e-openai-{unique_marker()}" + store.write(name, value) + resources.defer(lambda: store.destroy(name)) + return name + + +def _deploy(proxy: ProxyClient, resources: ResourceManager, secret_name: str) -> str: + model_name: Final = f"secret-manager-backed-{unique_marker()}" + model_id: Final = proxy.create_model( + model_name, + LiteLLMParamsBody(model=BACKEND_MODEL, api_key=f"os.environ/{secret_name}"), + provider_live=True, + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return model_name + + +def _chat(proxy: ProxyClient, key: str, model: str) -> Result[ChatResponse]: + return proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], + max_tokens=16, + ), + ) + + +def _eventually(proxy: ProxyClient, read: Callable[[], str | None], expected: str | None, context: str) -> None: + deadline: Final = time.monotonic() + proxy.poll_timeout + last: str | None = read() + while last != expected and time.monotonic() < deadline: + time.sleep(proxy.poll_interval) + last = read() + if last != expected: + pytest.fail( + f"{context}: the secret manager still holds {'a value' if last is not None else 'nothing'} after the deadline" + ) + + +class TestSecretManager: + @pytest.mark.covers("other.config.secret_resolution.kms_integration") + def test_deployment_key_resolves_from_the_manager( + self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str + ) -> None: + model: Final = _deploy(proxy, resources, _seed(store, resources, _provider_key())) + + response: Final = unwrap(_chat(proxy, scoped_key, model)) + + assert response.choices, f"the manager-backed deployment answered with no choices: {response}" + + @pytest.mark.covers("other.config.secret_resolution.manager_value_used") + def test_deployment_uses_the_value_the_manager_holds( + self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str + ) -> None: + bogus: Final = f"sk-litellm-e2e-not-a-key-{unique_marker()}" + model: Final = _deploy(proxy, resources, _seed(store, resources, bogus)) + + result: Final = _chat(proxy, scoped_key, model) + + match result: + case UnauthorizedError(body=body): + assert "AuthenticationError" in body, f"the 401 did not come from the provider: {body[:300]}" + case Success(): + pytest.fail("a deployment whose managed secret is not a real key still reached the provider") + case _: + pytest.fail(f"expected the provider to reject the manager-held key with 401, got {result}") + + @pytest.mark.covers("other.config.secret_manager.virtual_key_stored") + def test_generated_key_is_written_to_the_manager( + self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore + ) -> None: + alias: Final = f"litellm-e2e-vk-{unique_marker()}" + secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" + resources.defer(lambda: store.destroy(secret_name)) + key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) + resources.defer(lambda: proxy.delete_key(key)) + + _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") + + @pytest.mark.requires_capability("deletes_stored_keys") + @pytest.mark.covers("other.config.secret_manager.virtual_key_deleted") + def test_deleted_key_is_removed_from_the_manager( + self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore + ) -> None: + alias: Final = f"litellm-e2e-vk-{unique_marker()}" + secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" + resources.defer(lambda: store.destroy(secret_name)) + key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) + resources.defer(lambda: proxy.delete_key(key)) + _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") + + proxy.delete_key(key) + + _eventually(proxy, lambda: store.read(secret_name), None, f"the deleted key {alias}") From 2157351004d63e8a3db0f51c32bd8fe9c69ad6d5 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Tue, 22 Sep 2026 18:03:32 -0700 Subject: [PATCH 25/83] feat(ui): simplify auto-router setup and clarify feature limits (#42625) * feat(ui): simplify auto-router setup and clarify feature limits * fix(ui): validate auto-router drafts before saving * fix: keep auto-router allowances consistent after deletes and refreshes --- litellm/proxy/_types.py | 1 + .../auto_router_endpoints.py | 50 ++ .../model_management_endpoints.py | 13 +- .../auto_router_availability.py | 148 +++++ litellm/proxy/proxy_server.py | 9 + .../public_endpoints/autorouter_presets.json | 36 +- .../complexity_router/fuse_presets.json | 23 +- .../auto_router_tuning_baseline.py | 16 +- .../auto_router_endpoints.py | 19 + .../test_auto_router_endpoints.py | 153 ++++- .../test_model_management_endpoints.py | 145 ++++- .../test_auto_router_availability.py | 196 +++++++ .../proxy/proxy_server/test_lifecycle.py | 54 +- .../proxy/proxy_server/test_proxy_config.py | 41 +- .../public_endpoints/test_public_endpoints.py | 8 +- .../test_auto_router_tuning_baseline.py | 61 +- .../add_model/AutoRouterAvailability.tsx | 165 ++++++ ...oRouterClassifierTabs.integration.test.tsx | 255 ++++++-- .../add_model/AutoRouterClassifierTabs.tsx | 310 ++++++++-- .../add_model/ClassificationMethodConfig.tsx | 143 ++--- .../add_model/ClassifierPrimarySettings.tsx | 98 ++++ .../add_model/ClassifierTypeRadios.tsx | 2 +- .../ComplexityRouterAdvancedSections.tsx | 118 +++- ...mplexityRouterConfig.integration.test.tsx} | 461 ++++++++------- .../add_model/ComplexityRouterConfig.tsx | 36 +- ...plexityRouterFastMode.integration.test.tsx | 9 +- ...ecastClassifierConfig.integration.test.tsx | 23 +- .../add_model/ForecastClassifierConfig.tsx | 357 ++++++------ .../JevClassifierConfig.integration.test.tsx | 32 +- .../add_model/JevClassifierConfig.tsx | 45 +- .../JevConnectionTest.integration.test.tsx | 6 +- .../add_model/NonReasoningTierToggle.tsx | 2 +- .../components/add_model/RoutingOptions.tsx | 25 +- .../components/add_model/TierConfigIntro.tsx | 31 +- ... add_auto_router_tab.integration.test.tsx} | 442 ++++++++++---- .../add_model/add_auto_router_tab.tsx | 542 +++++++++--------- .../add_model/auto_router_connection_test.tsx | 10 +- ...dit_auto_router_modal.integration.test.tsx | 214 +++++-- .../edit_auto_router_modal.tsx | 267 +++++---- .../src/lib/autorouter_presets.test.ts | 30 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 87 +++ ui/litellm-dashboard/tests/autoRouterSetup.ts | 34 ++ 42 files changed, 3364 insertions(+), 1353 deletions(-) create mode 100644 litellm/proxy/management_helpers/auto_router_availability.py create mode 100644 tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py create mode 100644 ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx rename ui/litellm-dashboard/src/components/add_model/{ComplexityRouterConfig.test.tsx => ComplexityRouterConfig.integration.test.tsx} (85%) rename ui/litellm-dashboard/src/components/add_model/{add_auto_router_tab.test.tsx => add_auto_router_tab.integration.test.tsx} (77%) create mode 100644 ui/litellm-dashboard/tests/autoRouterSetup.ts diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 76df8790b92..c7273738fd0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum): # proxy admin, or team admin naming their own team via team_id "/auto_router/test_routing", "/auto_router/validate_complexity_router_config", + "/auto_router/availability", # Per-session auto-router read - the endpoint scopes the row to the caller's own key hash "/auto_router/session", "/cost/predict-cache", diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 03fc58622ce..2ae8639fe61 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import ( ) from litellm.types.management_endpoints.auto_router_endpoints import ( SHADOW_EVAL_TURN_VALVE, + AutoRouterAvailabilityRequest, + AutoRouterAvailabilityResponse, AutoRouterBenchmarkGroup, AutoRouterBenchmarksResponse, AutoRouterBenchmarkTotals, @@ -391,6 +393,54 @@ async def validate_complexity_router_config( return ComplexityRouterConfigValidationResponse(valid=error is None, error=error) +@router.post( + "/auto_router/availability", + tags=["model management"], # mutable-ok: FastAPI requires a list + response_model=AutoRouterAvailabilityResponse, +) +async def get_auto_router_availability( + data: AutoRouterAvailabilityRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> AutoRouterAvailabilityResponse: + from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability + from litellm.proxy.proxy_server import ( + _license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate + heuristic_v1_tuning_baselines, + llm_router, + proxy_config, + ) + + member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id) + rows: Final = proxy_config.auto_router_db_catalog + if rows is None or llm_router is None: + raise HTTPException(status_code=503, detail="Auto-router availability is unavailable") + saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None) + if data.saved_model_id is not None: + if saved is None: + raise HTTPException(status_code=404, detail="Saved auto router is unavailable") + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and ( + saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id) + ): + raise HTTPException(status_code=403, detail="Cannot check another user's auto router") + existing: Final = saved.deployment if saved is not None else None + others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments()) + candidate: Final = MappingProxyType( + { + "litellm_params": MappingProxyType( + {"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config} + ), + "model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}), + } + ) + return auto_router_availability( + others=others, + existing=existing, + candidate=candidate, + baselines=heuristic_v1_tuning_baselines, + limit=_license_check.auto_router_capability_limit(), + ) + + async def _resolve_saved_routing_test( data: AutoRouterRoutingTestRequest, user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fcadcfe2cae..615a528b552 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1782,10 +1782,11 @@ async def delete_team_models( # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back # onto this pod. The lock orders the eviction after any in-flight reconcile. - if llm_router is not None: - from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK + from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config - async with MODEL_RECONCILE_LOCK: + async with MODEL_RECONCILE_LOCK: + proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids)) + if llm_router is not None: for model_id in deleted_model_ids: llm_router.delete_deployment(id=model_id) @@ -2194,6 +2195,7 @@ async def delete_model( llm_router, premium_user, prisma_client, + proxy_config, proxy_logging_obj, store_model_in_db, user_api_key_cache, @@ -2245,8 +2247,9 @@ async def delete_model( # this pod serving a model the database no longer has, until the next # reconcile. Taking the lock orders this eviction after any such in-flight # reconcile's re-add, so the eviction is the last word. - if llm_router is not None: - async with MODEL_RECONCILE_LOCK: + async with MODEL_RECONCILE_LOCK: + proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id})) + if llm_router is not None: llm_router.delete_deployment(id=model_info.id) # Runs after the row delete so the sibling check sees post-delete state. diff --git a/litellm/proxy/management_helpers/auto_router_availability.py b/litellm/proxy/management_helpers/auto_router_availability.py new file mode 100644 index 00000000000..52cd9f0499b --- /dev/null +++ b/litellm/proxy/management_helpers/auto_router_availability.py @@ -0,0 +1,148 @@ +from collections.abc import Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, Json, TypeAdapter, ValidationError + +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.router_utils.auto_router_model_naming import ( + GATED_AUTO_ROUTER_CAPABILITIES, + capability_limit_violation, + classify_strategy_router_model, + count_capability_routers, + gated_capability_of, +) +from litellm.router_utils.auto_router_tuning_baseline import ( + is_mutable_tuned_candidate, + mutable_tuned_identities, + tuning_quota_violation, +) +from litellm.types.management_endpoints.auto_router_endpoints import ( + AutoRouterAllowance, + AutoRouterAvailabilityResponse, +) + + +class _CatalogModelInfo(BaseModel): + team_id: str | None = None + + +class _CatalogSource(BaseModel): + model_id: str + created_by: str | None = None + litellm_params: Json[dict[str, object]] | dict[str, object] + model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None + + +@dataclass(frozen=True, slots=True) +class AutoRouterCatalogEntry: + model_id: str + team_id: str | None + created_by: str | None + deployment: Mapping[str, object] + + +def _catalog_field(value: object, key: str) -> object: + if not isinstance(value, str): + return deepcopy(value) + return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True) + + +def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None: + try: + sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True) + except ValidationError: + return None + return tuple( + AutoRouterCatalogEntry( + model_id=row.model_id, + team_id=row.model_info.team_id if row.model_info is not None else None, + created_by=row.created_by, + deployment=MappingProxyType( + { + "litellm_params": MappingProxyType( + { + "model": model, + "complexity_router_config": _catalog_field( + row.litellm_params.get("complexity_router_config"), "complexity_router_config" + ), + } + ), + "model_info": MappingProxyType({"id": row.model_id, "db_model": True}), + } + ), + ) + for row in sources + if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str) + and classify_strategy_router_model(model) == "complexity" + ) + + +def auto_router_availability( + *, + others: Sequence[Mapping[str, object]], + existing: Mapping[str, object] | None, + candidate: Mapping[str, object], + baselines: Mapping[str, str] | None, + limit: int | None, +) -> AutoRouterAvailabilityResponse: + existing_params: Final = None if existing is None else existing.get("litellm_params") + candidate_params: Final = candidate.get("litellm_params") + owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None + claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None + counts: Final = tuple( + (capability, count_capability_routers(others, capability=capability)) + for capability in GATED_AUTO_ROUTER_CAPABILITIES + ) + tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0 + allowances: Final = tuple( + AutoRouterAllowance( + key=capability.key, + limit=limit, + remaining=None if limit is None else max(0, limit - held), + used_by_this_router=owned is capability, + ) + for capability, held in counts + ) + capability_error: Final = next( + ( + capability_limit_violation(capability=capability, held=held + 1, limit=limit) + for capability, held in counts + if capability is claimed + ), + None, + ) + tuning_error: Final = ( + tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit) + if baselines is not None + else None + ) + capability_labels: Final = { + "heuristic_v2": "Heuristic v2", + "capability": "Capability", + "llm_v2": "Fuse v2", + "tier_or_classifier_prompt": "Custom tiers or classifier instructions", + } + return AutoRouterAvailabilityResponse( + allowances=( + *allowances, + AutoRouterAllowance( + key="heuristic_tuning", + limit=limit, + remaining=None if limit is None or baselines is None else max(0, limit - tuned_count), + available=limit is None or baselines is not None, + used_by_this_router=bool( + existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines) + ), + ), + ), + error=( + f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance." + if capability_error is not None and claimed is not None + else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance." + if tuning_error is not None + else None + ), + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index acb36fc4a59..dab7decd4dc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import ( strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body +from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, @@ -5068,6 +5069,7 @@ class ProxyConfig: def __init__(self) -> None: self.config: Mapping[str, object] = MappingProxyType({}) + self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None self._last_semantic_filter_config: dict[str, object] | None = None self._last_websearch_interception_config: dict[str, object] | None = None self._last_hashicorp_vault_config: dict[str, object] | None = None @@ -7691,6 +7693,12 @@ class ProxyConfig: def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool: return should_load_db_object(object_type=object_type) + def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None: + if self.auto_router_db_catalog is not None: + self.auto_router_db_catalog = tuple( + row for row in self.auto_router_db_catalog if row.model_id not in model_ids + ) + async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None: """ Fetch all model deployments from the DB. @@ -7711,6 +7719,7 @@ class ProxyConfig: new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository( WriterPinnedClient(prisma_client.db) ).table.find_many() + self.auto_router_db_catalog = build_auto_router_catalog(new_models) return new_models except Exception as e: verbose_proxy_logger.exception( diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index 7a251afc076..1c2f96350c6 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -1,18 +1,18 @@ { "1m_context": { "label": "1M Context", - "description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.", + "description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], - "REASONING": ["claude-opus-5"] + "COMPLEX": ["gpt-6-sol"], + "REASONING": ["claude-opus-5-5"] }, "tier_model_configs": { "REASONING": [ { - "model_name": "claude-opus-5", + "model_name": "claude-opus-5-5", "litellm_params": { "reasoning_effort": "high" } } ] @@ -28,12 +28,12 @@ }, "anthropic_family": { "label": "Anthropic Family", - "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.", + "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.", "complexity_router_config": { "tiers": { "SIMPLE": ["claude-haiku-4-5"], "MEDIUM": ["claude-sonnet-5"], - "COMPLEX": ["claude-opus-5"], + "COMPLEX": ["claude-opus-5-5"], "REASONING": ["claude-fable-5-1"] }, "tier_model_configs": { @@ -55,12 +55,12 @@ }, "gemini_family": { "label": "Gemini Family", - "description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.", + "description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gemini-2.5-flash-lite"], - "MEDIUM": ["gemini-3.1-flash-lite"], - "COMPLEX": ["gemini-3.7-flash"], + "SIMPLE": ["gemini-3.5-flash-lite"], + "MEDIUM": ["gemini-3.8-flash"], + "COMPLEX": ["gemini-3.8-flash"], "REASONING": ["gemini-3.1-pro-preview"] }, "classifier_type": "heuristic", @@ -74,18 +74,18 @@ }, "lite": { "label": "Lite", - "description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.", + "description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.", "complexity_router_config": { "tiers": { "SIMPLE": ["deepseek-v4-flash"], - "MEDIUM": ["muse-spark-1.2"], + "MEDIUM": ["muse-spark-1.3"], "COMPLEX": ["kimi-k3"], - "REASONING": ["claude-opus-5"] + "REASONING": ["claude-opus-5-5"] }, "tier_model_configs": { "MEDIUM": [ { - "model_name": "muse-spark-1.2", + "model_name": "muse-spark-1.3", "litellm_params": { "reasoning_effort": "xhigh" } } ], @@ -113,12 +113,12 @@ }, "openai_family": { "label": "OpenAI Family", - "description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.", + "description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.", "complexity_router_config": { "tiers": { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], + "COMPLEX": ["gpt-6-sol"], "REASONING": ["gpt-6-astra"] }, "tier_model_configs": { diff --git a/litellm/router_strategy/complexity_router/fuse_presets.json b/litellm/router_strategy/complexity_router/fuse_presets.json index 4006366dc25..d3f9c48c0d4 100644 --- a/litellm/router_strategy/complexity_router/fuse_presets.json +++ b/litellm/router_strategy/complexity_router/fuse_presets.json @@ -1,5 +1,5 @@ { - "version": "2026-09-17-v1", + "version": "2026-09-22-v1", "models": [ { "id": "gpt-6-astra-v1", @@ -8,6 +8,20 @@ "text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks", "sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"] }, + { + "id": "gpt-6-sol-v1", + "label": "GPT-6 Sol", + "model": "gpt-6-sol", + "text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API", + "sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"] + }, + { + "id": "gpt-6-luna-v1", + "label": "GPT-6 Luna", + "model": "gpt-6-luna", + "text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API", + "sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"] + }, { "id": "gpt-5.6-sol-v1", "label": "GPT-5.6 Sol", @@ -63,6 +77,13 @@ "model": "claude-fable-5-1", "text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking", "sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"] + }, + { + "id": "claude-opus-5-5-v1", + "label": "Claude Opus 5.5", + "model": "claude-opus-5-5", + "text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use", + "sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"] } ], "harnesses": [ diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py index 9699ab886b9..e87548bf6de 100644 --- a/litellm/router_utils/auto_router_tuning_baseline.py +++ b/litellm/router_utils/auto_router_tuning_baseline.py @@ -10,12 +10,10 @@ from pydantic import ValidationError from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig -TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2" +# v2 hashes combine models and scoring rules; a new snapshot is required to separate them. +TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3" HEURISTIC_V1_TUNING_FIELDS: Final = ( - "tiers", - "tier_model_configs", - "classifier_type", "tier_boundaries", "reasoning_override_min_score", "token_thresholds", @@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None: validated: Final = ComplexityRouterConfig.model_validate(raw) except ValidationError: return None - supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | ( - frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset() + # The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes. + default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"]) + supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - ( + frozenset(("escalation_keywords",)) if default_escalation else frozenset() ) payload: Final = validated.model_dump( mode="json", @@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None: if limit is None or held <= limit: return None return ( - f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified " + f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified " "without an auto-router license. Keep this router on its recorded settings, or revert the other changed " - "router to its baseline, or remove one of them." + "router to its baseline, or remove one of them. Selecting models does not use this allowance." ) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 93ea925bd9e..334ca0dfb08 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel): error: str | None = None +class AutoRouterAvailabilityRequest(BaseModel): + team_id: str | None = None + saved_model_id: str | None = None + complexity_router_config: Mapping[str, object] | None = None + + +class AutoRouterAllowance(BaseModel): + key: str + limit: int | None + remaining: int | None + used_by_this_router: bool = False + available: bool = True + + +class AutoRouterAvailabilityResponse(BaseModel): + allowances: tuple[AutoRouterAllowance, ...] + error: str | None = None + + class AutoRouterRoutingTestRequest(BaseModel): """A single request to classify against a complexity-router config that need not be saved yet. diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index a282ae731dd..ff3d19e8637 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -723,11 +723,13 @@ class TestAutoRouterBenchmarks: def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals - row: Final = self.ROW.model_copy(update={ - "savings_estimated_turns": estimated_turns, - "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, - "savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0, - }) + row: Final = self.ROW.model_copy( + update={ + "savings_estimated_turns": estimated_turns, + "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, + "savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0, + } + ) totals: Final = _benchmark_totals(row) assert totals.spend == 10.0 assert totals.savings_estimated_turns == estimated_turns @@ -755,10 +757,16 @@ class TestAutoRouterBenchmarks: _summed_agg_row, ) - other = self.ROW.model_copy(update={ - "router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0, - "savings_estimated_turns": 10, "savings_estimated_actual_spend": 0.0, - }) + other = self.ROW.model_copy( + update={ + "router_name": "auto-2", + "sessions": 1, + "turns": 10, + "spend": 0.0, + "savings_estimated_turns": 10, + "savings_estimated_actual_spend": 0.0, + } + ) summed = _summed_agg_row([self.ROW, other]) totals = _benchmark_totals(summed) assert summed.sessions == 5 @@ -1091,18 +1099,27 @@ class TestAutoRouterSession: return lookups @pytest.mark.asyncio - @pytest.mark.parametrize("turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"]) + @pytest.mark.parametrize( + "turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"] + ) async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against( - self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool, + self, + monkeypatch: pytest.MonkeyPatch, + turns: int, + estimated: bool, ) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session caller = UserAPIKeyAuth(api_key="sk-caller") - row: Final = {key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")} + row: Final = { + key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_") + } spend: Final = 0.14 if turns == 3 else 10.0 if estimated and turns != 3: row["savings_estimated_saved_spend"] = -0.04 - self._rig(monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}]) + self._rig( + monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}] + ) response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1") assert response.model_dump() == { "session_id": "sess-1", @@ -1159,10 +1176,18 @@ class TestAutoRouterSession: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1} - self._rig(monkeypatch, [{ - **self.ROW, "api_key": ADMIN.api_key, "session_id": "s", - "baseline_models": {"old-baseline": 100}, "savings_estimated_baseline_models": priced, - }]) + self._rig( + monkeypatch, + [ + { + **self.ROW, + "api_key": ADMIN.api_key, + "session_id": "s", + "baseline_models": {"old-baseline": 100}, + "savings_estimated_baseline_models": priced, + } + ], + ) response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s") assert response.baseline_model == "anthropic/claude-opus-5" assert response.baseline_models == priced @@ -3562,3 +3587,97 @@ async def test_start_shadow_eval_seeds_a_zero_funnel_row_per_leg(monkeypatch: py if "group_id" in call.kwargs.get("where", {}) ] assert group_reads == [] + + +@pytest.mark.asyncio +async def test_availability_counts_db_and_yaml_without_disclosing_router_names(monkeypatch): + from litellm.models.model import LiteLLM_ProxyModelTable + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + row = LiteLLM_ProxyModelTable( + model_id="db-router", + model_name="private-team-router", + created_by="someone-else", + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + yaml_row = { + "model_name": "private-yaml-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "capability"}, + }, + } + find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table")) + monkeypatch.setattr( + proxy_server, + "prisma_client", + SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))), + ) + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,))) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: (yaml_row,))) + monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1)) + monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {}) + result = await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN) + assert {slot.key: slot.remaining for slot in result.allowances} == { + "heuristic_v2": 0, + "capability": 0, + "llm_v2": 1, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + assert "private" not in result.model_dump_json() + edit = await auto_router_endpoints.get_auto_router_availability( + AutoRouterAvailabilityRequest( + saved_model_id="db-router", complexity_router_config={"classifier_type": "heuristic_v2"} + ), + ADMIN, + ) + assert edit.allowances[0].used_by_this_router + assert edit.allowances[0].remaining == 1 + assert edit.error is None + find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_availability_denies_another_teams_edit_exemption(monkeypatch): + from litellm.models.model import LiteLLM_ProxyModelTable + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + row = LiteLLM_ProxyModelTable( + model_id="other-router", + model_name="other", + created_by="other", + model_info={"team_id": "other-team"}, + litellm_params={"model": "auto_router/complexity_router"}, + ) + find_many = AsyncMock(side_effect=AssertionError("Availability must not query the model table")) + monkeypatch.setattr( + proxy_server, + "prisma_client", + SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))), + ) + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", build_auto_router_catalog((row,))) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ())) + monkeypatch.setattr(auto_router_endpoints, "_authorize_router_dry_run", AsyncMock(return_value=None)) + with pytest.raises(HTTPException) as error: + await auto_router_endpoints.get_auto_router_availability( + AutoRouterAvailabilityRequest(team_id="own-team", saved_model_id="other-router"), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="owner"), + ) + assert error.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_availability_waits_for_the_first_complete_catalog(monkeypatch): + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + monkeypatch.setattr(proxy_server.proxy_config, "auto_router_db_catalog", None) + monkeypatch.setattr(proxy_server, "llm_router", SimpleNamespace(config_deployments=lambda: ())) + with pytest.raises(HTTPException) as error: + await auto_router_endpoints.get_auto_router_availability(AutoRouterAvailabilityRequest(), ADMIN) + assert error.value.status_code == 503 diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index bd252169131..8d618f4699b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3,6 +3,7 @@ import asyncio import contextlib import json from collections.abc import Iterator, Mapping +from types import SimpleNamespace from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -1222,6 +1223,117 @@ class TestDeleteModelClearsRouterRegistry: assert mock_router.complexity_routers.get("shared-name") is config_router +@pytest.fixture +def deleted_auto_router_catalog(monkeypatch): + from litellm.proxy import proxy_server + from litellm.proxy.management_helpers.auto_router_availability import build_auto_router_catalog + + rows = tuple( + LiteLLM_ProxyModelTable( + model_id=model_id, + model_name=f"model_name_{team_id}_{model_id}", + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": classifier}, + }, + model_info={"id": model_id, "team_id": team_id}, + created_by="admin", + updated_by="admin", + blocked=True, + ) + for model_id, team_id, classifier in ( + ("deleted-router", "deleted-team", "heuristic_v2"), + ("surviving-router", "surviving-team", "llm_v2"), + ) + ) + config = proxy_server.ProxyConfig() + config.auto_router_db_catalog = build_auto_router_catalog(rows) + monkeypatch.setattr(proxy_server, "proxy_config", config) + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", asyncio.Lock()) + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[])) + monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1)) + monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {}) + return config, rows + + +class TestDeletedAutoRouterAvailability: + @pytest.mark.asyncio + @pytest.mark.parametrize("delete_succeeds,has_router", ((True, True), (True, False), (False, True))) + async def test_single_delete_releases_allowance_only_after_success( + self, monkeypatch, deleted_auto_router_catalog, delete_succeeds, has_router + ): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability + from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete, delete_model + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + config, rows = deleted_auto_router_catalog + original = config.auto_router_db_catalog + row = rows[0].model_copy(update={"model_info": {"id": rows[0].model_id}}) + table = SimpleNamespace( + find_unique=AsyncMock(return_value=row), + delete=AsyncMock(return_value=row, side_effect=None if delete_succeeds else RuntimeError("delete failed")), + ) + prisma = SimpleNamespace( + db=SimpleNamespace(litellm_proxymodeltable=table, query_raw=AsyncMock(return_value=[])) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + request = AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"}) + before = await get_auto_router_availability(request, admin) + assert before.error is not None + if not has_router: + monkeypatch.setattr(proxy_server, "llm_router", None) + + if not delete_succeeds: + with pytest.raises(ProxyException, match="delete failed"): + await delete_model(ModelInfoDelete(id=row.model_id), admin) + assert config.auto_router_db_catalog == original + return + + await delete_model(ModelInfoDelete(id=row.model_id), admin) + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[])) + after = await get_auto_router_availability(request, admin) + assert after.error is None + assert {slot.key: slot.remaining for slot in after.allowances} == { + "heuristic_v2": 1, + "capability": 1, + "llm_v2": 0, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("has_router", (True, False)) + async def test_team_delete_releases_only_its_routers_allowance( + self, monkeypatch, deleted_auto_router_catalog, has_router + ): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_availability + from litellm.types.management_endpoints.auto_router_endpoints import AutoRouterAvailabilityRequest + + _, rows = deleted_auto_router_catalog + prisma = _TxPrismaClient(rows) + deleted = await delete_team_models( + team_ids=["deleted-team"], prisma_client=prisma, llm_router=proxy_server.llm_router if has_router else None + ) + + assert deleted == ["deleted-router"] + after = await get_auto_router_availability( + AutoRouterAvailabilityRequest(complexity_router_config={"classifier_type": "heuristic_v2"}), + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert after.error is None + assert {slot.key: slot.remaining for slot in after.allowances} == { + "heuristic_v2": 1, + "capability": 1, + "llm_v2": 0, + "tier_or_classifier_prompt": 1, + "heuristic_tuning": 1, + } + + class TestUpdateModel: """ Tests for the update_model (POST /model/update) handler. @@ -5274,7 +5386,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: """ @staticmethod - async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str) -> None: + async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str, config) -> None: """Run ``call_endpoint`` with the lock already held and assert it blocks. Holding MODEL_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If @@ -5291,6 +5403,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: """ lock = asyncio.Lock() monkeypatch.setattr("litellm.proxy.proxy_server.MODEL_RECONCILE_LOCK", lock) + stale_catalog = config.auto_router_db_catalog async with lock: task = asyncio.create_task(call_endpoint()) @@ -5301,16 +5414,19 @@ class TestDeleteEvictionsHoldTheReconcileLock: f"deleting {model_id} did not wait for MODEL_RECONCILE_LOCK -- an " f"in-flight reconcile can resurrect the deployment it just evicted" ) + config.auto_router_db_catalog = stale_catalog await asyncio.wait_for(task, timeout=5) + assert tuple(row.model_id for row in config.auto_router_db_catalog) == ("surviving-router",) @pytest.mark.asyncio - async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch): + async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog): from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelInfoDelete, delete_model, ) - model_id = "m-doomed" + config, rows = deleted_auto_router_catalog + model_id = rows[0].model_id row = MagicMock() row.model_dump.return_value = { "model_name": "gpt-4o", @@ -5347,16 +5463,17 @@ class TestDeleteEvictionsHoldTheReconcileLock: ), ) - await self._assert_evicts_under_lock(monkeypatch, call, model_id) + await self._assert_evicts_under_lock(monkeypatch, call, model_id, config) router.delete_deployment.assert_called_once_with(id=model_id) @pytest.mark.asyncio - async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch): + async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch, deleted_auto_router_catalog): from litellm.proxy.management_endpoints.model_management_endpoints import ( delete_team_models, ) - model_id = "m-team-doomed" + config, rows = deleted_auto_router_catalog + model_id = rows[0].model_id router = MagicMock() router.delete_deployment = MagicMock(return_value=True) @@ -5392,7 +5509,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: team_ids=["team-1"], prisma_client=prisma, llm_router=router ) - await self._assert_evicts_under_lock(monkeypatch, call, model_id) + await self._assert_evicts_under_lock(monkeypatch, call, model_id, config) router.delete_deployment.assert_called_once_with(id=model_id) @@ -6092,7 +6209,8 @@ class TestStrategyRouterWriteValidation: _TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}} _TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}} _TUNED_B = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4.1"}} - _TUNED_B_EDITED = {**_TUNED_B, "tiers": {"SIMPLE": "gpt-4o", "MEDIUM": "gpt-4.1"}} + _TUNED_B_EDITED = {**_TUNED_B, "code_keywords": ["internal-api"]} + _MODELS_ONLY_B = {**_TUNED_B, "tiers": {"SIMPLE": "fast-model", "MEDIUM": "capable-model"}} @staticmethod def _db_router_row(model_id: str, config: Mapping[str, object]) -> dict[str, object]: @@ -6110,7 +6228,9 @@ class TestStrategyRouterWriteValidation: (1, ["a", "b"], {"a": "_TUNED_A", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "a", "_TUNED_A_EDITED", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "refused"), - (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "refused"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B_EDITED", "refused"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "c", "_TUNED_B", "allowed"), + (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_MODELS_ONLY_B", "allowed"), (1, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B", "allowed"), (None, ["a", "b"], {"a": "_TUNED_A_EDITED", "b": "_TUNED_B"}, "b", "_TUNED_B_EDITED", "allowed"), (1, [], {}, "c", "_TUNED_B", "allowed"), @@ -6138,6 +6258,7 @@ class TestStrategyRouterWriteValidation: "_TUNED_A_EDITED": self._TUNED_A_EDITED, "_TUNED_B": self._TUNED_B, "_TUNED_B_EDITED": self._TUNED_B_EDITED, + "_MODELS_ONLY_B": self._MODELS_ONLY_B, } baselines = snapshot_tuning_baselines( [self._db_router_row(row_id, configs["_TUNED_A" if row_id == "a" else "_TUNED_B"]) for row_id in baseline_rows] @@ -6172,7 +6293,7 @@ class TestStrategyRouterWriteValidation: async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id): pass assert exc_info.value.status_code == 403 - assert "changed heuristic scorer settings or tier models" in str(exc_info.value.detail) + assert "changed heuristic scoring rules" in str(exc_info.value.detail) assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail) return async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=candidate_id) as table: @@ -6222,13 +6343,13 @@ class TestStrategyRouterWriteValidation: model_params=Deployment( model_name="second-tuned", litellm_params=LiteLLM_Params( - model="auto_router/complexity_router", complexity_router_config=self._TUNED_B + model="auto_router/complexity_router", complexity_router_config=self._TUNED_B_EDITED ), ), user_api_key_dict=admin, ) assert exc_info.value.code == "403" - assert "changed heuristic scorer settings or tier models" in str(exc_info.value.message) + assert "changed heuristic scoring rules" in str(exc_info.value.message) fake.tx_obj.litellm_proxymodeltable.create.assert_not_awaited() fake.litellm_proxymodeltable.create.assert_not_awaited() diff --git a/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py b/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py new file mode 100644 index 00000000000..027930d9ebb --- /dev/null +++ b/tests/test_litellm/proxy/management_helpers/test_auto_router_availability.py @@ -0,0 +1,196 @@ +from collections.abc import Mapping +from typing import Final +from types import SimpleNamespace + +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + +import pytest + +from litellm.proxy.management_helpers.auto_router_availability import ( + auto_router_availability, + build_auto_router_catalog, +) +from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines + + +def deployment( + model_id: str, + classifier: str, + *, + model: str = "solver", + tuned: bool = False, + config: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + return { + "model_name": model_id, + "model_info": {"id": model_id, "db_model": True}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": classifier, + "tiers": {"SIMPLE": [model]}, + **({"code_keywords": ["internal-api"]} if tuned else {}), + **(config or {}), + }, + }, + } + + +@pytest.mark.parametrize("classifier", ("heuristic_v2", "capability", "llm_v2")) +def test_occupied_allowance_blocks_new_router_but_not_owner(classifier: str) -> None: + existing: Final = deployment("existing", classifier) + candidate: Final = deployment("new", classifier) + new: Final = auto_router_availability(others=(existing,), existing=None, candidate=candidate, baselines={}, limit=1) + edit: Final = auto_router_availability(others=(), existing=existing, candidate=existing, baselines={}, limit=1) + new_slot: Final = next(slot for slot in new.allowances if slot.key == classifier) + edit_slot: Final = next(slot for slot in edit.allowances if slot.key == classifier) + assert (new_slot.remaining, new_slot.used_by_this_router, new.error is not None) == (0, False, True) + assert (edit_slot.remaining, edit_slot.used_by_this_router, edit.error) == (1, True, None) + + +def test_edit_does_not_exempt_another_classifier_allowance() -> None: + existing: Final = deployment("existing", "capability") + result: Final = auto_router_availability( + others=(deployment("other", "llm_v2"),), + existing=existing, + candidate=deployment("existing", "llm_v2"), + baselines={}, + limit=1, + ) + assert result.error is not None + assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0 + + +def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None: + original: Final = deployment("legacy", "heuristic") + changed: Final = deployment("other", "heuristic", tuned=True) + baselines: Final = snapshot_tuning_baselines((original,)) + unchanged: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=original, + baselines=baselines, + limit=1, + ) + edited: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=deployment("legacy", "heuristic", model="new"), + baselines=baselines, + limit=1, + ) + assert unchanged.error is None + assert next(slot for slot in unchanged.allowances if slot.key == "heuristic_tuning").remaining == 0 + assert edited.error is None + tuned: Final = auto_router_availability( + others=(changed,), + existing=original, + candidate=deployment("legacy", "heuristic", tuned=True), + baselines=baselines, + limit=1, + ) + assert tuned.error is not None + assert "weights, thresholds, keywords, and custom dimensions" in tuned.error + + +def test_missing_baselines_are_reported_as_unknown() -> None: + result: Final = auto_router_availability( + others=(), + existing=None, + candidate=deployment("new", "heuristic"), + baselines=None, + limit=1, + ) + slot: Final = next(slot for slot in result.allowances if slot.key == "heuristic_tuning") + assert (slot.available, slot.remaining, slot.limit) == (False, None, 1) + + +def test_unlimited_entitlement_does_not_report_exhausted_allowances() -> None: + result: Final = auto_router_availability( + others=(deployment("other", "heuristic_v2"),), + existing=None, + candidate=deployment("new", "heuristic_v2"), + baselines=None, + limit=None, + ) + assert all(slot.available and slot.limit is None and slot.remaining is None for slot in result.allowances) + assert result.error is None + + +@pytest.mark.parametrize( + "customization", + ( + {"tier_definitions": [{"name": "SIMPLE"}, {"name": "AUDIT", "description": "Review risks"}]}, + {"classification_prompt": "Use the simplest sufficient tier"}, + {"classification_examples": "Review this code -> COMPLEX"}, + {"classifier_llm_config": {"model": "judge", "system_prompt": "Route by urgency"}}, + ), +) +def test_customization_owner_can_edit_models_and_restoring_defaults_clears_the_gate( + customization: Mapping[str, object], +) -> None: + owner: Final = deployment("owner", "llm", config=customization) + blocked: Final = auto_router_availability( + others=(owner,), existing=None, candidate=deployment("new", "llm", config=customization), baselines={}, limit=1 + ) + assert blocked.error is not None + assert "Custom tiers or classifier instructions" in blocked.error + edited: Final = auto_router_availability( + others=(), + existing=owner, + candidate=deployment("owner", "llm", model="new", config=customization), + baselines={}, + limit=1, + ) + assert edited.error is None + assert next(slot for slot in edited.allowances if slot.key == "tier_or_classifier_prompt").used_by_this_router + restored: Final = auto_router_availability( + others=(owner,), existing=None, candidate=deployment("new", "llm"), baselines={}, limit=1 + ) + assert restored.error is None + assert next(slot for slot in restored.allowances if slot.key == "tier_or_classifier_prompt").remaining == 0 + + +def test_restoring_tiers_does_not_exempt_a_retained_custom_prompt() -> None: + prompt: Final = {"classification_prompt": "Use the simplest sufficient tier"} + result: Final = auto_router_availability( + others=(deployment("owner", "llm", config=prompt),), + existing=None, + candidate=deployment("new", "llm", config=prompt), + baselines={}, + limit=1, + ) + assert result.error is not None + assert "Custom tiers or classifier instructions" in result.error + + +@pytest.mark.parametrize("blocked", (False, True)) +def test_catalog_keeps_unloaded_routers_and_ownership_without_provider_credentials(blocked: bool, monkeypatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "catalog-test-key") + source: Final = SimpleNamespace( + model_id="saved", + created_by="owner", + model_info={"team_id": "team"}, + blocked=blocked, + litellm_params={ + "model": encrypt_value_helper("auto_router/complexity_router"), + "api_key": "private-key", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + provider: Final = SimpleNamespace(model_id="provider", litellm_params={"model": "openai/model"}) + catalog: Final = build_auto_router_catalog((source, provider)) + assert catalog is not None and len(catalog) == 1 + assert (catalog[0].model_id, catalog[0].team_id, catalog[0].created_by) == ("saved", "team", "owner") + assert catalog[0].deployment == { + "model_info": {"id": "saved", "db_model": True}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + } + + +def test_catalog_distinguishes_missing_data_from_an_empty_model_table() -> None: + assert build_auto_router_catalog(()) == () + assert build_auto_router_catalog((SimpleNamespace(model_id="incomplete"),)) is None diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 9954351fa2e..36d2e16d261 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -920,7 +920,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database(): @pytest.mark.asyncio -async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row(): +async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row(): from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT prisma_client = MagicMock() @@ -935,11 +935,61 @@ async def test_tuning_baseline_v2_is_created_alongside_the_legacy_row(): assert result == {'yaml:["a",[]]': DEFAULT_TUNING_FINGERPRINT} assert prisma_client.db.litellm_config.create.await_args.kwargs["data"] == { - "param_name": "auto_router_tuning_baseline_v2", + "param_name": "auto_router_tuning_baseline_v3", "param_value": json.dumps(dict(result)), } +@pytest.mark.asyncio +async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart(): + from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines + + deployments = [ + { + "model_name": name, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": name}, "code_keywords": [name]}, + }, + } + for name in ("a", "b") + ] + prisma_client = MagicMock() + prisma_client.db.litellm_config.find_unique = AsyncMock( + side_effect=lambda where: ( + MagicMock(param_value='{"legacy-router":"old-combined-hash"}') + if where["param_name"] == "auto_router_tuning_baseline_v2" + else None + ) + ) + prisma_client.db.litellm_config.create = AsyncMock() + + baseline = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, deployments) + + assert baseline == snapshot_tuning_baselines(deployments) + assert mutable_tuned_identities(deployments, baseline) == frozenset() + prisma_client.db.litellm_config.create.assert_awaited_once_with( + data={"param_name": "auto_router_tuning_baseline_v3", "param_value": json.dumps(dict(baseline))} + ) + prisma_client.db.litellm_config.find_unique.side_effect = None + prisma_client.db.litellm_config.find_unique.return_value = MagicMock(param_value=json.dumps(dict(baseline))) + changed = [ + { + "model_name": "a", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "different-model"}, "code_keywords": ["new-rule"]}, + }, + } + ] + + reloaded = await ProxyStartupEvent._load_heuristic_v1_tuning_baselines(prisma_client, changed) + + assert reloaded == baseline + assert mutable_tuned_identities(changed, reloaded) == frozenset({'yaml:["a",[]]'}) + prisma_client.db.litellm_config.create.assert_awaited_once() + + @pytest.mark.asyncio async def test_tuning_baseline_waits_for_a_complete_db_model_census(monkeypatch): prisma_client = MagicMock() diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index b47cce43dcc..89cb8356289 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -875,9 +875,7 @@ class _ConfigTable: await asyncio.sleep(0) return _ConfigRow(param_value=value) if value is not None else None - async def upsert( - self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]] - ) -> _ConfigRow: + async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> _ConfigRow: param_name: Final = where["param_name"] value: Final = _CONFIG_VALUE.validate_json(data["update"]["param_value"]) self.rows[param_name] = value @@ -926,7 +924,9 @@ class _ConfigPrisma: self.db.litellm_config.upserted_param_names.append(param_name) -def _db_backed_proxy_config(monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]]) -> tuple[ProxyConfig, _ConfigTable]: +def _db_backed_proxy_config( + monkeypatch, rows: Mapping[str, Mapping[str, JsonValue]] +) -> tuple[ProxyConfig, _ConfigTable]: table: Final = _ConfigTable(rows) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _ConfigPrisma(db=_ConfigDb(litellm_config=table))) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) @@ -4750,9 +4750,7 @@ def test_validate_deployment_access_windows_rejects_malformed_time(): "model_name": "gpt-4o-shared", "litellm_params": {"model": "gpt-4o"}, "model_info": { - "access_windows": [ - {"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} - ] + "access_windows": [{"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]}] }, } @@ -4767,9 +4765,7 @@ def test_validate_deployment_access_windows_rejects_unknown_timezone(): "model_name": "gpt-4o-shared", "litellm_params": {"model": "gpt-4o"}, "model_info": { - "access_windows": [ - {"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]} - ] + "access_windows": [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}] }, } @@ -4799,3 +4795,28 @@ def test_validate_deployment_access_windows_accepts_valid_and_absent(): ) is None ) + + +@pytest.mark.asyncio +async def test_model_refresh_updates_availability_catalog_and_retains_it_on_db_failure(): + pc = ProxyConfig() + row = SimpleNamespace( + model_id="gated", + created_by="owner", + model_info={}, + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": {"classifier_type": "heuristic_v2"}, + }, + ) + find_many = AsyncMock(side_effect=[[row], RuntimeError("database unavailable"), []]) + client = SimpleNamespace(db=SimpleNamespace(litellm_proxymodeltable=SimpleNamespace(find_many=find_many))) + assert pc.auto_router_db_catalog is None + assert await pc._get_models_from_db(client) == [row] + loaded = pc.auto_router_db_catalog + assert loaded is not None and loaded[0].model_id == "gated" + assert await pc._get_models_from_db(client) is None + assert pc.auto_router_db_catalog == loaded + assert await pc._get_models_from_db(client) == [] + assert pc.auto_router_db_catalog == () + assert find_many.await_count == 3 diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0f1ff3b024d..0dec44af402 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -1218,13 +1218,13 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog( assert "anthropic_family" in payload assert payload["1m_context"]["complexity_router_config"]["classifier_type"] == "heuristic_v2" assert payload["1m_context"]["complexity_router_config"]["tiers"] == { - "SIMPLE": ["gpt-5.6-luna"], + "SIMPLE": ["gpt-6-luna"], "MEDIUM": ["gpt-5.6-terra"], - "COMPLEX": ["gpt-5.6-sol"], - "REASONING": ["claude-opus-5"], + "COMPLEX": ["gpt-6-sol"], + "REASONING": ["claude-opus-5-5"], } assert payload["1m_context"]["complexity_router_config"]["tier_model_configs"] == { - "REASONING": [{"model_name": "claude-opus-5", "litellm_params": {"reasoning_effort": "high"}}] + "REASONING": [{"model_name": "claude-opus-5-5", "litellm_params": {"reasoning_effort": "high"}}] } for preset in payload.values(): assert isinstance(preset["label"], str) diff --git a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py index 75c115cd3ab..9307668d66f 100644 --- a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py +++ b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py @@ -33,21 +33,6 @@ _HISTORICAL_FINGERPRINTS: Final = ( {"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]}, "814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950", ), - ( - { - "tiers": _TIERS, - "dimension_weights": {"codePresence": 0.3}, - "custom_dimensions": [ - { - "name": "internalFrameworks", - "weight": 0.2, - "keywords": ["orbitmesh", "fluxgate"], - "patterns": [r"\bALTER\s{1,4}TABLE\b"], - } - ], - }, - "38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39", - ), ) @@ -78,11 +63,9 @@ class TestTuningFingerprint: {"tiers": {"SIMPLE": "x"}} ) - @pytest.mark.parametrize("field", sorted(set(HEURISTIC_V1_TUNING_FIELDS) - {"tier_model_configs"})) + @pytest.mark.parametrize("field", HEURISTIC_V1_TUNING_FIELDS) def test_every_tuning_field_changes_the_fingerprint(self, field: str) -> None: samples: dict[str, object] = { - "tiers": _ALT_TIERS, - "classifier_type": "heuristic_first", "tier_boundaries": {"simple_medium": 0.2, "medium_complex": 0.4, "complex_reasoning": 0.7}, "reasoning_override_min_score": 0.05, "token_thresholds": {"simple": 20, "complex": 500}, @@ -97,9 +80,6 @@ class TestTuningFingerprint: "keyword_tier_rules": [{"keywords": ["urgent"], "tier": "COMPLEX"}], } config: dict[str, object] = {field: samples[field]} - if field == "classifier_type": - config["heuristic_first_max_tier"] = "MEDIUM" - config["classifier_llm_config"] = {"model": "judge"} assert tuning_fingerprint(config) != DEFAULT_TUNING_FINGERPRINT def test_explicit_empty_tier_model_configs_follow_omission(self) -> None: @@ -126,12 +106,33 @@ class TestTuningFingerprint: != historical ) - def test_tier_model_overrides_change_the_fingerprint(self) -> None: + def test_tier_model_overrides_do_not_change_the_fingerprint(self) -> None: plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}}) with_override = tuning_fingerprint( {"tiers": {"SIMPLE": {"model_name": "x", "litellm_params": {"temperature": 0.1}}}} ) - assert plain != with_override + assert plain == with_override == DEFAULT_TUNING_FINGERPRINT + + @pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_first", "hybrid")) + def test_model_selection_and_classifier_switching_do_not_claim_tuning(self, classifier_type: str) -> None: + config: Final = { + "classifier_type": classifier_type, + **({"classifier_llm_config": {"model": "judge"}} if classifier_type != "heuristic" else {}), + **({"heuristic_first_max_tier": "MEDIUM"} if classifier_type == "heuristic_first" else {}), + **({"hybrid_boundary_margin": 0.1} if classifier_type == "hybrid" else {}), + "tiers": _ALT_TIERS, + "escalation_keywords": ["LITELLM ESCALATE"], + "tier_model_configs": {"COMPLEX": [{"model_name": "other-strong", "litellm_params": {"temperature": 0.1}}]}, + } + tuned: Final = _router("tuned", {"dimension_weights": {"codePresence": 0.9}}) + model_only: Final = _router("model-only", {"tiers": _TIERS}) + candidate: Final = _router("another", config) + assert tuning_fingerprint(config) == DEFAULT_TUNING_FINGERPRINT + assert tuning_quota_violation(candidate=candidate, others=(tuned, model_only), baselines={}, limit=1) is None + + def test_disabling_or_replacing_escalation_is_still_a_custom_rule(self) -> None: + assert tuning_fingerprint({"escalation_keywords": []}) != DEFAULT_TUNING_FINGERPRINT + assert tuning_fingerprint({"escalation_keywords": ["USE A STRONGER MODEL"]}) != DEFAULT_TUNING_FINGERPRINT def test_non_tuning_fields_do_not_change_the_fingerprint(self) -> None: assert ( @@ -230,17 +231,18 @@ class TestQuota: def test_router_added_after_snapshot_is_mutable_only_when_tuned(self) -> None: baselines = snapshot_tuning_baselines([_router("a", {"tiers": _TIERS})]) assert mutable_tuned_identities([_router("new", {})], baselines) == frozenset() - assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == { - router_identity(_router("new", {})) - } + assert mutable_tuned_identities([_router("new", {"tiers": _TIERS})], baselines) == frozenset() + assert mutable_tuned_identities( + [_router("new", {"tiers": _TIERS, "code_keywords": ["internal-api"]})], baselines + ) == {router_identity(_router("new", {}))} def test_quota_matrix(self) -> None: legacy_a = _router("a", {"tiers": _TIERS}) legacy_b = _router("b", {"tiers": _ALT_TIERS}) baselines = snapshot_tuning_baselines([legacy_a, legacy_b]) edited_a = _router("a", {"tiers": _TIERS, "dimension_weights": {"codePresence": 0.9}}) - edited_b = _router("b", {"tiers": _TIERS}) - new_c = _router("c", {"tiers": _TIERS}) + edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) + new_c = _router("c", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) assert tuning_quota_violation(candidate=edited_a, others=[legacy_b], baselines=baselines, limit=1) is None assert ( @@ -260,7 +262,7 @@ class TestQuota: legacy_a = _router("a", {"tiers": _TIERS}) legacy_b = _router("b", {"tiers": _ALT_TIERS}) baselines = snapshot_tuning_baselines([legacy_a, legacy_b]) - edited_b = _router("b", {"tiers": _TIERS}) + edited_b = _router("b", {"tiers": _TIERS, "code_keywords": ["internal-api"]}) assert tuning_quota_violation(candidate=edited_b, others=[legacy_a], baselines=baselines, limit=1) is None assert ( tuning_quota_violation(candidate=edited_b, others=[legacy_a, edited_b], baselines=baselines, limit=1) @@ -304,5 +306,6 @@ class TestQuota: assert message is not None assert "At most 1 auto-router(s)" in message assert "revert the other changed router to its baseline" in message + assert "Selecting models does not use this allowance" in message assert tuning_limit_violation(held=1, limit=1) is None assert tuning_limit_violation(held=5, limit=None) is None diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx new file mode 100644 index 00000000000..b1233ef03b8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterAvailability.tsx @@ -0,0 +1,165 @@ +import { createContext, useContext, useEffect, useState } from "react"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import { apiClient } from "@/components/networking"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import type { components } from "@/lib/http/schema"; + +type Availability = components["schemas"]["AutoRouterAvailabilityResponse"]; +type Request = components["schemas"]["AutoRouterAvailabilityRequest"]; +export type Allowance = components["schemas"]["AutoRouterAllowance"]; + +type AvailabilityState = { + data?: Availability; + isPending: boolean; + isError: boolean; + isChecking?: boolean; + refetch?: () => unknown; +}; + +export const AutoRouterAvailabilityContext = createContext({ isPending: true, isError: false }); + +export const useAutoRouterAvailability = (accessToken: string, body: Request, enabled = true) => { + const serialized = JSON.stringify(body.complexity_router_config ?? null); + const [debounced, setDebounced] = useState(serialized); + useEffect(() => { + const timeout = setTimeout(() => setDebounced(serialized), 300); + return () => clearTimeout(timeout); + }, [serialized]); + const options: UseQueryOptions = { + queryKey: ["autoRouterAvailability", accessToken, body.team_id, body.saved_model_id, debounced], + queryFn: ({ signal }) => + apiClient.post("/auto_router/availability", { + accessToken, + body: { ...body, complexity_router_config: JSON.parse(debounced) }, + signal, + }), + enabled: enabled && Boolean(accessToken), + placeholderData: (previous, previousQuery) => { + const key = previousQuery?.queryKey; + return key?.[1] === accessToken && key[2] === body.team_id && key[3] === body.saved_model_id + ? previous + : undefined; + }, + refetchOnMount: "always", + staleTime: 0, + retry: false, + }; + const query = useQuery(options); + const isChecking = query.isFetching || query.isPlaceholderData || serialized !== debounced; + const saveBlockedReason = () => { + if (!enabled) return null; + if (query.isPending || isChecking) return "Checking availability"; + if (query.isError || !query.data) return "Could not check availability. Retry before saving."; + return query.data.error ?? null; + }; + return { + ...query, + isPending: query.isPending || (query.isFetching && !query.isFetchedAfterMount), + isChecking, + saveBlockedReason: saveBlockedReason(), + }; +}; + +export const allowanceLabel = (allowance?: Allowance): string | null => { + if (!allowance?.available) return "Availability unavailable"; + if (allowance.limit == null) return null; + if (allowance.used_by_this_router) return "Used by this router"; + return `${allowance.remaining} of ${allowance.limit} available`; +}; + +const availabilityLabel = (state: AvailabilityState, key: string) => { + if (state.isPending || state.isChecking) return "Checking availability"; + if (state.isError) return "Availability unavailable"; + return allowanceLabel(state.data?.allowances.find((entry) => entry.key === key)); +}; + +export const useAllowanceLabel = (key: string) => availabilityLabel(useContext(AutoRouterAvailabilityContext), key); + +export const isAllowanceExhausted = (allowance?: Allowance) => + Boolean(allowance?.available && allowance.limit != null && allowance.remaining === 0) && + !allowance?.used_by_this_router; + +export const AUTO_ROUTER_CONTACT_URL = "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"; + +export const AutoRouterContactLink = ({ features, message }: { features?: string[]; message?: string }) => { + const state = useContext(AutoRouterAvailabilityContext); + if (state.isPending || state.isError || state.isChecking) return null; + const exhausted = state.data?.allowances.some( + (entry) => (!features || features.includes(entry.key)) && isAllowanceExhausted(entry), + ); + if (!exhausted) return null; + return ( + + {message} + + Talk to our team + + + ); +}; + +export const AutoRouterAllowanceLabel = ({ feature }: { feature: string }) => { + const label = useAllowanceLabel(feature); + return label ? ( + {label} + ) : null; +}; + +export const AutoRouterAllowanceNote = ({ feature, label }: { feature: string; label: string }) => { + const availability = useAllowanceLabel(feature); + return availability ? ( +

+ {label}: {availability} +

+ ) : null; +}; + +export const AutoRouterLimits = () => { + const state = useContext(AutoRouterAvailabilityContext); + const limits = [ + ["heuristic_v2", "Heuristic v2 routers"], + ["capability", "Capability routers"], + ["llm_v2", "Fuse v2 routers"], + ["tier_or_classifier_prompt", "Custom tiers or prompts"], + ["heuristic_tuning", "Rule-based tuning"], + ]; + return ( + + + View limits + + + Routing and customization limits +

+ Rule-based, Complexity, and Jev are unlimited with built-in settings. Choose or change tier models freely. + Customization allowances are shared across this proxy. +

+
+ {limits.map(([key, label]) => ( +
+
{label}
+
+ {availabilityLabel(state, key) ?? "Unlimited"} +
+
+ ))} +
+

+ Custom tier definitions and written classifier instructions share one allowance. Built-in prompts and + display-name changes do not use it. +

+

+ Changing scoring rules, such as weights, thresholds, keywords, or custom dimensions, uses the Rule-based + tuning allowance. It also applies to Heuristic first and Hybrid. Recorded settings on existing routers are + preserved; new routers start from built-in rules. +

+ + + + ); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index ac6851349ea..f843a472d15 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -1,7 +1,9 @@ import React, { useState } from "react"; import { describe, expect, it, vi } from "vitest"; -import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; +import { selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +import { AutoRouterAllowanceNote, AutoRouterAvailabilityContext } from "./AutoRouterAvailability"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; const initial: ComplexityRouterConfigValue = { @@ -9,28 +11,67 @@ const initial: ComplexityRouterConfigValue = { tiers: { SIMPLE: ["efficient"], MEDIUM: [], COMPLEX: [], REASONING: ["capable"] }, }; -function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) { +function Form({ + initialValue = initial, + remaining = 1, + limit = 1, + ownedFeature, + availabilityState, +}: { + initialValue?: ComplexityRouterConfigValue; + remaining?: number; + limit?: number | null; + ownedFeature?: string; + availabilityState?: Partial>; +}) { const [value, setValue] = useState(initialValue); return ( - - {value.classifier_type} - + ({ + key, + limit, + remaining, + available: true, + used_by_this_router: key === ownedFeature, + }), + ), + error: null, + }, + ...availabilityState, + }} + > + + {value.classifier_type} + + ); } -describe("AutoRouterClassifierTabs", () => { - it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid"] as const)( - "groups %s under Complexity without resetting its configuration", - (classifier_type) => { +describe("Auto-router classifier selection", () => { + it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid", "jev"] as const)( + "shows saved %s without changing its configuration", + async (classifier_type) => { const onChange = vi.fn(); renderWithProviders( - Existing classifier settings + Existing settings , ); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Existing classifier settings"); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + const family = { + heuristic: "Heuristics", + heuristic_v2: "Heuristics", + llm: "LLM", + heuristic_first: "LLM", + hybrid: "LLM", + jev: "Jev", + }[classifier_type]; + expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked(); + fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })); expect(onChange).not.toHaveBeenCalled(); }, ); @@ -38,38 +79,186 @@ describe("AutoRouterClassifierTabs", () => { it.each([ ["capability", "Capability"], ["llm_v2", "Fuse v2"], - ] as const)("opens saved %s settings and switches back to local Complexity", (classifier_type, label) => { - renderWithProviders(
); - expect(screen.getByRole("tab", { name: label })).toHaveAttribute("aria-selected", "true"); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("heuristic"); + ] as const)( + "opens saved %s and retains the LLM family when switching to Complexity", + async (classifier_type, label) => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent(label); + await selectAutoRouterOption("Routing approach", "Complexity"); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm"); + }, + ); + + it.each([ + [1, "heuristic"], + [0, "heuristic"], + ])("defaults to Rule-based when %s v2 slots remain", async (remaining, classifier) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("radio", { name: /^Heuristics$/ })); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(String(classifier)); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent( + `${remaining} of 1 available`, + ); }); - it("keeps custom tiers editable under Complexity and explains why forecast tabs are disabled", () => { + it.each([ + { data: undefined }, + { isPending: true }, + { isError: true }, + { isChecking: true }, + { data: { allowances: [], error: null } }, + { data: { allowances: [{ key: "heuristic_v2", limit: 1, remaining: null, available: false }], error: null } }, + ])("uses Rule-based when v2 availability is unverified: %j", async (availabilityState) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("radio", { name: "Heuristics" })); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(/^heuristic$/); + }); + + it("does not present Rule-based as having a classifier quota", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent(/^Rule-based/); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getAllByRole("menuitemradio")[0]).toHaveTextContent(/^Rule-based/); + expect(screen.getByRole("menuitemradio", { name: /^Rule-based/ })).not.toHaveTextContent("of 1 available"); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).toHaveTextContent("0 of 1 available"); + }); + + it("omits allowance labels with an unlimited entitlement", async () => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.getByRole("menuitemradio", { name: /^Heuristic v2/ })).not.toHaveTextContent("available"); + }); + + it.each([ + ["heuristic", "Heuristic", "Heuristic v2"], + ["llm", "Routing approach", "Capability"], + ["llm", "Routing approach", "Fuse v2"], + ] as const)("blocks exhausted %s options: %s / %s", (classifier_type, field, option) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: field })); + const unavailable = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) }); + expect(unavailable).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(unavailable); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type); + }); + + it.each([ + ["heuristic_v2", "heuristic", "Heuristic", "Heuristic v2"], + ["capability", "llm", "Routing approach", "Capability"], + ["llm_v2", "llm", "Routing approach", "Fuse v2"], + ] as const)("lets a saved router reselect its own %s allowance", async (feature, classifier_type, field, option) => { + renderWithProviders(); + await selectAutoRouterOption(field, option); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(feature); + expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router"); + }); + + it("shows Jev's single Complexity approach without changing saved configuration", () => { const onChange = vi.fn(); renderWithProviders( - + Existing settings + , + ); + expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited"); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); + expect(screen.getAllByRole("menuitemradio")).toHaveLength(1); + fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ })); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("keeps custom tiers editable and disables incompatible choices", async () => { + renderWithProviders( + - Custom tiers - , + />, ); - expect(screen.getByRole("tabpanel", { name: "Complexity" })).toHaveTextContent("Custom tiers"); + expect(screen.getByRole("radio", { name: /^Heuristics$/ })).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); for (const name of ["Capability", "Fuse v2"]) { - const tab = screen.getByRole("tab", { name }); - expect(tab).toHaveAttribute("aria-disabled", "true"); - expect(tab).toHaveAccessibleDescription("Restore standard tiers to use Capability or Fuse v2."); - fireEvent.click(tab); + expect(screen.getByRole("menuitemradio", { name: new RegExp(`^${name}`) })).toHaveAttribute( + "aria-disabled", + "true", + ); } - expect(onChange).not.toHaveBeenCalled(); - expect(screen.getByText("Restore standard tiers to use Capability or Fuse v2.")).toBeVisible(); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent("llm"); + }); +}); + +describe("Gated routing contact action", () => { + it("offers a pricing discussion in View limits", async () => { + renderWithProviders(); + expect(screen.queryByRole("link", { name: "Talk to our team" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "View limits" })); + const link = within(screen.getByRole("dialog")).getByRole("link", { name: "Talk to our team" }); + await waitFor(() => expect(link).toBeVisible()); + expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it.each([ + ["heuristic", "Heuristic", "Heuristic v2"], + ["llm", "Routing approach", "Capability"], + ] as const)( + "keeps the contact action available beside the disabled %s choice", + async (classifier_type, field, option) => { + renderWithProviders(); + expect(screen.queryByText(/Need more/)).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: field })); + const disabled = screen.getByRole("menuitemradio", { name: new RegExp(`^${option}`) }); + expect(disabled).toHaveAttribute("aria-disabled", "true"); + const link = screen.getByRole("menuitem", { name: `Talk to our team about ${option}` }); + await waitFor(() => expect(link).toBeVisible()); + expect(link).toHaveAttribute("href", "https://calendly.com/tin-berri/litellm-auto-router-pricing-discussion"); + expect(link).toHaveAttribute("target", "_blank"); + fireEvent.click(link); + expect(screen.getByRole("status", { name: "Classifier type" })).toHaveTextContent(classifier_type); + }, + ); + + it.each([ + { remaining: 1 }, + { remaining: 0, limit: null }, + { remaining: 0, availabilityState: { isPending: true } }, + { remaining: 0, availabilityState: { isError: true } }, + { remaining: 0, availabilityState: { isChecking: true } }, + ])("does not pitch an upgrade for a free or unverified option: %j", (props) => { + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); + expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument(); + }); + + it("does not pitch an upgrade for the saved heuristic's own slot", () => { + renderWithProviders( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); + expect(screen.queryByRole("menuitem", { name: /Talk to our team/ })).not.toBeInTheDocument(); + }); + + it("includes the sales action beside customization limits and blocked changes", () => { + const allowance = { key: "tier_or_classifier_prompt", limit: 1, remaining: 0, available: true }; + const state = { + isPending: false, + isError: false, + data: { allowances: [allowance], error: "Custom tiers have no available allowance" }, + }; + renderWithProviders( + + + + + , + ); + expect(screen.getByText(/Custom tiers: 0 of 1 available/)).toHaveTextContent("Talk to our team"); + expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 98c0d4aab2f..92fc8d2a335 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -1,8 +1,119 @@ -import React, { useId } from "react"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { effectiveClassifierType, type ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import React, { useContext, useId } from "react"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { ChevronDownIcon } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { + effectiveClassifierType, + type ClassifierType, + type ComplexityRouterConfigValue, +} from "./ComplexityRouterConfig"; import { transitionClassifierType } from "./classifier_type_transition"; import { isForecastClassifier } from "./forecast_classifier_config"; +import { + AutoRouterAllowanceLabel, + AutoRouterAvailabilityContext, + AutoRouterLimits, + AutoRouterContactLink, + isAllowanceExhausted, + AUTO_ROUTER_CONTACT_URL, +} from "./AutoRouterAvailability"; + +function ClassifierOption({ + value, + label, + description, + feature, + disabled, + unlimited = true, +}: { + value: string; + label: string; + description: string; + feature?: string; + disabled?: boolean; + unlimited?: boolean; +}) { + const state = useContext(AutoRouterAvailabilityContext); + const allowance = state.data?.allowances.find((entry) => entry.key === feature); + const fresh = !state.isPending && !state.isError && !state.isChecking; + const exhausted = isAllowanceExhausted(allowance); + return ( +
+ + + + {label} + {feature ? ( + + ) : ( + unlimited && Unlimited + )} + + {description} + + + {fresh && exhausted && ( + } + aria-label={`Talk to our team about ${label}`} + className="absolute top-9 right-8 cursor-pointer px-0 py-0 text-xs leading-5 font-medium text-blue-600 focus:text-blue-600 hover:underline dark:text-blue-400 dark:focus:text-blue-400" + > + Talk to our team + + )} +
+ ); +} + +function ClassifierMenu({ + id, + label, + value, + selectedLabel, + feature, + onValueChange, + children, +}: { + id: string; + label: string; + value: string; + selectedLabel: string; + feature?: string; + onValueChange: (value: string) => void; + children: React.ReactNode; +}) { + return ( + + } + > + {selectedLabel} + {feature ? ( + + ) : ( + Unlimited + )} + + + + + {children} + + + + ); +} interface AutoRouterClassifierTabsProps { value: ComplexityRouterConfigValue; @@ -11,47 +122,174 @@ interface AutoRouterClassifierTabsProps { } const AutoRouterClassifierTabs: React.FC = ({ value, onChange, children }) => { - const restrictionId = useId(); + const id = useId(); + const availability = useContext(AutoRouterAvailabilityContext); const classifierType = effectiveClassifierType(value); - const selected = isForecastClassifier(classifierType) ? classifierType : "complexity"; + const familyByType: Record = { + heuristic: "heuristics", + heuristic_v2: "heuristics", + llm: "llm", + heuristic_first: "llm", + hybrid: "llm", + capability: "llm", + llm_v2: "llm", + jev: "jev", + custom: "custom", + }; + const family = familyByType[classifierType]; const hasCustomTiers = Boolean(value.custom_tier_set); - - const handleChange = (tab: unknown) => { - if (tab === selected) return; - if (tab === "complexity") { - onChange(transitionClassifierType(value, isForecastClassifier(classifierType) ? "heuristic" : classifierType)); - } else if (!hasCustomTiers && (tab === "capability" || tab === "llm_v2")) { - onChange(transitionClassifierType(value, tab)); - } + const changeType = (next: ClassifierType) => { + if (next !== classifierType) onChange(transitionClassifierType(value, next)); }; + const changeFamily = (next: unknown) => { + if (next === family) return; + if (next === "heuristics") changeType("heuristic"); + if (next === "llm") changeType("llm"); + if (next === "jev") changeType("jev"); + }; + const approachLabels: Partial> = { capability: "Capability", llm_v2: "Fuse v2" }; + const approachDescription: Partial> = { + capability: "Use the efficient model when it is likely to succeed", + llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model", + }; return ( - -

Classifier type

- - Complexity - - Capability - - - Fuse v2 - - +
+
+ + What classifies your requests? + + + + {[ + { value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" }, + { value: "llm", label: "LLM", description: "Use a judge model to choose a solver" }, + { value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" }, + ].map((option) => ( + + ))} + +
+ {family === "custom" && ( +

This router uses a custom classifier plugin

+ )} + {family === "heuristics" && ( +
+ + { + if (next === "heuristic" || next === "heuristic_v2") changeType(next); + }} + > + + + +

+ {classifierType === "heuristic_v2" + ? "Use calibrated probabilities to match requests to a tier" + : "Match requests using scoring rules. Choose or change tier models freely"} +

+
+ )} + {(family === "llm" || family === "jev") && ( +
+ + { + if (next === "llm" || next === "capability" || next === "llm_v2") { + if (next === "llm" && !isForecastClassifier(classifierType)) return; + changeType(next); + } + }} + > + + {family === "llm" && ( + <> + + + + )} + +

+ {approachDescription[classifierType] ?? "Match task difficulty to a tier"} +

+
+ )} {hasCustomTiers && ( -

- Restore standard tiers to use Capability or Fuse v2. +

+ Restore standard tiers to use Heuristics, Capability, or Fuse v2

)} - {children} - + {availability.data?.error && !availability.isChecking && ( +
+

{availability.data.error}

+ +
+ )} + {availability.isError && ( +

+ Could not check availability.{" "} + +

+ )} + {children} +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index b7a0fd67443..ccd204aa521 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -1,9 +1,10 @@ +import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import { transitionClassifierType } from "./classifier_type_transition"; import JevClassifierConfig from "./JevClassifierConfig"; import { Info } from "lucide-react"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; -import { SearchSelect } from "@/components/shared/SearchSelect"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Card, CardContent } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; @@ -25,13 +26,10 @@ import ClassifierTypeRadios from "./ClassifierTypeRadios"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { - ClassificationFrequency, ClassifierFallback, ClassifierLLMConfig, ClassifierType, ComplexityRouterConfigValue, - classificationFrequency, - withClassificationFrequency, DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS, MIN_QUOTED_CONTEXT_TURN_CHARS, DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, @@ -171,6 +169,7 @@ interface ClassificationMethodConfigProps { showValidationErrors?: boolean; /** The resolved default model - see resolveComplexityDefaultModel. Names and gates the radio. */ defaultModel?: string; + advancedOnly?: boolean; } export const InactiveHeuristicV2Threshold: React.FC> = ({ @@ -215,13 +214,11 @@ const ClassificationMethodConfig: React.FC = ({ onCustomTechnicalKeywordsChange, showValidationErrors = false, defaultModel, + advancedOnly = false, }) => { const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); const classifierType = effectiveClassifierType(value); - const sessionFrequencyRestriction = restrictedBy(value, "sessionAffinity"); - const classifierModelMissing = - showValidationErrors && usesLlmClassifier(classifierType) && !value.classifier_llm_config?.model; const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; @@ -282,23 +279,6 @@ const ClassificationMethodConfig: React.FC = ({ onChange(nextValue); }; - const handleClassifierModelChange = (model: string | null) => { - if (model === null) return; - if (model === value.classifier_llm_config?.model) return; - const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config ?? { - model: "", - timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS, - }; - onChange({ - ...value, - classifier_llm_config: { - ...classifierLlmConfig, - model, - timeout_ms: classifierLlmConfig.timeout_ms, - }, - }); - }; - const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => { if (!value.classifier_llm_config) return; const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config; @@ -350,10 +330,6 @@ const ClassificationMethodConfig: React.FC = ({ onChange({ ...value, classifier_fallback: fallback }); }; - const handleClassificationFrequencyChange = (frequency: ClassificationFrequency) => { - onChange(withClassificationFrequency(value, frequency)); - }; - const handleClassifierContextWindowSizeChange = (windowSize: number) => { onChange({ ...value, @@ -389,7 +365,50 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> - + {!advancedOnly && ( + <> + + + + )} + {advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType) && ( +
+ + +
+ )} {classifierType === "custom" && ( @@ -474,66 +493,9 @@ const ClassificationMethodConfig: React.FC = ({ )} -
- How often to classify - - handleClassificationFrequencyChange(frequency as ClassificationFrequency) - } - > -
- - - -
- -

- Holding the tier keeps an agent on one model for a whole tool loop and cuts scoring cost. A turn the router - cannot match to a held decision, such as one with no session id or an expired one, is scored again -

-
- {classifierType === "jev" && } {usesLlmClassifier(classifierType) && (
-
- Classifier Model - - {classifierModelMissing && A classifier model is required} -
= ({
+ {!value.custom_tier_set && usesCustomPrompt ? ( = ({ /> Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders. - LLM and JEV default to 3 turns; JEV sends them to the configured TypeSafe endpoint. Set to 0 to omit + LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit conversation history. The current message and selected system text are still sent. @@ -769,6 +735,9 @@ const ClassificationMethodConfig: React.FC = ({ )} + {["heuristic", "heuristic_first", "hybrid"].includes(classifierType) && ( + + )} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx new file mode 100644 index 00000000000..32cb4851580 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx @@ -0,0 +1,98 @@ +import React from "react"; +import { Label } from "@/components/ui/label"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { + classificationFrequency, + withClassificationFrequency, + effectiveClassifierType, + usesLlmClassifier, + DEFAULT_CLASSIFIER_TIMEOUT_MS, + type ComplexityRouterConfigValue, + type ClassificationFrequency, +} from "./ComplexityRouterConfig"; +import { restrictedBy } from "./TierRestrictions"; + +export default function ClassifierPrimarySettings({ + value, + onChange, + modelOptions, + showValidationErrors = false, +}: { + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + modelOptions: { value: string; label: string }[]; + showValidationErrors?: boolean; +}) { + const id = React.useId(); + const restriction = restrictedBy(value, "sessionAffinity"); + const frequency = classificationFrequency(value); + const frequencyDescription = { + every_request: "Choose a model again for every request", + user_turn: "Reclassify when the user sends a new message", + session: "Keep the same tier for the session. Requires a client session ID", + }[frequency]; + const usesJudge = usesLlmClassifier(effectiveClassifierType(value)); + const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model; + return ( +
+
+ + +

{restriction?.reason ?? frequencyDescription}

+
+ {usesJudge && ( +
+ + { + if (!model || model === value.classifier_llm_config?.model) return; + onChange({ + ...value, + classifier_llm_config: { + ...value.classifier_llm_config, + model, + timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS, + reasoning_effort: undefined, + }, + }); + }} + /> + {missingJudge && ( +

+ A judge model is required +

+ )} +
+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx index 1602e19069a..1fd6dfa6a20 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx @@ -54,7 +54,7 @@ const ClassifierTypeRadios: React.FC = ({ value, clas diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx index 44adecb9e1d..a4e833152a9 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx @@ -6,6 +6,7 @@ import type { ModelGroup } from "@/components/llm_calls/fetch_models"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; +import ForecastClassifierConfig from "./ForecastClassifierConfig"; import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig"; import ResponseFormatControls from "./ResponseFormatControls"; import StallEscalationConfig from "./StallEscalationConfig"; @@ -79,13 +80,31 @@ const ComplexityRouterAdvancedSections: React.FC { const sections = [ + ...(forecast + ? [ + { + key: "classifier", + label: Classifier tuning, + children: ( + + ), + }, + ] + : []), ...(!forecast ? [ { key: "classifier", - label: Advanced: Classification Method, + label: Classification Method, children: ( Advanced: Heuristic Keyword Overrides, + label: Heuristic Keyword Overrides, children: , }, ] : []), { key: "adaptive", - label: Advanced: Adaptive Routing, + label: Adaptive Routing, children: ( @@ -119,39 +138,39 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Affinity, + label: Affinity, children: , }, { key: "modality", - label: Advanced: Modality Routing, + label: Modality Routing, children: , }, { key: "plan-mode", - label: Advanced: Plan-Mode Override, + label: Plan-Mode Override, children: ( ), }, { key: "housekeeping", - label: Advanced: Housekeeping Routing, + label: Housekeeping Routing, children: , }, { key: "reminder-markers", - label: Advanced: Ignore Custom Tags, + label: Ignore Custom Tags, children: , }, { key: "context-window", - label: Advanced: Context Window Escalation, + label: Context Window Escalation, children: , }, { key: "stall-escalation", - label: Advanced: Stalled Task Escalation, + label: Stalled Task Escalation, children: ( @@ -160,14 +179,14 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Response Format, + label: Response Format, children: , }, ...(onEscalationKeywordsChange ? [ { key: "escalation", - label: Advanced: Escalation Keywords, + label: Escalation Keywords, children: ( @@ -180,7 +199,7 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Compression, + label: Compression, children: , }, ] @@ -189,7 +208,7 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Keyword/Semantic Matching, + label: Keyword/Semantic Matching, children: ( <> {onKeywordTierRulesChange && ( @@ -220,20 +239,65 @@ const ComplexityRouterAdvancedSections: React.FC(() => + showValidationErrors ? groups.map((group) => group.label) : [], + ); + const [previousValidation, setPreviousValidation] = React.useState(showValidationErrors); + if (previousValidation !== showValidationErrors) { + setPreviousValidation(showValidationErrors); + if (showValidationErrors) setOpenGroups(groups.map((group) => group.label)); + } return ( - <> - {sections - .filter(({ key }) => !forecast || !["adaptive", "context-window", "escalation"].includes(key)) - .map(({ key, label, children }) => ( - - - - {label} - - {children} - - ))} - +
+ {groups.map((group) => ( + + setOpenGroups((current) => + open ? [...current, group.label] : current.filter((label) => label !== group.label), + ) + } + className="border-b border-border last:border-b-0" + > + + + {group.label} + + + {sections + .filter( + ({ key }) => + group.keys.includes(key) && + (!forecast || !["adaptive", "context-window", "escalation"].includes(key)), + ) + .map(({ key, label, children }) => ( +
+ {key !== "classifier" &&

{label}

} + {children} +
+ ))} + + + ))} +
); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx similarity index 85% rename from ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx rename to ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx index 756e505997c..4a00a469f97 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx @@ -1,8 +1,15 @@ +import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import { fireEvent, renderWithProviders, screen, within } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import React from "react"; -import { vi, type Mock } from "vitest"; -import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { describe, it, expect, vi, type Mock } from "vitest"; +import ComplexityRouterConfigView, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +const ComplexityRouterConfig = (props: React.ComponentProps) => ( + + + +); vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), @@ -45,12 +52,12 @@ const baseProps = { }; describe("ComplexityRouterConfig", () => { - it("should render", () => { + it("should render", async () => { renderWithProviders(); - expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + expect(screen.getByText("Models by tier")).toBeInTheDocument(); }); - it("should display all four tier labels", () => { + it("should display all four tier labels", async () => { renderWithProviders(); expect(screen.getByText("Simple Tier")).toBeInTheDocument(); expect(screen.getByText("Medium Tier")).toBeInTheDocument(); @@ -58,7 +65,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText("Reasoning Tier")).toBeInTheDocument(); }); - it("should show example queries for each tier", () => { + it("should show example queries for each tier", async () => { renderWithProviders(); expect(screen.getByText(/Hello!/)).toBeInTheDocument(); expect(screen.getByText(/Explain how REST APIs work/)).toBeInTheDocument(); @@ -66,46 +73,51 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText(/Think step by step/)).toBeInTheDocument(); }); - it("should display the how classification works section", () => { + it("should display the how classification works section", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); }); - it("should show score thresholds in the classification section", () => { + it("should show score thresholds in the classification section", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/Score < 0.15/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.15 - 0.35/)).toBeInTheDocument(); expect(screen.getByText(/Score 0.35 - 0.60/)).toBeInTheDocument(); expect(screen.getByText(/Score > 0.60/)).toBeInTheDocument(); }); - it("leaves the score threshold list color to the theme instead of an inline style", () => { + it("leaves the score threshold list color to the theme instead of an inline style", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const list = screen.getByText(/Score < 0.15/).closest("ul"); expect(list).toBeInTheDocument(); expect(list).toHaveClass("text-muted-foreground"); expect(list?.style.color).toBe(""); }); - it("should default to heuristic and hide classifier model/timeout fields", () => { + it("should default to heuristic and hide classifier model/timeout fields", async () => { renderWithProviders(); - expect(screen.getByText("Advanced: Classification Method")).toBeInTheDocument(); - expect(screen.queryByText("Classifier Model")).not.toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByText("Classifier tuning")).toBeInTheDocument(); + expect(screen.queryByText("Judge model")).not.toBeInTheDocument(); }); - it("shows heuristic advanced sections and hides keyword overrides for capability classifiers", () => { + it("shows heuristic advanced sections and hides keyword overrides for capability classifiers", async () => { const { rerender } = renderWithProviders(); - expect(screen.getByText("Advanced: Heuristic Keyword Overrides")).toBeInTheDocument(); - expect(screen.getByText("Advanced: Housekeeping Routing")).toBeInTheDocument(); - expect(screen.getByText("Advanced: Ignore Custom Tags")).toBeInTheDocument(); + openAutoRouterAdvanced("Heuristic Keyword Overrides"); + + expect(screen.getByText("Heuristic Keyword Overrides")).toBeInTheDocument(); + openAutoRouterAdvanced("Housekeeping Routing"); + expect(screen.getByText("Housekeeping Routing")).toBeInTheDocument(); + openAutoRouterAdvanced("Ignore Custom Tags"); + expect(screen.getByText("Ignore Custom Tags")).toBeInTheDocument(); const capabilityValue = { ...defaultValue, classifier_type: "capability" as const }; rerender(); - expect(screen.queryByText("Advanced: Heuristic Keyword Overrides")).not.toBeInTheDocument(); + expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument(); }); it.each([ @@ -115,7 +127,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); if (visible) { expect(screen.getByLabelText("Classifier plugin timeout (ms)")).toBeInTheDocument(); } else { @@ -128,7 +140,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Ignore Custom Tags")); + openAutoRouterAdvanced("Ignore Custom Tags"); const validation = screen.queryByText(/needs both/i); if (showValidationErrors) { expect(validation).toBeInTheDocument(); @@ -137,11 +149,11 @@ describe("ComplexityRouterConfig", () => { } }); - it("disables housekeeping sentinels when cheapest-tier routing is off", () => { + it("disables housekeeping sentinels when cheapest-tier routing is off", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Housekeeping Routing")); + openAutoRouterAdvanced("Housekeeping Routing"); const sentinelInput = screen.getByRole("combobox", { name: "e.g., conversation title" }); expect(sentinelInput).toBeDisabled(); }); @@ -151,7 +163,7 @@ describe("ComplexityRouterConfig", () => { const onChange = vi.fn(); renderWithProviders(); - await user.click(screen.getByText("Advanced: Response Format")); + openAutoRouterAdvanced("Response Format"); await user.click(screen.getByRole("switch", { name: "Return raw model name" })); expect(onChange).toHaveBeenCalledWith({ @@ -160,13 +172,13 @@ describe("ComplexityRouterConfig", () => { }); }); - it("should reveal classifier model and timeout fields when llm is selected", () => { + it("should reveal classifier model and timeout fields when llm is selected", async () => { const onChange = vi.fn(); renderWithProviders(); // Collapse panel content isn't rendered until first expanded. - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("LLM Classifier")); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^LLM$/ })); const expectedValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -178,14 +190,14 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenCalledWith(expectedValue); }); - it("selects heuristic v2 without requiring a classifier model or showing weighted scoring", () => { + it("selects heuristic v2 without requiring a classifier model or showing weighted scoring", async () => { const onChange = vi.fn(); const { rerender } = renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("Heuristic v2")); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("Heuristic", "Heuristic v2"); expect(onChange).toHaveBeenCalledWith( expect.objectContaining({ @@ -197,7 +209,7 @@ describe("ComplexityRouterConfig", () => { const heuristicV2Value: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "heuristic_v2" }; rerender(); - expect(screen.queryByText("Classifier Model")).not.toBeInTheDocument(); + expect(screen.queryByText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument(); expect(screen.getByText(/estimates success probability for all four tiers/)).toBeInTheDocument(); expect(screen.queryByText(/Score < 0.15/)).not.toBeInTheDocument(); @@ -233,7 +245,7 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenCalledWith({ ...value, heuristic_v2_success_threshold: undefined }); }); - it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", () => { + it("shows an inactive zero threshold until explicitly cleared and hides the summary for active or absent values", async () => { const onChange = vi.fn(); const value = { ...defaultValue, heuristic_v2_success_threshold: 0 }; const { rerender } = renderWithProviders( @@ -248,7 +260,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); }); - it("should show classifier fields and use the configured values when classifier_type is llm", () => { + it("should show classifier fields and use the configured values when classifier_type is llm", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -258,9 +270,9 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); - expect(screen.getByText("Classifier Model")).toBeInTheDocument(); + expect(screen.getByText("Judge model")).toBeInTheDocument(); expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).toBeChecked(); expect(screen.getByLabelText("Circuit breaker cooldown (seconds)")).toHaveValue("30"); @@ -268,7 +280,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); - it("should allow the default-on classifier circuit breaker to be disabled", () => { + it("should allow the default-on classifier circuit breaker to be disabled", async () => { const onChange = vi.fn(); const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -276,7 +288,7 @@ describe("ComplexityRouterConfig", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); @@ -287,7 +299,7 @@ describe("ComplexityRouterConfig", () => { ); }); - it("should default the context window and budget when llm is selected", () => { + it("should default the context window and budget when llm is selected", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -295,13 +307,13 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByLabelText("Context Window Size")).toHaveValue("3"); expect(screen.getByLabelText("Context Character Budget")).toHaveValue("8000"); }); - it("should warn when the budget is too small to quote any turn that does not already fit", () => { + it("should warn when the budget is too small to quote any turn that does not already fit", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -310,12 +322,12 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/no room to quote a turn/i)).toBeInTheDocument(); }); - it("should not warn on a budget large enough to quote a turn, nor on a deliberate zero", () => { + it("should not warn on a budget large enough to quote a turn, nor on a deliberate zero", async () => { for (const budget of [120, 8000, 0]) { const { unmount } = renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText(/no room to quote a turn/i)).not.toBeInTheDocument(); unmount(); } }); - it("should show the assistant-turns switch with its configured value when classifier_type is llm", () => { + it("should show the assistant-turns switch with its configured value when classifier_type is llm", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -344,13 +356,13 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Include Assistant Turns")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).toBeChecked(); }); - it("should render the assistant-turns switch off when it is not set", () => { + it("should render the assistant-turns switch off when it is not set", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -358,18 +370,18 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("switch", { name: "Include Assistant Turns" })).not.toBeChecked(); }); - it("should hide the assistant-turns switch when classifier_type is heuristic", () => { + it("should hide the assistant-turns switch when classifier_type is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Include Assistant Turns")).not.toBeInTheDocument(); }); - it("should call onChange when the assistant-turns switch is toggled", () => { + it("should call onChange when the assistant-turns switch is toggled", async () => { const onChange = vi.fn(); const llmValue: ComplexityRouterConfigValue = { ...defaultValue, @@ -378,7 +390,7 @@ describe("ComplexityRouterConfig", () => { }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Include Assistant Turns" })); expect(onChange).toHaveBeenCalledWith( @@ -386,9 +398,9 @@ describe("ComplexityRouterConfig", () => { ); }); - it("should hide classifier context fields when classifier_type is heuristic", () => { + it("should hide classifier context fields when classifier_type is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Context Window Size")).not.toBeInTheDocument(); expect(screen.queryByText("Context Per-Turn Character Limit")).not.toBeInTheDocument(); }); @@ -416,7 +428,7 @@ describe("ComplexityRouterConfig", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const input = screen.getByLabelText(label); fireEvent.change(input, { target: { value: "" } }); @@ -429,14 +441,14 @@ describe("ComplexityRouterConfig", () => { expect(onChange).toHaveBeenLastCalledWith({ ...llmValue, ...expected }); }); - it("restores the committed context window size after an empty field loses focus", () => { + it("restores the committed context window size after an empty field loses focus", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const input = screen.getByLabelText("Context Window Size"); fireEvent.change(input, { target: { value: "" } }); @@ -445,13 +457,13 @@ describe("ComplexityRouterConfig", () => { expect(input).toHaveValue("3"); }); - it("should render the custom technical keywords field", () => { + it("should render the custom technical keywords field", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); }); - it("should display existing custom technical keywords as tags", () => { + it("should display existing custom technical keywords as tags", async () => { renderWithProviders( { onCustomTechnicalKeywordsChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("udp")).toBeInTheDocument(); expect(screen.getByText("kafka")).toBeInTheDocument(); }); @@ -474,7 +486,7 @@ describe("ComplexityRouterConfig", () => { onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; await user.type(within(keywordsSection).getByRole("combobox"), "udp"); await user.click(await screen.findByText('Create "udp"')); @@ -491,35 +503,35 @@ describe("ComplexityRouterConfig", () => { onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const keywordsSection = screen.getByText("Custom Technical Keywords").closest("div")?.parentElement as HTMLElement; await user.type(within(keywordsSection).getByRole("combobox"), "udp, kafka ,terraform"); await user.click(await screen.findByText('Create "udp, kafka ,terraform"')); expect(onCustomTechnicalKeywordsChange).toHaveBeenCalledWith(["udp", "kafka", "terraform"]); }); - it("should render an empty state when no keyword tier rules exist", () => { + it("should render an empty state when no keyword tier rules exist", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Keyword Tier Overrides")).toBeInTheDocument(); expect(screen.getByText("No keyword tier overrides configured")).toBeInTheDocument(); }); - it("hides the keyword-tier and semantic sections when their change handlers are absent (edit modal)", () => { + it("hides the keyword-tier and semantic sections when their change handlers are absent (edit modal)", async () => { // The edit-auto-router modal renders ComplexityRouterConfig without these handlers; // the sections must stay hidden rather than render interactive-but-dead controls. renderWithProviders(); expect(screen.queryByText("Keyword Tier Overrides")).not.toBeInTheDocument(); expect(screen.queryByText("Semantic keyword matching")).not.toBeInTheDocument(); // Core tier config still renders. - expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); + expect(screen.getByText("Models by tier")).toBeInTheDocument(); }); it("should call onKeywordTierRulesChange with a new rule when 'Add keyword rule' is clicked", async () => { const user = userEvent.setup(); const onKeywordTierRulesChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); expect(onKeywordTierRulesChange).toHaveBeenCalledTimes(1); const newRules = onKeywordTierRulesChange.mock.calls[0][0]; @@ -537,7 +549,7 @@ describe("ComplexityRouterConfig", () => { onKeywordTierRulesChange={onKeywordTierRulesChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); const field = screen.getByText("Keywords 1").closest("div") as HTMLElement; await user.type(within(field).getByRole("combobox"), "invoice"); @@ -556,7 +568,7 @@ describe("ComplexityRouterConfig", () => { onKeywordTierRulesChange={onKeywordTierRulesChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("invoice")).toBeInTheDocument(); expect(screen.getByText("refund")).toBeInTheDocument(); @@ -564,17 +576,17 @@ describe("ComplexityRouterConfig", () => { expect(onKeywordTierRulesChange).toHaveBeenCalledWith([]); }); - it("should not show embedding model or match score fields when semantic matching is disabled", () => { + it("should not show embedding model or match score fields when semantic matching is disabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Semantic keyword matching")).toBeInTheDocument(); expect(screen.queryByText("Embedding model")).not.toBeInTheDocument(); expect(screen.queryByText("Minimum match score")).not.toBeInTheDocument(); }); - it("should show embedding model and match score fields when semantic matching is enabled", () => { + it("should show embedding model and match score fields when semantic matching is enabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByText("Embedding model")).toBeInTheDocument(); expect(screen.getByText("Minimum match score")).toBeInTheDocument(); }); @@ -589,7 +601,7 @@ describe("ComplexityRouterConfig", () => { onSemanticMatchingEnabledChange={onSemanticMatchingEnabledChange} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("switch", { name: "Semantic keyword matching" })); expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); }); @@ -606,34 +618,34 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryAllByText("text-embedding-3-small")).toHaveLength(0); }); - it("does not show tier validation errors by default", () => { + it("does not show tier validation errors by default", async () => { renderWithProviders(); expect(screen.queryByText("This tier is required")).not.toBeInTheDocument(); }); - it("shows an inline error on the classifier model select when llm is selected without a model", () => { + it("shows an inline error on the classifier model select when llm is selected without a model", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByText("A classifier model is required")).toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByText("A judge model is required")).toBeInTheDocument(); }); - it("does not show the classifier model error once a classifier model is set", () => { + it("does not show the classifier model error once a classifier model is set", async () => { const llmValue: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.queryByText("A classifier model is required")).not.toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.queryByText("A judge model is required")).not.toBeInTheDocument(); }); - it("shows a validation error only under unfilled tiers when showValidationErrors is true", () => { + it("shows a validation error only under unfilled tiers when showValidationErrors is true", async () => { renderWithProviders( { expect(screen.getAllByText(/tier is required/)).toHaveLength(1); }); - it("renders the escalation keywords section with current keywords when the handler is provided", () => { + it("renders the escalation keywords section with current keywords when the handler is provided", async () => { renderWithProviders( { onEscalationKeywordsChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Escalation Keywords")); - expect(screen.getByText("Escalation Keywords")).toBeInTheDocument(); + openAutoRouterAdvanced("Escalation Keywords"); + expect(screen.getAllByText("Escalation Keywords")).not.toHaveLength(0); expect(screen.getByText("LITELLM ESCALATE")).toBeInTheDocument(); }); - it("hides the escalation keywords section when no handler is provided", () => { + it("hides the escalation keywords section when no handler is provided", async () => { renderWithProviders(); - expect(screen.queryByText("Advanced: Escalation Keywords")).not.toBeInTheDocument(); + expect(screen.queryByText("Escalation Keywords")).not.toBeInTheDocument(); }); }); @@ -671,21 +683,21 @@ describe("ComplexityRouterConfig classifier fallback", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; - it("defaults the fallback to the heuristic, matching the backend field default", () => { + it("defaults the fallback to the heuristic, matching the backend field default", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeChecked(); }); - it("records a switch to the default model fallback", () => { + it("records a switch to the default model fallback", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("radio", { name: /Route to the default model/ })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classifier_fallback: "default_model" })); }); - it("disables the default model fallback when no tier would produce one", () => { + it("disables the default model fallback when no tier would produce one", async () => { // The deployment's default model is derived from the tiers on submit, so offering the option // with no tiers picked would save a config the backend rejects at startup. const noTiers: ComplexityRouterConfigValue = { @@ -693,17 +705,17 @@ describe("ComplexityRouterConfig classifier fallback", () => { tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model/ })).toHaveAttribute("aria-disabled", "true"); }); - it("hides the fallback choice for the heuristic classifier, which has nothing to fall back from", () => { + it("hides the fallback choice for the heuristic classifier, which has nothing to fall back from", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("If the classifier fails")).not.toBeInTheDocument(); }); - it("stops describing the heuristic as the fallback once a custom prompt routes failures to the default model", () => { + it("stops describing the heuristic as the fallback once a custom prompt routes failures to the default model", async () => { // With both set, the heuristic scorer never runs, so the panel must not keep implying a // score decides anything on this router. renderWithProviders( @@ -717,11 +729,11 @@ describe("ComplexityRouterConfig classifier fallback", () => { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/no longer runs at all/)).toBeInTheDocument(); }); - it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", () => { + it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", async () => { renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText(/only when the classifier call fails/)).toBeInTheDocument(); }); - it("clears a stored fallback when switching back to the heuristic classifier", () => { + it("clears a stored fallback when switching back to the heuristic classifier", async () => { const onChange = vi.fn(); renderWithProviders( { onChange={onChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /rule-based scoring/ })); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^Heuristics$/ })); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classifier_fallback: undefined })); }); }); @@ -758,19 +770,21 @@ describe("ComplexityRouterConfig classification frequency", () => { classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, }; - it("defaults to every request, matching both backend field defaults", () => { + it("defaults to every request, matching both backend field defaults", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every request/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); - expect(screen.getByRole("radio", { name: /Once per session/ })).not.toBeChecked(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( + "Every new user message", + ); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent("Once per session"); }); - it("writes both wire fields when the frequency moves to every new user message", () => { + it("writes both wire fields when the frequency moves to every new user message", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /Every new user message/ })); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("How often to classify", "Every new user message"); expect(onChange).toHaveBeenCalledWith({ ...llmValue, classification_mode: "user_turn", @@ -778,11 +792,11 @@ describe("ComplexityRouterConfig classification frequency", () => { }); }); - it("writes session affinity, not a classification mode, when the frequency moves to once per session", () => { + it("writes session affinity, not a classification mode, when the frequency moves to once per session", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByRole("radio", { name: /Once per session/ })); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("How often to classify", "Once per session"); expect(onChange).toHaveBeenCalledWith({ ...llmValue, classification_mode: "every_request", @@ -790,7 +804,7 @@ describe("ComplexityRouterConfig classification frequency", () => { }); }); - it("shows a hand-authored config that sets both fields as once per session, matching the backend", () => { + it("shows a hand-authored config that sets both fields as once per session, matching the backend", async () => { renderWithProviders( { onChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Once per session/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: /Every new user message/ })).not.toBeChecked(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Once per session"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( + "Every new user message", + ); }); - it("records a switch back to every request", () => { + it("records a switch back to every request", async () => { const onChange = vi.fn(); renderWithProviders( { onChange={onChange} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeChecked(); - fireEvent.click(screen.getByRole("radio", { name: /Every request/ })); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every new user message"); + await selectAutoRouterOption("How often to classify", "Every request"); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ classification_mode: "every_request" })); }); - it("offers the frequency on a heuristic router, where holding the tier still pins the model", () => { + it("offers the frequency on a heuristic router, where holding the tier still pins the model", async () => { // The backend pin is gated on the two fields alone, so a heuristic router that switches models // mid tool loop is fixed by this control too. renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - expect(screen.getByRole("radio", { name: /Every new user message/ })).toBeInTheDocument(); + openAutoRouterAdvanced("Classification Method"); + expect(screen.getByRole("combobox", { name: "How often to classify" })).toBeVisible(); }); }); @@ -836,11 +852,11 @@ describe("ComplexityRouterConfig classifier rubric", () => { const openClassificationPanel = (value: ComplexityRouterConfigValue, onChange = vi.fn()) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); return onChange; }; - it("shows an existing router with no stored preset as legacy in the prompt control", () => { + it("shows an existing router with no stored preset as legacy in the prompt control", async () => { // This router predates the setting. Displaying a calibrated preset it does not have would tell the // operator their traffic is graded by examples the classifier never receives, and saving the form // unchanged would then move its tier decisions. @@ -849,19 +865,19 @@ describe("ComplexityRouterConfig classifier rubric", () => { expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); }); - it("stamps the calibrated preset on a classifier being switched on for the first time", () => { + it("stamps the calibrated preset on a classifier being switched on for the first time", async () => { // A heuristic router turning on the LLM classifier has no prior tier behaviour to preserve, so a // newly configured classifier starts on the calibrated rubric rather than the legacy one. const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - fireEvent.click(screen.getByText("LLM Classifier")); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("radio", { name: /^LLM$/ })); expect(onChange).toHaveBeenCalledWith( expect.objectContaining({ classifier_llm_config: expect.objectContaining({ classification_rubric: "agentic" }) }), ); }); - it("shows the calibrated preset when a router stores one", () => { + it("shows the calibrated preset when a router stores one", async () => { openClassificationPanel({ ...llmValue, classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000, classification_rubric: "agentic" }, @@ -906,7 +922,7 @@ describe("ComplexityRouterConfig classifier rubric", () => { ); }); - it("keeps the rubric out of the legacy whole-prompt editor, which replaces it entirely", () => { + it("keeps the rubric out of the legacy whole-prompt editor, which replaces it entirely", async () => { // The backend rejects both together, so the legacy editor must not offer a rubric to pick. openClassificationPanel({ ...llmValue, @@ -916,7 +932,7 @@ describe("ComplexityRouterConfig classifier rubric", () => { expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); - it("hides the prompt control for the heuristic classifier, which sends no prompt at all", () => { + it("hides the prompt control for the heuristic classifier, which sends no prompt at all", async () => { openClassificationPanel(defaultValue); expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); @@ -928,7 +944,7 @@ describe("ComplexityRouterConfig tier labels", () => { tier_labels: { SIMPLE: "Cheap", MEDIUM: "Standard", COMPLEX: "Premium", REASONING: "Deep" }, }; - it("shows the operator's names in the tier headers instead of the defaults", () => { + it("shows the operator's names in the tier headers instead of the defaults", async () => { renderWithProviders(); expect(screen.getByText("Cheap Tier")).toBeInTheDocument(); expect(screen.getByText("Deep Tier")).toBeInTheDocument(); @@ -936,13 +952,13 @@ describe("ComplexityRouterConfig tier labels", () => { expect(screen.queryByText("Reasoning Tier")).not.toBeInTheDocument(); }); - it("keeps the rung ordinal and canonical name visible under a rename", () => { + it("keeps the rung ordinal and canonical name visible under a rename", async () => { renderWithProviders(); expect(screen.getByText(/Tier 1 of 4/)).toHaveTextContent("Tier 1 of 4 · SIMPLE"); expect(screen.getByText(/Tier 4 of 4/)).toHaveTextContent("Tier 4 of 4 · REASONING"); }); - it("names the renamed tier in the required-field error", () => { + it("names the renamed tier in the required-field error", async () => { renderWithProviders( { expect(screen.getByText("The Deep tier is required")).toBeInTheDocument(); }); - it("reports a typed label back to the caller under its canonical tier key", () => { + it("reports a typed label back to the caller under its canonical tier key", async () => { const onChange = vi.fn(); renderWithProviders(); fireEvent.change(screen.getByLabelText("Display name for the Simple tier"), { target: { value: "Cheap" } }); expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ tier_labels: { SIMPLE: "Cheap" } })); }); - it("shows a stored label in its input so an edit round-trips", () => { + it("shows a stored label in its input so an edit round-trips", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Reasoning tier")).toHaveValue("Deep"); }); - it("leaves the label inputs empty when nothing was renamed", () => { + it("leaves the label inputs empty when nothing was renamed", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toHaveValue(""); }); - it("uses the operator's names in the classification score table", () => { + it("uses the operator's names in the classification score table", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("Cheap")).toBeInTheDocument(); expect(screen.getByText("Deep")).toBeInTheDocument(); }); - it("uses the operator's names in the keyword rule tier picker", () => { + it("uses the operator's names in the keyword rule tier picker", async () => { renderWithProviders( { keywordTierRules={[{ id: "r1", keywords: ["invoice"], tier: "REASONING" }]} />, ); - fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByRole("combobox", { name: "Route keyword rule 1 to tier" })).toHaveTextContent("Deep"); }); }); describe("ComplexityRouterConfig modality panel", () => { - it("defaults the image-routing switch off and writes modality_routing through onChange", () => { + it("defaults the image-routing switch off and writes modality_routing through onChange", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const toggle = screen.getByRole("switch", { name: "Route image requests to vision-capable models" }); expect(toggle).not.toBeChecked(); @@ -1003,19 +1019,19 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).toHaveBeenCalledWith({ ...defaultValue, modality_routing: true }); }); - it("renders a stored modality_routing=true as on", () => { + it("renders a stored modality_routing=true as on", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); expect(screen.getByRole("switch", { name: "Route image requests to vision-capable models" })).toBeChecked(); }); // The backend ignores modality_pin_override unless modality_routing is on, so offering it while // image routing is off would let an operator save a flag that does nothing. - it("disables the pin-override switch while image routing is off", () => { + it("disables the pin-override switch while image routing is off", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); expect(override).toHaveAttribute("aria-disabled", "true"); @@ -1023,11 +1039,11 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).not.toHaveBeenCalled(); }); - it("writes modality_pin_override through onChange once image routing is on", () => { + it("writes modality_pin_override through onChange once image routing is on", async () => { const onChange = vi.fn(); const value = { ...defaultValue, modality_routing: true }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); const override = screen.getByRole("switch", { name: "Override session pin for image requests" }); expect(override).not.toBeChecked(); @@ -1036,51 +1052,51 @@ describe("ComplexityRouterConfig modality panel", () => { expect(onChange).toHaveBeenCalledWith({ ...value, modality_pin_override: true }); }); - it("renders a stored modality_pin_override=true as on", () => { + it("renders a stored modality_pin_override=true as on", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Modality Routing")); + openAutoRouterAdvanced("Modality Routing"); expect(screen.getByRole("switch", { name: "Override session pin for image requests" })).toBeChecked(); }); }); describe("ComplexityRouterConfig affinity panel", () => { - it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", () => { + it("holds the deployment switch at its backend default, session pinning having moved to the frequency choice", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); expect(screen.getByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked(); expect(screen.queryByRole("switch", { name: "Pin a session to its first model" })).not.toBeInTheDocument(); }); - it("writes deployment_affinity through onChange without touching other keys", () => { + it("writes deployment_affinity through onChange without touching other keys", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); fireEvent.click(screen.getByRole("switch", { name: "Pin one model deployment per tier" })); expect(onChange).toHaveBeenCalledWith({ ...defaultValue, deployment_affinity: false }); }); - it("renders a stored deployment_affinity=false as off", () => { + it("renders a stored deployment_affinity=false as off", async () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); expect(screen.getByRole("switch", { name: "Pin one model deployment per tier" })).not.toBeChecked(); }); - it("writes an idle TTL on blur and keeps the partial input as a draft while typing", () => { + it("writes an idle TTL on blur and keeps the partial input as a draft while typing", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); expect(ttl).toHaveAttribute("placeholder", "3600"); @@ -1091,11 +1107,11 @@ describe("ComplexityRouterConfig affinity panel", () => { expect(onChange).toHaveBeenCalledWith({ ...defaultValue, session_affinity_ttl_seconds: 300 }); }); - it("clearing the idle TTL returns the router to its backend default", () => { + it("clearing the idle TTL returns the router to its backend default", async () => { const onChange = vi.fn(); const value = { ...defaultValue, session_affinity_ttl_seconds: 300 }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); expect(ttl).toHaveValue("300"); @@ -1105,10 +1121,10 @@ describe("ComplexityRouterConfig affinity panel", () => { expect(onChange).toHaveBeenCalledWith({ ...value, session_affinity_ttl_seconds: undefined }); }); - it("clamps a non-positive idle TTL to the backend's minimum", () => { + it("clamps a non-positive idle TTL to the backend's minimum", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Affinity")); + openAutoRouterAdvanced("Affinity"); const ttl = screen.getByLabelText("How long a pin survives idle (seconds)"); fireEvent.change(ttl, { target: { value: "0" } }); @@ -1121,12 +1137,12 @@ describe("ComplexityRouterConfig affinity panel", () => { describe("ComplexityRouterConfig default model", () => { const getDefaultModelSelect = () => screen.getByRole("combobox", { name: "Default model" }); - it("shows what the tiers currently imply, so an untouched router still names its default", () => { + it("shows what the tiers currently imply, so an untouched router still names its default", async () => { renderWithProviders(); expect(getDefaultModelSelect()).toHaveAttribute("placeholder", "Derived from tiers: gpt-3.5-turbo"); }); - it("asks for a model rather than naming a derived one when no tier holds one", () => { + it("asks for a model rather than naming a derived one when no tier holds one", async () => { const noTiers: ComplexityRouterConfigValue = { ...defaultValue, tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -1157,13 +1173,13 @@ describe("ComplexityRouterConfig default model", () => { expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ default_model: undefined })); }); - it("shows a pinned model as the selection instead of the tier-derived one", () => { + it("shows a pinned model as the selection instead of the tier-derived one", async () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, default_model: "claude-3-opus" }; renderWithProviders(); expect(getDefaultModelSelect()).toHaveValue("claude-3-opus"); }); - it("unlocks the default model fallback on a pin alone, with no tier to derive from", () => { + it("unlocks the default model fallback on a pin alone, with no tier to derive from", async () => { const pinnedNoTiers: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -1172,11 +1188,11 @@ describe("ComplexityRouterConfig default model", () => { default_model: "claude-3-opus", }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model/ })).not.toHaveAttribute("aria-disabled"); }); - it("names the resolved default on the fallback option, so the destination is not a guess", () => { + it("names the resolved default on the fallback option, so the destination is not a guess", async () => { const pinned: ComplexityRouterConfigValue = { ...defaultValue, classifier_type: "llm", @@ -1184,13 +1200,13 @@ describe("ComplexityRouterConfig default model", () => { default_model: "claude-3-opus", }; renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("radio", { name: /Route to the default model \(claude-3-opus\)/ })).toBeInTheDocument(); }); }); describe("plan-mode override", () => { - const openPanel = () => fireEvent.click(screen.getByText("Advanced: Plan-Mode Override")); + const openPanel = () => openAutoRouterAdvanced("Plan-Mode Override"); const switchName = "Route plan-mode requests to a minimum tier"; it("toggling on floors at the highest tier that has models", async () => { @@ -1248,13 +1264,13 @@ describe("plan-mode override", () => { }); describe("ComplexityRouterConfig per-model reasoning effort", () => { - it("renders one effort select per selected model, defaulting to Default", () => { + it("renders one effort select per selected model, defaulting to Default", async () => { renderWithProviders(); const select = screen.getByRole("combobox", { name: "Reasoning effort for gpt-4 in the Complex tier" }); expect(select).toHaveTextContent("Default"); }); - it("shows the hydrated effort for a model that has one stored", () => { + it("shows the hydrated effort for a model that has one stored", async () => { renderWithProviders( { const renderClassifier = (value: ComplexityRouterConfigValue = llmValue, onChange = vi.fn()) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); return onChange; }; @@ -1348,7 +1364,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" }, }); const user = userEvent.setup(); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); await user.click(await screen.findByRole("option", { name: "gpt-3.5-turbo" })); expect(onChange).toHaveBeenCalledWith({ ...llmValue, @@ -1364,7 +1380,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" }, }); const user = userEvent.setup(); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); if (action === "click") await user.click(await screen.findByRole("option", { name: "gpt-4" })); else await user.keyboard("{Enter}"); expect(onChange).not.toHaveBeenCalled(); @@ -1397,7 +1413,7 @@ describe("ComplexityRouterConfig classifier reasoning effort", () => { }); describe("ComplexityRouterConfig reasoning effort gating", () => { - it("offers no effort select for a model group without reasoning support", () => { + it("offers no effort select for a model group without reasoning support", async () => { renderWithProviders(); expect( screen.queryByRole("combobox", { name: "Reasoning effort for gpt-3.5-turbo in the Simple tier" }), @@ -1406,7 +1422,7 @@ describe("ComplexityRouterConfig reasoning effort gating", () => { // A stored effort on a model the group info calls non-reasoning must stay visible, or the // operator has no way to clear it. - it("keeps the select for a non-reasoning model that already has a stored effort", () => { + it("keeps the select for a non-reasoning model that already has a stored effort", async () => { renderWithProviders( { // An empty list is the group's own answer that its deployments share no level, which is different // from the field being absent, so the control is dropped rather than falling back to every level. - it("offers no effort at all when the group intersects to nothing", () => { + it("offers no effort at all when the group intersects to nothing", async () => { renderWithProviders( { // Hand-authored configs can carry a level outside the supported set (e.g. max); it must render // and stay clearable rather than being masked as Default. - it("keeps showing a stored effort outside the supported set", () => { + it("keeps showing a stored effort outside the supported set", async () => { renderWithProviders( { describe("ComplexityRouterConfig custom technical keywords", () => { const openClassificationPanel = (value: ComplexityRouterConfigValue) => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); }; const llmConfig = { model: "gpt-3.5-turbo", timeout_ms: 3000 }; @@ -1503,7 +1519,7 @@ describe("ComplexityRouterConfig custom technical keywords", () => { expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); }); - it("hides the keywords when the scorer never runs, so they cannot imply an effect they have none", () => { + it("hides the keywords when the scorer never runs, so they cannot imply an effect they have none", async () => { const llmWithDefaultFallback = { ...defaultValue, classifier_type: "llm" as const, @@ -1547,19 +1563,19 @@ describe("ComplexityRouterConfig tier editing", () => { }, }; - it("offers Edit tiers only when the parent owns the editor flag", () => { + it("offers Edit tiers only when the parent owns the editor flag", async () => { renderWithProviders(); expect(screen.queryByRole("button", { name: "Edit tiers" })).not.toBeInTheDocument(); }); - it("surfaces the caller's orphaned-rule verdict while editing, so Done is not a silent exit", () => { + it("surfaces the caller's orphaned-rule verdict while editing, so Done is not a silent exit", async () => { renderEditor(customValue, { keywordRulesError: "Keyword rule(s) 1 route to a tier this router no longer has" }); expect( screen.getByText("Keyword rule(s) 1 route to a tier this router no longer has", { exact: false }), ).toBeInTheDocument(); }); - it("keeps the orphaned-rule verdict out of the collapsed view, where the submit tooltip owns it", () => { + it("keeps the orphaned-rule verdict out of the collapsed view, where the submit tooltip owns it", async () => { renderWithProviders( { expect(screen.queryByText("route to a tier this router no longer has", { exact: false })).not.toBeInTheDocument(); }); - it("renders the four built-in tiers before any edit, unchanged", () => { + it("renders the four built-in tiers before any edit, unchanged", async () => { renderWithProviders(); expect(screen.getByRole("button", { name: "Edit tiers" })).toBeInTheDocument(); expect(screen.getByText("Tier 1 of 4", { exact: false })).toHaveTextContent("SIMPLE"); }); - it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", () => { + it("adds a row and moves the form into an edited tier set, which the built-in record never leaves", async () => { const { committed } = renderEditor(); fireEvent.click(screen.getByRole("button", { name: "Add tier" })); const next = committed(); @@ -1585,7 +1601,7 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.tiers).toEqual(defaultValue.tiers); }); - it("renames a built-in tier straight from the editor, which is what makes the set custom", () => { + it("renames a built-in tier straight from the editor, which is what makes the set custom", async () => { const { committed } = renderEditor(); fireEvent.change(screen.getByLabelText("Name for tier 3"), { target: { value: "SECURITY_REVIEW" } }); const next = committed(); @@ -1598,13 +1614,13 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.tiers).toEqual(defaultValue.tiers); }); - it("opening the editor and changing nothing leaves the router on the built-in tiers", () => { + it("opening the editor and changing nothing leaves the router on the built-in tiers", async () => { const { onChange } = renderEditor(); expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); expect(onChange).not.toHaveBeenCalled(); }); - it("swaps the display-name field for the tier-name field while the editor is open", () => { + it("swaps the display-name field for the tier-name field while the editor is open", async () => { const { rerender } = renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); rerender(); @@ -1612,23 +1628,23 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByLabelText("Name for tier 1")).toBeInTheDocument(); }); - it("drops the scorer card entirely once an edited tier set replaces the heuristic", () => { + it("drops the scorer card entirely once an edited tier set replaces the heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); expect( screen.queryByText("scores each request across 7 built-in dimensions", { exact: false }), ).not.toBeInTheDocument(); }); - it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => { + it("keeps the scorer card on a built-in router, whose tiers the score still decides", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("How Classification Works")).toBeInTheDocument(); expect(screen.getByText("scores each request across 7 built-in dimensions", { exact: false })).toBeInTheDocument(); }); - it("says why a custom row is blocked instead of only reddening its border", () => { + it("says why a custom row is blocked instead of only reddening its border", async () => { const missingDefinition: ComplexityRouterConfigValue = { ...customValue, custom_tier_set: { @@ -1652,24 +1668,24 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.getByRole("button", { name: "Done" })).toBeDisabled(); }); - it("enables Done once every row carries a name, a definition and a model", () => { + it("enables Done once every row carries a name, a definition and a model", async () => { renderEditor(customValue); expect(screen.getByRole("button", { name: "Done" })).toBeEnabled(); }); - it("refuses to remove a row that would take the set below the backend's minimum", () => { + it("refuses to remove a row that would take the set below the backend's minimum", async () => { renderEditor(customValue); expect(screen.getByRole("button", { name: "Remove the CASUAL tier" })).toBeDisabled(); }); - it("keeps a definition on one line, because the backend rejects a newline in it", () => { + it("keeps a definition on one line, because the backend rejects a newline in it", async () => { const { committed } = renderEditor(customValue); fireEvent.change(screen.getByLabelText("Definition for tier 2"), { target: { value: "audits\nand reviews" } }); const next = committed(); expect(next.custom_tier_set?.tiers[1].definition).toBe("audits and reviews"); }); - it("moves a keyword rule with the tier it points at when that tier is renamed", () => { + it("moves a keyword rule with the tier it points at when that tier is renamed", async () => { const onKeywordTierRulesChange = vi.fn(); renderWithProviders( { expect(onKeywordTierRulesChange).toHaveBeenCalledWith([{ id: "r1", keywords: ["audit"], tier: "AUDIT" }]); }); - it("re-points the fallback tier when the row it named is removed, never leaving it dangling", () => { + it("re-points the fallback tier when the row it named is removed, never leaving it dangling", async () => { const threeRows: ComplexityRouterConfigValue = { ...customValue, custom_tier_set: { @@ -1702,7 +1718,7 @@ describe("ComplexityRouterConfig tier editing", () => { expect(next.custom_tier_set?.tiers.some((row) => row.id === next.custom_tier_set?.fallback_tier_id)).toBe(true); }); - it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", () => { + it("turns off a plan-mode floor whose row was removed, rather than leaving it pointing at nothing", async () => { const withFloor: ComplexityRouterConfigValue = { ...customValue, plan_mode_min_tier: "sec", @@ -1719,14 +1735,14 @@ describe("ComplexityRouterConfig tier editing", () => { expect(committed().plan_mode_min_tier).toBeUndefined(); }); - it("replaces the display-name inputs with the reason an edited tier set forbids them", () => { + it("replaces the display-name inputs with the reason an edited tier set forbids them", async () => { renderWithProviders(); expect(screen.queryByLabelText("Display name for the Simple tier")).not.toBeInTheDocument(); expect(screen.getByText("Display names rename the built-in tiers", { exact: false })).toBeInTheDocument(); expect(screen.getByLabelText("Fallback tier")).toBeInTheDocument(); }); - it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", () => { + it("disables the once-per-session frequency and says why, rather than letting a stripped value look saved", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); - const sessionOption = screen.getByRole("radio", { name: /Once per session/ }); + openAutoRouterAdvanced("Classification Method"); + fireEvent.click(screen.getByRole("combobox", { name: "How often to classify" })); + const sessionOption = screen.getByRole("option", { name: "Once per session" }); expect(sessionOption).toHaveAttribute("aria-disabled", "true"); expect(sessionOption).not.toBeChecked(); expect( @@ -1743,15 +1760,15 @@ describe("ComplexityRouterConfig tier editing", () => { ).toBeInTheDocument(); }); - it("lets an edited tier set write its own opening instructions instead of refusing a prompt outright", () => { + it("lets an edited tier set write its own opening instructions instead of refusing a prompt outright", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("your own calibration examples", { exact: false })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); expect(screen.queryByText("A replacement prompt drops the tier bullets", { exact: false })).not.toBeInTheDocument(); }); - it("gives built-in routers the opening-only editor, keeping the tier definitions derived", () => { + it("gives built-in routers the opening-only editor, keeping the tier definitions derived", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByText("The base rubric supplies", { exact: false })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Customize prompt" })).toBeInTheDocument(); expect(screen.queryByText("Replace the built-in complexity rubric", { exact: false })).not.toBeInTheDocument(); }); - it("keeps the legacy whole-prompt editor only on a router that already stored a replacement prompt", () => { + it("keeps the legacy whole-prompt editor only on a router that already stored a replacement prompt", async () => { renderWithProviders( { onEditingTiersChange={vi.fn()} />, ); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("button", { name: "Edit custom prompt" })).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Customize prompt" })).not.toBeInTheDocument(); }); - it("leaves built-in routers with their display-name inputs and no restriction copy", () => { + it("leaves built-in routers with their display-name inputs and no restriction copy", async () => { renderWithProviders(); expect(screen.getByLabelText("Display name for the Simple tier")).toBeInTheDocument(); expect(screen.queryByText("Display names rename the built-in tiers", { exact: false })).not.toBeInTheDocument(); @@ -1810,9 +1827,9 @@ describe("classifier vision settings", () => { ); }; - it("starts off and reveals the default cap when enabled", () => { + it("starts off and reveals the default cap when enabled", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); const vision = screen.getByRole("switch", { name: "Use images for classification" }); expect(vision).not.toBeChecked(); @@ -1823,10 +1840,10 @@ describe("classifier vision settings", () => { expect(screen.getByLabelText("Maximum images per request")).toHaveValue("1"); }); - it("writes the switch and a clamped image cap into the classifier config", () => { + it("writes the switch and a clamped image cap into the classifier config", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); expect(onChange).toHaveBeenLastCalledWith({ @@ -1841,10 +1858,10 @@ describe("classifier vision settings", () => { }); }); - it("keeps the image cap draft empty until a valid value is entered", () => { + it("keeps the image cap draft empty until a valid value is entered", async () => { const onChange = vi.fn(); renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); onChange.mockClear(); @@ -1861,9 +1878,9 @@ describe("classifier vision settings", () => { }); }); - it("is absent when the classifier is heuristic", () => { + it("is absent when the classifier is heuristic", async () => { renderWithProviders(); - fireEvent.click(screen.getByText("Advanced: Classification Method")); + openAutoRouterAdvanced("Classification Method"); expect(screen.queryByText("Use images for classification")).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index acf6b62a95a..32f9ebf97ad 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -1,4 +1,6 @@ import RoutingOptions from "./RoutingOptions"; +import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import type { JevClassifierConfig } from "./jev_classifier_config"; import { type ClassifierType } from "./classifier_types"; export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types"; @@ -6,11 +8,11 @@ import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassi import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config"; import { SimpleTooltip } from "@/components/ui/tooltip"; import { MultiSelect } from "@/components/shared/MultiSelect"; +import TierConfigIntro from "./TierConfigIntro"; import DefaultModelField from "./DefaultModelField"; import { Info, Plus, Trash2, X } from "lucide-react"; import NonReasoningTierToggle from "./NonReasoningTierToggle"; -import TierConfigIntro from "./TierConfigIntro"; import TierRowSelect from "./TierRowSelect"; import { Card, CardContent } from "@/components/ui/card"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; @@ -226,10 +228,16 @@ const TierSetToolbar: React.FC<{ ) )} + {editing && ( + + )} {editing && ( Add or remove tiers to define your own set. Every custom tier needs a definition the classifier routes on, and - an edited set requires the LLM or JEV classification method + an edited set requires the LLM or Jev classification method )} {editing && keywordRulesError && ( @@ -596,10 +604,14 @@ const ComplexityRouterConfig: React.FC = ({ return (
+
-

- {forecast ? "Solver models" : "Complexity Tier Configuration"} -

+

{forecast ? "Solver models" : "Models by tier"}

{!forecast && ( @@ -619,6 +631,7 @@ const ComplexityRouterConfig: React.FC = ({ fastModeByModel={fastModeByModel} /> = ({ ) : ( <> - {!customTierSet && ( @@ -750,13 +762,19 @@ const ComplexityRouterConfig: React.FC = ({ )} - {!forecast && } + - + {forecast && ( <> - void>(); renderWithProviders(); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + openAutoRouterAdvanced("Keyword/Semantic Matching"); expect(screen.getByRole("switch", { name: "Fast mode for secondary in the Medium routing pool tier" })).toBeChecked(); expect(onChange).not.toHaveBeenCalled(); await user.click(screen.getByRole("combobox", { name: "Select medium routing pool models" })); @@ -245,7 +246,7 @@ it.each(["capability", "llm_v2"] as const)( ); const view = renderWithProviders(editor(hydrateComplexityRouterConfig(stored, undefined))); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); + openAutoRouterAdvanced("Keyword/Semantic Matching"); const select = () => screen.getByRole("combobox", { name: "Default model" }); expect(select()).toHaveValue("legacy-default"); expect(onChange).not.toHaveBeenCalled(); @@ -284,8 +285,8 @@ it.each(["capability", "llm_v2"] as const)("offers only populated keyword target /> ); const view = renderWithProviders(editor([])); - await user.click(screen.getByRole("button", { name: "Advanced routing options" })); - await user.click(screen.getByText("Advanced: Keyword/Semantic Matching")); + openAutoRouterAdvanced("Keyword/Semantic Matching"); + openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: "Add keyword rule" })); const rules = onRulesChange.mock.lastCall![0]; expect(rules[0].tier).toBe("SIMPLE"); diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx index a03ccb11456..f243e63d387 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx @@ -1,3 +1,4 @@ +import { selectAutoRouterApproach } from "../../../tests/autoRouterSetup"; import React, { useState } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; @@ -306,7 +307,7 @@ describe("forecast classifier form", () => { ); }); - it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => { + it("switches a populated standard router to Capability without saving hidden pools or their overrides", async () => { renderWithProviders( { }} />, ); - fireEvent.click(screen.getByRole("tab", { name: "Capability" })); + await selectAutoRouterApproach("Capability"); fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.7" } }); expect(screen.getByRole("button", { name: "Save configuration" })).toBeEnabled(); fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); @@ -347,7 +348,7 @@ describe("forecast classifier form", () => { it.each(["capability", "llm_v2"] as const)( "carries non-default solver assignments when switching away from %s", - (source) => { + async (source) => { const pair = { efficient_tier: "MEDIUM", capable_tier: "COMPLEX" }; const previous: ComplexityRouterConfigValue = { ...(source === "capability" ? initial : fuseInitial), @@ -359,7 +360,7 @@ describe("forecast classifier form", () => { tier_model_params: { MEDIUM: { efficient: { max_tokens: 128 } }, COMPLEX: { capable: { speed: "fast" } } }, }; renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: source === "capability" ? "Fuse v2" : "Capability" })); + await selectAutoRouterApproach(source === "capability" ? "Fuse v2" : "Capability"); if (source === "capability") { fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Small solver" } }); fireEvent.change(screen.getByLabelText("Capable solver profile"), { target: { value: "Large solver" } }); @@ -402,20 +403,22 @@ describe("forecast classifier form", () => { ] as const)("restores the current rubric when switching %s through Complexity to %s", async (source, target) => { const user = userEvent.setup(); renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: "Complexity" })); + await selectAutoRouterApproach("Complexity"); fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${target}`) })); - await user.click(screen.getByRole("combobox", { name: "Classifier Model" })); + await user.click(screen.getByRole("combobox", { name: "Judge model" })); await user.click(screen.getByRole("option", { name: "judge" })); fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); const output = screen.getByRole("status", { name: "Saved configuration" }); expect(output).toHaveTextContent('"classification_rubric":"agentic"'); expect(output).toHaveTextContent('"model":"judge"'); - expect(output).toHaveTextContent('"timeout_ms":3000'); + expect(output).toHaveTextContent( + `"timeout_ms":${(source === "capability" ? initial : fuseInitial).classifier_llm_config?.timeout_ms}`, + ); expect(output).not.toHaveTextContent('"capability_classifier_config"'); expect(output).not.toHaveTextContent('"llm_v2_config"'); }); - it("saves capability threshold edits together with fitted calibration", () => { + it("saves capability threshold edits together with fitted calibration", async () => { renderWithProviders(); fireEvent.change(screen.getByLabelText("Solve probability threshold"), { target: { value: "0.6" } }); fireEvent.click(screen.getByRole("button", { name: "Classifier options" })); @@ -432,9 +435,9 @@ describe("forecast classifier form", () => { expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); }); - it("switches to Fuse, requires solver context, and saves the filled fields", () => { + it("switches to Fuse, requires solver context, and saves the filled fields", async () => { renderWithProviders(); - fireEvent.click(screen.getByRole("tab", { name: "Fuse v2" })); + await selectAutoRouterApproach("Fuse v2"); expect(screen.queryByLabelText("Solve probability threshold")).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Save configuration" })).toBeDisabled(); fireEvent.change(screen.getByLabelText("Efficient solver profile"), { diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx index c336423fe39..d759b870209 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.tsx @@ -36,6 +36,7 @@ interface Props { onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; effortOptionsByModel: Record; + section?: "all" | "required" | "advanced"; } const NumberField = ({ @@ -188,7 +189,7 @@ const CalibrationFields = ({ const emptyCoefficients = () => ({ slope: Number.NaN, intercept: Number.NaN }); -const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel }: Props) => { +const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptionsByModel, section = "all" }: Props) => { const id = React.useId(); const isCapability = value.classifier_type === "capability"; const capability = value.capability_classifier_config ?? newCapabilitySettings(); @@ -212,72 +213,195 @@ const ForecastClassifierConfig = ({ value, onChange, modelOptions, effortOptions ? "Forecasts whether the efficient solver can complete the task using the bundled capability card" : "Forecasts success for both solvers and selects efficient when the estimated quality gap is within your allowance"}

-
- - { - if (model === llm.model) return; - onChange({ ...value, classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined } }); - }} - /> -
- {isCapability ? ( + {section === "all" && ( <> - updateCapability({ ...capability, base_threshold })} - /> - - ) : ( - <> - - updateFuse({ ...fuse, max_quality_gap })} - /> +
+ + { + if (model === llm.model) return; + onChange({ + ...value, + classifier_llm_config: { ...llm, model: model ?? "", reasoning_effort: undefined }, + }); + }} + /> +
)} - - - - Classifier options - - - onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } })} - /> - onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })} - /> - onChange({ ...value, classifier_llm_config })} - /> - onChange({ ...value, classifier_llm_config })} - /> + {section !== "advanced" && ( + <> + {isCapability ? ( + <> + updateCapability({ ...capability, base_threshold })} + /> + + ) : ( + <> + + updateFuse({ ...fuse, max_quality_gap })} + /> + + )} + + )} + {section !== "required" && ( + + {section === "all" && ( + + + Classifier options + + )} + + + onChange({ ...value, classifier_llm_config: { ...llm, reasoning_effort } }) + } + /> + onChange({ ...value, classifier_llm_config: { ...llm, timeout_ms } })} + /> + onChange({ ...value, classifier_llm_config })} + /> + onChange({ ...value, classifier_llm_config })} + /> + {isCapability && ( + updateCapability({ ...capability, threshold_step })} + /> + )} + updateTransport({ max_output_tokens })} + /> +
+ + { + if (response_format === "json_schema" || response_format === "json_object") + updateTransport({ response_format }); + }} + /> +
+
+ +

+ Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts +

+ {config.calibration && ( +
+ + setCalibrationVersion(event.target.value)} + /> +
+ )} + {isCapability && capability.calibration && ( + + updateCapability({ + ...capability, + calibration: { version: capability.calibration?.version ?? "", ...next }, + }) + } + /> + )} + {!isCapability && + fuse.calibration && + (["efficient", "capable"] as const).map((role) => ( + { + if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } }); + }} + /> + ))} +
+ + + )} + {section === "all" && ( + <>
- {isCapability && ( - updateCapability({ ...capability, threshold_step })} - /> - )} - updateTransport({ max_output_tokens })} - /> -
- - { - if (response_format === "json_schema" || response_format === "json_object") - updateTransport({ response_format }); - }} - /> -
-
- -

- Optional coefficients fitted for your judge, solvers, and harness. Leave off to use raw forecasts -

- {config.calibration && ( -
- - setCalibrationVersion(event.target.value)} - /> -
- )} - {isCapability && capability.calibration && ( - - updateCapability({ - ...capability, - calibration: { version: capability.calibration?.version ?? "", ...next }, - }) - } - /> - )} - {!isCapability && - fuse.calibration && - (["efficient", "capable"] as const).map((role) => ( - { - if (fuse.calibration) updateFuse({ ...fuse, calibration: { ...fuse.calibration, [role]: next } }); - }} - /> - ))} -
- - + + )}

The classifier uses its bundled prompt and always falls back to the capable solver

- {error && ( + {section !== "advanced" && error && (

{error}

diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index 896fde3a446..7da8b12c2d7 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -98,28 +98,28 @@ describe("JEV classifier editor", () => { afterEach(() => vi.mocked(useAuthorized).mockReset()); it("uses built-in JEV without a license and preserves custom tiers and context through reload", () => { renderWithProviders(); - expect(screen.getByLabelText("Classifier Model")).toBeInTheDocument(); + expect(screen.getByLabelText("Judge model")).toBeInTheDocument(); expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); expect(screen.getByText("Classifier Prompt")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument(); - fireEvent.click(screen.getByRole("radio", { name: /JEV Classifier/ })); - expect(screen.getByRole("tab", { name: "Complexity" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-latest"); - expect(screen.getByLabelText("JEV Instructions")).toBeDisabled(); - expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ })); + expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked(); + expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest"); + expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); + expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument(); expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument(); expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument(); - fireEvent.change(screen.getByLabelText("JEV Model"), { target: { value: "jev-test" } }); - fireEvent.change(screen.getByLabelText("JEV Timeout (ms)"), { target: { value: "4200" } }); + fireEvent.change(screen.getByLabelText("Jev Model"), { target: { value: "jev-test" } }); + fireEvent.change(screen.getByLabelText("Jev Timeout (ms)"), { target: { value: "4200" } }); fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } }); fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } }); fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); fireEvent.click(screen.getByRole("button", { name: "Customize tiers" })); fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); - expect(screen.getByRole("radio", { name: /JEV Classifier/ })).toBeChecked(); - expect(screen.getByLabelText("JEV Model")).toHaveValue("jev-test"); - expect(screen.getByLabelText("JEV Timeout (ms)")).toHaveValue(4200); + expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked(); + expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test"); + expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200); expect(screen.getByLabelText("Context Window Size")).toHaveValue("6"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked(); fireEvent.click(screen.getByRole("button", { name: "Probe current config" })); @@ -152,10 +152,10 @@ describe("JEV classifier editor", () => { return ; }; renderWithProviders(); - expect(screen.getByLabelText("JEV Instructions")).toBeEnabled(); - fireEvent.change(screen.getByLabelText("JEV Instructions"), { target: { value: "New instructions" } }); - expect(screen.getByLabelText("JEV Instructions")).toHaveValue("New instructions"); - fireEvent.click(screen.getByRole("button", { name: "Restore built-in JEV instructions" })); - expect(screen.getByLabelText("JEV Instructions")).toHaveValue(""); + expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); + fireEvent.change(screen.getByLabelText("Jev Instructions"), { target: { value: "New instructions" } }); + expect(screen.getByLabelText("Jev Instructions")).toHaveValue("New instructions"); + fireEvent.click(screen.getByRole("button", { name: "Restore built-in Jev instructions" })); + expect(screen.getByLabelText("Jev Instructions")).toHaveValue(""); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 25286eaef07..97609bcd8c3 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -1,10 +1,9 @@ import React, { useId } from "react"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; -import { SimpleTooltip } from "@/components/ui/tooltip"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { defaultJevClassifierConfig } from "./jev_classifier_config"; @@ -17,7 +16,6 @@ export default function JevClassifierConfig({ onChange: (value: ComplexityRouterConfigValue) => void; }) { const id = useId(); - const { premiumUser } = useAuthorized(); const config = value.jev_classifier_config ?? defaultJevClassifierConfig(); const update = (patch: Partial) => onChange({ ...value, jev_classifier_config: { ...config, ...patch } }); @@ -28,11 +26,11 @@ export default function JevClassifierConfig({ Uses TypeSafe System One Choice evaluation with your configured tiers

- + update({ model: event.target.value })} />
- +
- - -
-