From 888686d858571b102391bab7ff4dc3ee2819ddf3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:29:46 -0700 Subject: [PATCH 1/7] test(headroom): pin that SDK-echoed null tool_calls never reach /v1/compress (#39667) * fix(headroom): drop null tool_calls before calling /v1/compress headroom-ai 0.27.0 through 0.30.0 iterate msg.get("tool_calls", []) and fail with 'NoneType' object is not iterable when an assistant row carries an explicit tool_calls: null, which the fail-closed guardrail then surfaces as a 502. Strip the null key from rows sent to the compression service; real tool_calls lists are forwarded unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(headroom): pin that SDK-echoed null tool_calls never reach /v1/compress Main now withholds every assistant row from /v1/compress (#39974), so the null tool_calls strip added here never runs on a row headroom-ai 0.27-0.30 fails on. Drop the strip and pin the customer's history shape through the public OpenAI chat translation handler instead. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../guardrail_hooks/test_headroom.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 6f4b1816150..5bdcdddf4b7 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2008,6 +2008,45 @@ async def test_apply_guardrail_sends_textless_parts_rows_unflattened( assert wire_messages[1]["content"] == "D" * 5000 +@pytest.mark.asyncio +async def test_sdk_echoed_null_tool_calls_never_reach_compression_service(guardrail: HeadroomGuardrail): + """LIT-6921: OpenAI SDK assistant messages echoed back as history carry + ``"tool_calls": null``, which headroom-ai 0.27-0.30 answer with a 503. The + chat handler sends no row carrying that null and forwards the echoed rows + unchanged.""" + from openai.types.chat import ChatCompletionMessage + + from litellm.llms.openai.chat.guardrail_translation.handler import OpenAIChatCompletionsHandler + + def _sdk_assistant(**fields: object) -> dict: + return ChatCompletionMessage.model_validate({"role": "assistant", **fields}).model_dump() + + tool_call = {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + history = [ + {"role": "user", "content": "E" * 5000}, + _sdk_assistant(content=None, tool_calls=[tool_call]), + {"role": "tool", "tool_call_id": "call_1", "content": "R" * 5000}, + _sdk_assistant(content="summary"), + {"role": "user", "content": "F" * 5000}, + _sdk_assistant(content="noted"), + {"role": "user", "content": "and now?"}, + ] + assert history[3]["tool_calls"] is None + sent: dict = {} + + def _echo(**kwargs): + sent["messages"] = kwargs["json"]["messages"] + return _make_compress_response(json.loads(json.dumps(sent["messages"]))) + + data = {"model": "claude-sonnet-5", "messages": json.loads(json.dumps(history))} + with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): + result = await OpenAIChatCompletionsHandler().process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert [row["role"] for row in sent["messages"]] == ["user", "tool", "user"] + assert all(row.get("tool_calls", []) is not None for row in sent["messages"]) + assert [result["messages"][index] for index in (1, 3, 5)] == [history[index] for index in (1, 3, 5)] + + @pytest.mark.asyncio async def test_fail_open_returns_original_parts_shapes(): guardrail = _make_guardrail(unreachable_fallback="fail_open") From 541e3ed5305a530ebef74d5684d3aca2632c1009 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sat, 10 Oct 2026 18:00:16 -0700 Subject: [PATCH 2/7] fix(mcp): exclude gateway credentials from guardrail bearers (#45897) Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../mcp_server/auth/user_api_key_auth_mcp.py | 25 +++++++ .../mcp_server/rest_endpoints.py | 6 +- .../proxy/_experimental/mcp_server/server.py | 2 +- .../auth/test_user_api_key_auth_mcp.py | 36 ++++++++++ .../mcp_server/test_operations.py | 40 ++++++++++- .../mcp_server/test_rest_endpoints.py | 67 +++++++++++++++++++ 6 files changed, 172 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2c4c942ebeb..18b7c279606 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -1546,6 +1546,31 @@ class MCPRequestHandler: authorization: Final = next((value for key, value in headers.items() if key.lower() == "authorization"), "") return SecretStr(authorization[len("bearer ") :]) if authorization.lower().startswith("bearer ") else None + @staticmethod + def get_guardrail_bearer_token(headers: Headers, auth: UserAPIKeyAuth) -> SecretStr | None: + from litellm.proxy._experimental.mcp_server.utils import ( + _custom_litellm_key_header_name, # pyright: ignore[reportPrivateUsage] # reuse the shared credential-header setting + ) + + bearer: Final = MCPRequestHandler.get_incoming_bearer_token(headers) + if bearer is None: + return None + value: Final = bearer.get_secret_value() + if MCPRequestHandler._is_gateway_admission_credential(value): + return None + if MCPRequestHandler._is_caller_admission_key( + value, auth.api_key + ) or MCPRequestHandler._is_caller_admission_key(hash_token(value), auth.api_key): + return None + admitted_credential: Final = MCPRequestHandler.caller_admission_credential( + headers, + auth, + custom_key_header_name=_custom_litellm_key_header_name(), + ) + if not auth.jwt_claims and MCPRequestHandler._is_caller_admission_key(value, admitted_credential): + return None + return bearer + @staticmethod def get_oauth2_headers_from_headers(headers: Headers) -> dict[str, str]: """ diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 38f526bcd13..38d9ca52abb 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -412,7 +412,7 @@ if MCP_AVAILABLE: raw_headers=virtual_raw_headers, litellm_logging_obj=virtual_logging_obj, guardrail_context=MCPRequestContext.resolve_guardrail_context(virtual_data), - incoming_bearer_token=MCPRequestHandler.get_incoming_bearer_token(request.headers), + incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token(request.headers, user_api_key_dict), ) except Exception as e: virtual_request_data: Final = virtual_processor.data @@ -1346,7 +1346,9 @@ if MCP_AVAILABLE: litellm_logging_obj=data.get("litellm_logging_obj"), guardrail_context=MCPRequestContext.resolve_guardrail_context(data), requested_server_id=canonical_server_id, - incoming_bearer_token=MCPRequestHandler.get_incoming_bearer_token(request.headers), + incoming_bearer_token=MCPRequestHandler.get_guardrail_bearer_token( + request.headers, user_api_key_dict + ), ) result: Final = complete_call_tool_result(executed, WireCompat.LEGACY) except Exception as e: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index d299f470f12..1916a19a2c4 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -855,7 +855,7 @@ if MCP_AVAILABLE: wire_compat_for(ctx.protocol_version), ctx.protocol_version, incoming_bearer_token=( - MCPRequestHandler.get_incoming_bearer_token(request.headers) + MCPRequestHandler.get_guardrail_bearer_token(request.headers, auth) if isinstance(request, StarletteRequest) else None ), diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 9f717f522c0..6254b1bc44c 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -10814,3 +10814,39 @@ async def test_admission_request_body_serves_stashed_peek_callable(): {"type": "http", "method": "POST", "path": "/mcp", "headers": []} ) assert await without_peek.body() == b"{}" + + +@pytest.mark.parametrize("bearer,api_key,claims,explicit_key,expected", [ + (None, "sk-key", None, None, None), + ("sk-key", "sk-key", None, None, None), + ("sk-key", "sk-key", {"sub": "previous-jwt"}, "sk-key", None), + ("caller.jwt.assertion", "sk-key", {"sub": "caller"}, None, "caller.jwt.assertion"), + ("caller.jwt.assertion", None, {"sub": "caller"}, None, "caller.jwt.assertion"), + ("upstream", "sk-key", None, "sk-key", "upstream"), + ("llm_session_test", None, {"sub": "caller"}, None, None), + ("llm_srefresh_test", None, {"sub": "caller"}, None, None), + ("opaque-master", "litellm_proxy_master_key", None, None, None), +]) +def test_guardrail_bearer_preserves_identity_without_exposing_gateway_credentials( + bearer: str | None, api_key: str | None, claims: dict[str, str] | None, + explicit_key: str | None, expected: str | None, +) -> None: + headers: Final = Headers({ + **({"Authorization": f"Bearer {bearer}"} if bearer is not None else {}), + **({"x-litellm-api-key": explicit_key} if explicit_key else {}), + }) + token: Final = MCPRequestHandler.get_guardrail_bearer_token( + headers, UserAPIKeyAuth(api_key=api_key, jwt_claims=claims), + ) + assert (token.get_secret_value() if token is not None else None) == expected + + +def test_guardrail_bearer_respects_custom_admission_header(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "litellm_key_header_name", "x-gateway-key") + token: Final = MCPRequestHandler.get_guardrail_bearer_token( + Headers({"x-gateway-key": "opaque-master", "authorization": "Bearer opaque-master"}), + UserAPIKeyAuth(api_key="litellm_proxy_master_key"), + ) + assert token is None diff --git a/tests/unit/proxy/_experimental/mcp_server/test_operations.py b/tests/unit/proxy/_experimental/mcp_server/test_operations.py index 32a7c3fbe58..dcc4eb1a570 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_operations.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_operations.py @@ -71,7 +71,7 @@ async def test_mapped_caller_bearer_reaches_agent_evaluation( "headers": [] if verdict == "missing" else [(b"authorization", f"Bearer {bearer}".encode())], }) auth: Final = UserAPIKeyAuth( - api_key="stored-key-hash", user_id="mapped-caller", + api_key="stored-key-hash", user_id="mapped-caller", jwt_claims={"sub": "mapped-caller"}, object_permission={"object_permission_id": "mapped", "mcp_servers": ["guarded"], "mcp_tool_search_enabled": mode == "virtual"}, ) oauth, raw, _, _ = MCPRequestHandler.scrub_gateway_admission_credentials( @@ -1552,3 +1552,41 @@ def test_discovery_extra_headers_exclude_gateway_admission_key(header_name: str) assert auth_header is None assert extra_headers == {"X-Tenant": "tenant-control"} assert raw_headers == {"x-litellm-api-key": caller_key, "x-tenant": "tenant-control"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kind", ["absent", "non_starlette", "without_bearer", "admission_key"]) +async def test_guardrail_context_never_recovers_a_previous_request_bearer( + _mcp_request_ctx: Callable[..., ServerRequestContext], request_kind: str, monkeypatch: pytest.MonkeyPatch, +) -> None: + from mcp.server.auth.middleware.auth_context import auth_context_var + from starlette.requests import Request + from litellm.proxy._experimental.mcp_server import server as gateway + + request: Final = ( + None if request_kind == "absent" else object() if request_kind == "non_starlette" else Request({ + "type": "http", "method": "POST", "path": "/mcp", + "headers": [(b"authorization", b"Bearer sk-current-key")] if request_kind == "admission_key" else [], + }) + ) + previous: Final = auth_context_var.set(None) + try: + gateway.set_auth_context( + UserAPIKeyAuth(api_key="sk-current-key"), + raw_headers={"authorization": "Bearer previous.caller.assertion"}, + ) + capture: Final = _CatalogHookCapture() + monkeypatch.setattr(litellm, "callbacks", [capture]) + async with gateway._legacy_operation_context(_mcp_request_ctx(request=request), trace=False) as context: + await operations.global_mcp_server_manager.pre_call_tool_check( + name="echo", arguments={}, server_name="guarded", + user_api_key_auth=context.user_api_key_auth, + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + server=MCPServer(server_id="guarded", name="guarded", transport=MCPTransport.http, allow_all_keys=True), + raw_headers=dict(context.raw_headers or {}), + incoming_bearer_token=context.incoming_bearer_token, + ) + assert capture.data is not None + assert capture.data["incoming_bearer_token"] is None + finally: + auth_context_var.reset(previous) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py index fc61ec66b30..2a04c8cda2a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -4862,3 +4862,70 @@ async def test_saved_preview_protocol_omission_and_explicit_edits( assert result == {"protocol_version": expected} assert saved.protocol_version == "2025-11-25" assert payload.mcp_info == metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("virtual", [False, True]) +@pytest.mark.parametrize("bearer,claims,expected", [ + ("sk-guardrail-admission", None, None), + ("caller.jwt.assertion", {"sub": "caller"}, "caller.jwt.assertion"), + ("upstream-token", None, "upstream-token"), +]) +async def test_rest_guardrail_bearer_excludes_admission_key( + monkeypatch: pytest.MonkeyPatch, virtual: bool, + bearer: str, claims: dict[str, str] | None, expected: str | None, +) -> None: + import litellm + from litellm.caching.dual_cache import DualCache + from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + observed: Final[asyncio.Queue[object]] = asyncio.Queue() + + class CaptureBearer(CustomLogger): + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, + data: dict[str, object], call_type: str, + ) -> None: + if call_type == "call_mcp_tool" and "incoming_bearer_token" in data: + observed.put_nowait(data["incoming_bearer_token"]) + + async def echo() -> str: + return "guarded-result" + + auth: Final = UserAPIKeyAuth( + api_key="sk-guardrail-admission", jwt_claims=claims, + object_permission={"object_permission_id": "guarded", "mcp_servers": ["guarded"], "mcp_tool_search_enabled": virtual}, + ) + managed: Final = MCPServer( + server_id="guarded", name="guarded", transport=MCPTransport.http, + spec_path="/catalog.yaml", allow_all_keys=True, + ) + monkeypatch.setitem(rest_endpoints.global_mcp_server_manager.registry, managed.server_id, managed) + monkeypatch.setattr(litellm, "callbacks", [CaptureBearer()]) + global_mcp_tool_registry.register_tool( + name="guarded-echo", description="Echo", input_schema={"type": "object"}, handler=echo, + ) + headers: Final = {"authorization": f"Bearer {bearer}", **( + {} if claims else {"x-litellm-api-key": "sk-guardrail-admission"} + )} + payload: Final = { + "server_id": "guarded", "name": "mcp_tool_call" if virtual else "guarded-echo", + "arguments": {"tool_name": "guarded-echo", "arguments": {}} if virtual else {}, + } + try: + result: Final = await rest_endpoints.call_tool_rest_api( + _build_request(headers, path="/mcp-rest/tools/call", json_body=payload), + user_api_key_dict=auth, + ) + assert result.is_error is False + assert observed.qsize() == 1 + assert observed.get_nowait() == expected + if claims is None: + list_headers: Final = rest_endpoints._extract_mcp_headers_from_request( + _build_request(headers), auth, + )[2] + authorization: Final = list_headers.get("authorization", "") + assert (authorization.removeprefix("Bearer ") or None) == expected + finally: + global_mcp_tool_registry.unregister_tools_with_prefix("guarded-") From 4c532a571364cfb0a8e9a2b9a35324b46bc898d4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:18:53 -0700 Subject: [PATCH 3/7] feat(decisions): accept decisions as a model mode alongside evaluation (#45474) * refactor(decisions): rename the evaluation model mode to decisions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): canonicalize the health check mode without rebinding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): replace canonical_model_mode with is_decisions_model_mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): preserve evaluation mode compatibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: drop the rebuilt dashboard bundle from this PR Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(dashboard): format decision mode helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/model-catalog/src/capabilities.rs | 1 + .../crates/model-catalog/tests/catalog.rs | 16 +++++++++++++++- .../litellm_core_utils/health_check_helpers.py | 4 ++-- litellm/main.py | 6 ++++-- .../proxy/health_endpoints/_health_endpoints.py | 1 + litellm/types/utils.py | 5 +++++ .../management/test_model_health_check.py | 10 ++++++---- .../test_health_check_helpers.py | 8 +++++--- .../health_endpoints/test_health_endpoints.py | 12 +++++++----- .../test_typesafe_passthrough_logging_handler.py | 5 +++-- tests/unit/test_model_prices_schema.py | 2 +- tests/unit/test_utils.py | 2 +- tests/unit/types/test_types_utils.py | 9 +++++++++ .../components/provider_info_helpers.test.tsx | 6 +++--- .../src/lib/decisionModels.test.ts | 3 ++- ui/litellm-dashboard/src/lib/decisionModels.ts | 3 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 17 files changed, 68 insertions(+), 27 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/capabilities.rs b/litellm-rust/crates/model-catalog/src/capabilities.rs index 3df68fabc4d..30724766aeb 100644 --- a/litellm-rust/crates/model-catalog/src/capabilities.rs +++ b/litellm-rust/crates/model-catalog/src/capabilities.rs @@ -9,6 +9,7 @@ pub enum Mode { AudioTranscription, Chat, Completion, + Decisions, Embedding, Evaluation, Guardrail, diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index d87de3c5b3b..d384dfea9a8 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance}; +use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance}; use rstest::{fixture, rstest}; use serde_json::json; @@ -42,6 +42,20 @@ fn fixture_catalog() -> Catalog { .unwrap() } +#[rstest] +#[case::decisions("decisions", Mode::Decisions)] +#[case::legacy_evaluation("evaluation", Mode::Evaluation)] +fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) { + let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string(); + let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap(); + + assert_eq!( + catalog.lookup("Test").unwrap().entry.info().mode, + Some(expected) + ); + assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode)); +} + #[rstest] #[ignore] fn preserves_fields_and_metadata(fixture_catalog: Catalog) { diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 3cbc2d770cc..03af9076c79 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -214,7 +214,7 @@ class HealthCheckHelpers: "responses", "anthropic_messages", "ocr", - "evaluation", + "decisions", ], Callable, ]: @@ -311,7 +311,7 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), - "evaluation": lambda: litellm.asystemone( + "decisions": lambda: litellm.asystemone( **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", diff --git a/litellm/main.py b/litellm/main.py index c2c6d5affe8..474d82e8ef1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -141,6 +141,7 @@ from litellm.types.utils import ( ModelResponseStream, RawRequestTypedDict, StreamingChoices, + is_decisions_model_mode, ) from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS from litellm.utils import ( @@ -8821,8 +8822,9 @@ async def ahealth_check( input=input, ) - if mode in mode_handlers: - _response: Final = await mode_handlers[mode]() + handler_mode: Final = "decisions" if is_decisions_model_mode(mode) else mode + if handler_mode in mode_handlers: + _response: Final = await mode_handlers[handler_mode]() _response_headers: Final = cast( # cast-ok: provider headers are stored as a string-keyed mapping Mapping[str, object], (get_hidden_params(_response) or {}).get("headers", {}) or {} ) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 261bf99a6f2..fa58fda8f47 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2091,6 +2091,7 @@ async def test_model_connection( "anthropic_messages", "ocr", "evaluation", + "decisions", ] | None = fastapi.Body( None, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f89ac614e8..f7385a6d92d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -160,6 +160,10 @@ class LiteLLMCommonStrings(Enum): llm_provider_not_provided = "Unmapped LLM provider for this endpoint. You passed model={model}, custom_llm_provider={custom_llm_provider}. Check supported provider and route: https://docs.litellm.ai/docs/providers" +def is_decisions_model_mode(mode: str | None) -> bool: + return mode in ("decisions", "evaluation") + + SupportedCacheControls: Final = ["ttl", "s-maxage", "no-cache", "no-store"] @@ -430,6 +434,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "audio_transcription", "audio_speech", "responses", + "decisions", "evaluation", "ocr", "realtime", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 075c6051353..ef8412a6ed1 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -3,6 +3,7 @@ import uuid from typing import Final import httpx +import pytest from integration._support.client import Gateway, object_value, string_value from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse @@ -93,7 +94,7 @@ def _probes_sent_to(gateway: Gateway, handle: ScenarioHandle) -> list[tuple[str, ] -def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe( +def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe( gateway: Gateway, ) -> None: with gateway.scenario() as scenario: @@ -114,7 +115,8 @@ def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_se ] -def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway) -> None: +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None: with gateway.scenario() as scenario: handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _CONFIGURED_PROBE_REPLY) scenario.cleanups.callback(delete_scenario, handle) @@ -122,7 +124,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g model="typesafe/jev-custom", api_base=handle.api_base(), model_info={ - "mode": "evaluation", + "mode": mode, "health_check_params": {"state": _CONFIGURED_STATE, "questions": _CONFIGURED_QUESTIONS}, }, ) @@ -136,7 +138,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g ] -def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map( +def test_decisions_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map( gateway: Gateway, ) -> None: with gateway.scenario() as scenario: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 9903ab71876..123be0bd427 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -589,7 +589,7 @@ def test_ocr_health_check_document_raises_without_the_extension(): ("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"), ), ) -async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( +async def test_ahealth_check_probes_decisions_models_through_the_decisions_api( model: str, upstream_url: str, monkeypatch: pytest.MonkeyPatch, @@ -617,7 +617,9 @@ async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( @pytest.mark.asyncio -async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions( +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +async def test_ahealth_check_decisions_uses_configured_probe_state_and_questions( + mode: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, ) -> None: @@ -638,7 +640,7 @@ async def test_ahealth_check_evaluation_uses_configured_probe_state_and_question "state": "custom probe", "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}}, }, - mode=None, + mode=mode, ) assert "error" not in result, result diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 14e5e76c692..1ca07eafb7d 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -882,7 +882,8 @@ async def test_test_model_connection_request_mode_wins_over_resolved_mode(): @pytest.mark.asyncio -async def test_test_model_connection_evaluation_mode_uses_decisions_handler(): +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +async def test_test_model_connection_decision_modes_are_forwarded(mode: str): deployment: Final = MappingProxyType( { "model_name": "typesafe/jev-latest", @@ -893,14 +894,14 @@ async def test_test_model_connection_evaluation_mode_uses_decisions_handler(): with _test_connection_probe(deployment) as ahealth_check: result: Final = await health_test_model_connection( request=MagicMock(), - mode="evaluation", + mode=mode, litellm_params={"model": "typesafe/jev-latest"}, model_info={"id": "typesafe-jev-id"}, user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), ) assert result["status"] == "success" - assert ahealth_check.call_args.kwargs["mode"] == "evaluation" + assert ahealth_check.call_args.kwargs["mode"] == mode @pytest.mark.asyncio @@ -5011,7 +5012,8 @@ def test_test_model_connection_accepts_image_edit_mode(monkeypatch): assert response.json()["status"] == "success" -def test_test_model_connection_accepts_evaluation_mode(monkeypatch): +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +def test_test_model_connection_accepts_decision_modes(monkeypatch, mode: str): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() @@ -5036,7 +5038,7 @@ def test_test_model_connection_accepts_evaluation_mode(monkeypatch): response = client.post( "/health/test_connection", json={ - "mode": "evaluation", + "mode": mode, "litellm_params": {"model": "typesafe/jev-latest", "api_key": "sk-test"}, }, ) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 07b52711dc3..7d9f4ae33c8 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -140,6 +140,7 @@ def test_success_handler_dispatches_to_typesafe_handler(): @pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) @pytest.mark.parametrize("guardrail_cost", [0.0, 0.25]) @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) @pytest.mark.parametrize("provider,requested,routing_model", [ @@ -149,7 +150,7 @@ def test_success_handler_dispatches_to_typesafe_handler(): ]) async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float, - provider: str, requested: str + provider: str, requested: str, mode: str ) -> None: checkpoint: Final = routing_model or requested model: Final = f"{provider}/{checkpoint}" @@ -157,7 +158,7 @@ async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( output_rate: Final = 0.005 monkeypatch.setitem(litellm.model_cost, model, { "input_cost_per_token": input_rate, "output_cost_per_token": output_rate, - "litellm_provider": provider, "mode": "evaluation", + "litellm_provider": provider, "mode": mode, }) start: Final = datetime.now() logging_obj: Final = Logging( diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 37df91d22af..50c9643683b 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -637,7 +637,7 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path): PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) -MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) +MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "decisions", "evaluation"}) VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset( { "vertex_ai-ai21_models", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6b63f69fd2e..466f3c988e4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8403,7 +8403,7 @@ def test_models_by_provider(): or v["litellm_provider"] == "bedrock_converse" ): continue - elif v.get("mode") in ("search", "evaluation"): + elif v.get("mode") in ("search", "decisions", "evaluation"): continue else: providers.add(v["litellm_provider"]) diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index ebec1b1b924..fbe00ca260b 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -29,6 +29,7 @@ from litellm.types.utils import ( ModelResponseStream, TextCompletionResponse, all_litellm_params, + is_decisions_model_mode, text_tokens_without_nested_reasoning, ) @@ -41,6 +42,14 @@ class _DictHiddenParamsAccessor(Protocol): def hidden_params(self, hidden_params: dict[str, object]) -> None: ... +@pytest.mark.parametrize( + ("mode", "expected"), + (("decisions", True), ("evaluation", True), ("chat", False), (None, False)), +) +def test_is_decisions_model_mode(mode: str | None, expected: bool) -> None: + assert is_decisions_model_mode(mode) is expected + + def test_rust_is_a_known_litellm_param(): assert "rust" in all_litellm_params diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 728f95a2225..52e13642020 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -506,9 +506,9 @@ describe("provider_info_helpers", () => { it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { const modelMap = { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, - "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" }, + "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "decisions" }, "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, - "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" }, }; expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([ "typesafe/jev-latest", @@ -518,7 +518,7 @@ describe("provider_info_helpers", () => { it("should list only strands_decider models for the 'StrandsDecider' provider key", () => { const modelMap = { - "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" }, "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, }; diff --git a/ui/litellm-dashboard/src/lib/decisionModels.test.ts b/ui/litellm-dashboard/src/lib/decisionModels.test.ts index 0876d88b261..7552cc0cfd8 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.test.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.test.ts @@ -17,7 +17,8 @@ const COST_MAP = { }; describe("isDecisionMode", () => { - it("is true only for the evaluation mode", () => { + it("is true for both the decisions and evaluation modes", () => { + expect(isDecisionMode("decisions")).toBe(true); expect(isDecisionMode("evaluation")).toBe(true); expect(isDecisionMode("chat")).toBe(false); expect(isDecisionMode(null)).toBe(false); diff --git a/ui/litellm-dashboard/src/lib/decisionModels.ts b/ui/litellm-dashboard/src/lib/decisionModels.ts index 51de8d62ace..a14ce1948a9 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.ts @@ -25,7 +25,8 @@ export interface DecisionCatalog { export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), providers: new Map() }; -export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "evaluation"; +export const isDecisionMode = (mode: string | null | undefined): boolean => + mode === "decisions" || mode === "evaluation"; const isDecisionEntry = (entry: CostMapEntry): boolean => isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint)); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9853a71a2ed..c56a4b3786c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27208,7 +27208,7 @@ export interface components { * Mode * @description The mode to test the model with. If not provided, resolved the way /health does: the deployment's model_info.mode (only while the request tests the deployment's own model), then the mode the provider requires for that model, then the model cost map. */ - mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation") | null; + mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation" | "decisions") | null; /** * Model Info * @description Model info for the health check From 55c89977d7fb6d746acb7f29a5b63c9489e62fa3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:42:18 -0700 Subject: [PATCH 4/7] feat(ui): load Home page What's new launches from /public/whats_new at runtime (#45912) * feat(ui): load the Home page What's new launches from /public/whats_new at runtime Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): show the three newest What's new launches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * build: ship whats_new.json in the wheel Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 4 + .../public_endpoints/public_endpoints.py | 76 +++++- litellm/proxy/public_endpoints/whats_new.json | 25 ++ .../public_endpoints/public_endpoints.py | 19 ++ pyproject.toml | 1 + .../public_endpoints/test_public_endpoints.py | 229 ++++++++++++------ .../_components/HomePage.integration.test.tsx | 57 ++++- .../(dashboard)/home/_components/HomePage.tsx | 67 ++--- .../home/_components/homeContent.test.ts | 40 +++ .../home/_components/homeContent.ts | 54 ++--- .../home/_components/useWhatsNew.ts | 10 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 69 ++++++ 12 files changed, 505 insertions(+), 146 deletions(-) create mode 100644 litellm/proxy/public_endpoints/whats_new.json create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/home/_components/useWhatsNew.ts diff --git a/litellm/__init__.py b/litellm/__init__.py index a611b90bc37..34f08da3c89 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -440,6 +440,10 @@ autorouter_presets_url: str = os.getenv( "LITELLM_AUTOROUTER_PRESETS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json", ) +whats_new_url: str = os.getenv( + "LITELLM_WHATS_NEW_URL", + "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/whats_new.json", +) suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 47caacdcfff..75e8a5af6a5 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -6,8 +6,9 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from importlib.resources import files from typing import TYPE_CHECKING, Final, Protocol +import httpx from fastapi import APIRouter, HTTPException, Request -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -37,6 +38,7 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ( ProviderCreateInfo, PublicModelHubInfo, SupportedEndpointsResponse, + WhatsNewResponse, ) from litellm.types.utils import LlmProviders @@ -505,14 +507,18 @@ def _load_bundled_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: ) -async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]: +async def _fetch_remote_bytes(url: str) -> bytes: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.UI) response: Final = await client.get(url, timeout=5.0) response.raise_for_status() - presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_python(response.json()) + return response.content + + +async def _fetch_remote_autorouter_presets(url: str) -> Mapping[str, AutoRouterPresetRecord]: + presets: Final = _AUTOROUTER_PRESETS_ADAPTER.validate_json(await _fetch_remote_bytes(url)) if not presets: raise ValueError("remote auto-router preset catalog is empty") return presets @@ -575,6 +581,70 @@ async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord return await get_autorouter_presets(url=litellm.autorouter_presets_url) +def _load_bundled_whats_new() -> WhatsNewResponse: + return WhatsNewResponse.model_validate_json( + files("litellm.proxy.public_endpoints").joinpath("whats_new.json").read_text(encoding="utf-8") + ) + + +async def _fetch_remote_whats_new(url: str) -> WhatsNewResponse: + return WhatsNewResponse.model_validate_json(await _fetch_remote_bytes(url)) + + +async def _resolve_whats_new(url: str, fetch: Callable[[str], Awaitable[WhatsNewResponse]]) -> WhatsNewResponse: + if os.getenv("LITELLM_LOCAL_WHATS_NEW", "").lower() == "true": + return _load_bundled_whats_new() + try: + return await fetch(url) + except (httpx.HTTPError, ValidationError, OSError) as e: + verbose_logger.warning( + "LiteLLM: failed to fetch What's new launches from %s: %s. Serving the bundled list for the life of this process.", + url, + str(e), + ) + return _load_bundled_whats_new() + + +class _WhatsNewCache: + launches: WhatsNewResponse | None = None + lock: asyncio.Lock | None = None + + +async def get_whats_new( + url: str, + fetch: Callable[[str], Awaitable[WhatsNewResponse]] = _fetch_remote_whats_new, +) -> WhatsNewResponse: + cached: Final = _WhatsNewCache.launches + if cached is not None: + return cached + if _WhatsNewCache.lock is None: + _WhatsNewCache.lock = asyncio.Lock() + async with _WhatsNewCache.lock: + held: Final = _WhatsNewCache.launches + if held is not None: + return held + resolved: Final = await _resolve_whats_new(url=url, fetch=fetch) + _WhatsNewCache.launches = resolved + return resolved + + +@router.get( + "/public/whats_new", + tags=["public"], + response_model=WhatsNewResponse, +) +async def get_public_whats_new() -> WhatsNewResponse: + """ + Return the launches the dashboard Home page shows under What's new. + + Resolved once per process: fetched from ``litellm.whats_new_url`` (override with ``LITELLM_WHATS_NEW_URL``) + on the first request, falling back to the list bundled with the package on any failure, so an airgapped + proxy serves the bundled list without retrying. Set ``LITELLM_LOCAL_WHATS_NEW=True`` to skip the fetch. + A restart picks up a newly published list. + """ + return await get_whats_new(url=litellm.whats_new_url) + + @router.get( "/public/endpoints", tags=["public"], diff --git a/litellm/proxy/public_endpoints/whats_new.json b/litellm/proxy/public_endpoints/whats_new.json new file mode 100644 index 00000000000..40affb71e98 --- /dev/null +++ b/litellm/proxy/public_endpoints/whats_new.json @@ -0,0 +1,25 @@ +{ + "launches": [ + { + "icon": "box", + "title": "Claude Haiku 5.5", + "description": "Anthropic's newest small model, with day 0 pricing on Anthropic, Bedrock and Vertex AI.", + "href": "https://docs.litellm.ai/blog/claude-haiku-5-5", + "published_on": "2026-10-07" + }, + { + "icon": "scale", + "title": "Decision Models", + "description": "Call decision models at /v1/decisions and try them in the Decisions playground.", + "href": "https://docs.litellm.ai/docs/decisions", + "published_on": "2026-10-07" + }, + { + "icon": "zap", + "title": "OpenAI Ultrafast", + "description": "service_tier: ultrafast on GPT-6 Astra and GPT-6.1 Sol, including /ultrafast in Codex.", + "href": "https://docs.litellm.ai/docs/providers/openai/ultrafast", + "published_on": "2026-10-06" + } + ] +} diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index c487ce8b6c5..6e4adb4bbe7 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,4 +1,5 @@ from collections.abc import Mapping, Sequence +from datetime import date from typing import Any, Literal from pydantic import ConfigDict @@ -113,6 +114,24 @@ class AutoRouterPresetRecord(LiteLLMBaseModel): complexity_router_config: AutoRouterPresetConfig +class WhatsNewLaunch(LiteLLMBaseModel): + """One launch card in the dashboard Home page's What's new section.""" + + model_config = ConfigDict(frozen=True) + + icon: str + title: str + description: str + href: str + published_on: date + + +class WhatsNewResponse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + launches: tuple[WhatsNewLaunch, ...] + + class ComplexityScorerDefaults(LiteLLMBaseModel): """The complexity router's shipped heuristic scorer defaults. diff --git a/pyproject.toml b/pyproject.toml index 133fc735df1..0ed8b8757ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -321,6 +321,7 @@ include = [ "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", + "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", ] diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index b720ba24c3e..0779f692b17 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,6 +1,8 @@ import json import re from datetime import datetime, timezone +from importlib.resources import files +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -17,7 +19,7 @@ from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_pres from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) -from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo +from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo, WhatsNewResponse from litellm.types.utils import LlmProviders @@ -67,13 +69,10 @@ def test_get_provider_create_fields(): assert isinstance(first_provider["credential_fields"], list) has_detailed_fields = any( - provider.get("credential_fields") - and len(provider.get("credential_fields", [])) > 0 + provider.get("credential_fields") and len(provider.get("credential_fields", [])) > 0 for provider in response_data ) - assert ( - has_detailed_fields - ), "Expected at least one provider to have detailed credential fields" + assert has_detailed_fields, "Expected at least one provider to have detailed credential fields" def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_entries( @@ -124,10 +123,7 @@ def test_get_litellm_model_cost_map_returns_cost_map(): sample_model_data = payload[sample_model] assert isinstance(sample_model_data, dict) # Check for common cost fields that should be present - assert ( - "input_cost_per_token" in sample_model_data - or "output_cost_per_token" in sample_model_data - ) + assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data def test_public_ai_hub_info_is_public_by_default(monkeypatch): @@ -218,9 +214,9 @@ def test_anthropic_provider_fields_support_byok(): "Anthropic api_key must be optional so admins can configure BYOK models " "without entering a key. See BYOK tutorial." ) - assert fields_by_key["api_key"].get( - "tooltip" - ), "Anthropic api_key must have a tooltip explaining the BYOK use case." + assert fields_by_key["api_key"].get("tooltip"), ( + "Anthropic api_key must have a tooltip explaining the BYOK use case." + ) assert "api_base" in fields_by_key, ( "Anthropic provider form must expose api_base so cloud customers " "can override the upstream URL without env var access." @@ -228,16 +224,14 @@ def test_anthropic_provider_fields_support_byok(): api_base_field = fields_by_key["api_base"] assert api_base_field["required"] is False assert api_base_field["field_type"] == "text" - assert api_base_field.get( - "tooltip" - ), "api_base should have a tooltip explaining it is optional." + assert api_base_field.get("tooltip"), "api_base should have a tooltip explaining it is optional." # UI forms render fields in credential_fields order; api_base should come first # so an admin sees the URL override before the key field. field_order = [f["key"] for f in anthropic["credential_fields"]] - assert field_order.index("api_base") < field_order.index( - "api_key" - ), "api_base must appear before api_key in credential_fields (matches AI21 and ANTHROPIC_TEXT convention)." + assert field_order.index("api_base") < field_order.index("api_key"), ( + "api_base must appear before api_key in credential_fields (matches AI21 and ANTHROPIC_TEXT convention)." + ) def test_bedrock_mantle_provider_fields(): @@ -554,9 +548,7 @@ def test_google_ai_studio_provider_fields_expose_api_base(): assert response.status_code == 200 providers = response.json() - google_ai = next( - (p for p in providers if p["provider"] == "Google_AI_Studio"), None - ) + google_ai = next((p for p in providers if p["provider"] == "Google_AI_Studio"), None) assert google_ai is not None, "Google_AI_Studio provider entry not found" assert google_ai["litellm_provider"] == "gemini" @@ -576,18 +568,15 @@ def test_google_ai_studio_provider_fields_expose_api_base(): # placeholder shows the canonical URL so users still get the visual hint. # (See greptileai threads on PR #30419.) assert api_base_field["default_value"] is None - assert ( - api_base_field["placeholder"] - == "https://generativelanguage.googleapis.com/v1beta" - ) + assert api_base_field["placeholder"] == "https://generativelanguage.googleapis.com/v1beta" # UI forms render fields in credential_fields order; api_base should come # first so an admin sees the URL override before the key field (matches # OpenAI and Anthropic conventions). field_order = [f["key"] for f in google_ai["credential_fields"]] - assert field_order.index("api_base") < field_order.index( - "api_key" - ), "api_base must appear before api_key in credential_fields." + assert field_order.index("api_base") < field_order.index("api_key"), ( + "api_base must appear before api_key in credential_fields." + ) def test_public_model_hub_with_healthy_model(): @@ -615,20 +604,15 @@ def test_public_model_hub_with_healthy_model(): mock_llm_router = MagicMock() mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock( - return_value=[mock_health_check] - ) + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[mock_health_check]) with ( patch("litellm.public_model_groups", ["gpt-3.5-turbo"]), patch("litellm.proxy.proxy_server.get_model_group_info") as mock_get_info, patch("litellm.proxy.proxy_server.llm_router", mock_llm_router), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), - patch( - "litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict" - ) as mock_convert, + patch("litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict") as mock_convert, ): - mock_get_info.return_value = [mock_model_group] mock_convert.return_value = { "status": "healthy", @@ -673,20 +657,15 @@ def test_public_model_hub_with_unhealthy_model(): mock_llm_router = MagicMock() mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock( - return_value=[mock_health_check] - ) + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[mock_health_check]) with ( patch("litellm.public_model_groups", ["gpt-4"]), patch("litellm.proxy.proxy_server.get_model_group_info") as mock_get_info, patch("litellm.proxy.proxy_server.llm_router", mock_llm_router), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), - patch( - "litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict" - ) as mock_convert, + patch("litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict") as mock_convert, ): - mock_get_info.return_value = [mock_model_group] mock_convert.return_value = { "status": "unhealthy", @@ -732,7 +711,6 @@ def test_public_model_hub_without_health_check(): patch("litellm.proxy.proxy_server.llm_router", mock_llm_router), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), ): - mock_get_info.return_value = [mock_model_group] response = client.get( @@ -789,9 +767,7 @@ def test_public_model_hub_mixed_health_statuses(): mock_llm_router = MagicMock() mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock( - return_value=[healthy_check, unhealthy_check] - ) + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[healthy_check, unhealthy_check]) def convert_side_effect(check): if check.model_name == "gpt-3.5-turbo": @@ -813,11 +789,8 @@ def test_public_model_hub_mixed_health_statuses(): patch("litellm.proxy.proxy_server.get_model_group_info") as mock_get_info, patch("litellm.proxy.proxy_server.llm_router", mock_llm_router), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), - patch( - "litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict" - ) as mock_convert, + patch("litellm.proxy.health_endpoints._health_endpoints.convert_health_check_to_dict") as mock_convert, ): - mock_get_info.return_value = [ healthy_model, unhealthy_model, @@ -1066,9 +1039,7 @@ def test_get_supported_endpoints_provider_fields(reset_endpoints_cache): def test_get_supported_endpoints_paths_start_with_slash(reset_endpoints_cache): endpoints = _make_client().get("/public/endpoints").json()["endpoints"] for item in endpoints: - assert item["endpoint"].startswith( - "/" - ), f"Expected path starting with /, got: {item['endpoint']}" + assert item["endpoint"].startswith("/"), f"Expected path starting with /, got: {item['endpoint']}" def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache): @@ -1092,9 +1063,9 @@ def test_get_supported_endpoints_display_names_have_no_slug_suffix( endpoints = _make_client().get("/public/endpoints").json()["endpoints"] for item in endpoints: for provider in item["providers"]: - assert not suffix_re.search( - provider["display_name"] - ), f"display_name still contains slug suffix: {provider['display_name']!r}" + assert not suffix_re.search(provider["display_name"]), ( + f"display_name still contains slug suffix: {provider['display_name']!r}" + ) def test_get_supported_endpoints_is_cached(reset_endpoints_cache): @@ -1320,7 +1291,6 @@ def test_public_mcp_hub_does_not_expose_upstream_url(): app.dependency_overrides.clear() - @pytest.fixture def reset_autorouter_presets_cache(): from litellm.proxy.public_endpoints.public_endpoints import _AutoRouterPresetsCache @@ -1332,9 +1302,7 @@ def reset_autorouter_presets_cache(): _AutoRouterPresetsCache.lock = None -def test_get_autorouter_presets_local_mode_serves_bundled_catalog( - monkeypatch, reset_autorouter_presets_cache -): +def test_get_autorouter_presets_local_mode_serves_bundled_catalog(monkeypatch, reset_autorouter_presets_cache): monkeypatch.setenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "True") app = FastAPI() app.include_router(router) @@ -1362,9 +1330,7 @@ def test_get_autorouter_presets_local_mode_serves_bundled_catalog( @pytest.mark.asyncio -async def test_get_autorouter_presets_fetches_once_per_process( - monkeypatch, reset_autorouter_presets_cache -): +async def test_get_autorouter_presets_fetches_once_per_process(monkeypatch, reset_autorouter_presets_cache): from litellm.proxy.public_endpoints.public_endpoints import ( _AUTOROUTER_PRESETS_ADAPTER, get_autorouter_presets, @@ -1376,7 +1342,9 @@ async def test_get_autorouter_presets_fetches_once_per_process( "remote_only": { "label": "Remote Only", "description": "from the remote catalog", - "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]} + }, } } ) @@ -1411,7 +1379,9 @@ async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( "remote_only": { "label": "Remote Only", "description": "from the remote catalog", - "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]} + }, } } ) @@ -1507,9 +1477,7 @@ async def test_autorouter_presets_adapter_rejects_wrong_shapes(): ) -def test_get_autorouter_presets_passes_unknown_catalog_fields_through( - monkeypatch, reset_autorouter_presets_cache -): +def test_get_autorouter_presets_passes_unknown_catalog_fields_through(monkeypatch, reset_autorouter_presets_cache): from litellm.proxy.public_endpoints.public_endpoints import ( _AUTOROUTER_PRESETS_ADAPTER, _AutoRouterPresetsCache, @@ -1551,12 +1519,14 @@ async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypa "remote_only": { "label": "Remote Only", "description": "from the remote catalog", - "complexity_router_config": {"tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]}}, + "complexity_router_config": { + "tiers": {"SIMPLE": ["m1"], "MEDIUM": ["m2"], "COMPLEX": ["m3"], "REASONING": ["m4"]} + }, } } response = MagicMock() response.raise_for_status = MagicMock() - response.json = MagicMock(return_value=catalog) + response.content = json.dumps(catalog).encode() client = MagicMock() client.get = AsyncMock(return_value=response) monkeypatch.setattr(http_handler_module, "get_async_httpx_client", lambda llm_provider: client) @@ -1565,6 +1535,121 @@ async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypa assert presets["remote_only"].label == "Remote Only" response.raise_for_status.assert_called_once() - response.json = MagicMock(return_value={}) + response.content = b"{}" with pytest.raises(ValueError, match="empty"): await _fetch_remote_autorouter_presets("https://example.test/presets.json") + + +@pytest.fixture +def reset_whats_new_cache() -> Iterator[None]: + from litellm.proxy.public_endpoints.public_endpoints import _WhatsNewCache + + _WhatsNewCache.launches = None + _WhatsNewCache.lock = None + yield + _WhatsNewCache.launches = None + _WhatsNewCache.lock = None + + +def _remote_whats_new() -> WhatsNewResponse: + return WhatsNewResponse.model_validate( + { + "launches": [ + { + "icon": "sparkles", + "title": "Remote Launch", + "description": "from the remote list", + "href": "https://docs.litellm.ai/blog/remote", + "published_on": "2026-10-11", + } + ] + } + ) + + +def test_get_whats_new_local_mode_serves_bundled_list( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_WHATS_NEW", "True") + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + response = client.get("/public/whats_new") + + assert response.status_code == 200 + served: Final = WhatsNewResponse.model_validate_json(response.content) + bundled: Final = WhatsNewResponse.model_validate_json( + files("litellm.proxy.public_endpoints").joinpath("whats_new.json").read_text(encoding="utf-8") + ) + assert served == bundled + assert len(served.launches) > 0 + + +@pytest.mark.asyncio +async def test_get_whats_new_fetches_once_per_process( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + from litellm.proxy.public_endpoints.public_endpoints import get_whats_new + + monkeypatch.delenv("LITELLM_LOCAL_WHATS_NEW", raising=False) + remote = _remote_whats_new() + calls: Final[list[str]] = [] + + async def fake_fetch(url: str) -> WhatsNewResponse: + calls.append(url) + return remote + + first = await get_whats_new(url="https://example.test/whats_new.json", fetch=fake_fetch) + second = await get_whats_new(url="https://example.test/whats_new.json", fetch=fake_fetch) + + assert first == remote + assert second == remote + assert calls == ["https://example.test/whats_new.json"] + + +@pytest.mark.asyncio +async def test_get_whats_new_caches_bundled_fallback_when_remote_is_unreachable( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + from litellm.proxy.public_endpoints.public_endpoints import _load_bundled_whats_new, get_whats_new + + monkeypatch.delenv("LITELLM_LOCAL_WHATS_NEW", raising=False) + calls: Final[list[str]] = [] + + async def unreachable_fetch(url: str) -> WhatsNewResponse: + calls.append(url) + raise OSError("network is unreachable") + + first = await get_whats_new(url="https://example.test/whats_new.json", fetch=unreachable_fetch) + second = await get_whats_new(url="https://example.test/whats_new.json", fetch=unreachable_fetch) + + assert first == _load_bundled_whats_new() + assert second == first + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_fetch_remote_whats_new_rejects_a_malformed_list_so_the_bundled_one_is_served( + monkeypatch: pytest.MonkeyPatch, reset_whats_new_cache: None +) -> None: + import httpx + + from litellm.proxy.public_endpoints import public_endpoints + + monkeypatch.delenv("LITELLM_LOCAL_WHATS_NEW", raising=False) + request = httpx.Request("GET", "https://example.test/whats_new.json") + client = MagicMock() + client.get = AsyncMock( + return_value=httpx.Response( + 200, + request=request, + content=b'{"launches": [{"icon": "box", "title": "No date", "description": "d", "href": "https://x"}]}', + ) + ) + + with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client", return_value=client): + served = await public_endpoints.get_whats_new(url="https://example.test/whats_new.json") + + assert served == public_endpoints._load_bundled_whats_new() + client.get.assert_awaited_once_with("https://example.test/whats_new.json", timeout=5.0) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx index ad3f80b7d78..a17a18beee5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx @@ -6,7 +6,6 @@ import type { DailyActivityAggregatedResponse } from "@/components/UsagePage/dai import { EMPTY_DAILY_ACTIVITY_METADATA } from "@/components/UsagePage/dailyActivityApi"; import { all_admin_roles } from "@/utils/roles"; import HomePage from "./HomePage"; -import { WHATS_NEW_ITEMS } from "./homeContent"; const auth = vi.hoisted(() => ({ accessToken: "sk-test", @@ -31,6 +30,28 @@ vi.mock("@/components/networking", async (importOriginal) => ({ const blogPosts = { posts: [{ title: "Post one", description: "d", date: "2026-10-09", url: "https://x/1" }] }; +const launch = (title: string, icon: string, published_on: string) => ({ + icon, + title, + description: `${title} description`, + href: `https://docs.litellm.ai/blog/${title}`, + published_on, +}); + +const whatsNew = vi.hoisted(() => ({ launches: [] as unknown[] })); + +const requestUrl = (input: RequestInfo | URL): string => { + if (typeof input === "string") return input; + return input instanceof URL ? input.href : input.url; +}; + +const fakeFetch = (input: RequestInfo | URL) => { + const body = requestUrl(input).endsWith("/public/whats_new") ? whatsNew : blogPosts; + return Promise.resolve( + new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }), + ); +}; + const dayResult = (date: string, model: string, tokens: number) => { const metrics = { spend: 1, @@ -86,7 +107,8 @@ describe("HomePage", () => { network.aggregated.mockReset().mockResolvedValue(activity); network.gateway.mockReset().mockResolvedValue({ total_successful_requests: 0, total_failed_requests: 0 }); network.userInfo.mockReset().mockResolvedValue({ user_info: { max_budget: null } }); - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify(blogPosts), { status: 200 }))); + whatsNew.launches = [launch("mid", "box", "2026-10-06"), launch("new", "zap", "2026-10-11")]; + vi.stubGlobal("fetch", vi.fn(fakeFetch)); }); afterEach(() => { vi.unstubAllGlobals(); @@ -121,13 +143,32 @@ describe("HomePage", () => { expect(network.aggregated).toHaveBeenCalledTimes(2); }); - it("lists the curated launches newest first with their links", () => { + it("renders the launches served by /public/whats_new newest first, whatever order the JSON lists them in", async () => { + whatsNew.launches = [ + launch("mid", "box", "2026-10-06"), + launch("new", "an-icon-from-a-newer-list", "2026-10-11"), + launch("old", "scale", "2026-09-30"), + ]; renderHome(); - const section = screen.getByTestId("home-whats-new"); - const links = within(section).getAllByRole("link"); - expect(links.map((link) => link.getAttribute("href"))).toEqual(WHATS_NEW_ITEMS.map((item) => item.href)); - const dates = WHATS_NEW_ITEMS.map((item) => item.publishedOn); - expect(dates).toEqual([...dates].sort().reverse()); + const section = await screen.findByTestId("home-whats-new"); + expect(await within(section).findByText("new")).toBeInTheDocument(); + expect( + within(section) + .getAllByRole("link") + .map((link) => link.getAttribute("href")), + ).toEqual([ + "https://docs.litellm.ai/blog/new", + "https://docs.litellm.ai/blog/mid", + "https://docs.litellm.ai/blog/old", + ]); + expect(within(section).getByText("Oct 11")).toBeInTheDocument(); + }); + + it("hides What's new when /public/whats_new has no launches", async () => { + whatsNew.launches = []; + renderHome(); + expect(await screen.findByRole("link", { name: /post one/i })).toBeInTheDocument(); + expect(screen.queryByTestId("home-whats-new")).not.toBeInTheDocument(); }); it("ranks model groups by token share on the leaderboard", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx index f4519958e9d..bcae30795c6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.tsx @@ -4,21 +4,24 @@ import { ArrowRight, Plus, Sparkles } from "lucide-react"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; import { Page } from "@/components/shared/Page"; import { PageHeader, PageHeaderControls, PageHeaderTitle } from "@/components/shared/PageHeader"; import { uiHref } from "@/utils/uiHref"; import { Leaderboard, useBreakdown } from "@/app/(dashboard)/usage/_components/components/overview/BreakdownChart"; import { Panel } from "@/app/(dashboard)/usage/_components/components/overview/Primitives"; import UsageStatStrip from "@/app/(dashboard)/usage/_components/components/overview/UsageStatStrip"; -import { formatPublishedOn, WHATS_NEW_ITEMS } from "./homeContent"; +import { formatPublishedOn, launchIcon } from "./homeContent"; import UpdatesFeed from "./UpdatesFeed"; import { HOME_USAGE_DAYS, useHomeUsage } from "./useHomeUsage"; +import { useWhatsNew } from "./useWhatsNew"; const LEADERBOARD = { metric: "tokens", dimension: "model_groups" } as const; export default function HomePage() { const { isViewOnly } = useAuthorized(); const usage = useHomeUsage(); + const whatsNew = useWhatsNew(); const { series, ranking } = useBreakdown(usage.results, LEADERBOARD, 8); return ( @@ -56,34 +59,40 @@ export default function HomePage() {
-
-
-

- - What's new in LiteLLM? -

-
-
- {WHATS_NEW_ITEMS.map((item) => { - const Icon = item.icon; - return ( - - -
- -
-
{item.title}
-

{item.description}

-
- {formatPublishedOn(item.publishedOn)} - -
-
-
- ); - })} -
-
+ {(whatsNew.isLoading || (whatsNew.data?.length ?? 0) > 0) && ( +
+
+

+ + What's new in LiteLLM? +

+
+
+ {whatsNew.isLoading && + [0, 1, 2].map((i) => ( + + ))} + {whatsNew.data?.map((item) => { + const Icon = launchIcon(item.icon); + return ( + + +
+ +
+
{item.title}
+

{item.description}

+
+ {formatPublishedOn(item.published_on)} + +
+
+
+ ); + })} +
+
+ )} ({ + icon: "box", + title, + description: "d", + href: `https://docs.litellm.ai/${title}`, + published_on, +}); + +describe("latestLaunches", () => { + it("orders launches by published_on, newest first, without mutating the input", () => { + const launches = [launch("old", "2026-09-01"), launch("new", "2026-10-11"), launch("mid", "2026-10-01")]; + expect(latestLaunches(launches).map((l) => l.title)).toEqual(["new", "mid", "old"]); + expect(launches.map((l) => l.title)).toEqual(["old", "new", "mid"]); + }); + + it("keeps only the three newest so the cards fill one row", () => { + const launches = [ + launch("oldest", "2026-08-01"), + launch("a", "2026-10-03"), + launch("b", "2026-10-02"), + launch("c", "2026-10-01"), + ]; + expect(latestLaunches(launches).map((l) => l.title)).toEqual(["a", "b", "c"]); + }); +}); + +describe("launchIcon", () => { + it("maps a known icon name to its lucide icon", () => { + expect(launchIcon("box")).toBe(Box); + }); + + it("falls back to Sparkles for an icon name this dashboard version does not know", () => { + expect(launchIcon("hologram")).toBe(Sparkles); + expect(launchIcon("constructor")).toBe(Sparkles); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts index 41d1aac25de..c3eb4b2c838 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/homeContent.ts @@ -1,40 +1,26 @@ -import { Box, Scale, Zap, type LucideIcon } from "lucide-react"; +import { Bot, Box, Gauge, Plug, Rocket, Scale, Shield, Sparkles, Zap, type LucideIcon } from "lucide-react"; +import type { components } from "@/lib/http/schema"; -export interface Launch { - readonly icon: LucideIcon; - readonly title: string; - readonly description: string; - readonly href: string; - readonly publishedOn: string; -} +export type WhatsNewLaunch = components["schemas"]["WhatsNewLaunch"]; -const LAUNCHES: readonly Launch[] = [ - { - icon: Box, - title: "Claude Haiku 5.5", - description: "Anthropic's newest small model, with day 0 pricing on Anthropic, Bedrock and Vertex AI.", - href: "https://docs.litellm.ai/blog/claude-haiku-5-5", - publishedOn: "2026-10-07", - }, - { - icon: Scale, - title: "Decision Models", - description: "Call decision models at /v1/decisions and try them in the Decisions playground.", - href: "https://docs.litellm.ai/docs/decisions", - publishedOn: "2026-10-07", - }, - { - icon: Zap, - title: "OpenAI Ultrafast", - description: "service_tier: ultrafast on GPT-6 Astra and GPT-6.1 Sol, including /ultrafast in Codex.", - href: "https://docs.litellm.ai/docs/providers/openai/ultrafast", - publishedOn: "2026-10-06", - }, -]; +const LAUNCH_ICONS: ReadonlyMap = new Map([ + ["bot", Bot], + ["box", Box], + ["gauge", Gauge], + ["plug", Plug], + ["rocket", Rocket], + ["scale", Scale], + ["shield", Shield], + ["sparkles", Sparkles], + ["zap", Zap], +]); -export const WHATS_NEW_ITEMS: readonly Launch[] = [...LAUNCHES].sort((a, b) => - b.publishedOn.localeCompare(a.publishedOn), -); +export const launchIcon = (name: string): LucideIcon => LAUNCH_ICONS.get(name) ?? Sparkles; + +export const WHATS_NEW_MAX = 3; + +export const latestLaunches = (launches: readonly WhatsNewLaunch[]): readonly WhatsNewLaunch[] => + [...launches].sort((a, b) => b.published_on.localeCompare(a.published_on)).slice(0, WHATS_NEW_MAX); export const formatPublishedOn = (isoDate: string): string => new Date(`${isoDate}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useWhatsNew.ts b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useWhatsNew.ts new file mode 100644 index 00000000000..c2171f3349a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/useWhatsNew.ts @@ -0,0 +1,10 @@ +import { $api } from "@/lib/http/api"; +import { latestLaunches } from "./homeContent"; + +export const useWhatsNew = () => + $api.useQuery( + "get", + "/public/whats_new", + {}, + { staleTime: 60 * 60 * 1000, select: (data) => latestLaunches(data.launches) }, + ); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c56a4b3786c..61354f8d9fa 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -13658,6 +13658,31 @@ export interface paths { patch?: never; trace?: never; }; + "/public/whats_new": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Public Whats New + * @description Return the launches the dashboard Home page shows under What's new. + * + * Resolved once per process: fetched from ``litellm.whats_new_url`` (override with ``LITELLM_WHATS_NEW_URL``) + * on the first request, falling back to the list bundled with the package on any failure, so an airgapped + * proxy serves the bundled list without retrying. Set ``LITELLM_LOCAL_WHATS_NEW=True`` to skip the fetch. + * A restart picks up a newly published list. + */ + get: operations["get_public_whats_new_public_whats_new_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/queue/chat/completions": { parameters: { query?: never; @@ -49245,6 +49270,30 @@ export interface components { type: "web_search" | "web_search_2025_08_26"; user_location?: components["schemas"]["openai__types__responses__web_search_tool_param__UserLocation"] | null; }; + /** + * WhatsNewLaunch + * @description One launch card in the dashboard Home page's What's new section. + */ + WhatsNewLaunch: { + /** Description */ + description: string; + /** Href */ + href: string; + /** Icon */ + icon: string; + /** + * Published On + * Format: date + */ + published_on: string; + /** Title */ + title: string; + }; + /** WhatsNewResponse */ + WhatsNewResponse: { + /** Launches */ + launches: components["schemas"]["WhatsNewLaunch"][]; + }; /** WorkerRegistryEntry */ WorkerRegistryEntry: { /** Name */ @@ -68630,6 +68679,26 @@ export interface operations { }; }; }; + get_public_whats_new_public_whats_new_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WhatsNewResponse"]; + }; + }; + }; + }; async_queue_request_queue_chat_completions_post: { parameters: { query?: { From 2ab80ba4facdfa52c0e231a7b38df0ad726a2ec3 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sat, 10 Oct 2026 20:49:59 -0700 Subject: [PATCH 5/7] feat(ui): streamline auto-router setup with deployed decision models (#45913) * refactor(ui): simplify Jev auto-router creation * refactor(ui): polish auto-router classifier presentation * refactor(ui): clarify auto-router configuration sections * refactor(ui): align router templates and optional settings * feat(router): support deployed decision classifiers * refactor(ui): select onboarded decision models * style(ui): remove divider before router advanced settings * test(ui): cover decision deployment catalog selection * fix(router): tighten decision adapter types and regression checks * fix(ui): preserve deployed decision model editing * test(ui): use supported decision picker query options * style(ui): format decision model selection regression * test(router): verify deployed classifier cost and provider accounting --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../auto_router_permissions.py | 1 + .../complexity_router/complexity_router.py | 20 +- .../complexity_router/config.py | 12 +- .../complexity_router/jev_classifier.py | 81 ++++- .../router_utils/auto_router_model_naming.py | 10 +- .../test_auto_router_permissions.py | 23 ++ .../complexity_router/test_jev_classifier.py | 150 ++++++++- .../test_auto_router_model_naming.py | 17 ++ .../AutoRouters/AutoRoutersPanel.tsx | 5 +- ...oRouterClassifierTabs.integration.test.tsx | 9 +- .../add_model/AutoRouterClassifierTabs.tsx | 87 +++--- .../add_model/ClassificationMethodConfig.tsx | 10 +- .../add_model/ClassifierPrimarySettings.tsx | 68 +++-- .../ComplexityRouterAdvancedSections.tsx | 7 +- .../add_model/ComplexityRouterConfig.tsx | 99 ++++-- .../add_model/DecisionModelSelect.tsx | 62 ++++ .../JevClassifierConfig.integration.test.tsx | 14 +- .../add_model/JevClassifierConfig.tsx | 146 +++++---- .../add_model/NonReasoningTierToggle.tsx | 2 +- .../components/add_model/RoutingOptions.tsx | 12 +- .../add_auto_router_tab.integration.test.tsx | 211 ++++++++++--- .../add_model/add_auto_router_tab.tsx | 285 +++++++++--------- .../components/add_model/auto_setup.test.ts | 44 ++- .../src/components/add_model/auto_setup.ts | 15 + .../add_model/jev_classifier_config.test.ts | 21 ++ .../add_model/jev_classifier_config.ts | 16 +- ...dit_auto_router_modal.integration.test.tsx | 55 ++++ .../edit_auto_router_modal.tsx | 51 +++- .../src/lib/autorouter_presets.test.ts | 18 ++ .../src/lib/autorouter_presets.ts | 24 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 31 files changed, 1188 insertions(+), 389 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/DecisionModelSelect.tsx diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index 603b36d9406..a4428eb3a85 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -76,6 +76,7 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid") + deployment_name: str | None = None provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" model: str api_key: None = None diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a5f00aa7e25..7ef31f7a0a2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -123,6 +123,7 @@ from .jev_classifier import ( HttpJevClassifierClient, JevClassifierClient, JevVerdict, + RouterJevClassifierClient, build_jev_request, jev_classifier_cost, ) @@ -1331,8 +1332,9 @@ class ComplexityRouter(CustomLogger): - Question complexity (multiple questions) """ - @staticmethod - def _build_jev_client(config: OpenSourceClassifierConfig) -> JevClassifierClient: + def _build_jev_client(self, config: OpenSourceClassifierConfig) -> JevClassifierClient: + if config.deployment_name is not None: + return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name) if config.provider in ("laya", "bespoke"): from litellm.llms.oss_decision import oss_connection @@ -2318,13 +2320,23 @@ class ComplexityRouter(CustomLogger): if not self._tier_pools().get(tier_name): raise ValueError(f"Jev classifier returned tier {tier_name!r}, which has no models configured") model: Final = response.model or config.model - accounting_provider: Final = "typesafe" if config.provider == "jev" else config.provider + accounting_provider: Final = ( + response.classifier_provider + if config.deployment_name and response.classifier_provider + else "typesafe" + if config.provider == "jev" + else config.provider + ) verdict: Final = JevVerdict( label=answer.choice, probabilities=answer.probabilities, confidence=answer.confidence, model=model, - cost=jev_classifier_cost(response, config.model, accounting_provider), + cost=response.classifier_cost + if config.deployment_name + else jev_classifier_cost( + response, config.model, "typesafe" if config.provider == "jev" else config.provider + ), provider=accounting_provider, ) if breaker is not None and permit is not None: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 36c19e37b8d..992c098bc80 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -716,6 +716,7 @@ _ENVIRONMENT_KEY_SCOPE: Final = MappingProxyType( class OpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True) + deployment_name: str | None = Field(default=None, min_length=1) provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" model: str = "jev-latest" api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers") @@ -754,6 +755,12 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): @model_validator(mode="after") def _keep_the_environment_key_on_the_environment_base(self) -> "OpenSourceClassifierConfig": + if self.deployment_name is not None: + if not self.deployment_name.strip(): + raise ValueError("deployment_name must name a configured decision model") + if self.api_key is not None or self.api_base is not None: + raise ValueError("deployment_name uses the deployment connection; omit api_key and api_base") + return self if self.provider in ("laya", "bespoke"): from litellm.llms.oss_decision import validate_oss_api_base, validate_oss_model @@ -833,7 +840,10 @@ def _resolve_normalized_complexity_router_config_write( { key: value for key, value in existing.items() - if same_provider and key in ("api_key", "api_base") and (key != "api_key" or same_base) + if same_provider + and not classifier.get("deployment_name") + and key in ("api_key", "api_base") + and (key != "api_key" or same_base) } ) return ComplexityRouterConfigWrite( diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 65483b0a631..b8b3b4aadba 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from types import MappingProxyType -from typing import Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias +from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias from uuid import uuid4 import httpx @@ -10,6 +10,7 @@ from pydantic import ConfigDict, Field, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_router_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.internal_call_metadata import ( effective_turn_off_message_logging, forwarded_internal_call_metadata, @@ -22,9 +23,13 @@ from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISION from litellm.llms.laya.common_utils import laya_response_model from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS +from litellm.types.decisions import DecisionsResponse from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +if TYPE_CHECKING: + from litellm.router import Router + JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"] DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS @@ -67,6 +72,8 @@ class JevSystemOneResponse(LiteLLMBaseModel): model: str | None = None answers: Mapping[str, JevChoiceAnswer] + classifier_cost: float | None = None + classifier_provider: str | None = None usage: JevUsage | None = None @@ -79,6 +86,76 @@ class JevClassifierClient(Protocol): ) -> JevSystemOneResponse: ... +class _DecisionsRouter(Protocol): + async def adecisions( + self, + *, + model: str, + state: str, + questions: Mapping[str, object], + timeout: float, + num_retries: int, + disable_fallbacks: bool, + metadata: Mapping[str, object], + proxy_server_request: Mapping[str, object], + turn_off_message_logging: bool | None, + litellm_session_id: str = "", + litellm_trace_id: str = "", + ) -> object: ... + + +_PARENT_METADATA: Final = TypeAdapter[Mapping[str, object] | None](Mapping[str, object] | None) + + +class RouterJevClassifierClient: + def __init__(self, router: "Router", deployment_name: str) -> None: + self._router: _DecisionsRouter = router + self._deployment_name = deployment_name + + async def evaluate( + self, + request: JevSystemOneRequest, + timeout_s: float, + request_kwargs: Mapping[str, object] | None = None, + ) -> JevSystemOneResponse: + parent: Final = request_kwargs or MappingProxyType({}) + metadata: Final = { + **forwarded_internal_call_metadata( + _PARENT_METADATA.validate_python(parent.get("litellm_metadata") or parent.get("metadata")), + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + ), + INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + } + payload: Final = { + "state": request.state, + "questions": {key: value.model_dump() for key, value in request.questions.items()}, + } + response: Final = DecisionsResponse.model_validate( + await self._router.adecisions( + model=self._deployment_name, + state=request.state, + questions={key: value.model_dump() for key, value in request.questions.items()}, + timeout=timeout_s, + num_retries=0, + disable_fallbacks=True, + metadata=metadata, + proxy_server_request={ + "originating_request_masked": masked_originating_request(request_kwargs), + "body": {"model": self._deployment_name, **payload}, + }, + turn_off_message_logging=effective_turn_off_message_logging(request_kwargs), + **parent_session_kwargs(request_kwargs), + ) + ) + return JevSystemOneResponse.model_validate( + { + **response.model_dump(), + "classifier_cost": response.hidden_params.get("response_cost"), + "classifier_provider": response.hidden_params.get("custom_llm_provider"), + } + ) + + class HttpJevClassifierClient: def __init__( self, @@ -212,7 +289,7 @@ class JevVerdict(NamedTuple): confidence: float model: str cost: float | None - provider: ClassifierProvider = "typesafe" + provider: str = "typesafe" class _RegistryPricing(LiteLLMBaseModel): diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index 7ace4094cf0..3d78af94cde 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -169,9 +169,13 @@ def strategy_router_dependencies( else () ) + ( - _named( - f"{accounting_provider}/{decision_classifier.get('model', 'jev-latest')}", - "evaluation", + ( + _named(decision_classifier.get("deployment_name"), "classifier") + if decision_classifier.get("deployment_name") + else _named( + f"{accounting_provider}/{decision_classifier.get('model', 'jev-latest')}", + "evaluation", + ) ) if complexity.get("classifier_type") == "oss_classifier" else () diff --git a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py index 5ef1399b0cb..16ddf009b0b 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py @@ -427,3 +427,26 @@ async def test_jev_evaluation_obeys_each_containing_scope( return await operation assert not catalog.get_model_list(f"{provider}/{model}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("restricted", ["key", "team", None]) +async def test_deployed_decision_classifier_requires_key_and_team_access(catalog: Router, restricted: str | None) -> None: + permitted: Final = ["allowed", "other"] + operation: Final = authorize_member_auto_router_dependencies( + config=validate_member_auto_router_config({ + "tiers": {"SIMPLE": "allowed"}, + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"deployment_name": "other"}, + }), + default_model=None, + user_api_key_dict=_actor(models=["allowed"] if restricted == "key" else permitted), + team=_team(models=["allowed"] if restricted == "team" else permitted), + prisma_client=_Client(), + llm_router=catalog, + ) + if restricted is not None: + with pytest.raises(ProxyException, match="other"): + await operation + return + await operation diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index fea283cbe9c..c1c96588004 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -573,7 +573,11 @@ async def test_databricks_routes_through_the_serving_endpoint_and_accounts_the_e "jev_classifier_config" if legacy else "opensource_classifier_config": { "provider": "databricks", "model": endpoint, - **({"api_key": "dapi-configured", "api_base": "https://workspace.test/serving-endpoints"} if configured_key else {}), + **( + {"api_key": "dapi-configured", "api_base": "https://workspace.test/serving-endpoints"} + if configured_key + else {} + ), }, "tiers": {"SIMPLE": "cheap"}, }, @@ -596,7 +600,9 @@ async def test_databricks_routes_through_the_serving_endpoint_and_accounts_the_e assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("databricks", endpoint) assert outcome.classifier_cost == pytest.approx(0.31) sent: Final = route.calls.last.request - assert sent.headers.get("authorization") == ("Bearer dapi-configured" if configured_key else "Bearer dapi-env-token") + assert sent.headers.get("authorization") == ( + "Bearer dapi-configured" if configured_key else "Bearer dapi-env-token" + ) assert json.loads(sent.content)["model"] == endpoint assert len(recorder.calls) == 1 assert recorder.calls[0]["response_cost"] == pytest.approx(0.31) @@ -765,3 +771,143 @@ async def test_http_jev_classifier_client_posts_to_system_one() -> None: assert captured["content_type"] == "application/json" assert captured["body"] == request.model_dump(mode="json") assert response.model == "jev-1.13.0" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [200, 503]) +@pytest.mark.parametrize("provider,model", [("typesafe", "jev-latest"), ("databricks", "my-endpoint")]) +async def test_deployed_decision_classifier_uses_registered_connection( + status_code: int, provider: str, model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem(litellm.model_cost, f"{provider}/{model}", {"input_cost_per_token": 0.01}) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decision-deployment", + "litellm_params": { + "model": f"{provider}/{model}", + "api_key": "deployment-key", + "api_base": "https://decision.test/serving-endpoints" + if provider == "databricks" + else "https://decision.test", + }, + } + ], + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "deployed-classifier", + router, + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as requests: + route: Final = requests.post( + "https://decision.test/serving-endpoints/my-endpoint/invocations" + if provider == "databricks" + else "https://decision.test/v1/systemone" + ).mock( + return_value=httpx.Response( + status_code, + json={ + "model": model, + "answers": {"tier": _answer().model_dump()}, + "usage": {"input_tokens": 31, "output_tokens": 0}, + }, + ) + ) + outcome: Final = await classifier.aclassify("What is two plus two?") + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(route.calls[0].request.content)["model"] == model + assert (outcome.cause == "jev_classifier") is (status_code == 200) + if status_code == 200: + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.label == _answer().choice + + assert outcome.jev_verdict.provider == provider + assert outcome.classifier_cost == pytest.approx(0.31) + + +@pytest.mark.parametrize("deployment_name", ["", " "]) +def test_deployed_decision_classifier_rejects_blank_deployment_name(deployment_name: str) -> None: + with pytest.raises(ValueError, match="deployment_name"): + JevClassifierConfig.model_validate({"deployment_name": deployment_name}) + + +@pytest.mark.parametrize("connection", [{"api_key": "secret"}, {"api_base": "https://override.test"}]) +def test_deployed_decision_classifier_rejects_connection_overrides(connection: Mapping[str, str]) -> None: + with pytest.raises(ValueError, match="deployment connection"): + JevClassifierConfig.model_validate({"deployment_name": "deployed", **connection}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["timeout", "cancelled", "invalid"]) +async def test_deployed_decision_classifier_preserves_failure_handling( + failure: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decision-deployment", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "test", + "api_base": "https://decision.test", + }, + } + ], + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "decision-router", + router, + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + ) + cancellation_reached: Final = asyncio.Event() + with respx.mock(assert_all_called=failure != "cancelled") as upstream: + route: Final = upstream.post("https://decision.test/v1/systemone") + if failure == "timeout": + route.mock(side_effect=httpx.ReadTimeout("classifier timeout")) + elif failure == "cancelled": + + async def cancel_request(request: httpx.Request) -> httpx.Response: + cancellation_reached.set() + raise asyncio.CancelledError() + + route.mock(side_effect=cancel_request) + else: + route.respond(json={"answers": {"tier": _answer("not-a-tier").model_dump()}}) + if failure == "cancelled": + with pytest.raises(asyncio.CancelledError): + await classifier.aclassify("short question") + assert cancellation_reached.is_set() + return + outcome: Final = await classifier.aclassify("short question") + assert outcome.cause != "jev_classifier" + assert outcome.jev_verdict is None + + +def test_switching_to_deployed_classifier_does_not_copy_legacy_credentials() -> None: + from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write + + write: Final = resolve_complexity_router_config_write( + {"classifier_type": "oss_classifier", "opensource_classifier_config": {"deployment_name": "registered"}}, + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"api_key": "legacy-secret", "api_base": "https://legacy.test"}, + }, + ) + assert write.effective is not None + assert write.effective["opensource_classifier_config"] == {"deployment_name": "registered"} diff --git a/tests/unit/router_utils/test_auto_router_model_naming.py b/tests/unit/router_utils/test_auto_router_model_naming.py index d7016cbc9ec..1cfa3aa5c91 100644 --- a/tests/unit/router_utils/test_auto_router_model_naming.py +++ b/tests/unit/router_utils/test_auto_router_model_naming.py @@ -755,3 +755,20 @@ def test_chained_heuristic_and_judge_prompt_claim_each_active_capability( assert count_capability_routers(({"litellm_params": params},), capability=capability) == ( 1 if capability.key in expected else 0 ) + + +def test_deployed_decision_classifier_is_a_model_access_dependency() -> None: + dependencies: Final = strategy_router_dependencies( + { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap"}, + }, + } + ) + assert tuple((dependency.model_name, dependency.role) for dependency in dependencies) == ( + ("cheap", "tier"), + ("decision-deployment", "classifier"), + ) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 607b676d304..6f7e04f92c6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -100,7 +100,10 @@ export function AutoRoutersPanel({ {/* The form is long, so the dialog caps its height and scrolls its body rather than growing past the viewport. */} - + Add Auto Router diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index 0d3e7cfe54b..3c5d555cff8 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -105,7 +105,7 @@ describe("Auto-router classifier selection", () => { llm: "LLM", heuristic_first: "LLM", hybrid: "LLM", - jev: "OSS Classifier", + jev: "Decisions Model", }[classifier_type]; expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked(); fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })); @@ -191,17 +191,14 @@ describe("Auto-router classifier selection", () => { expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router"); }); - it("shows Jev's single Complexity approach without changing saved configuration", () => { + it("omits Jev's redundant routing approach without changing saved configuration", () => { const onChange = vi.fn(); renderWithProviders( Existing settings , ); - expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited"); - fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); - expect(screen.getAllByRole("menuitemradio")).toHaveLength(1); - fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ })); + expect(screen.queryByRole("button", { name: "Routing approach" })).not.toBeInTheDocument(); expect(onChange).not.toHaveBeenCalled(); }); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index f21dc3d795a..509329bb948 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -1,7 +1,9 @@ import React, { useContext, useId } from "react"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Label } from "@/components/ui/label"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; -import { ChevronDownIcon } from "lucide-react"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { ChevronDownIcon, Info } from "lucide-react"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -124,9 +126,17 @@ interface AutoRouterClassifierTabsProps { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; children: React.ReactNode; + judgeModel?: React.ReactNode; + decisionModel?: React.ReactNode; } -const AutoRouterClassifierTabs: React.FC = ({ value, onChange, children }) => { +const AutoRouterClassifierTabs: React.FC = ({ + value, + onChange, + children, + judgeModel, + decisionModel, +}) => { const id = useId(); const availability = useContext(AutoRouterAvailabilityContext); const classifierType = effectiveClassifierType(value); @@ -167,10 +177,17 @@ const AutoRouterClassifierTabs: React.FC = ({ val llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model", }; return ( -
+
- - What classifies your requests? + +
+

Choose your classifier

+ + + +
@@ -179,8 +196,8 @@ const AutoRouterClassifierTabs: React.FC = ({ val { value: "llm", label: "LLM", description: "Use a judge model to choose a solver" }, { value: "jev", - label: "OSS Classifier", - description: "Use Jev, Laya, Bespoke Nimble, or Databricks to choose a tier", + label: "Decisions Model", + description: "Use a decision model to choose a tier", }, ].map((option) => (
- {family === "jev" && ( -
- OSS provider - - - - - - -
- )} + {family === "jev" && + (decisionModel ?? ( +
+ + +
+ ))} {family === "custom" && (

This router uses a custom classifier plugin

)} @@ -268,7 +284,8 @@ const AutoRouterClassifierTabs: React.FC = ({ val

)} - {(family === "llm" || family === "jev") && ( + {family === "llm" && judgeModel} + {family === "llm" && (
= }; interface ClassificationMethodConfigProps { + jevSection?: "all" | "advanced"; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; @@ -204,6 +205,7 @@ const ClassificationMethodConfig: React.FC = ({ showValidationErrors = false, defaultModel, advancedOnly = false, + jevSection, section, }) => { const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); @@ -522,7 +524,9 @@ const ClassificationMethodConfig: React.FC = ({
)} - {showClassifier && classifierType === "jev" && } + {showClassifier && classifierType === "jev" && ( + + )} {showClassifier && usesLlmClassifier(classifierType) && (
= ({ /> Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders. - LLM and OSS classifiers default to 3 turns. Set to 0 to omit conversation history. The current message and - selected system text are still sent. + LLM and decision model classifiers default to 3 turns. Set to 0 to omit conversation history. The current + message and selected system text are still sent.
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx index 32cb4851580..cf5e7e2db03 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx @@ -18,11 +18,13 @@ export default function ClassifierPrimarySettings({ onChange, modelOptions, showValidationErrors = false, + section = "all", }: { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; showValidationErrors?: boolean; + section?: "all" | "judge" | "frequency"; }) { const id = React.useId(); const restriction = restrictedBy(value, "sessionAffinity"); @@ -35,34 +37,44 @@ export default function ClassifierPrimarySettings({ const usesJudge = usesLlmClassifier(effectiveClassifierType(value)); const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model; return ( -
-
- - -

{restriction?.reason ?? frequencyDescription}

-
- {usesJudge && ( +
+ {section !== "judge" && ( +
+ + +

{restriction?.reason ?? frequencyDescription}

+
+ )} + {section !== "frequency" && usesJudge && (
void; forecast: boolean; @@ -59,6 +60,7 @@ interface ComplexityRouterAdvancedSectionsProps { } const ComplexityRouterAdvancedSections: React.FC = ({ + hideClassifier, value, onChange, forecast, @@ -268,7 +270,10 @@ const ComplexityRouterAdvancedSections: React.FC sections.some(({ key }) => group.keys.includes(key))); + ].filter( + (group) => + !(hideClassifier && group.keys.includes("classifier")) && sections.some(({ key }) => group.keys.includes(key)), + ); const [openGroups, setOpenGroups] = React.useState(() => showValidationErrors ? groups.map((group) => group.label) : [], ); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 4e8e981fd4a..72a08c53267 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -37,7 +37,7 @@ import { } from "./tier_rows"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig"; +import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig"; import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -472,6 +472,8 @@ export const withClassificationFrequency = ( }); interface ComplexityRouterConfigProps { + creationFlow?: boolean; + tierSetup?: React.ReactNode; modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -548,7 +550,60 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03; */ export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_ORDER.slice(0, -1); +const isJevCreation = (value: ComplexityRouterConfigValue, creationFlow: boolean | undefined) => + creationFlow && effectiveClassifierType(value) === "jev"; + +const CreationClassifierSettings = ({ + creationFlow, + value, + onChange, + modelOptions, + classifierEffortOptionsByModel, + defaultModel, + showValidationErrors, +}: Pick & { + creationFlow: boolean | undefined; + modelOptions: { value: string; label: string }[]; + classifierEffortOptionsByModel: Record; + defaultModel: string | undefined; + showValidationErrors: boolean; +}) => { + const jevCreation = isJevCreation(value, creationFlow); + return ( + <> + + {jevCreation && ( +
+ +
+ +
+
+
+ )} + + ); +}; + const ComplexityRouterConfig: React.FC = ({ + creationFlow, + tierSetup, modelInfo, value, onChange, @@ -572,6 +627,7 @@ const ComplexityRouterConfig: React.FC = ({ showValidationErrors = false, }) => { const forecast = isForecastClassifier(value.classifier_type); + const tierHeading = creationFlow ? "Define your tiers" : "Models by tier"; const customTierSet = value.custom_tier_set; const tierRows = activeTierRows(value); const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null; @@ -620,19 +676,25 @@ const ComplexityRouterConfig: React.FC = ({ return (
- -
-

{forecast ? "Solver models" : "Models by tier"}

- {!forecast && ( - - - - )} + {creationFlow && } +
+
+

{forecast ? "Solver models" : tierHeading}

+ {!forecast && ( + + + + )} +
@@ -657,16 +719,9 @@ const ComplexityRouterConfig: React.FC = ({ ) : ( <> + {tierSetup} - {!customTierSet && ( - - )} - {tierRows.map((row, index) => { const tierInfo = builtInTierInfo(row.id); const label = tierRowLabel(row, value.tier_labels); @@ -756,6 +811,14 @@ const ComplexityRouterConfig: React.FC = ({ ); })} + {!customTierSet && ( + + )} + = ({ )} - = ({ )}
void; + modelInfo: ModelGroup[]; + deployments: AutoRouterDeployment[] | undefined; +}) { + const id = React.useId(); + const { data: modelCostMapData } = useModelCostMap(true, true); + const decisionCatalog = React.useMemo(() => buildDecisionCatalog(modelCostMapData), [modelCostMapData]); + const decisionModels = modelInfo.filter( + (model) => + isDecisionMode(model.mode) || + (deployments ?? []).some( + (deployment) => + deployment.model_name === model.model_group && + isDecisionSelection( + decisionCatalog, + deployment.litellm_params?.custom_llm_provider ?? deployment.litellm_params?.model?.split("/")[0], + [deployment.litellm_params?.model ?? "", deployment.litellm_params?.base_model ?? ""], + ), + ), + ); + + return ( +
+ + ({ value: model.model_group, label: model.model_group }))} + value={value.jev_classifier_config?.deployment_name ?? ""} + placeholder="Select the decision model" + allowClear={false} + onValueChange={(deploymentName) => { + if (!deploymentName) return; + onChange({ + ...value, + jev_classifier_config: { + ...(value.jev_classifier_config ?? defaultJevClassifierConfig()), + deployment_name: deploymentName, + }, + }); + }} + /> +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index 15a736de7b2..3480a8e922a 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -111,19 +111,19 @@ describe("JEV classifier editor", () => { expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); expect(screen.getByText("Classifier Prompt")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument(); - fireEvent.click(screen.getByRole("radio", { name: /^OSS Classifier$/ })); - expect(screen.getByRole("radio", { name: /^OSS Classifier$/ })).toBeChecked(); + fireEvent.click(screen.getByRole("radio", { name: /^Decisions Model$/ })); + expect(screen.getByRole("radio", { name: /^Decisions Model$/ })).toBeChecked(); expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-latest"); expect(screen.getByLabelText("Classifier Instructions")).toBeEnabled(); expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument(); expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument(); expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument(); - fireEvent.click(screen.getByRole("radio", { name: "Laya" })); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Laya"); expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("english"); - fireEvent.click(screen.getByRole("radio", { name: "Jev" })); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Jev"); expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-latest"); - fireEvent.click(screen.getByRole("radio", { name: label })); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), label); if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest"); if (provider === "databricks") expect(screen.getByLabelText("Classifier Model")).toHaveValue(""); if (typesTheModel) { @@ -137,8 +137,8 @@ describe("JEV classifier editor", () => { fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); fireEvent.click(screen.getByRole("button", { name: "Customize tiers" })); fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); - expect(screen.getByRole("radio", { name: /^OSS Classifier$/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: label })).toBeChecked(); + expect(screen.getByRole("radio", { name: /^Decisions Model$/ })).toBeChecked(); + expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveTextContent(label); if (typesTheModel) expect(screen.getByLabelText("Classifier Model")).toHaveValue(model); else expect(screen.getByLabelText("Classifier Model")).toHaveTextContent(model); expect(screen.getByLabelText("Classifier Timeout (ms)")).toHaveValue(4200); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 86d2d0f8348..31f29dd9e9d 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -21,9 +21,11 @@ const providerDescriptions = { export default function JevClassifierConfig({ value, onChange, + section = "all", }: { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; + section?: "all" | "model" | "advanced"; }) { const id = useId(); const config = value.jev_classifier_config ?? defaultJevClassifierConfig(); @@ -32,75 +34,85 @@ export default function JevClassifierConfig({ onChange({ ...value, jev_classifier_config: { ...config, ...patch } }); return ( -
-

{providerDescriptions[config.provider ?? "jev"]}

-
- - {models ? ( - - ) : ( - + {section !== "advanced" && !config.deployment_name && ( + <> +

{providerDescriptions[config.provider ?? "jev"]}

+
+ + {models ? ( + + ) : ( + update({ model: event.target.value })} + /> + )} +
+ + )} + {section !== "model" && ( + <> +
+ + update({ timeout_ms: Number(event.target.value) })} + /> +
+ + update({ + circuit_breaker_enabled: next.circuit_breaker_enabled, + circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds, + }) } - onChange={(event) => update({ model: event.target.value })} /> - )} -
-
- - update({ timeout_ms: Number(event.target.value) })} - /> -
- - update({ - circuit_breaker_enabled: next.circuit_breaker_enabled, - circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds, - }) - } - /> -
- - -