feat(decisions): accept decisions as a model mode alongside evaluation (#45474)

* refactor(decisions): rename the evaluation model mode to decisions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(decisions): canonicalize the health check mode without rebinding

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(decisions): replace canonical_model_mode with is_decisions_model_mode

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(decisions): preserve evaluation mode compatibility

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore: drop the rebuilt dashboard bundle from this PR

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* style(dashboard): format decision mode helper

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm-rust/crates/model-catalog/src/capabilities.rs

@@ -9,6 +9,7 @@ pub enum Mode {
     AudioTranscription,
     Chat,
     Completion,
+    Decisions,
     Embedding,
     Evaluation,
     Guardrail,

litellm-rust/crates/model-catalog/tests/catalog.rs

@@ -1,6 +1,6 @@
 use std::path::{Path, PathBuf};
 
-use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance};
+use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance};
 use rstest::{fixture, rstest};
 use serde_json::json;
 
@@ -42,6 +42,20 @@ fn fixture_catalog() -> Catalog {
     .unwrap()
 }
 
+#[rstest]
+#[case::decisions("decisions", Mode::Decisions)]
+#[case::legacy_evaluation("evaluation", Mode::Evaluation)]
+fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) {
+    let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string();
+    let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap();
+
+    assert_eq!(
+        catalog.lookup("Test").unwrap().entry.info().mode,
+        Some(expected)
+    );
+    assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode));
+}
+
 #[rstest]
 #[ignore]
 fn preserves_fields_and_metadata(fixture_catalog: Catalog) {

litellm/litellm_core_utils/health_check_helpers.py

@@ -214,7 +214,7 @@ class HealthCheckHelpers:
             "responses",
             "anthropic_messages",
             "ocr",
-            "evaluation",
+            "decisions",
         ],
         Callable,
     ]:
@@ -311,7 +311,7 @@ class HealthCheckHelpers:
                 **_filter_model_params(model_params=model_params),
                 document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
             ),
-            "evaluation": lambda: litellm.asystemone(
+            "decisions": lambda: litellm.asystemone(
                 **DECISIONS_CALL_PARAMS.validate_python(
                     {
                         "state": prompt or "health check",

litellm/main.py

@@ -141,6 +141,7 @@ from litellm.types.utils import (
     ModelResponseStream,
     RawRequestTypedDict,
     StreamingChoices,
+    is_decisions_model_mode,
 )
 from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
 from litellm.utils import (
@@ -8821,8 +8822,9 @@ async def ahealth_check(
             input=input,
         )
 
-        if mode in mode_handlers:
-            _response: Final = await mode_handlers[mode]()
+        handler_mode: Final = "decisions" if is_decisions_model_mode(mode) else mode
+        if handler_mode in mode_handlers:
+            _response: Final = await mode_handlers[handler_mode]()
             _response_headers: Final = cast(  # cast-ok: provider headers are stored as a string-keyed mapping
                 Mapping[str, object], (get_hidden_params(_response) or {}).get("headers", {}) or {}
             )

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -2091,6 +2091,7 @@ async def test_model_connection(
         "anthropic_messages",
         "ocr",
         "evaluation",
+        "decisions",
     ]
     | None = fastapi.Body(
         None,

litellm/types/utils.py

@@ -160,6 +160,10 @@ class LiteLLMCommonStrings(Enum):
     llm_provider_not_provided = "Unmapped LLM provider for this endpoint. You passed model={model}, custom_llm_provider={custom_llm_provider}. Check supported provider and route: https://docs.litellm.ai/docs/providers"
 
 
+def is_decisions_model_mode(mode: str | None) -> bool:
+    return mode in ("decisions", "evaluation")
+
+
 SupportedCacheControls: Final = ["ttl", "s-maxage", "no-cache", "no-store"]
 
 
@@ -430,6 +434,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
             "audio_transcription",
             "audio_speech",
             "responses",
+            "decisions",
             "evaluation",
             "ocr",
             "realtime",

tests/integration/management/test_model_health_check.py

@@ -3,6 +3,7 @@ import uuid
 from typing import Final
 
 import httpx
+import pytest
 from integration._support.client import Gateway, object_value, string_value
 from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
 from integration.cost_calculation.cost_tracking_case import JsonResponse
@@ -93,7 +94,7 @@ def _probes_sent_to(gateway: Gateway, handle: ScenarioHandle) -> list[tuple[str,
     ]
 
 
-def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe(
+def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe(
     gateway: Gateway,
 ) -> None:
     with gateway.scenario() as scenario:
@@ -114,7 +115,8 @@ def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_se
         ]
 
 
-def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway) -> None:
+@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
+def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None:
     with gateway.scenario() as scenario:
         handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _CONFIGURED_PROBE_REPLY)
         scenario.cleanups.callback(delete_scenario, handle)
@@ -122,7 +124,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g
             model="typesafe/jev-custom",
             api_base=handle.api_base(),
             model_info={
-                "mode": "evaluation",
+                "mode": mode,
                 "health_check_params": {"state": _CONFIGURED_STATE, "questions": _CONFIGURED_QUESTIONS},
             },
         )
@@ -136,7 +138,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g
         ]
 
 
-def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map(
+def test_decisions_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map(
     gateway: Gateway,
 ) -> None:
     with gateway.scenario() as scenario:

tests/unit/litellm_core_utils/test_health_check_helpers.py

@@ -589,7 +589,7 @@ def test_ocr_health_check_document_raises_without_the_extension():
         ("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"),
     ),
 )
-async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api(
+async def test_ahealth_check_probes_decisions_models_through_the_decisions_api(
     model: str,
     upstream_url: str,
     monkeypatch: pytest.MonkeyPatch,
@@ -617,7 +617,9 @@ async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api(
 
 
 @pytest.mark.asyncio
-async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions(
+@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
+async def test_ahealth_check_decisions_uses_configured_probe_state_and_questions(
+    mode: str,
     monkeypatch: pytest.MonkeyPatch,
     respx_mock: respx.MockRouter,
 ) -> None:
@@ -638,7 +640,7 @@ async def test_ahealth_check_evaluation_uses_configured_probe_state_and_question
             "state": "custom probe",
             "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}},
         },
-        mode=None,
+        mode=mode,
     )
 
     assert "error" not in result, result

tests/unit/proxy/health_endpoints/test_health_endpoints.py

@@ -882,7 +882,8 @@ async def test_test_model_connection_request_mode_wins_over_resolved_mode():
 
 
 @pytest.mark.asyncio
-async def test_test_model_connection_evaluation_mode_uses_decisions_handler():
+@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
+async def test_test_model_connection_decision_modes_are_forwarded(mode: str):
     deployment: Final = MappingProxyType(
         {
             "model_name": "typesafe/jev-latest",
@@ -893,14 +894,14 @@ async def test_test_model_connection_evaluation_mode_uses_decisions_handler():
     with _test_connection_probe(deployment) as ahealth_check:
         result: Final = await health_test_model_connection(
             request=MagicMock(),
-            mode="evaluation",
+            mode=mode,
             litellm_params={"model": "typesafe/jev-latest"},
             model_info={"id": "typesafe-jev-id"},
             user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"),
         )
 
     assert result["status"] == "success"
-    assert ahealth_check.call_args.kwargs["mode"] == "evaluation"
+    assert ahealth_check.call_args.kwargs["mode"] == mode
 
 
 @pytest.mark.asyncio
@@ -5011,7 +5012,8 @@ def test_test_model_connection_accepts_image_edit_mode(monkeypatch):
     assert response.json()["status"] == "success"
 
 
-def test_test_model_connection_accepts_evaluation_mode(monkeypatch):
+@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
+def test_test_model_connection_accepts_decision_modes(monkeypatch, mode: str):
     monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
     litellm.in_memory_llm_clients_cache.flush_cache()
 
@@ -5036,7 +5038,7 @@ def test_test_model_connection_accepts_evaluation_mode(monkeypatch):
         response = client.post(
             "/health/test_connection",
             json={
-                "mode": "evaluation",
+                "mode": mode,
                 "litellm_params": {"model": "typesafe/jev-latest", "api_key": "sk-test"},
             },
         )

tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py

@@ -140,6 +140,7 @@ def test_success_handler_dispatches_to_typesafe_handler():
 
 
 @pytest.mark.asyncio
+@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
 @pytest.mark.parametrize("guardrail_cost", [0.0, 0.25])
 @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"])
 @pytest.mark.parametrize("provider,requested,routing_model", [
@@ -149,7 +150,7 @@ def test_success_handler_dispatches_to_typesafe_handler():
 ])
 async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost(
     monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float,
-    provider: str, requested: str
+    provider: str, requested: str, mode: str
 ) -> None:
     checkpoint: Final = routing_model or requested
     model: Final = f"{provider}/{checkpoint}"
@@ -157,7 +158,7 @@ async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost(
     output_rate: Final = 0.005
     monkeypatch.setitem(litellm.model_cost, model, {
         "input_cost_per_token": input_rate, "output_cost_per_token": output_rate,
-        "litellm_provider": provider, "mode": "evaluation",
+        "litellm_provider": provider, "mode": mode,
     })
     start: Final = datetime.now()
     logging_obj: Final = Logging(

tests/unit/test_model_prices_schema.py

@@ -637,7 +637,7 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path):
 
 
 PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"})
-MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"})
+MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "decisions", "evaluation"})
 VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset(
     {
         "vertex_ai-ai21_models",

tests/unit/test_utils.py

@@ -8403,7 +8403,7 @@ def test_models_by_provider():
             or v["litellm_provider"] == "bedrock_converse"
         ):
             continue
-        elif v.get("mode") in ("search", "evaluation"):
+        elif v.get("mode") in ("search", "decisions", "evaluation"):
             continue
         else:
             providers.add(v["litellm_provider"])

tests/unit/types/test_types_utils.py

@@ -29,6 +29,7 @@ from litellm.types.utils import (
     ModelResponseStream,
     TextCompletionResponse,
     all_litellm_params,
+    is_decisions_model_mode,
     text_tokens_without_nested_reasoning,
 )
 
@@ -41,6 +42,14 @@ class _DictHiddenParamsAccessor(Protocol):
     def hidden_params(self, hidden_params: dict[str, object]) -> None: ...
 
 
+@pytest.mark.parametrize(
+    ("mode", "expected"),
+    (("decisions", True), ("evaluation", True), ("chat", False), (None, False)),
+)
+def test_is_decisions_model_mode(mode: str | None, expected: bool) -> None:
+    assert is_decisions_model_mode(mode) is expected
+
+
 def test_rust_is_a_known_litellm_param():
     assert "rust" in all_litellm_params
 

ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx

@@ -506,9 +506,9 @@ describe("provider_info_helpers", () => {
     it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => {
       const modelMap = {
         "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
-        "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" },
+        "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "decisions" },
         "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" },
-        "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" },
+        "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" },
       };
       expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([
         "typesafe/jev-latest",
@@ -518,7 +518,7 @@ describe("provider_info_helpers", () => {
 
     it("should list only strands_decider models for the 'StrandsDecider' provider key", () => {
       const modelMap = {
-        "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" },
+        "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" },
         "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
         "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" },
       };

ui/litellm-dashboard/src/lib/decisionModels.test.ts

@@ -17,7 +17,8 @@ const COST_MAP = {
 };
 
 describe("isDecisionMode", () => {
-  it("is true only for the evaluation mode", () => {
+  it("is true for both the decisions and evaluation modes", () => {
+    expect(isDecisionMode("decisions")).toBe(true);
     expect(isDecisionMode("evaluation")).toBe(true);
     expect(isDecisionMode("chat")).toBe(false);
     expect(isDecisionMode(null)).toBe(false);

ui/litellm-dashboard/src/lib/decisionModels.ts

@@ -25,7 +25,8 @@ export interface DecisionCatalog {
 
 export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), providers: new Map() };
 
-export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "evaluation";
+export const isDecisionMode = (mode: string | null | undefined): boolean =>
+  mode === "decisions" || mode === "evaluation";
 
 const isDecisionEntry = (entry: CostMapEntry): boolean =>
   isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint));

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -27208,7 +27208,7 @@ export interface components {
              * Mode
              * @description The mode to test the model with. If not provided, resolved the way /health does: the deployment's model_info.mode (only while the request tests the deployment's own model), then the mode the provider requires for that model, then the model cost map.
              */
-            mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation") | null;
+            mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation" | "decisions") | null;
             /**
              * Model Info
              * @description Model info for the health check