From 4c532a571364cfb0a8e9a2b9a35324b46bc898d4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:18:53 -0700 Subject: [PATCH] feat(decisions): accept decisions as a model mode alongside evaluation (#45474) * refactor(decisions): rename the evaluation model mode to decisions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): canonicalize the health check mode without rebinding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(decisions): replace canonical_model_mode with is_decisions_model_mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): preserve evaluation mode compatibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: drop the rebuilt dashboard bundle from this PR Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(dashboard): format decision mode helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/model-catalog/src/capabilities.rs | 1 + .../crates/model-catalog/tests/catalog.rs | 16 +++++++++++++++- .../litellm_core_utils/health_check_helpers.py | 4 ++-- litellm/main.py | 6 ++++-- .../proxy/health_endpoints/_health_endpoints.py | 1 + litellm/types/utils.py | 5 +++++ .../management/test_model_health_check.py | 10 ++++++---- .../test_health_check_helpers.py | 8 +++++--- .../health_endpoints/test_health_endpoints.py | 12 +++++++----- .../test_typesafe_passthrough_logging_handler.py | 5 +++-- tests/unit/test_model_prices_schema.py | 2 +- tests/unit/test_utils.py | 2 +- tests/unit/types/test_types_utils.py | 9 +++++++++ .../components/provider_info_helpers.test.tsx | 6 +++--- .../src/lib/decisionModels.test.ts | 3 ++- ui/litellm-dashboard/src/lib/decisionModels.ts | 3 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 17 files changed, 68 insertions(+), 27 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/capabilities.rs b/litellm-rust/crates/model-catalog/src/capabilities.rs index 3df68fabc4d..30724766aeb 100644 --- a/litellm-rust/crates/model-catalog/src/capabilities.rs +++ b/litellm-rust/crates/model-catalog/src/capabilities.rs @@ -9,6 +9,7 @@ pub enum Mode { AudioTranscription, Chat, Completion, + Decisions, Embedding, Evaluation, Guardrail, diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index d87de3c5b3b..d384dfea9a8 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance}; +use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance}; use rstest::{fixture, rstest}; use serde_json::json; @@ -42,6 +42,20 @@ fn fixture_catalog() -> Catalog { .unwrap() } +#[rstest] +#[case::decisions("decisions", Mode::Decisions)] +#[case::legacy_evaluation("evaluation", Mode::Evaluation)] +fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) { + let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string(); + let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap(); + + assert_eq!( + catalog.lookup("Test").unwrap().entry.info().mode, + Some(expected) + ); + assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode)); +} + #[rstest] #[ignore] fn preserves_fields_and_metadata(fixture_catalog: Catalog) { diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 3cbc2d770cc..03af9076c79 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -214,7 +214,7 @@ class HealthCheckHelpers: "responses", "anthropic_messages", "ocr", - "evaluation", + "decisions", ], Callable, ]: @@ -311,7 +311,7 @@ class HealthCheckHelpers: **_filter_model_params(model_params=model_params), document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider), ), - "evaluation": lambda: litellm.asystemone( + "decisions": lambda: litellm.asystemone( **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", diff --git a/litellm/main.py b/litellm/main.py index c2c6d5affe8..474d82e8ef1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -141,6 +141,7 @@ from litellm.types.utils import ( ModelResponseStream, RawRequestTypedDict, StreamingChoices, + is_decisions_model_mode, ) from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS from litellm.utils import ( @@ -8821,8 +8822,9 @@ async def ahealth_check( input=input, ) - if mode in mode_handlers: - _response: Final = await mode_handlers[mode]() + handler_mode: Final = "decisions" if is_decisions_model_mode(mode) else mode + if handler_mode in mode_handlers: + _response: Final = await mode_handlers[handler_mode]() _response_headers: Final = cast( # cast-ok: provider headers are stored as a string-keyed mapping Mapping[str, object], (get_hidden_params(_response) or {}).get("headers", {}) or {} ) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 261bf99a6f2..fa58fda8f47 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2091,6 +2091,7 @@ async def test_model_connection( "anthropic_messages", "ocr", "evaluation", + "decisions", ] | None = fastapi.Body( None, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f89ac614e8..f7385a6d92d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -160,6 +160,10 @@ class LiteLLMCommonStrings(Enum): llm_provider_not_provided = "Unmapped LLM provider for this endpoint. You passed model={model}, custom_llm_provider={custom_llm_provider}. Check supported provider and route: https://docs.litellm.ai/docs/providers" +def is_decisions_model_mode(mode: str | None) -> bool: + return mode in ("decisions", "evaluation") + + SupportedCacheControls: Final = ["ttl", "s-maxage", "no-cache", "no-store"] @@ -430,6 +434,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "audio_transcription", "audio_speech", "responses", + "decisions", "evaluation", "ocr", "realtime", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 075c6051353..ef8412a6ed1 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -3,6 +3,7 @@ import uuid from typing import Final import httpx +import pytest from integration._support.client import Gateway, object_value, string_value from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse @@ -93,7 +94,7 @@ def _probes_sent_to(gateway: Gateway, handle: ScenarioHandle) -> list[tuple[str, ] -def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe( +def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_default_probe( gateway: Gateway, ) -> None: with gateway.scenario() as scenario: @@ -114,7 +115,8 @@ def test_evaluation_mode_health_check_resolves_the_mode_from_the_cost_map_and_se ] -def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway) -> None: +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None: with gateway.scenario() as scenario: handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _CONFIGURED_PROBE_REPLY) scenario.cleanups.callback(delete_scenario, handle) @@ -122,7 +124,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g model="typesafe/jev-custom", api_base=handle.api_base(), model_info={ - "mode": "evaluation", + "mode": mode, "health_check_params": {"state": _CONFIGURED_STATE, "questions": _CONFIGURED_QUESTIONS}, }, ) @@ -136,7 +138,7 @@ def test_evaluation_mode_health_check_sends_the_configured_state_and_questions(g ] -def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map( +def test_decisions_mode_health_check_of_the_self_hosted_strands_model_resolves_the_mode_from_the_cost_map( gateway: Gateway, ) -> None: with gateway.scenario() as scenario: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 9903ab71876..123be0bd427 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -589,7 +589,7 @@ def test_ocr_health_check_document_raises_without_the_extension(): ("cloudflare/clef", "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clef"), ), ) -async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( +async def test_ahealth_check_probes_decisions_models_through_the_decisions_api( model: str, upstream_url: str, monkeypatch: pytest.MonkeyPatch, @@ -617,7 +617,9 @@ async def test_ahealth_check_probes_evaluation_models_through_the_decisions_api( @pytest.mark.asyncio -async def test_ahealth_check_evaluation_uses_configured_probe_state_and_questions( +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +async def test_ahealth_check_decisions_uses_configured_probe_state_and_questions( + mode: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, ) -> None: @@ -638,7 +640,7 @@ async def test_ahealth_check_evaluation_uses_configured_probe_state_and_question "state": "custom probe", "questions": {"ok": {"type": "noul", "instructions": "Is it ok?"}}, }, - mode=None, + mode=mode, ) assert "error" not in result, result diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 14e5e76c692..1ca07eafb7d 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -882,7 +882,8 @@ async def test_test_model_connection_request_mode_wins_over_resolved_mode(): @pytest.mark.asyncio -async def test_test_model_connection_evaluation_mode_uses_decisions_handler(): +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +async def test_test_model_connection_decision_modes_are_forwarded(mode: str): deployment: Final = MappingProxyType( { "model_name": "typesafe/jev-latest", @@ -893,14 +894,14 @@ async def test_test_model_connection_evaluation_mode_uses_decisions_handler(): with _test_connection_probe(deployment) as ahealth_check: result: Final = await health_test_model_connection( request=MagicMock(), - mode="evaluation", + mode=mode, litellm_params={"model": "typesafe/jev-latest"}, model_info={"id": "typesafe-jev-id"}, user_api_key_dict=UserAPIKeyAuth(user_id="test-user", token="test-token"), ) assert result["status"] == "success" - assert ahealth_check.call_args.kwargs["mode"] == "evaluation" + assert ahealth_check.call_args.kwargs["mode"] == mode @pytest.mark.asyncio @@ -5011,7 +5012,8 @@ def test_test_model_connection_accepts_image_edit_mode(monkeypatch): assert response.json()["status"] == "success" -def test_test_model_connection_accepts_evaluation_mode(monkeypatch): +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) +def test_test_model_connection_accepts_decision_modes(monkeypatch, mode: str): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() @@ -5036,7 +5038,7 @@ def test_test_model_connection_accepts_evaluation_mode(monkeypatch): response = client.post( "/health/test_connection", json={ - "mode": "evaluation", + "mode": mode, "litellm_params": {"model": "typesafe/jev-latest", "api_key": "sk-test"}, }, ) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 07b52711dc3..7d9f4ae33c8 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -140,6 +140,7 @@ def test_success_handler_dispatches_to_typesafe_handler(): @pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) @pytest.mark.parametrize("guardrail_cost", [0.0, 0.25]) @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) @pytest.mark.parametrize("provider,requested,routing_model", [ @@ -149,7 +150,7 @@ def test_success_handler_dispatches_to_typesafe_handler(): ]) async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float, - provider: str, requested: str + provider: str, requested: str, mode: str ) -> None: checkpoint: Final = routing_model or requested model: Final = f"{provider}/{checkpoint}" @@ -157,7 +158,7 @@ async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( output_rate: Final = 0.005 monkeypatch.setitem(litellm.model_cost, model, { "input_cost_per_token": input_rate, "output_cost_per_token": output_rate, - "litellm_provider": provider, "mode": "evaluation", + "litellm_provider": provider, "mode": mode, }) start: Final = datetime.now() logging_obj: Final = Logging( diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 37df91d22af..50c9643683b 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -637,7 +637,7 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path): PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) -MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) +MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "decisions", "evaluation"}) VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset( { "vertex_ai-ai21_models", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6b63f69fd2e..466f3c988e4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8403,7 +8403,7 @@ def test_models_by_provider(): or v["litellm_provider"] == "bedrock_converse" ): continue - elif v.get("mode") in ("search", "evaluation"): + elif v.get("mode") in ("search", "decisions", "evaluation"): continue else: providers.add(v["litellm_provider"]) diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index ebec1b1b924..fbe00ca260b 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -29,6 +29,7 @@ from litellm.types.utils import ( ModelResponseStream, TextCompletionResponse, all_litellm_params, + is_decisions_model_mode, text_tokens_without_nested_reasoning, ) @@ -41,6 +42,14 @@ class _DictHiddenParamsAccessor(Protocol): def hidden_params(self, hidden_params: dict[str, object]) -> None: ... +@pytest.mark.parametrize( + ("mode", "expected"), + (("decisions", True), ("evaluation", True), ("chat", False), (None, False)), +) +def test_is_decisions_model_mode(mode: str | None, expected: bool) -> None: + assert is_decisions_model_mode(mode) is expected + + def test_rust_is_a_known_litellm_param(): assert "rust" in all_litellm_params diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 728f95a2225..52e13642020 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -506,9 +506,9 @@ describe("provider_info_helpers", () => { it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { const modelMap = { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, - "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" }, + "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "decisions" }, "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, - "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" }, }; expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([ "typesafe/jev-latest", @@ -518,7 +518,7 @@ describe("provider_info_helpers", () => { it("should list only strands_decider models for the 'StrandsDecider' provider key", () => { const modelMap = { - "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "decisions" }, "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, }; diff --git a/ui/litellm-dashboard/src/lib/decisionModels.test.ts b/ui/litellm-dashboard/src/lib/decisionModels.test.ts index 0876d88b261..7552cc0cfd8 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.test.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.test.ts @@ -17,7 +17,8 @@ const COST_MAP = { }; describe("isDecisionMode", () => { - it("is true only for the evaluation mode", () => { + it("is true for both the decisions and evaluation modes", () => { + expect(isDecisionMode("decisions")).toBe(true); expect(isDecisionMode("evaluation")).toBe(true); expect(isDecisionMode("chat")).toBe(false); expect(isDecisionMode(null)).toBe(false); diff --git a/ui/litellm-dashboard/src/lib/decisionModels.ts b/ui/litellm-dashboard/src/lib/decisionModels.ts index 51de8d62ace..a14ce1948a9 100644 --- a/ui/litellm-dashboard/src/lib/decisionModels.ts +++ b/ui/litellm-dashboard/src/lib/decisionModels.ts @@ -25,7 +25,8 @@ export interface DecisionCatalog { export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), providers: new Map() }; -export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "evaluation"; +export const isDecisionMode = (mode: string | null | undefined): boolean => + mode === "decisions" || mode === "evaluation"; const isDecisionEntry = (entry: CostMapEntry): boolean => isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint)); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9853a71a2ed..c56a4b3786c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27208,7 +27208,7 @@ export interface components { * Mode * @description The mode to test the model with. If not provided, resolved the way /health does: the deployment's model_info.mode (only while the request tests the deployment's own model), then the mode the provider requires for that model, then the model cost map. */ - mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation") | null; + mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation" | "decisions") | null; /** * Model Info * @description Model info for the health check