diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index 449a1032b35..1573e8e7729 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -71,6 +71,7 @@ class _MemberJevClassifierConfig(BaseModel): model_config = ConfigDict(extra="forbid") + deployment_name: str | None = None model: str api_key: None = None api_base: None = None diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 0f252952a9d..6e58bd2dd67 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -115,6 +115,7 @@ from .jev_classifier import ( HttpJevClassifierClient, JevClassifierClient, JevVerdict, + RouterJevClassifierClient, build_jev_request, jev_classifier_cost, ) @@ -1271,8 +1272,9 @@ class ComplexityRouter(CustomLogger): - Question complexity (multiple questions) """ - @staticmethod - def _build_jev_client(config: JevClassifierConfig) -> JevClassifierClient: + def _build_jev_client(self, config: JevClassifierConfig) -> JevClassifierClient: + if config.deployment_name is not None: + return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name) api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY") if not api_key: raise ValueError("jev_classifier_config.api_key or TYPESAFE_API_KEY is required for classifier_type 'jev'") @@ -2181,7 +2183,9 @@ class ComplexityRouter(CustomLogger): probabilities=answer.probabilities, confidence=answer.confidence, model=model, - cost=jev_classifier_cost(response, config.model), + cost=response.classifier_cost + if config.deployment_name + else jev_classifier_cost(response, config.model), ) if breaker is not None and permit is not None: breaker.record_success(permit) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index dbc70631298..7b97a59a974 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -681,6 +681,7 @@ class CapabilityClassifierConfig(BaseModel): class JevClassifierConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) + deployment_name: str | None = Field(default=None, min_length=1) model: str = "jev-latest" api_key: str | None = Field(default=None, description="TypeSafe API key, falling back to TYPESAFE_API_KEY") api_base: str | None = Field( @@ -711,6 +712,12 @@ class JevClassifierConfig(BaseModel): @model_validator(mode="after") def _keep_the_environment_key_on_the_environment_base(self) -> "JevClassifierConfig": + if self.deployment_name is not None: + if not self.deployment_name.strip(): + raise ValueError("deployment_name must name a configured decision model") + if self.api_key is not None or self.api_base is not None: + raise ValueError("deployment_name uses the deployment connection; omit api_key and api_base") + return self if self.api_base is not None and self.api_key is None: raise ValueError( "jev_classifier_config.api_base requires jev_classifier_config.api_key: TYPESAFE_API_KEY is only sent " diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 02e57975626..44e831ab283 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from types import MappingProxyType -from typing import Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias +from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias from uuid import uuid4 import httpx @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_router_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.classifier_logging import masked_originating_request from litellm.litellm_core_utils.internal_call_metadata import ( effective_turn_off_message_logging, forwarded_internal_call_metadata, @@ -22,8 +23,12 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthr TypeSafePassthroughLoggingHandler, ) from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS +from litellm.types.decisions import DecisionsResponse from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +if TYPE_CHECKING: + from litellm.router import Router + JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS @@ -65,6 +70,8 @@ class JevSystemOneResponse(BaseModel): model: str | None = None answers: Mapping[str, JevChoiceAnswer] + classifier_cost: float | None = None + classifier_provider: str | None = None usage: JevUsage | None = None @@ -77,6 +84,74 @@ class JevClassifierClient(Protocol): ) -> JevSystemOneResponse: ... +class _DecisionsRouter(Protocol): + async def adecisions( + self, + *, + model: str, + state: str, + questions: Mapping[str, object], + timeout: float, + num_retries: int, + disable_fallbacks: bool, + metadata: Mapping[str, object], + proxy_server_request: Mapping[str, object], + turn_off_message_logging: bool | None, + litellm_session_id: str = "", + litellm_trace_id: str = "", + ) -> object: ... + + +_PARENT_METADATA: Final = TypeAdapter[Mapping[str, object] | None](Mapping[str, object] | None) + + +class RouterJevClassifierClient: + def __init__(self, router: "Router", deployment_name: str) -> None: + self._router: _DecisionsRouter = router + self._deployment_name = deployment_name + + async def evaluate( + self, + request: JevSystemOneRequest, + timeout_s: float, + request_kwargs: Mapping[str, object] | None = None, + ) -> JevSystemOneResponse: + parent: Final = request_kwargs or MappingProxyType({}) + metadata: Final = { # mutable-ok: Router logging enriches the SDK metadata dictionary in place + **forwarded_internal_call_metadata( + _PARENT_METADATA.validate_python(parent.get("litellm_metadata") or parent.get("metadata")), + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + ), + INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + } + response: Final = DecisionsResponse.model_validate( + await self._router.adecisions( + model=self._deployment_name, + state=request.state, + questions=request.model_dump()["questions"], + timeout=timeout_s, + num_retries=0, + disable_fallbacks=True, + metadata=metadata, + proxy_server_request={ # mutable-ok: SDK logging consumes the proxy request as a JSON dictionary + "originating_request_masked": masked_originating_request(request_kwargs), + "body": request.model_copy(update=MappingProxyType({"model": self._deployment_name})).model_dump(), + }, + turn_off_message_logging=effective_turn_off_message_logging(request_kwargs), + **parent_session_kwargs(request_kwargs), + ) + ) + return JevSystemOneResponse.model_validate( + MappingProxyType( + { + **response.model_dump(), + "classifier_cost": response.hidden_params.get("response_cost"), + "classifier_provider": response.hidden_params.get("custom_llm_provider"), + } + ) + ) + + class HttpJevClassifierClient: def __init__(self, api_key: str, api_base: str, http_client: AsyncHTTPHandler) -> None: self._api_key = api_key @@ -189,6 +264,7 @@ class JevVerdict(NamedTuple): confidence: float model: str cost: float | None + provider: str = "typesafe" class _RegistryPricing(BaseModel): diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index 6b589c3bfc0..430eec56b75 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -164,9 +164,13 @@ def strategy_router_dependencies( else () ) + ( - _named( - f"typesafe/{_mapping(complexity.get('jev_classifier_config')).get('model', 'jev-latest')}", - "evaluation", + ( + _named(_mapping(complexity.get("jev_classifier_config")).get("deployment_name"), "classifier") + if _mapping(complexity.get("jev_classifier_config")).get("deployment_name") + else _named( + f"typesafe/{_mapping(complexity.get('jev_classifier_config')).get('model', 'jev-latest')}", + "evaluation", + ) ) if complexity.get("classifier_type") == "jev" else () diff --git a/tests/test_litellm/proxy/management_helpers/test_auto_router_permissions.py b/tests/test_litellm/proxy/management_helpers/test_auto_router_permissions.py index e16271a5189..0e6974835fd 100644 --- a/tests/test_litellm/proxy/management_helpers/test_auto_router_permissions.py +++ b/tests/test_litellm/proxy/management_helpers/test_auto_router_permissions.py @@ -308,3 +308,26 @@ async def test_jev_evaluation_obeys_each_containing_scope(catalog: Router, restr return await operation assert not catalog.get_model_list("typesafe/jev-latest") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("restricted", ["key", "team", None]) +async def test_deployed_decision_classifier_requires_key_and_team_access(catalog: Router, restricted: str | None) -> None: + permitted: Final = ["allowed", "other"] + operation: Final = authorize_member_auto_router_dependencies( + config=validate_member_auto_router_config({ + "tiers": {"SIMPLE": "allowed"}, + "classifier_type": "jev", + "jev_classifier_config": {"deployment_name": "other"}, + }), + default_model=None, + user_api_key_dict=_actor(models=["allowed"] if restricted == "key" else permitted), + team=_team(models=["allowed"] if restricted == "team" else permitted), + prisma_client=_Client(), + llm_router=catalog, + ) + if restricted is not None: + with pytest.raises(ProxyException, match="other"): + await operation + return + await operation diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 45070dfd3a7..515c9bb872f 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -8,6 +8,7 @@ from unittest.mock import create_autospec import httpx import pytest +import respx import litellm from litellm._logging import verbose_router_logger @@ -550,3 +551,122 @@ async def test_http_jev_classifier_client_posts_to_system_one() -> None: assert captured["content_type"] == "application/json" assert captured["body"] == request.model_dump(mode="json") assert response.model == "jev-1.13.0" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code", [200, 503]) +async def test_deployed_decision_classifier_uses_registered_connection( + status_code: int, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem(litellm.model_cost, "typesafe/jev-latest", {"input_cost_per_token": 0.01}) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decision-deployment", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "deployment-key", + "api_base": "https://decision.test", + }, + } + ], + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "deployed-classifier", + router, + { + "classifier_type": "jev", + "jev_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as requests: + route: Final = requests.post("https://decision.test/v1/systemone").mock( + return_value=httpx.Response( + status_code, + json={ + "model": "jev-latest", + "answers": {"tier": _answer().model_dump()}, + "usage": {"input_tokens": 31, "output_tokens": 0}, + }, + ) + ) + outcome: Final = await classifier.aclassify("What is two plus two?") + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(route.calls[0].request.content)["model"] == "jev-latest" + assert (outcome.cause == "jev_classifier") is (status_code == 200) + if status_code == 200: + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.label == _answer().choice + + assert outcome.jev_verdict.provider == "typesafe" + assert outcome.classifier_cost == pytest.approx(0.31) + + +@pytest.mark.parametrize("deployment_name", ["", " "]) +def test_deployed_decision_classifier_rejects_blank_deployment_name(deployment_name: str) -> None: + with pytest.raises(ValueError, match="deployment_name"): + JevClassifierConfig.model_validate({"deployment_name": deployment_name}) + + +@pytest.mark.parametrize("connection", [{"api_key": "secret"}, {"api_base": "https://override.test"}]) +def test_deployed_decision_classifier_rejects_connection_overrides(connection: Mapping[str, str]) -> None: + with pytest.raises(ValueError, match="deployment connection"): + JevClassifierConfig.model_validate({"deployment_name": "deployed", **connection}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["timeout", "cancelled", "invalid"]) +async def test_deployed_decision_classifier_preserves_failure_handling( + failure: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "decision-deployment", + "litellm_params": { + "model": "typesafe/jev-latest", + "api_key": "test", + "api_base": "https://decision.test", + }, + } + ], + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "decision-router", + router, + { + "classifier_type": "jev", + "jev_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + ) + cancellation_reached: Final = asyncio.Event() + with respx.mock(assert_all_called=failure != "cancelled") as upstream: + route: Final = upstream.post("https://decision.test/v1/systemone") + if failure == "timeout": + route.mock(side_effect=httpx.ReadTimeout("classifier timeout")) + elif failure == "cancelled": + + async def cancel_request(request: httpx.Request) -> httpx.Response: + cancellation_reached.set() + raise asyncio.CancelledError() + + route.mock(side_effect=cancel_request) + else: + route.respond(json={"answers": {"tier": _answer("not-a-tier").model_dump()}}) + if failure == "cancelled": + with pytest.raises(asyncio.CancelledError): + await classifier.aclassify("short question") + assert cancellation_reached.is_set() + return + outcome: Final = await classifier.aclassify("short question") + assert outcome.cause != "jev_classifier" + assert outcome.jev_verdict is None diff --git a/tests/unit/router_utils/test_auto_router_model_naming.py b/tests/unit/router_utils/test_auto_router_model_naming.py index 645f9e5e62a..175b33a50a0 100644 --- a/tests/unit/router_utils/test_auto_router_model_naming.py +++ b/tests/unit/router_utils/test_auto_router_model_naming.py @@ -6,11 +6,11 @@ import pytest from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS from litellm.router_utils.auto_router_model_naming import ( - carries_complexity_router_settings, - classify_strategy_router_model, GATED_AUTO_ROUTER_CAPABILITIES, capability_limit_violation, + carries_complexity_router_settings, claimed_capability, + classify_strategy_router_model, count_capability_routers, gated_capability_of, strategy_router_dependencies, @@ -687,3 +687,20 @@ def test_capabilities_are_mutually_exclusive_on_one_config(config: Mapping[str, and system_prompt only counts for the classifier types heuristic_v2 is not one of. """ assert sum(1 for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(config)) <= 1 + + +def test_deployed_decision_classifier_is_a_model_access_dependency() -> None: + dependencies: Final = strategy_router_dependencies( + { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "jev", + "jev_classifier_config": {"deployment_name": "decision-deployment"}, + "tiers": {"SIMPLE": "cheap"}, + }, + } + ) + assert tuple((dependency.model_name, dependency.role) for dependency in dependencies) == ( + ("cheap", "tier"), + ("decision-deployment", "classifier"), + ) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 1625e0cbfb8..e9c418e0b64 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -100,7 +100,10 @@ export function AutoRoutersPanel({ {/* The form is long, so the dialog caps its height and scrolls its body rather than growing past the viewport. */} - + Add Auto Router diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index f843a472d15..0d40fee7a2b 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -68,7 +68,7 @@ describe("Auto-router classifier selection", () => { llm: "LLM", heuristic_first: "LLM", hybrid: "LLM", - jev: "Jev", + jev: "Decisions Model", }[classifier_type]; expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked(); fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })); @@ -154,17 +154,14 @@ describe("Auto-router classifier selection", () => { expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router"); }); - it("shows Jev's single Complexity approach without changing saved configuration", () => { + it("omits Jev's redundant routing approach without changing saved configuration", () => { const onChange = vi.fn(); renderWithProviders( Existing settings , ); - expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited"); - fireEvent.click(screen.getByRole("button", { name: "Routing approach" })); - expect(screen.getAllByRole("menuitemradio")).toHaveLength(1); - fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ })); + expect(screen.queryByRole("button", { name: "Routing approach" })).not.toBeInTheDocument(); expect(onChange).not.toHaveBeenCalled(); }); diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 92fc8d2a335..b3116f9682c 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -1,7 +1,8 @@ import React, { useContext, useId } from "react"; import { Label } from "@/components/ui/label"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; -import { ChevronDownIcon } from "lucide-react"; +import { SimpleTooltip } from "@/components/ui/tooltip"; +import { ChevronDownIcon, Info } from "lucide-react"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -119,9 +120,17 @@ interface AutoRouterClassifierTabsProps { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; children: React.ReactNode; + judgeModel?: React.ReactNode; + decisionModel?: React.ReactNode; } -const AutoRouterClassifierTabs: React.FC = ({ value, onChange, children }) => { +const AutoRouterClassifierTabs: React.FC = ({ + value, + onChange, + children, + judgeModel, + decisionModel, +}) => { const id = useId(); const availability = useContext(AutoRouterAvailabilityContext); const classifierType = effectiveClassifierType(value); @@ -154,17 +163,28 @@ const AutoRouterClassifierTabs: React.FC = ({ val llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model", }; return ( -
+
- - What classifies your requests? + +
+

Choose your classifier

+ + + +
{[ { value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" }, { value: "llm", label: "LLM", description: "Use a judge model to choose a solver" }, - { value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" }, + { + value: "jev", + label: "Decisions Model", + description: "Use a decision model to choose a tier", + }, ].map((option) => (
+ {family === "jev" && decisionModel} {family === "custom" && (

This router uses a custom classifier plugin

)} @@ -224,7 +245,8 @@ const AutoRouterClassifierTabs: React.FC = ({ val

)} - {(family === "llm" || family === "jev") && ( + {family === "llm" && judgeModel} + {family === "llm" && (
= }; interface ClassificationMethodConfigProps { + jevSection?: "all" | "advanced"; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; @@ -215,6 +216,7 @@ const ClassificationMethodConfig: React.FC = ({ showValidationErrors = false, defaultModel, advancedOnly = false, + jevSection, }) => { const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); @@ -493,7 +495,7 @@ const ClassificationMethodConfig: React.FC = ({
)} - {classifierType === "jev" && } + {classifierType === "jev" && } {usesLlmClassifier(classifierType) && (
= ({ /> Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders. - LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit - conversation history. The current message and selected system text are still sent. + LLM and decision model classifiers default to 3 turns. Set to 0 to omit conversation history. The current + message and selected system text are still sent.
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx index 32cb4851580..cf5e7e2db03 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierPrimarySettings.tsx @@ -18,11 +18,13 @@ export default function ClassifierPrimarySettings({ onChange, modelOptions, showValidationErrors = false, + section = "all", }: { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; modelOptions: { value: string; label: string }[]; showValidationErrors?: boolean; + section?: "all" | "judge" | "frequency"; }) { const id = React.useId(); const restriction = restrictedBy(value, "sessionAffinity"); @@ -35,34 +37,44 @@ export default function ClassifierPrimarySettings({ const usesJudge = usesLlmClassifier(effectiveClassifierType(value)); const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model; return ( -
-
- - -

{restriction?.reason ?? frequencyDescription}

-
- {usesJudge && ( +
+ {section !== "judge" && ( +
+ + +

{restriction?.reason ?? frequencyDescription}

+
+ )} + {section !== "frequency" && usesJudge && (
void; forecast: boolean; @@ -53,6 +54,7 @@ interface ComplexityRouterAdvancedSectionsProps { } const ComplexityRouterAdvancedSections: React.FC = ({ + hideClassifier, value, onChange, forecast, @@ -255,7 +257,10 @@ const ComplexityRouterAdvancedSections: React.FC + !(hideClassifier && group.keys.includes("classifier")) && sections.some(({ key }) => group.keys.includes(key)), + ); const [openGroups, setOpenGroups] = React.useState(() => showValidationErrors ? groups.map((group) => group.label) : [], ); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 32f9ebf97ad..11bf6cd7361 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -37,7 +37,7 @@ import { } from "./tier_rows"; import React from "react"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig"; +import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig"; import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections"; import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions"; import { @@ -456,6 +456,8 @@ export const withClassificationFrequency = ( }); interface ComplexityRouterConfigProps { + creationFlow?: boolean; + tierSetup?: React.ReactNode; modelInfo: ModelGroup[]; value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; @@ -532,7 +534,59 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03; */ export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_ORDER.slice(0, -1); +const isJevCreation = (value: ComplexityRouterConfigValue, creationFlow: boolean | undefined) => + creationFlow && effectiveClassifierType(value) === "jev"; + +const CreationClassifierSettings = ({ + creationFlow, + value, + onChange, + modelOptions, + classifierEffortOptionsByModel, + defaultModel, + showValidationErrors, +}: Pick & { + creationFlow: boolean | undefined; + modelOptions: { value: string; label: string }[]; + classifierEffortOptionsByModel: Record; + defaultModel: string | undefined; + showValidationErrors: boolean; +}) => { + const jevCreation = isJevCreation(value, creationFlow); + return ( + <> + + {jevCreation && ( +
+ +
+ +
+
+
+ )} + + ); +}; + const ComplexityRouterConfig: React.FC = ({ + creationFlow, + tierSetup, modelInfo, value, onChange, @@ -556,6 +610,7 @@ const ComplexityRouterConfig: React.FC = ({ showValidationErrors = false, }) => { const forecast = isForecastClassifier(value.classifier_type); + const tierHeading = creationFlow ? "Define your tiers" : "Models by tier"; const customTierSet = value.custom_tier_set; const tierRows = activeTierRows(value); const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null; @@ -604,19 +659,25 @@ const ComplexityRouterConfig: React.FC = ({ return (
- -
-

{forecast ? "Solver models" : "Models by tier"}

- {!forecast && ( - - - - )} + {creationFlow && } +
+
+

{forecast ? "Solver models" : tierHeading}

+ {!forecast && ( + + + + )} +
@@ -641,16 +702,9 @@ const ComplexityRouterConfig: React.FC = ({ ) : ( <> + {tierSetup} - {!customTierSet && ( - - )} - {tierRows.map((row, index) => { const tierInfo = builtInTierInfo(row.id); const label = tierRowLabel(row, value.tier_labels); @@ -740,6 +794,14 @@ const ComplexityRouterConfig: React.FC = ({ ); })} + {!customTierSet && ( + + )} + = ({ )} - = ({ )}
void; + modelInfo: ModelGroup[]; + deployments: AutoRouterDeployment[] | undefined; +}) { + const id = React.useId(); + const { data: modelCostMapData } = useModelCostMap(); + const decisionCatalog = React.useMemo(() => buildDecisionCatalog(modelCostMapData), [modelCostMapData]); + const decisionModels = modelInfo.filter( + (model) => + isDecisionMode(model.mode) || + (deployments ?? []).some( + (deployment) => + deployment.model_name === model.model_group && + isDecisionSelection( + decisionCatalog, + deployment.litellm_params?.custom_llm_provider ?? deployment.litellm_params?.model?.split("/")[0], + [deployment.litellm_params?.model ?? "", deployment.litellm_params?.base_model ?? ""], + ), + ), + ); + + return ( +
+ + ({ value: model.model_group, label: model.model_group }))} + value={value.jev_classifier_config?.deployment_name ?? ""} + placeholder="Select the decision model" + allowClear={false} + onValueChange={(deploymentName) => { + if (!deploymentName) return; + onChange({ + ...value, + jev_classifier_config: { + ...(value.jev_classifier_config ?? defaultJevClassifierConfig()), + deployment_name: deploymentName, + }, + }); + }} + /> +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index 7da8b12c2d7..2132ed39964 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -102,24 +102,24 @@ describe("JEV classifier editor", () => { expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); expect(screen.getByText("Classifier Prompt")).toBeInTheDocument(); expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument(); - fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ })); - expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked(); - expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest"); - expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); + fireEvent.click(screen.getByRole("radio", { name: /Decisions Model/ })); + expect(screen.getByRole("radio", { name: /^Decisions Model/ })).toBeChecked(); + expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-latest"); + expect(screen.getByLabelText("Classifier Instructions")).toBeEnabled(); expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument(); expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument(); expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument(); expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument(); - fireEvent.change(screen.getByLabelText("Jev Model"), { target: { value: "jev-test" } }); - fireEvent.change(screen.getByLabelText("Jev Timeout (ms)"), { target: { value: "4200" } }); + fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: "jev-test" } }); + fireEvent.change(screen.getByLabelText("Classifier Timeout (ms)"), { target: { value: "4200" } }); fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } }); fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } }); fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" })); fireEvent.click(screen.getByRole("button", { name: "Customize tiers" })); fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); - expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked(); - expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test"); - expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200); + expect(screen.getByRole("radio", { name: /Decisions Model/ })).toBeChecked(); + expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-test"); + expect(screen.getByLabelText("Classifier Timeout (ms)")).toHaveValue(4200); expect(screen.getByLabelText("Context Window Size")).toHaveValue("6"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked(); fireEvent.click(screen.getByRole("button", { name: "Probe current config" })); @@ -152,10 +152,10 @@ describe("JEV classifier editor", () => { return ; }; renderWithProviders(); - expect(screen.getByLabelText("Jev Instructions")).toBeEnabled(); - fireEvent.change(screen.getByLabelText("Jev Instructions"), { target: { value: "New instructions" } }); - expect(screen.getByLabelText("Jev Instructions")).toHaveValue("New instructions"); - fireEvent.click(screen.getByRole("button", { name: "Restore built-in Jev instructions" })); - expect(screen.getByLabelText("Jev Instructions")).toHaveValue(""); + expect(screen.getByLabelText("Classifier Instructions")).toBeEnabled(); + fireEvent.change(screen.getByLabelText("Classifier Instructions"), { target: { value: "New instructions" } }); + expect(screen.getByLabelText("Classifier Instructions")).toHaveValue("New instructions"); + fireEvent.click(screen.getByRole("button", { name: "Restore built-in instructions" })); + expect(screen.getByLabelText("Classifier Instructions")).toHaveValue(""); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 97609bcd8c3..fe2999afbfc 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -11,9 +11,11 @@ import { defaultJevClassifierConfig } from "./jev_classifier_config"; export default function JevClassifierConfig({ value, onChange, + section = "all", }: { value: ComplexityRouterConfigValue; onChange: (value: ComplexityRouterConfigValue) => void; + section?: "all" | "model" | "advanced"; }) { const id = useId(); const config = value.jev_classifier_config ?? defaultJevClassifierConfig(); @@ -21,55 +23,67 @@ export default function JevClassifierConfig({ onChange({ ...value, jev_classifier_config: { ...config, ...patch } }); return ( -
-

- Uses TypeSafe System One Choice evaluation with your configured tiers -

-
- - update({ model: event.target.value })} /> -
-
- - update({ timeout_ms: Number(event.target.value) })} - /> -
- - update({ - circuit_breaker_enabled: next.circuit_breaker_enabled, - circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds, - }) - } - /> -
- - -