mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
Merge pull request #45914 from BerriAI/litellm_jev_router_creation_ui_104
feat(ui): backport decision auto-router setup to 1.104
This commit is contained in:
joshua-berri
GitHub
commit
ebcabd96d0
34 changed files with 1270 additions and 345 deletions
|
|
@ -71,6 +71,7 @@ class _MemberJevClassifierConfig(BaseModel):
|
|||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
deployment_name: str | None = None
|
||||
model: str
|
||||
api_key: None = None
|
||||
api_base: None = None
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ from .jev_classifier import (
|
|||
HttpJevClassifierClient,
|
||||
JevClassifierClient,
|
||||
JevVerdict,
|
||||
RouterJevClassifierClient,
|
||||
build_jev_request,
|
||||
jev_classifier_cost,
|
||||
)
|
||||
|
|
@ -1271,8 +1272,9 @@ class ComplexityRouter(CustomLogger):
|
|||
- Question complexity (multiple questions)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _build_jev_client(config: JevClassifierConfig) -> JevClassifierClient:
|
||||
def _build_jev_client(self, config: JevClassifierConfig) -> JevClassifierClient:
|
||||
if config.deployment_name is not None:
|
||||
return RouterJevClassifierClient(self.litellm_router_instance, config.deployment_name)
|
||||
api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError("jev_classifier_config.api_key or TYPESAFE_API_KEY is required for classifier_type 'jev'")
|
||||
|
|
@ -2181,7 +2183,9 @@ class ComplexityRouter(CustomLogger):
|
|||
probabilities=answer.probabilities,
|
||||
confidence=answer.confidence,
|
||||
model=model,
|
||||
cost=jev_classifier_cost(response, config.model),
|
||||
cost=response.classifier_cost
|
||||
if config.deployment_name
|
||||
else jev_classifier_cost(response, config.model),
|
||||
)
|
||||
if breaker is not None and permit is not None:
|
||||
breaker.record_success(permit)
|
||||
|
|
|
|||
|
|
@ -681,6 +681,7 @@ class CapabilityClassifierConfig(BaseModel):
|
|||
class JevClassifierConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
deployment_name: str | None = Field(default=None, min_length=1)
|
||||
model: str = "jev-latest"
|
||||
api_key: str | None = Field(default=None, description="TypeSafe API key, falling back to TYPESAFE_API_KEY")
|
||||
api_base: str | None = Field(
|
||||
|
|
@ -711,6 +712,12 @@ class JevClassifierConfig(BaseModel):
|
|||
|
||||
@model_validator(mode="after")
|
||||
def _keep_the_environment_key_on_the_environment_base(self) -> "JevClassifierConfig":
|
||||
if self.deployment_name is not None:
|
||||
if not self.deployment_name.strip():
|
||||
raise ValueError("deployment_name must name a configured decision model")
|
||||
if self.api_key is not None or self.api_base is not None:
|
||||
raise ValueError("deployment_name uses the deployment connection; omit api_key and api_base")
|
||||
return self
|
||||
if self.api_base is not None and self.api_key is None:
|
||||
raise ValueError(
|
||||
"jev_classifier_config.api_base requires jev_classifier_config.api_key: TYPESAFE_API_KEY is only sent "
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
|
@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
effective_turn_off_message_logging,
|
||||
forwarded_internal_call_metadata,
|
||||
|
|
@ -22,8 +23,12 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthr
|
|||
TypeSafePassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS
|
||||
from litellm.types.decisions import DecisionsResponse
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)]
|
||||
DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS
|
||||
|
||||
|
|
@ -65,6 +70,8 @@ class JevSystemOneResponse(BaseModel):
|
|||
|
||||
model: str | None = None
|
||||
answers: Mapping[str, JevChoiceAnswer]
|
||||
classifier_cost: float | None = None
|
||||
classifier_provider: str | None = None
|
||||
usage: JevUsage | None = None
|
||||
|
||||
|
||||
|
|
@ -77,6 +84,74 @@ class JevClassifierClient(Protocol):
|
|||
) -> JevSystemOneResponse: ...
|
||||
|
||||
|
||||
class _DecisionsRouter(Protocol):
|
||||
async def adecisions(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
state: str,
|
||||
questions: Mapping[str, object],
|
||||
timeout: float,
|
||||
num_retries: int,
|
||||
disable_fallbacks: bool,
|
||||
metadata: Mapping[str, object],
|
||||
proxy_server_request: Mapping[str, object],
|
||||
turn_off_message_logging: bool | None,
|
||||
litellm_session_id: str = "",
|
||||
litellm_trace_id: str = "",
|
||||
) -> object: ...
|
||||
|
||||
|
||||
_PARENT_METADATA: Final = TypeAdapter[Mapping[str, object] | None](Mapping[str, object] | None)
|
||||
|
||||
|
||||
class RouterJevClassifierClient:
|
||||
def __init__(self, router: "Router", deployment_name: str) -> None:
|
||||
self._router: _DecisionsRouter = router
|
||||
self._deployment_name = deployment_name
|
||||
|
||||
async def evaluate(
|
||||
self,
|
||||
request: JevSystemOneRequest,
|
||||
timeout_s: float,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
) -> JevSystemOneResponse:
|
||||
parent: Final = request_kwargs or MappingProxyType({})
|
||||
metadata: Final = { # mutable-ok: Router logging enriches the SDK metadata dictionary in place
|
||||
**forwarded_internal_call_metadata(
|
||||
_PARENT_METADATA.validate_python(parent.get("litellm_metadata") or parent.get("metadata")),
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
),
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
}
|
||||
response: Final = DecisionsResponse.model_validate(
|
||||
await self._router.adecisions(
|
||||
model=self._deployment_name,
|
||||
state=request.state,
|
||||
questions=request.model_dump()["questions"],
|
||||
timeout=timeout_s,
|
||||
num_retries=0,
|
||||
disable_fallbacks=True,
|
||||
metadata=metadata,
|
||||
proxy_server_request={ # mutable-ok: SDK logging consumes the proxy request as a JSON dictionary
|
||||
"originating_request_masked": masked_originating_request(request_kwargs),
|
||||
"body": request.model_copy(update=MappingProxyType({"model": self._deployment_name})).model_dump(),
|
||||
},
|
||||
turn_off_message_logging=effective_turn_off_message_logging(request_kwargs),
|
||||
**parent_session_kwargs(request_kwargs),
|
||||
)
|
||||
)
|
||||
return JevSystemOneResponse.model_validate(
|
||||
MappingProxyType(
|
||||
{
|
||||
**response.model_dump(),
|
||||
"classifier_cost": response.hidden_params.get("response_cost"),
|
||||
"classifier_provider": response.hidden_params.get("custom_llm_provider"),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class HttpJevClassifierClient:
|
||||
def __init__(self, api_key: str, api_base: str, http_client: AsyncHTTPHandler) -> None:
|
||||
self._api_key = api_key
|
||||
|
|
@ -189,6 +264,7 @@ class JevVerdict(NamedTuple):
|
|||
confidence: float
|
||||
model: str
|
||||
cost: float | None
|
||||
provider: str = "typesafe"
|
||||
|
||||
|
||||
class _RegistryPricing(BaseModel):
|
||||
|
|
|
|||
|
|
@ -164,9 +164,13 @@ def strategy_router_dependencies(
|
|||
else ()
|
||||
)
|
||||
+ (
|
||||
_named(
|
||||
f"typesafe/{_mapping(complexity.get('jev_classifier_config')).get('model', 'jev-latest')}",
|
||||
"evaluation",
|
||||
(
|
||||
_named(_mapping(complexity.get("jev_classifier_config")).get("deployment_name"), "classifier")
|
||||
if _mapping(complexity.get("jev_classifier_config")).get("deployment_name")
|
||||
else _named(
|
||||
f"typesafe/{_mapping(complexity.get('jev_classifier_config')).get('model', 'jev-latest')}",
|
||||
"evaluation",
|
||||
)
|
||||
)
|
||||
if complexity.get("classifier_type") == "jev"
|
||||
else ()
|
||||
|
|
|
|||
|
|
@ -308,3 +308,26 @@ async def test_jev_evaluation_obeys_each_containing_scope(catalog: Router, restr
|
|||
return
|
||||
await operation
|
||||
assert not catalog.get_model_list("typesafe/jev-latest")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("restricted", ["key", "team", None])
|
||||
async def test_deployed_decision_classifier_requires_key_and_team_access(catalog: Router, restricted: str | None) -> None:
|
||||
permitted: Final = ["allowed", "other"]
|
||||
operation: Final = authorize_member_auto_router_dependencies(
|
||||
config=validate_member_auto_router_config({
|
||||
"tiers": {"SIMPLE": "allowed"},
|
||||
"classifier_type": "jev",
|
||||
"jev_classifier_config": {"deployment_name": "other"},
|
||||
}),
|
||||
default_model=None,
|
||||
user_api_key_dict=_actor(models=["allowed"] if restricted == "key" else permitted),
|
||||
team=_team(models=["allowed"] if restricted == "team" else permitted),
|
||||
prisma_client=_Client(),
|
||||
llm_router=catalog,
|
||||
)
|
||||
if restricted is not None:
|
||||
with pytest.raises(ProxyException, match="other"):
|
||||
await operation
|
||||
return
|
||||
await operation
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from unittest.mock import create_autospec
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
|
@ -550,3 +551,122 @@ async def test_http_jev_classifier_client_posts_to_system_one() -> None:
|
|||
assert captured["content_type"] == "application/json"
|
||||
assert captured["body"] == request.model_dump(mode="json")
|
||||
assert response.model == "jev-1.13.0"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status_code", [200, 503])
|
||||
async def test_deployed_decision_classifier_uses_registered_connection(
|
||||
status_code: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setitem(litellm.model_cost, "typesafe/jev-latest", {"input_cost_per_token": 0.01})
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "decision-deployment",
|
||||
"litellm_params": {
|
||||
"model": "typesafe/jev-latest",
|
||||
"api_key": "deployment-key",
|
||||
"api_base": "https://decision.test",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
classifier: Final = ComplexityRouter(
|
||||
"deployed-classifier",
|
||||
router,
|
||||
{
|
||||
"classifier_type": "jev",
|
||||
"jev_classifier_config": {"deployment_name": "decision-deployment"},
|
||||
"tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
|
||||
},
|
||||
derive_savings_baseline=False,
|
||||
)
|
||||
with respx.mock(assert_all_called=True) as requests:
|
||||
route: Final = requests.post("https://decision.test/v1/systemone").mock(
|
||||
return_value=httpx.Response(
|
||||
status_code,
|
||||
json={
|
||||
"model": "jev-latest",
|
||||
"answers": {"tier": _answer().model_dump()},
|
||||
"usage": {"input_tokens": 31, "output_tokens": 0},
|
||||
},
|
||||
)
|
||||
)
|
||||
outcome: Final = await classifier.aclassify("What is two plus two?")
|
||||
assert route.call_count == 1
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer deployment-key"
|
||||
assert json.loads(route.calls[0].request.content)["model"] == "jev-latest"
|
||||
assert (outcome.cause == "jev_classifier") is (status_code == 200)
|
||||
if status_code == 200:
|
||||
assert outcome.jev_verdict is not None
|
||||
assert outcome.jev_verdict.label == _answer().choice
|
||||
|
||||
assert outcome.jev_verdict.provider == "typesafe"
|
||||
assert outcome.classifier_cost == pytest.approx(0.31)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_name", ["", " "])
|
||||
def test_deployed_decision_classifier_rejects_blank_deployment_name(deployment_name: str) -> None:
|
||||
with pytest.raises(ValueError, match="deployment_name"):
|
||||
JevClassifierConfig.model_validate({"deployment_name": deployment_name})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("connection", [{"api_key": "secret"}, {"api_base": "https://override.test"}])
|
||||
def test_deployed_decision_classifier_rejects_connection_overrides(connection: Mapping[str, str]) -> None:
|
||||
with pytest.raises(ValueError, match="deployment connection"):
|
||||
JevClassifierConfig.model_validate({"deployment_name": "deployed", **connection})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["timeout", "cancelled", "invalid"])
|
||||
async def test_deployed_decision_classifier_preserves_failure_handling(
|
||||
failure: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "decision-deployment",
|
||||
"litellm_params": {
|
||||
"model": "typesafe/jev-latest",
|
||||
"api_key": "test",
|
||||
"api_base": "https://decision.test",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
classifier: Final = ComplexityRouter(
|
||||
"decision-router",
|
||||
router,
|
||||
{
|
||||
"classifier_type": "jev",
|
||||
"jev_classifier_config": {"deployment_name": "decision-deployment"},
|
||||
"tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
|
||||
},
|
||||
derive_savings_baseline=False,
|
||||
)
|
||||
cancellation_reached: Final = asyncio.Event()
|
||||
with respx.mock(assert_all_called=failure != "cancelled") as upstream:
|
||||
route: Final = upstream.post("https://decision.test/v1/systemone")
|
||||
if failure == "timeout":
|
||||
route.mock(side_effect=httpx.ReadTimeout("classifier timeout"))
|
||||
elif failure == "cancelled":
|
||||
|
||||
async def cancel_request(request: httpx.Request) -> httpx.Response:
|
||||
cancellation_reached.set()
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
route.mock(side_effect=cancel_request)
|
||||
else:
|
||||
route.respond(json={"answers": {"tier": _answer("not-a-tier").model_dump()}})
|
||||
if failure == "cancelled":
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await classifier.aclassify("short question")
|
||||
assert cancellation_reached.is_set()
|
||||
return
|
||||
outcome: Final = await classifier.aclassify("short question")
|
||||
assert outcome.cause != "jev_classifier"
|
||||
assert outcome.jev_verdict is None
|
||||
|
|
|
|||
|
|
@ -6,11 +6,11 @@ import pytest
|
|||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
carries_complexity_router_settings,
|
||||
classify_strategy_router_model,
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
claimed_capability,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capability_of,
|
||||
strategy_router_dependencies,
|
||||
|
|
@ -687,3 +687,20 @@ def test_capabilities_are_mutually_exclusive_on_one_config(config: Mapping[str,
|
|||
and system_prompt only counts for the classifier types heuristic_v2 is not one of.
|
||||
"""
|
||||
assert sum(1 for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(config)) <= 1
|
||||
|
||||
|
||||
def test_deployed_decision_classifier_is_a_model_access_dependency() -> None:
|
||||
dependencies: Final = strategy_router_dependencies(
|
||||
{
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "jev",
|
||||
"jev_classifier_config": {"deployment_name": "decision-deployment"},
|
||||
"tiers": {"SIMPLE": "cheap"},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert tuple((dependency.model_name, dependency.role) for dependency in dependencies) == (
|
||||
("cheap", "tier"),
|
||||
("decision-deployment", "classifier"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -100,7 +100,10 @@ export function AutoRoutersPanel({
|
|||
<Dialog open={isCreating} onOpenChange={setIsCreating}>
|
||||
{/* The form is long, so the dialog caps its height and scrolls its body rather than
|
||||
growing past the viewport. */}
|
||||
<DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-4xl">
|
||||
<DialogContent
|
||||
overlayClassName="bg-black/40 backdrop-blur-sm"
|
||||
className="max-h-[90vh] overflow-y-auto sm:max-w-4xl"
|
||||
>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Add Auto Router</DialogTitle>
|
||||
<DialogDescription>
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ describe("Auto-router classifier selection", () => {
|
|||
llm: "LLM",
|
||||
heuristic_first: "LLM",
|
||||
hybrid: "LLM",
|
||||
jev: "Jev",
|
||||
jev: "Decisions Model",
|
||||
}[classifier_type];
|
||||
expect(screen.getByRole("radio", { name: new RegExp(`^${family}$`) })).toBeChecked();
|
||||
fireEvent.click(screen.getByRole("radio", { name: new RegExp(`^${family}$`) }));
|
||||
|
|
@ -154,17 +154,14 @@ describe("Auto-router classifier selection", () => {
|
|||
expect(screen.getByRole("button", { name: field })).toHaveTextContent("Used by this router");
|
||||
});
|
||||
|
||||
it("shows Jev's single Complexity approach without changing saved configuration", () => {
|
||||
it("omits Jev's redundant routing approach without changing saved configuration", () => {
|
||||
const onChange = vi.fn();
|
||||
renderWithProviders(
|
||||
<AutoRouterClassifierTabs value={{ ...initial, classifier_type: "jev" }} onChange={onChange}>
|
||||
Existing settings
|
||||
</AutoRouterClassifierTabs>,
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("ComplexityUnlimited");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Routing approach" }));
|
||||
expect(screen.getAllByRole("menuitemradio")).toHaveLength(1);
|
||||
fireEvent.click(screen.getByRole("menuitemradio", { name: /^Complexity/ }));
|
||||
expect(screen.queryByRole("button", { name: "Routing approach" })).not.toBeInTheDocument();
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import React, { useContext, useId } from "react";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
|
||||
import { ChevronDownIcon } from "lucide-react";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { ChevronDownIcon, Info } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
|
|
@ -119,9 +120,17 @@ interface AutoRouterClassifierTabsProps {
|
|||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
children: React.ReactNode;
|
||||
judgeModel?: React.ReactNode;
|
||||
decisionModel?: React.ReactNode;
|
||||
}
|
||||
|
||||
const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ value, onChange, children }) => {
|
||||
const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({
|
||||
value,
|
||||
onChange,
|
||||
children,
|
||||
judgeModel,
|
||||
decisionModel,
|
||||
}) => {
|
||||
const id = useId();
|
||||
const availability = useContext(AutoRouterAvailabilityContext);
|
||||
const classifierType = effectiveClassifierType(value);
|
||||
|
|
@ -154,17 +163,28 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ val
|
|||
llm_v2: "Use the efficient model when its predicted quality is close enough to the capable model",
|
||||
};
|
||||
return (
|
||||
<div className="flex flex-col gap-6">
|
||||
<div className="flex flex-col gap-4">
|
||||
<fieldset>
|
||||
<legend className="mb-3 flex w-full items-center justify-between gap-3 text-sm font-medium">
|
||||
What classifies your requests?
|
||||
<legend className="mb-4 flex w-full items-center justify-between gap-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<h4 className="m-0 text-xl font-semibold text-foreground">Choose your classifier</h4>
|
||||
<SimpleTooltip content="Choose how requests are evaluated and routed to your tiers">
|
||||
<button type="button" aria-label="About classifiers" className="text-muted-foreground">
|
||||
<Info className="size-4" />
|
||||
</button>
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
<AutoRouterLimits />
|
||||
</legend>
|
||||
<RadioGroup value={family} onValueChange={changeFamily} className="grid gap-3 sm:grid-cols-3">
|
||||
{[
|
||||
{ value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" },
|
||||
{ value: "llm", label: "LLM", description: "Use a judge model to choose a solver" },
|
||||
{ value: "jev", label: "Jev", description: "Use TypeSafe System One Choice to choose a tier" },
|
||||
{
|
||||
value: "jev",
|
||||
label: "Decisions Model",
|
||||
description: "Use a decision model to choose a tier",
|
||||
},
|
||||
].map((option) => (
|
||||
<Label
|
||||
key={option.value}
|
||||
|
|
@ -189,6 +209,7 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ val
|
|||
))}
|
||||
</RadioGroup>
|
||||
</fieldset>
|
||||
{family === "jev" && decisionModel}
|
||||
{family === "custom" && (
|
||||
<p className="text-sm text-muted-foreground">This router uses a custom classifier plugin</p>
|
||||
)}
|
||||
|
|
@ -224,7 +245,8 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({ val
|
|||
</p>
|
||||
</div>
|
||||
)}
|
||||
{(family === "llm" || family === "jev") && (
|
||||
{family === "llm" && judgeModel}
|
||||
{family === "llm" && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-approach`}>Routing approach</Label>
|
||||
<ClassifierMenu
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
|
|||
};
|
||||
|
||||
interface ClassificationMethodConfigProps {
|
||||
jevSection?: "all" | "advanced";
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
|
|
@ -215,6 +216,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
showValidationErrors = false,
|
||||
defaultModel,
|
||||
advancedOnly = false,
|
||||
jevSection,
|
||||
}) => {
|
||||
const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null);
|
||||
const hasDefaultModel = Boolean(defaultModel);
|
||||
|
|
@ -493,7 +495,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
|
||||
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} section={jevSection} />}
|
||||
{usesLlmClassifier(classifierType) && (
|
||||
<div className="mt-4 space-y-3">
|
||||
<ClassifierReasoningEffortSelect
|
||||
|
|
@ -642,8 +644,8 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Number of prior user turns sent to the classifier provider, excluding tool output and harness reminders.
|
||||
LLM and Jev default to 3 turns; Jev sends them to the configured TypeSafe endpoint. Set to 0 to omit
|
||||
conversation history. The current message and selected system text are still sent.
|
||||
LLM and decision model classifiers default to 3 turns. Set to 0 to omit conversation history. The current
|
||||
message and selected system text are still sent.
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -18,11 +18,13 @@ export default function ClassifierPrimarySettings({
|
|||
onChange,
|
||||
modelOptions,
|
||||
showValidationErrors = false,
|
||||
section = "all",
|
||||
}: {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
showValidationErrors?: boolean;
|
||||
section?: "all" | "judge" | "frequency";
|
||||
}) {
|
||||
const id = React.useId();
|
||||
const restriction = restrictedBy(value, "sessionAffinity");
|
||||
|
|
@ -35,34 +37,44 @@ export default function ClassifierPrimarySettings({
|
|||
const usesJudge = usesLlmClassifier(effectiveClassifierType(value));
|
||||
const missingJudge = showValidationErrors && usesJudge && !value.classifier_llm_config?.model;
|
||||
return (
|
||||
<div className="mb-6 grid gap-4 sm:grid-cols-2">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-frequency`}>How often to classify</Label>
|
||||
<Select
|
||||
items={[
|
||||
{ value: "every_request", label: "Every request" },
|
||||
{ value: "user_turn", label: "Every new user message" },
|
||||
{ value: "session", label: "Once per session" },
|
||||
]}
|
||||
value={frequency}
|
||||
onValueChange={(frequency) => {
|
||||
if (frequency) onChange(withClassificationFrequency(value, frequency as ClassificationFrequency));
|
||||
}}
|
||||
>
|
||||
<SelectTrigger id={`${id}-frequency`} className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="every_request">Every request</SelectItem>
|
||||
<SelectItem value="user_turn">Every new user message</SelectItem>
|
||||
<SelectItem value="session" disabled={Boolean(restriction)}>
|
||||
Once per session
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<p className="text-xs text-muted-foreground">{restriction?.reason ?? frequencyDescription}</p>
|
||||
</div>
|
||||
{usesJudge && (
|
||||
<div
|
||||
className={
|
||||
{
|
||||
all: "mb-6 grid gap-4 sm:grid-cols-2",
|
||||
frequency: "mb-6 grid gap-4",
|
||||
judge: "grid gap-4",
|
||||
}[section]
|
||||
}
|
||||
>
|
||||
{section !== "judge" && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-frequency`}>How often to classify</Label>
|
||||
<Select
|
||||
items={[
|
||||
{ value: "every_request", label: "Every request" },
|
||||
{ value: "user_turn", label: "Every new user message" },
|
||||
{ value: "session", label: "Once per session" },
|
||||
]}
|
||||
value={frequency}
|
||||
onValueChange={(frequency) => {
|
||||
if (frequency) onChange(withClassificationFrequency(value, frequency as ClassificationFrequency));
|
||||
}}
|
||||
>
|
||||
<SelectTrigger id={`${id}-frequency`} className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="every_request">Every request</SelectItem>
|
||||
<SelectItem value="user_turn">Every new user message</SelectItem>
|
||||
<SelectItem value="session" disabled={Boolean(restriction)}>
|
||||
Once per session
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<p className="text-xs text-muted-foreground">{restriction?.reason ?? frequencyDescription}</p>
|
||||
</div>
|
||||
)}
|
||||
{section !== "frequency" && usesJudge && (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-judge`}>Judge model</Label>
|
||||
<SearchSelect
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import type { AutoRouterCompressionState } from "./buildAutoRouterCompression";
|
|||
import { activeTierName, type TierRow } from "./tier_rows";
|
||||
|
||||
interface ComplexityRouterAdvancedSectionsProps {
|
||||
hideClassifier?: boolean;
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
forecast: boolean;
|
||||
|
|
@ -53,6 +54,7 @@ interface ComplexityRouterAdvancedSectionsProps {
|
|||
}
|
||||
|
||||
const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSectionsProps> = ({
|
||||
hideClassifier,
|
||||
value,
|
||||
onChange,
|
||||
forecast,
|
||||
|
|
@ -255,7 +257,10 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
|
|||
},
|
||||
{ label: "Sessions and efficiency", keys: ["affinity", "adaptive", "compression"] },
|
||||
{ label: "Compatibility", keys: ["response"] },
|
||||
];
|
||||
].filter(
|
||||
(group) =>
|
||||
!(hideClassifier && group.keys.includes("classifier")) && sections.some(({ key }) => group.keys.includes(key)),
|
||||
);
|
||||
const [openGroups, setOpenGroups] = React.useState<string[]>(() =>
|
||||
showValidationErrors ? groups.map((group) => group.label) : [],
|
||||
);
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import {
|
|||
} from "./tier_rows";
|
||||
import React from "react";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
|
||||
import ClassificationMethodConfig, { InactiveHeuristicV2Threshold } from "./ClassificationMethodConfig";
|
||||
import ComplexityRouterAdvancedSections from "./ComplexityRouterAdvancedSections";
|
||||
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
|
||||
import {
|
||||
|
|
@ -456,6 +456,8 @@ export const withClassificationFrequency = (
|
|||
});
|
||||
|
||||
interface ComplexityRouterConfigProps {
|
||||
creationFlow?: boolean;
|
||||
tierSetup?: React.ReactNode;
|
||||
modelInfo: ModelGroup[];
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
|
|
@ -532,7 +534,59 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03;
|
|||
*/
|
||||
export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_ORDER.slice(0, -1);
|
||||
|
||||
const isJevCreation = (value: ComplexityRouterConfigValue, creationFlow: boolean | undefined) =>
|
||||
creationFlow && effectiveClassifierType(value) === "jev";
|
||||
|
||||
const CreationClassifierSettings = ({
|
||||
creationFlow,
|
||||
value,
|
||||
onChange,
|
||||
modelOptions,
|
||||
classifierEffortOptionsByModel,
|
||||
defaultModel,
|
||||
showValidationErrors,
|
||||
}: Pick<ComplexityRouterConfigProps, "value" | "onChange"> & {
|
||||
creationFlow: boolean | undefined;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
classifierEffortOptionsByModel: Record<string, string[] | null | undefined>;
|
||||
defaultModel: string | undefined;
|
||||
showValidationErrors: boolean;
|
||||
}) => {
|
||||
const jevCreation = isJevCreation(value, creationFlow);
|
||||
return (
|
||||
<>
|
||||
<ClassifierPrimarySettings
|
||||
section={creationFlow ? "frequency" : "all"}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
{jevCreation && (
|
||||
<div className="mb-6">
|
||||
<RoutingOptions label="Advanced classifier settings" showValidationErrors={showValidationErrors}>
|
||||
<div className="px-4 pb-4">
|
||||
<ClassificationMethodConfig
|
||||
advancedOnly
|
||||
jevSection="advanced"
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
effortOptionsByModel={classifierEffortOptionsByModel}
|
||||
defaultModel={defaultModel}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
</div>
|
||||
</RoutingOptions>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
||||
creationFlow,
|
||||
tierSetup,
|
||||
modelInfo,
|
||||
value,
|
||||
onChange,
|
||||
|
|
@ -556,6 +610,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
showValidationErrors = false,
|
||||
}) => {
|
||||
const forecast = isForecastClassifier(value.classifier_type);
|
||||
const tierHeading = creationFlow ? "Define your tiers" : "Models by tier";
|
||||
const customTierSet = value.custom_tier_set;
|
||||
const tierRows = activeTierRows(value);
|
||||
const tierRowsError = customTierSet ? getCustomTierRowsError(customTierSet) : null;
|
||||
|
|
@ -604,19 +659,25 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
|
||||
return (
|
||||
<div className="w-full max-w-none">
|
||||
<ClassifierPrimarySettings
|
||||
<CreationClassifierSettings
|
||||
creationFlow={creationFlow}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
classifierEffortOptionsByModel={classifierEffortOptionsByModel}
|
||||
defaultModel={defaultModel}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
<div className="inline-flex items-center gap-2 mb-4">
|
||||
<h4 className="m-0 text-xl font-semibold text-foreground">{forecast ? "Solver models" : "Models by tier"}</h4>
|
||||
{!forecast && (
|
||||
<SimpleTooltip content="Map each complexity tier to one or more models. Simple queries use cheaper/faster models, complex queries use more capable models.">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
)}
|
||||
{creationFlow && <Separator className="my-6" />}
|
||||
<div className="mb-4">
|
||||
<div className="inline-flex items-center gap-2">
|
||||
<h4 className="m-0 text-xl font-semibold text-foreground">{forecast ? "Solver models" : tierHeading}</h4>
|
||||
{!forecast && (
|
||||
<SimpleTooltip content="Map each complexity tier to one or more models. Simple queries use cheaper/faster models, complex queries use more capable models.">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<InactiveHeuristicV2Threshold value={value} onChange={onChange} />
|
||||
|
|
@ -641,16 +702,9 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
) : (
|
||||
<>
|
||||
<TierConfigIntro value={value} />
|
||||
{tierSetup}
|
||||
<Card>
|
||||
<CardContent>
|
||||
{!customTierSet && (
|
||||
<NonReasoningTierToggle
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
available={value.classifier_type === "llm" || value.classifier_type === "jev"}
|
||||
/>
|
||||
)}
|
||||
|
||||
{tierRows.map((row, index) => {
|
||||
const tierInfo = builtInTierInfo(row.id);
|
||||
const label = tierRowLabel(row, value.tier_labels);
|
||||
|
|
@ -740,6 +794,14 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
);
|
||||
})}
|
||||
|
||||
{!customTierSet && (
|
||||
<NonReasoningTierToggle
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
available={value.classifier_type === "llm" || value.classifier_type === "jev"}
|
||||
/>
|
||||
)}
|
||||
|
||||
<TierSetToolbar
|
||||
editing={editingTiers}
|
||||
isCustomSet={Boolean(customTierSet)}
|
||||
|
|
@ -763,7 +825,6 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
</>
|
||||
)}
|
||||
<DefaultModelField value={value} onChange={onChange} modelOptions={modelOptions} />
|
||||
<Separator className="my-6" />
|
||||
|
||||
<RoutingOptions
|
||||
showValidationErrors={showValidationErrors}
|
||||
|
|
@ -787,6 +848,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
)}
|
||||
<div className="rounded-lg border border-border bg-muted">
|
||||
<ComplexityRouterAdvancedSections
|
||||
hideClassifier={isJevCreation(value, creationFlow)}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
forecast={forecast}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
import React from "react";
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import { buildDecisionCatalog, isDecisionMode, isDecisionSelection } from "@/lib/decisionModels";
|
||||
import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
export default function DecisionModelSelect({
|
||||
value,
|
||||
onChange,
|
||||
modelInfo,
|
||||
deployments,
|
||||
}: {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelInfo: ModelGroup[];
|
||||
deployments: AutoRouterDeployment[] | undefined;
|
||||
}) {
|
||||
const id = React.useId();
|
||||
const { data: modelCostMapData } = useModelCostMap();
|
||||
const decisionCatalog = React.useMemo(() => buildDecisionCatalog(modelCostMapData), [modelCostMapData]);
|
||||
const decisionModels = modelInfo.filter(
|
||||
(model) =>
|
||||
isDecisionMode(model.mode) ||
|
||||
(deployments ?? []).some(
|
||||
(deployment) =>
|
||||
deployment.model_name === model.model_group &&
|
||||
isDecisionSelection(
|
||||
decisionCatalog,
|
||||
deployment.litellm_params?.custom_llm_provider ?? deployment.litellm_params?.model?.split("/")[0],
|
||||
[deployment.litellm_params?.model ?? "", deployment.litellm_params?.base_model ?? ""],
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={id}>Decision model</Label>
|
||||
<SearchSelect
|
||||
inputId={id}
|
||||
aria-label="Decision model"
|
||||
options={decisionModels.map((model) => ({ value: model.model_group, label: model.model_group }))}
|
||||
value={value.jev_classifier_config?.deployment_name ?? ""}
|
||||
placeholder="Select the decision model"
|
||||
allowClear={false}
|
||||
onValueChange={(deploymentName) => {
|
||||
if (!deploymentName) return;
|
||||
onChange({
|
||||
...value,
|
||||
jev_classifier_config: {
|
||||
...(value.jev_classifier_config ?? defaultJevClassifierConfig()),
|
||||
deployment_name: deploymentName,
|
||||
},
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -102,24 +102,24 @@ describe("JEV classifier editor", () => {
|
|||
expect(screen.getByText("Reasoning Effort")).toBeInTheDocument();
|
||||
expect(screen.getByText("Classifier Prompt")).toBeInTheDocument();
|
||||
expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /Jev Classifier/ }));
|
||||
expect(screen.getByRole("radio", { name: /^Jev Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("Jev Instructions")).toBeEnabled();
|
||||
fireEvent.click(screen.getByRole("radio", { name: /Decisions Model/ }));
|
||||
expect(screen.getByRole("radio", { name: /^Decisions Model/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-latest");
|
||||
expect(screen.getByLabelText("Classifier Instructions")).toBeEnabled();
|
||||
expect(screen.queryByLabelText("Judge model")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Reasoning Effort")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Classifier Prompt")).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("switch", { name: "Use images for classification" })).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Jev Model"), { target: { value: "jev-test" } });
|
||||
fireEvent.change(screen.getByLabelText("Jev Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: "jev-test" } });
|
||||
fireEvent.change(screen.getByLabelText("Classifier Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } });
|
||||
fireEvent.change(screen.getByLabelText("Circuit breaker cooldown (seconds)"), { target: { value: "50" } });
|
||||
fireEvent.click(screen.getByRole("switch", { name: "Classifier circuit breaker" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Customize tiers" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Save and reload" }));
|
||||
expect(screen.getByRole("radio", { name: /Jev Classifier/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Jev Model")).toHaveValue("jev-test");
|
||||
expect(screen.getByLabelText("Jev Timeout (ms)")).toHaveValue(4200);
|
||||
expect(screen.getByRole("radio", { name: /Decisions Model/ })).toBeChecked();
|
||||
expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-test");
|
||||
expect(screen.getByLabelText("Classifier Timeout (ms)")).toHaveValue(4200);
|
||||
expect(screen.getByLabelText("Context Window Size")).toHaveValue("6");
|
||||
expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Probe current config" }));
|
||||
|
|
@ -152,10 +152,10 @@ describe("JEV classifier editor", () => {
|
|||
return <JevEditor value={value} onChange={setValue} />;
|
||||
};
|
||||
renderWithProviders(<LicensedForm />);
|
||||
expect(screen.getByLabelText("Jev Instructions")).toBeEnabled();
|
||||
fireEvent.change(screen.getByLabelText("Jev Instructions"), { target: { value: "New instructions" } });
|
||||
expect(screen.getByLabelText("Jev Instructions")).toHaveValue("New instructions");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Restore built-in Jev instructions" }));
|
||||
expect(screen.getByLabelText("Jev Instructions")).toHaveValue("");
|
||||
expect(screen.getByLabelText("Classifier Instructions")).toBeEnabled();
|
||||
fireEvent.change(screen.getByLabelText("Classifier Instructions"), { target: { value: "New instructions" } });
|
||||
expect(screen.getByLabelText("Classifier Instructions")).toHaveValue("New instructions");
|
||||
fireEvent.click(screen.getByRole("button", { name: "Restore built-in instructions" }));
|
||||
expect(screen.getByLabelText("Classifier Instructions")).toHaveValue("");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,9 +11,11 @@ import { defaultJevClassifierConfig } from "./jev_classifier_config";
|
|||
export default function JevClassifierConfig({
|
||||
value,
|
||||
onChange,
|
||||
section = "all",
|
||||
}: {
|
||||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
section?: "all" | "model" | "advanced";
|
||||
}) {
|
||||
const id = useId();
|
||||
const config = value.jev_classifier_config ?? defaultJevClassifierConfig();
|
||||
|
|
@ -21,55 +23,67 @@ export default function JevClassifierConfig({
|
|||
onChange({ ...value, jev_classifier_config: { ...config, ...patch } });
|
||||
|
||||
return (
|
||||
<div className="mt-4 space-y-3">
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Uses TypeSafe System One Choice evaluation with your configured tiers
|
||||
</p>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-model`}>Jev Model</Label>
|
||||
<Input id={`${id}-model`} value={config.model} onChange={(event) => update({ model: event.target.value })} />
|
||||
</div>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-timeout`}>Jev Timeout (ms)</Label>
|
||||
<Input
|
||||
id={`${id}-timeout`}
|
||||
type="number"
|
||||
min={1}
|
||||
step={1}
|
||||
value={config.timeout_ms}
|
||||
onChange={(event) => update({ timeout_ms: Number(event.target.value) })}
|
||||
/>
|
||||
</div>
|
||||
<ClassifierCircuitBreakerConfig
|
||||
value={config}
|
||||
onChange={(next) =>
|
||||
update({
|
||||
circuit_breaker_enabled: next.circuit_breaker_enabled,
|
||||
circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds,
|
||||
})
|
||||
}
|
||||
/>
|
||||
<div>
|
||||
<Label htmlFor={`${id}-instructions`}>Jev Instructions</Label>
|
||||
<AutoRouterAllowanceNote
|
||||
feature="tier_or_classifier_prompt"
|
||||
label="Custom instructions share the custom-tier allowance"
|
||||
/>
|
||||
<Textarea
|
||||
id={`${id}-instructions`}
|
||||
value={config.instructions ?? ""}
|
||||
placeholder="Leave blank to use the built-in instructions"
|
||||
onChange={(event) => update({ instructions: event.target.value || undefined })}
|
||||
/>
|
||||
{config.instructions && (
|
||||
<Button variant="outline" type="button" onClick={() => update({ instructions: undefined })}>
|
||||
Restore built-in Jev instructions
|
||||
</Button>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Built-in Jev is available without a license and uses the shipped tier criteria
|
||||
</p>
|
||||
</div>
|
||||
<div className={section === "model" ? "space-y-3" : "mt-4 space-y-3"}>
|
||||
{section !== "advanced" && !config.deployment_name && (
|
||||
<>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Uses TypeSafe System One Choice evaluation with your configured tiers
|
||||
</p>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-model`}>Classifier Model</Label>
|
||||
<Input
|
||||
id={`${id}-model`}
|
||||
value={config.model}
|
||||
onChange={(event) => update({ model: event.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{section !== "model" && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-timeout`}>Classifier Timeout (ms)</Label>
|
||||
<Input
|
||||
id={`${id}-timeout`}
|
||||
type="number"
|
||||
min={1}
|
||||
step={1}
|
||||
value={config.timeout_ms}
|
||||
onChange={(event) => update({ timeout_ms: Number(event.target.value) })}
|
||||
/>
|
||||
</div>
|
||||
<ClassifierCircuitBreakerConfig
|
||||
value={config}
|
||||
onChange={(next) =>
|
||||
update({
|
||||
circuit_breaker_enabled: next.circuit_breaker_enabled,
|
||||
circuit_breaker_cooldown_seconds: next.circuit_breaker_cooldown_seconds,
|
||||
})
|
||||
}
|
||||
/>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor={`${id}-instructions`}>Classifier Instructions</Label>
|
||||
<AutoRouterAllowanceNote
|
||||
feature="tier_or_classifier_prompt"
|
||||
label="Custom instructions share the custom-tier allowance"
|
||||
/>
|
||||
<Textarea
|
||||
id={`${id}-instructions`}
|
||||
value={config.instructions ?? ""}
|
||||
placeholder="Leave blank to use the built-in instructions"
|
||||
onChange={(event) => update({ instructions: event.target.value || undefined })}
|
||||
/>
|
||||
{config.instructions && (
|
||||
<Button variant="outline" type="button" onClick={() => update({ instructions: undefined })}>
|
||||
Restore built-in instructions
|
||||
</Button>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Built-in decision model classification is available without a license and uses the shipped tier criteria
|
||||
</p>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ const NonReasoningTierToggle: React.FC<{
|
|||
|
||||
return (
|
||||
<>
|
||||
<Separator className="my-4" />
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<Switch
|
||||
checked={value.enable_non_reasoning_tier === true}
|
||||
|
|
@ -41,7 +42,6 @@ const NonReasoningTierToggle: React.FC<{
|
|||
reasoning about it. Escalation still moves up out of it when a request needs more.
|
||||
{!available && " Requires the LLM or Jev classification method"}
|
||||
</span>
|
||||
<Separator className="my-4" />
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,12 +3,18 @@ import { ChevronRight } from "lucide-react";
|
|||
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
|
||||
|
||||
interface RoutingOptionsProps {
|
||||
label?: string;
|
||||
showValidationErrors?: boolean;
|
||||
summary?: string;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
const RoutingOptions = ({ showValidationErrors = false, summary, children }: RoutingOptionsProps) => {
|
||||
const RoutingOptions = ({
|
||||
label = "Advanced settings",
|
||||
showValidationErrors = false,
|
||||
summary,
|
||||
children,
|
||||
}: RoutingOptionsProps) => {
|
||||
const [open, setOpen] = React.useState(false);
|
||||
const [previousValidation, setPreviousValidation] = React.useState(showValidationErrors);
|
||||
if (previousValidation !== showValidationErrors) {
|
||||
|
|
@ -16,10 +22,10 @@ const RoutingOptions = ({ showValidationErrors = false, summary, children }: Rou
|
|||
if (showValidationErrors) setOpen(true);
|
||||
}
|
||||
return (
|
||||
<Collapsible open={open} onOpenChange={setOpen} className="rounded-lg border">
|
||||
<Collapsible open={open} onOpenChange={setOpen} className="rounded-lg border bg-muted/40">
|
||||
<CollapsibleTrigger className="group flex w-full flex-wrap items-center gap-2 px-4 py-3 text-left font-medium">
|
||||
<ChevronRight className="size-4 transition-transform group-data-panel-open:rotate-90" />
|
||||
Advanced settings
|
||||
{label}
|
||||
{summary && <span className="text-xs font-normal text-muted-foreground">{summary}</span>}
|
||||
</CollapsibleTrigger>
|
||||
<CollapsibleContent className="space-y-4">{children}</CollapsibleContent>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
|||
import { getMissingTiersError } from "./build_complexity_router_config";
|
||||
import { getSubmitBlockedReason } from "./add_auto_router_tab";
|
||||
import { buildModelAvailability } from "@/lib/autorouter_presets";
|
||||
import { apiClient, modelCreateCall, testAutoRouterRouting } from "../networking";
|
||||
import { apiClient, modelCostMap, modelCreateCall, testAutoRouterRouting } from "../networking";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
|
||||
import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets";
|
||||
|
|
@ -54,7 +54,7 @@ const openTemplateDropdown = (): void => {
|
|||
};
|
||||
|
||||
const expandDetailedConfiguration = (): void => {
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
expect(screen.getByText("Define your tiers")).toBeVisible();
|
||||
};
|
||||
|
||||
const visibleOptions = (): HTMLElement[] => screen.queryAllByRole("option");
|
||||
|
|
@ -78,6 +78,14 @@ const expectTierModel = (tier: string, model: string): void => {
|
|||
expect(chips.map((chip) => chip.getAttribute("aria-label"))).toEqual([model]);
|
||||
};
|
||||
|
||||
const waitForAutomaticTiers = async () => {
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
within(tierChips("Simple")).getAllByLabelText(/.+/, { selector: '[data-slot="combobox-chip"]' }).length,
|
||||
).toBeGreaterThan(0),
|
||||
);
|
||||
};
|
||||
|
||||
const selectTemplate = async (label: string): Promise<void> => {
|
||||
await userEvent.click(optionByLabel(label)!);
|
||||
};
|
||||
|
|
@ -114,6 +122,7 @@ vi.mock("../networking", () => ({
|
|||
error: null,
|
||||
}),
|
||||
},
|
||||
modelCostMap: vi.fn().mockResolvedValue({}),
|
||||
modelCreateCall: vi.fn().mockResolvedValue({}),
|
||||
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
testAutoRouterRouting: vi.fn(),
|
||||
|
|
@ -179,6 +188,104 @@ describe("AddAutoRouterTab", () => {
|
|||
testQueryClient.clear();
|
||||
mockFetchAvailableModels.mockResolvedValue([]);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
vi.mocked(modelCostMap).mockResolvedValue({});
|
||||
});
|
||||
|
||||
it("shows Jev essentials before tiers and saves tuning from its own disclosure", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
...ALL_FAMILY_MODELS,
|
||||
{ model_group: "my-decision", mode: "evaluation" },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitForAutomaticTiers();
|
||||
await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" }));
|
||||
expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument();
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
expect(screen.queryByRole("option", { name: ALL_FAMILY_MODELS[0].model_group })).not.toBeInTheDocument();
|
||||
await userEvent.click(await screen.findByRole("option", { name: "my-decision" }));
|
||||
expect(screen.queryByRole("button", { name: "Routing approach" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByRole("button", { name: "Choose models for me" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByLabelText("Classifier Timeout (ms)")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
await userEvent.click(screen.getByRole("button", { name: "Advanced classifier settings" }));
|
||||
fireEvent.change(screen.getByLabelText("Classifier Timeout (ms)"), { target: { value: "4200" } });
|
||||
fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "5" } });
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
await userEvent.click(screen.getByRole("button", { name: "Advanced classifier settings" }));
|
||||
await userEvent.click(screen.getByRole("button", { name: "Advanced settings" }));
|
||||
expect(screen.queryByRole("button", { name: "Classifier tuning" })).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "jev-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
const expected = {
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { deployment_name: "my-decision", timeout_ms: 4200 },
|
||||
classifier_context_window_size: 5,
|
||||
classification_mode: "user_turn",
|
||||
};
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ mode: "evaluation" },
|
||||
{ mode: "chat", supported_endpoints: ["/v1/decisions"] },
|
||||
{ supported_endpoints: ["/v1/systemone"] },
|
||||
])("selects and saves an onboarded alias using decision catalog metadata %j", async (metadata) => {
|
||||
const alias = "onboarded-decider";
|
||||
const underlyingModel = "mixed-provider/decision-model";
|
||||
vi.mocked(modelCostMap).mockResolvedValue({
|
||||
[underlyingModel]: { litellm_provider: "mixed-provider", ...metadata },
|
||||
"mixed-provider/chat-model": { litellm_provider: "mixed-provider", mode: "chat" },
|
||||
"catalog-only/decision-model": { litellm_provider: "catalog-only", mode: "evaluation" },
|
||||
});
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
...ALL_FAMILY_MODELS,
|
||||
{ model_group: alias },
|
||||
{ model_group: "ordinary-chat", mode: "chat" },
|
||||
]);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([
|
||||
{ model_name: alias, litellm_params: { model: underlyingModel } },
|
||||
{ model_name: "ordinary-chat", litellm_params: { model: "mixed-provider/chat-model" } },
|
||||
{ model_name: "unavailable-decider", litellm_params: { model: underlyingModel } },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitForAutomaticTiers();
|
||||
await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" }));
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
const option = await screen.findByRole("option", { name: alias });
|
||||
expect(screen.getAllByRole("option")).toHaveLength(1);
|
||||
await userEvent.click(option);
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "decision-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce());
|
||||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject({
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { deployment_name: alias },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the judge picker empty state when no decision deployments exist", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitForAutomaticTiers();
|
||||
await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" }));
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
expect(await screen.findByText("No results")).toBeVisible();
|
||||
await userEvent.keyboard("{Escape}");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it("does not refill a tier cleared by the user when models refresh", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await waitForAutomaticTiers();
|
||||
await userEvent.click(within(tierChips("Simple")).getByRole("button", { name: "" }));
|
||||
await act(async () => {
|
||||
await testQueryClient.refetchQueries({ queryKey: ["availableModels"] });
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
});
|
||||
|
||||
it.each([1, 0])("defaults to Rule-based with %s v2 slots remaining", async (remaining) => {
|
||||
|
|
@ -271,10 +378,15 @@ describe("AddAutoRouterTab", () => {
|
|||
error: body?.complexity_router_config?.tier_definitions ? blocked : null,
|
||||
};
|
||||
});
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
...ALL_FAMILY_MODELS,
|
||||
{ model_group: "my-decision", mode: "evaluation" },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
|
||||
await user.click(screen.getByRole("radio", { name: "Jev" }));
|
||||
await waitForAutomaticTiers();
|
||||
await user.click(screen.getByRole("radio", { name: "Decisions Model" }));
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
await userEvent.click(await screen.findByRole("option", { name: "my-decision" }));
|
||||
await waitFor(() =>
|
||||
expect(apiClient.post).toHaveBeenLastCalledWith(
|
||||
"/auto_router/availability",
|
||||
|
|
@ -301,7 +413,7 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(within(screen.getByRole("alert")).getByRole("link", { name: "Talk to our team" })).toBeVisible();
|
||||
await user.click(screen.getByRole("button", { name: "Restore defaults" }));
|
||||
await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument());
|
||||
expect(screen.getByRole("radio", { name: "Jev" })).toBeChecked();
|
||||
expect(screen.getByRole("radio", { name: "Decisions Model" })).toBeChecked();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
|
|
@ -314,10 +426,15 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
it("blocks button and Enter submissions until the edited draft is checked", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
...ALL_FAMILY_MODELS,
|
||||
{ model_group: "my-decision", mode: "evaluation" },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
await user.click(await screen.findByRole("button", { name: "Choose models for me" }));
|
||||
await user.click(screen.getByRole("radio", { name: "Jev" }));
|
||||
await waitForAutomaticTiers();
|
||||
await user.click(screen.getByRole("radio", { name: "Decisions Model" }));
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
await userEvent.click(await screen.findByRole("option", { name: "my-decision" }));
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "checked-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
let complete: ((result: unknown) => void) | undefined;
|
||||
|
|
@ -357,17 +474,29 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByRole("button", { name: "Routing approach" })).toHaveTextContent("Complexity");
|
||||
});
|
||||
|
||||
it.each(["LLM", "Jev"])("keeps %s and the frequency when choosing models automatically", async (family) => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
const automatic = await screen.findByRole("button", { name: "Choose models for me" });
|
||||
await userEvent.click(screen.getByRole("radio", { name: family }));
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
await userEvent.click(automatic);
|
||||
expect(screen.getByRole("radio", { name: family })).toBeChecked();
|
||||
expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every new user message");
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
});
|
||||
it.each(["LLM", "Decisions Model"])(
|
||||
"keeps %s and the frequency selected before automatic models finish loading",
|
||||
async (family) => {
|
||||
let complete: ((models: ModelGroup[]) => void) | undefined;
|
||||
mockFetchAvailableModels.mockReturnValue(
|
||||
new Promise<ModelGroup[]>((resolve) => {
|
||||
complete = resolve;
|
||||
}),
|
||||
);
|
||||
renderWithProviders(<Harness />);
|
||||
await userEvent.click(screen.getByRole("radio", { name: family }));
|
||||
await selectAutoRouterOption("How often to classify", "Every new user message");
|
||||
await act(async () => {
|
||||
complete?.(ALL_FAMILY_MODELS);
|
||||
});
|
||||
await waitForAutomaticTiers();
|
||||
expect(screen.getByRole("radio", { name: family })).toBeChecked();
|
||||
expect(screen.getByRole("combobox", { name: "How often to classify" })).toHaveTextContent(
|
||||
"Every new user message",
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["Capability", "Fuse v2"])(
|
||||
"creates %s from its dedicated tab without complexity templates",
|
||||
|
|
@ -446,14 +575,14 @@ describe("AddAutoRouterTab", () => {
|
|||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
await screen.findByTestId("configure-automatically-button");
|
||||
await waitForAutomaticTiers();
|
||||
await selectAutoRouterApproach("Capability");
|
||||
expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument();
|
||||
await selectAutoRouterApproach("Complexity");
|
||||
expect(screen.getByTestId("configure-automatically-button")).toBeInTheDocument();
|
||||
expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument();
|
||||
expect(screen.getByTestId("template-selector")).toBeInTheDocument();
|
||||
expandDetailedConfiguration();
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
expect(screen.getByText("Define your tiers")).toBeInTheDocument();
|
||||
for (const label of ["Adaptive Routing", "Context Window Escalation", "Escalation Keywords"]) {
|
||||
openAutoRouterAdvanced(label);
|
||||
expect(screen.getAllByText(label)).not.toHaveLength(0);
|
||||
|
|
@ -501,9 +630,9 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
expect(screen.getByText("Define your tiers")).toBeVisible();
|
||||
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
expect(screen.getByText("Define your tiers")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides automatic setup when no available model is recommended", async () => {
|
||||
|
|
@ -529,8 +658,7 @@ describe("AddAutoRouterTab", () => {
|
|||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = await screen.findByTestId("configure-automatically-button");
|
||||
await userEvent.click(button);
|
||||
await waitForAutomaticTiers();
|
||||
|
||||
expectTierModel("Simple", "gpt-5.6-luna");
|
||||
expectTierModel("Medium", "claude-sonnet-5");
|
||||
|
|
@ -549,8 +677,7 @@ describe("AddAutoRouterTab", () => {
|
|||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = await screen.findByTestId("configure-automatically-button");
|
||||
await userEvent.click(button);
|
||||
await waitForAutomaticTiers();
|
||||
|
||||
expectTierModel("Simple", "gpt-5.6-luna");
|
||||
expectTierModel("Medium", "claude-sonnet-5");
|
||||
|
|
@ -565,9 +692,9 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false");
|
||||
|
||||
await userEvent.click(await screen.findByTestId("configure-automatically-button"));
|
||||
await waitForAutomaticTiers();
|
||||
|
||||
expect(screen.getByText("Models by tier")).toBeInTheDocument();
|
||||
expect(screen.getByText("Define your tiers")).toBeInTheDocument();
|
||||
expectTierModel("Simple", simpleModel);
|
||||
});
|
||||
|
||||
|
|
@ -929,16 +1056,16 @@ describe("AddAutoRouterTab", () => {
|
|||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
renderWithProviders(<Harness />);
|
||||
const automaticSetup = await screen.findByRole("button", { name: "Choose models for me" });
|
||||
await waitFor(() => expect(automaticSetup).toBeEnabled());
|
||||
await user.click(automaticSetup);
|
||||
await waitForAutomaticTiers();
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } });
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
await selectAutoRouterOption("Heuristic", "Heuristic v2");
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
|
||||
await user.click(automaticSetup);
|
||||
await act(async () => {
|
||||
await testQueryClient.refetchQueries({ queryKey: ["availableModels"] });
|
||||
});
|
||||
expect(screen.getByRole("textbox", { name: "Success threshold" })).toHaveValue("1.1");
|
||||
expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled();
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "" } });
|
||||
|
|
@ -1281,7 +1408,7 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
// Custom is the escape hatch, not the headline choice, so it's listed after every bundled preset
|
||||
// rather than first.
|
||||
it("lists Custom Configuration after the bundled presets", async () => {
|
||||
it("lists Start from scratch after the bundled presets", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
|
|
@ -1293,7 +1420,7 @@ describe("AddAutoRouterTab", () => {
|
|||
"Gemini Family",
|
||||
"Lite",
|
||||
"OpenAI Family",
|
||||
"Custom Configuration",
|
||||
"Start from scratch",
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -1451,11 +1578,11 @@ describe("AddAutoRouterTab", () => {
|
|||
expectTierModel("reasoning", ANTHROPIC_TIERS.REASONING[0]);
|
||||
});
|
||||
|
||||
it("expands detailed configuration when Custom Configuration is chosen", async () => {
|
||||
it("expands detailed configuration when Start from scratch is chosen", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
await selectTemplate("Custom Configuration");
|
||||
await selectTemplate("Start from scratch");
|
||||
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
|
|
@ -1469,7 +1596,7 @@ describe("AddAutoRouterTab", () => {
|
|||
await selectTemplate("Anthropic Family");
|
||||
expect(screen.queryByText("Keyword/Semantic Matching")).not.toBeInTheDocument();
|
||||
|
||||
expect(screen.getByText("Models by tier")).toBeVisible();
|
||||
expect(screen.getByText("Define your tiers")).toBeVisible();
|
||||
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
|
|
@ -1738,7 +1865,7 @@ describe("AddAutoRouterTab", () => {
|
|||
"Gemini Family",
|
||||
"Lite",
|
||||
"OpenAI Family",
|
||||
"Custom Configuration",
|
||||
"Start from scratch",
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -1876,7 +2003,10 @@ describe("preset catalog fetch states", () => {
|
|||
vi.clearAllMocks();
|
||||
testQueryClient.clear();
|
||||
vi.mocked(handleAddAutoRouterSubmit).mockReset();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
...ALL_FAMILY_MODELS,
|
||||
{ model_group: "my-decision", mode: "evaluation" },
|
||||
]);
|
||||
vi.mocked(useAutoRouterPresets).mockReturnValue({
|
||||
...LOADED_PRESETS_QUERY,
|
||||
data: [
|
||||
|
|
@ -1896,6 +2026,8 @@ describe("preset catalog fetch states", () => {
|
|||
renderWithProviders(<Harness />);
|
||||
await waitForPresetEnabled("Bounded JEV");
|
||||
await selectTemplate("Bounded JEV");
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Decision model" }));
|
||||
await userEvent.click(await screen.findByRole("option", { name: "my-decision" }));
|
||||
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "bounded-router" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
|
||||
fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
|
||||
|
|
@ -1931,7 +2063,7 @@ describe("preset catalog fetch states", () => {
|
|||
expect(screen.getByText("Loading templates...")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("degrades to Custom Configuration with a retry hint that refetches the catalog", async () => {
|
||||
it("degrades to Start from scratch with a retry hint that refetches the catalog", async () => {
|
||||
const refetch = vi.fn();
|
||||
vi.mocked(useAutoRouterPresets).mockReturnValue({
|
||||
...LOADED_PRESETS_QUERY,
|
||||
|
|
@ -1946,7 +2078,7 @@ describe("preset catalog fetch states", () => {
|
|||
openTemplateDropdown();
|
||||
const options = screen.queryAllByRole("option");
|
||||
expect(options).toHaveLength(1);
|
||||
expect(options[0]).toHaveTextContent("Custom Configuration");
|
||||
expect(options[0]).toHaveTextContent("Start from scratch");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Retry" }));
|
||||
expect(refetch).toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import { AutoRouterAvailabilityContext, useAutoRouterAvailability } from "./AutoRouterAvailability";
|
||||
import { decisionDeploymentName } from "./jev_classifier_config";
|
||||
import DecisionModelSelect from "./DecisionModelSelect";
|
||||
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
|
||||
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
|
||||
import { getForecastConfigError, isForecastClassifier } from "../add_model/forecast_classifier_config";
|
||||
import React, { useEffect, useState } from "react";
|
||||
|
|
@ -10,7 +13,7 @@ import { FormField } from "@/components/shared/form/FormField";
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Select, SelectContent, SelectItem, SelectSeparator, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
import { useZodForm } from "@/lib/forms/useZodForm";
|
||||
|
|
@ -55,7 +58,6 @@ import {
|
|||
} from "./build_complexity_router_config";
|
||||
import { builderParamsFromValue } from "./complexity_router_builder_params";
|
||||
import { activeTierName, activeTierRows, getCustomTierRowsError, resolveComplexityDefaultModel } from "./tier_rows";
|
||||
import { tierRowLabel } from "./complexity_router_tiers";
|
||||
import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
|
||||
import { AutoRouterConnectionTestDialog } from "./auto_router_connection_test";
|
||||
import {
|
||||
|
|
@ -77,7 +79,7 @@ import {
|
|||
} from "@/lib/autorouter_presets";
|
||||
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import { buildAutomaticRouterConfig, buildPreferredTierModels } from "./auto_setup";
|
||||
import { buildAutomaticRouterConfig, buildPreferredTierModels, prefillEmptyTiers } from "./auto_setup";
|
||||
|
||||
interface AddAutoRouterTabProps {
|
||||
handleOk: () => void;
|
||||
|
|
@ -120,15 +122,6 @@ const isPresetHintAlarming = (availability: PresetAvailability): boolean => avai
|
|||
|
||||
const NO_PRESETS: AutoRouterPreset[] = [];
|
||||
|
||||
// A one-line summary of what's configured, shown when the detailed section is collapsed so a
|
||||
// caller can see the shape of the config without opening it.
|
||||
const tierConfigSummary = (config: ComplexityRouterConfigValue): string => {
|
||||
const parts = activeTierRows(config)
|
||||
.filter((row) => row.models.length > 0)
|
||||
.map((row) => `${tierRowLabel(row, config.tier_labels)}: ${row.models.join(", ")}`);
|
||||
return parts.length > 0 ? parts.join(" · ") : "No tiers configured yet";
|
||||
};
|
||||
|
||||
// Why the submit is unavailable, or null when it is available. The button reads this to disable
|
||||
// itself and to say what is missing, so the two can never give different answers. Checks the
|
||||
// config actually being built, not which preset (if any) it came from: a preset only ever
|
||||
|
|
@ -149,6 +142,9 @@ export const getSubmitBlockedReason = (
|
|||
: getMissingTiersError(activeTierRows(config))) ??
|
||||
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
|
||||
(effectiveClassifierType(config) === "jev" && !config.jev_classifier_config?.deployment_name
|
||||
? "Please select a decision model"
|
||||
: null) ??
|
||||
getClassifierModelError(config) ??
|
||||
getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ??
|
||||
getReminderMarkersError(config.reminder_markers) ??
|
||||
|
|
@ -337,7 +333,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const templateItems = React.useMemo(
|
||||
() => [
|
||||
...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })),
|
||||
{ value: "custom", label: "Custom Configuration" },
|
||||
{ value: "custom", label: "Start from scratch" },
|
||||
],
|
||||
[sortedPresetOptions],
|
||||
);
|
||||
|
|
@ -353,20 +349,16 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
setEscalationKeywords(prefill.escalationKeywords);
|
||||
};
|
||||
|
||||
const handleAutomaticSetup = () => {
|
||||
if (automaticRouterConfig === null) return;
|
||||
setSelectedPreset(undefined);
|
||||
setComplexityRouterConfig({
|
||||
...complexityRouterConfig,
|
||||
tiers: { ...complexityRouterConfig.tiers, ...automaticRouterConfig.tiers },
|
||||
tier_model_params: { ...complexityRouterConfig.tier_model_params, ...automaticRouterConfig.tier_model_params },
|
||||
});
|
||||
|
||||
toast.success("Models selected", { description: tierConfigSummary(automaticRouterConfig) });
|
||||
};
|
||||
const [prefillAttempted, setPrefillAttempted] = useState(false);
|
||||
const automaticPrefill = automaticSetupLoading || modelsUnverifiable ? null : automaticRouterConfig;
|
||||
if (automaticPrefill && !prefillAttempted) {
|
||||
setPrefillAttempted(true);
|
||||
setComplexityRouterConfig((current) => prefillEmptyTiers(current, automaticPrefill));
|
||||
}
|
||||
|
||||
const handlePresetChange = (presetKey: string | undefined) => {
|
||||
if (!presetKey || presetKey === "custom") {
|
||||
setPrefillAttempted(true);
|
||||
setSelectedPreset(presetKey);
|
||||
applyPrefill(buildEmptyPrefill());
|
||||
|
||||
|
|
@ -380,6 +372,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const presetState = presetAvailability(preset);
|
||||
if (presetState.kind !== "available") return;
|
||||
|
||||
setPrefillAttempted(true);
|
||||
setSelectedPreset(presetKey);
|
||||
applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability));
|
||||
};
|
||||
|
|
@ -388,6 +381,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
tiers: Object.fromEntries(activeTierRows(complexityRouterConfig).map((row) => [activeTierName(row), row.models])),
|
||||
classifierType: effectiveClassifierType(complexityRouterConfig),
|
||||
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
|
||||
decisionDeploymentName: decisionDeploymentName(complexityRouterConfig.jev_classifier_config),
|
||||
semanticMatchingEnabled,
|
||||
embeddingModel,
|
||||
defaultModel: complexityRouterConfig.default_model,
|
||||
|
|
@ -546,8 +540,71 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
setIsTestModalVisible(true);
|
||||
};
|
||||
|
||||
const tierSetup = (
|
||||
<div className="mb-4 w-full space-y-2">
|
||||
<div className="space-y-2">
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger aria-label="Use a template" data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Use a template" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="[&_[data-slot=select-item]>div]:min-w-0 [&_[data-slot=select-item]>div]:shrink [&_[data-slot=select-item]>div]:whitespace-normal [&_[data-slot=select-item]>div]:break-words">
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState) ? "text-destructive" : "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments ? "Matches your deployments" : null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
<div className="text-xs text-muted-foreground">Replaces classifier and model choices</div>
|
||||
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectSeparator className="mx-0" />
|
||||
<SelectItem value="custom" label="Start from scratch">
|
||||
<div>
|
||||
<div className="font-medium">Start from scratch</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Clears tier models, switches to Heuristics, and resets routing settings
|
||||
</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<p className="text-xs text-muted-foreground">Replaces classifier and model choices</p>
|
||||
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Start from scratch is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
const configurationForm = (
|
||||
<ComplexityRouterConfig
|
||||
creationFlow
|
||||
tierSetup={tierSetup}
|
||||
editingTiers={editingTiers}
|
||||
onEditingTiersChange={setEditingTiers}
|
||||
modelInfo={modelInfo}
|
||||
|
|
@ -571,7 +628,6 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
);
|
||||
const forecast = isForecastClassifier(complexityRouterConfig.classifier_type);
|
||||
|
||||
return (
|
||||
<AutoRouterAvailabilityContext.Provider value={routerAvailability}>
|
||||
|
|
@ -579,7 +635,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
<Card>
|
||||
<CardContent>
|
||||
<form onSubmit={form.handleSubmit(() => handleAutoRouterSubmit())} noValidate>
|
||||
<div className="mb-6">
|
||||
<div className="mb-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="auto_router_name"
|
||||
|
|
@ -590,6 +646,31 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
)}
|
||||
</FormField>
|
||||
</div>
|
||||
<div className="mb-6 border-b pb-6">
|
||||
{isAdmin && (
|
||||
<div>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint(
|
||||
"Model Access Group",
|
||||
"Use model access groups to control who can access this auto router",
|
||||
)}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{requiresTeamScope && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
|
|
@ -610,129 +691,39 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
</FormField>
|
||||
)}
|
||||
|
||||
<AutoRouterClassifierTabs value={complexityRouterConfig} onChange={handleClassifierChange}>
|
||||
<AutoRouterClassifierTabs
|
||||
value={complexityRouterConfig}
|
||||
onChange={handleClassifierChange}
|
||||
decisionModel={
|
||||
<DecisionModelSelect
|
||||
value={complexityRouterConfig}
|
||||
onChange={setComplexityRouterConfig}
|
||||
modelInfo={modelInfo}
|
||||
deployments={deployments}
|
||||
/>
|
||||
}
|
||||
judgeModel={
|
||||
<ClassifierPrimarySettings
|
||||
section="judge"
|
||||
value={complexityRouterConfig}
|
||||
onChange={setComplexityRouterConfig}
|
||||
modelOptions={modelInfo
|
||||
.filter((model) => model.mode !== "embedding")
|
||||
.map((model) => ({ value: model.model_group, label: model.model_group }))}
|
||||
showValidationErrors={showValidationErrors}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<FieldGroup>
|
||||
<div className="space-y-4 empty:hidden">
|
||||
{!forecast && (
|
||||
<>
|
||||
{!automaticSetupLoading && automaticRouterConfig && !complexityRouterConfig.custom_tier_set && (
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-border bg-muted px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">Choose models quickly</p>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Fill the tiers while keeping your classifier and settings.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
data-testid="configure-automatically-button"
|
||||
onClick={handleAutomaticSetup}
|
||||
>
|
||||
Choose models for me
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-2">
|
||||
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
|
||||
<Select
|
||||
items={templateItems}
|
||||
value={selectedPreset ?? null}
|
||||
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
|
||||
>
|
||||
<SelectTrigger data-testid="template-selector" className="w-full">
|
||||
<SelectValue placeholder="Choose a template or select Custom to define your own" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
|
||||
const disabledHint = presetDisabledHint(presetState);
|
||||
const hintClass = isPresetHintAlarming(presetState)
|
||||
? "text-destructive"
|
||||
: "text-muted-foreground";
|
||||
const matchedHint =
|
||||
presetState.kind === "available" && presetState.viaDeployments
|
||||
? "Matches your deployments"
|
||||
: null;
|
||||
|
||||
return (
|
||||
<SelectItem
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={disabledHint !== null}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-muted-foreground">{preset.description}</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Replaces classifier and model choices
|
||||
</div>
|
||||
{disabledHint && (
|
||||
<div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>
|
||||
)}
|
||||
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
|
||||
</div>
|
||||
</SelectItem>
|
||||
);
|
||||
})}
|
||||
<SelectItem value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Define your auto router from scratch
|
||||
</div>
|
||||
</div>
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{presetsPending && (
|
||||
<div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>
|
||||
)}
|
||||
{presetsUnavailable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load templates, so only Custom Configuration is shown.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchPresets()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{modelsUnverifiable && (
|
||||
<div className="text-xs mt-1 text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{configurationForm}
|
||||
|
||||
{isAdmin && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="model_access_group"
|
||||
label={labelWithHint(
|
||||
"Model Access Group",
|
||||
"Use model access groups to control who can access this auto router",
|
||||
)}
|
||||
>
|
||||
{({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => (
|
||||
<AccessGroupTagsCombobox
|
||||
id={id}
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
options={modelAccessGroups}
|
||||
ariaInvalid={ariaInvalid}
|
||||
ariaDescribedBy={ariaDescribedBy}
|
||||
/>
|
||||
)}
|
||||
</FormField>
|
||||
{modelsUnverifiable && (
|
||||
<p className="text-sm text-destructive">
|
||||
Could not load available models.{" "}
|
||||
<button type="button" className="underline" onClick={() => void refetchModels()}>
|
||||
Retry
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
{configurationForm}
|
||||
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<Tooltip>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets";
|
||||
import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup";
|
||||
import {
|
||||
buildAutomaticRouterConfig,
|
||||
buildPreferredTierModels,
|
||||
prefillEmptyTiers,
|
||||
type PreferredTierModels,
|
||||
} from "./auto_setup";
|
||||
|
||||
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
|
||||
const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({
|
||||
|
|
@ -230,3 +236,39 @@ describe("buildAutomaticRouterConfig", () => {
|
|||
).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefillEmptyTiers", () => {
|
||||
const empty: ComplexityRouterConfigValue = {
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-custom", timeout_ms: 4200 },
|
||||
classification_mode: "user_turn",
|
||||
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
};
|
||||
const automatic: ComplexityRouterConfigValue = {
|
||||
classifier_type: "heuristic_v2",
|
||||
tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["strong"] },
|
||||
tier_model_params: { REASONING: { strong: { reasoning_effort: "high" } } },
|
||||
};
|
||||
it("fills tiers while preserving a classifier configured before models load", () => {
|
||||
expect(prefillEmptyTiers(empty, automatic)).toEqual({
|
||||
...empty,
|
||||
tiers: automatic.tiers,
|
||||
tier_model_params: automatic.tier_model_params,
|
||||
});
|
||||
});
|
||||
it("leaves the draft alone when no recommended models are available", () => {
|
||||
expect(prefillEmptyTiers(empty, null)).toBe(empty);
|
||||
});
|
||||
it("preserves a partial manual assignment", () => {
|
||||
const partial = { ...empty, tiers: { ...empty.tiers, SIMPLE: ["manual"] } };
|
||||
expect(prefillEmptyTiers(partial, automatic)).toBe(partial);
|
||||
});
|
||||
it("preserves a custom tier set created before models load", () => {
|
||||
const custom = { ...empty, custom_tier_set: { tiers: [], fallback_tier_id: "" } };
|
||||
expect(prefillEmptyTiers(custom, automatic)).toBe(custom);
|
||||
});
|
||||
it.each(["capability", "llm_v2"] as const)("does not prefill %s solver pools", (classifier_type) => {
|
||||
const solver = { ...empty, classifier_type };
|
||||
expect(prefillEmptyTiers(solver, automatic)).toBe(solver);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { activeTierRows } from "./tier_rows";
|
||||
import { isForecastClassifier } from "./forecast_classifier_config";
|
||||
import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets";
|
||||
|
|
@ -93,3 +95,16 @@ export const buildAutomaticRouterConfig = (
|
|||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export const prefillEmptyTiers = (
|
||||
current: ComplexityRouterConfigValue,
|
||||
automatic: ComplexityRouterConfigValue | null,
|
||||
): ComplexityRouterConfigValue => {
|
||||
if (!automatic || current.custom_tier_set || isForecastClassifier(current.classifier_type)) return current;
|
||||
if (activeTierRows(current).some((row) => row.models.length > 0)) return current;
|
||||
return {
|
||||
...current,
|
||||
tiers: { ...current.tiers, ...automatic.tiers },
|
||||
tier_model_params: { ...current.tier_model_params, ...automatic.tier_model_params },
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
import { expect, it } from "vitest";
|
||||
import { jevClassifierConfigSchema } from "./jev_classifier_config";
|
||||
|
||||
it("preserves legacy classifier tuning when the backend returns a null deployment reference", () => {
|
||||
const legacyConfig = {
|
||||
deployment_name: null,
|
||||
model: "english",
|
||||
timeout_ms: 4100,
|
||||
};
|
||||
const parsed = jevClassifierConfigSchema.parse(legacyConfig);
|
||||
expect(parsed).toMatchObject({ model: "english", timeout_ms: 4100 });
|
||||
expect(parsed.deployment_name).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves a deployment alias independently of legacy provider model validation", () => {
|
||||
const deploymentConfig = {
|
||||
deployment_name: "decider",
|
||||
model: "custom-deployed-model",
|
||||
timeout_ms: 4100,
|
||||
};
|
||||
const parsed = jevClassifierConfigSchema.parse(deploymentConfig);
|
||||
expect(parsed.deployment_name).toBe("decider");
|
||||
});
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { z } from "zod";
|
||||
|
||||
const jevClassifierConfigFields = {
|
||||
deployment_name: z.preprocess((value) => value ?? undefined, z.string().trim().min(1).optional()),
|
||||
model: z.string().trim().min(1).default("jev-latest"),
|
||||
timeout_ms: z.number().int().positive().default(3000),
|
||||
instructions: z
|
||||
|
|
@ -20,6 +21,7 @@ export const defaultJevClassifierConfig = (): JevClassifierConfig => jevClassifi
|
|||
export const normalizeJevClassifierConfig = (
|
||||
config: JevClassifierConfig = defaultJevClassifierConfig(),
|
||||
): JevClassifierConfig => ({
|
||||
...(config.deployment_name !== undefined && { deployment_name: config.deployment_name }),
|
||||
model: config.model.trim(),
|
||||
timeout_ms: config.timeout_ms,
|
||||
...(config.instructions?.trim() && { instructions: config.instructions.trim() }),
|
||||
|
|
@ -28,3 +30,6 @@ export const normalizeJevClassifierConfig = (
|
|||
circuit_breaker_cooldown_seconds: config.circuit_breaker_cooldown_seconds,
|
||||
}),
|
||||
});
|
||||
|
||||
export const decisionDeploymentName = (config: JevClassifierConfig | undefined): string | undefined =>
|
||||
config?.deployment_name;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
|
||||
import { openAutoRouterAdvanced, selectAutoRouterOption } from "../../../tests/autoRouterSetup";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
|
@ -27,6 +28,12 @@ const {
|
|||
}));
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
modelInfoCall: vi.fn().mockResolvedValue({
|
||||
data: [
|
||||
{ model_name: "decision-one", litellm_params: { model: "typesafe/jev-latest" } },
|
||||
{ model_name: "decision-two", litellm_params: { model: "typesafe/jev-latest" } },
|
||||
],
|
||||
}),
|
||||
apiClient: { post: vi.fn().mockResolvedValue({ allowances: [], error: null }) },
|
||||
modelPatchUpdateCall,
|
||||
modelAvailableCall,
|
||||
|
|
@ -1502,3 +1509,51 @@ describe("EditAutoRouterModal availability checks", () => {
|
|||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
|
||||
});
|
||||
});
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
||||
useModelCostMap: () => ({ data: { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" } } }),
|
||||
}));
|
||||
|
||||
describe("EditAutoRouterModal deployed decision classifier", () => {
|
||||
it.each(["evaluation", undefined])(
|
||||
"shows and changes the saved deployment without ignored legacy model controls (mode=%s)",
|
||||
async (mode) => {
|
||||
vi.mocked(fetchAvailableModels).mockResolvedValueOnce([
|
||||
{ model_group: "decision-one", mode },
|
||||
{ model_group: "decision-two", mode },
|
||||
{ model_group: "gpt-4o-mini", mode: "chat" },
|
||||
]);
|
||||
modelPatchUpdateCall.mockClear();
|
||||
renderModal({
|
||||
modelData: {
|
||||
...MODEL_DATA,
|
||||
litellm_params: {
|
||||
...MODEL_DATA.litellm_params,
|
||||
complexity_router_config: {
|
||||
...STORED_CONFIG,
|
||||
classifier_type: "jev",
|
||||
jev_classifier_config: { model: "jev-latest", deployment_name: "decision-one" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const picker = await screen.findByRole("combobox", { name: "Decision model" });
|
||||
expect(picker).toHaveValue("decision-one");
|
||||
await userEvent.click(picker);
|
||||
await userEvent.click(await screen.findByRole("option", { name: "decision-two" }));
|
||||
expect(picker).toHaveValue("decision-two");
|
||||
openAutoRouterAdvanced("Classification Method");
|
||||
expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument();
|
||||
fireEvent.change(screen.getByLabelText("Classifier Timeout (ms)"), { target: { value: "4500" } });
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
|
||||
await userEvent.click(screen.getByRole("button", { name: "Save Changes" }));
|
||||
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled());
|
||||
expect(savedConfig().jev_classifier_config).toEqual(
|
||||
expect.objectContaining({
|
||||
deployment_name: "decision-two",
|
||||
timeout_ms: 4500,
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
import { useQuery } from "@tanstack/react-query";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import DecisionModelSelect from "../add_model/DecisionModelSelect";
|
||||
import { AutoRouterAvailabilityContext, useAutoRouterAvailability } from "../add_model/AutoRouterAvailability";
|
||||
import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs";
|
||||
import { usesClassifierContext } from "../add_model/classifier_types";
|
||||
|
|
@ -231,6 +235,42 @@ export const buildUpdatedComplexityRouterConfig = (
|
|||
};
|
||||
};
|
||||
|
||||
const EditClassifierTabs = ({
|
||||
value,
|
||||
onChange,
|
||||
children,
|
||||
modelInfo,
|
||||
accessToken,
|
||||
userRole,
|
||||
isVisible,
|
||||
}: React.ComponentProps<typeof AutoRouterClassifierTabs> & {
|
||||
modelInfo: ModelGroup[];
|
||||
accessToken: string;
|
||||
userRole: string;
|
||||
isVisible: boolean;
|
||||
}) => {
|
||||
const { userId } = useAuthorized();
|
||||
const hasDeployment = Boolean(value.jev_classifier_config?.deployment_name);
|
||||
const { data: deployments } = useQuery({
|
||||
queryKey: autoRouterListKey(userId ?? "", userRole),
|
||||
queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole),
|
||||
enabled: Boolean(isVisible && accessToken && hasDeployment),
|
||||
});
|
||||
return (
|
||||
<AutoRouterClassifierTabs
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
decisionModel={
|
||||
hasDeployment ? (
|
||||
<DecisionModelSelect value={value} onChange={onChange} modelInfo={modelInfo} deployments={deployments} />
|
||||
) : undefined
|
||||
}
|
||||
>
|
||||
{children}
|
||||
</AutoRouterClassifierTabs>
|
||||
);
|
||||
};
|
||||
|
||||
const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
||||
isVisible,
|
||||
onCancel,
|
||||
|
|
@ -630,7 +670,14 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
{isComplexityRouterModel ? (
|
||||
/* Complexity Router Configuration */
|
||||
<div className="w-full">
|
||||
<AutoRouterClassifierTabs value={complexityRouterConfig} onChange={setComplexityRouterConfig}>
|
||||
<EditClassifierTabs
|
||||
value={complexityRouterConfig}
|
||||
onChange={setComplexityRouterConfig}
|
||||
modelInfo={modelInfo}
|
||||
accessToken={accessToken}
|
||||
userRole={userRole}
|
||||
isVisible={isVisible}
|
||||
>
|
||||
<ComplexityRouterConfig
|
||||
editingTiers={editingTiers}
|
||||
onEditingTiersChange={setEditingTiers}
|
||||
|
|
@ -659,7 +706,7 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
autoRouterCompression={autoRouterCompression}
|
||||
onAutoRouterCompressionChange={isMemberManaged ? undefined : setAutoRouterCompression}
|
||||
/>
|
||||
</AutoRouterClassifierTabs>
|
||||
</EditClassifierTabs>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ function DialogOverlay({ className, ...props }: DialogPrimitive.Backdrop.Props)
|
|||
<DialogPrimitive.Backdrop
|
||||
data-slot="dialog-overlay"
|
||||
className={cn(
|
||||
"fixed inset-0 isolate z-popup bg-black/10 duration-100 supports-backdrop-filter:backdrop-blur-xs data-open:animate-in data-open:fade-in-0 data-closed:animate-out data-closed:fade-out-0",
|
||||
"fixed inset-0 isolate z-popup bg-black/10 duration-100",
|
||||
!(typeof className === "string" && className.includes("backdrop-blur-")) &&
|
||||
"supports-backdrop-filter:backdrop-blur-xs",
|
||||
"data-open:animate-in data-open:fade-in-0 data-closed:animate-out data-closed:fade-out-0",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
|
|
@ -38,15 +41,17 @@ function DialogOverlay({ className, ...props }: DialogPrimitive.Backdrop.Props)
|
|||
|
||||
function DialogContent({
|
||||
className,
|
||||
overlayClassName,
|
||||
children,
|
||||
showCloseButton = true,
|
||||
...props
|
||||
}: DialogPrimitive.Popup.Props & {
|
||||
showCloseButton?: boolean;
|
||||
overlayClassName?: string;
|
||||
}) {
|
||||
return (
|
||||
<DialogPortal>
|
||||
<DialogOverlay />
|
||||
<DialogOverlay className={overlayClassName} forceRender={Boolean(overlayClassName)} />
|
||||
<DialogPrimitive.Popup
|
||||
data-slot="dialog-content"
|
||||
className={cn(
|
||||
|
|
|
|||
|
|
@ -815,6 +815,24 @@ describe("autorouter_presets", () => {
|
|||
});
|
||||
|
||||
describe("getReferencedModelsError", () => {
|
||||
it.each(["jev", "heuristic"] as const)(
|
||||
"only checks a removed decision deployment when %s is active",
|
||||
(classifierType) => {
|
||||
const config = {
|
||||
tiers: { SIMPLE: ["solver"], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifierType,
|
||||
classifierLlmConfig: undefined,
|
||||
decisionDeploymentName: "removed-decision",
|
||||
semanticMatchingEnabled: false,
|
||||
embeddingModel: undefined,
|
||||
};
|
||||
const availability = groupsOnly(["solver"]);
|
||||
expect(getReferencedModelsError(config, availability)).toBe(
|
||||
classifierType === "jev" ? "Model(s) no longer available: removed-decision" : null,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] };
|
||||
const available = groupsOnly(["gpt-5-nano"]);
|
||||
// Both fields are always populated with a model missing from `available`; only the
|
||||
|
|
|
|||
|
|
@ -246,21 +246,27 @@ export const getReferencedModelsError = (
|
|||
tiers: ComplexityRouterConfigPayload["tiers"];
|
||||
classifierType: ClassifierType;
|
||||
classifierLlmConfig: ClassifierLLMConfig | undefined;
|
||||
decisionDeploymentName?: string;
|
||||
semanticMatchingEnabled: boolean;
|
||||
embeddingModel: string | undefined;
|
||||
defaultModel?: string;
|
||||
},
|
||||
availability: ModelAvailability,
|
||||
): string | null => {
|
||||
const missing = getMissingModels(
|
||||
{
|
||||
tiers: params.tiers,
|
||||
default_model: params.defaultModel,
|
||||
classifier_llm_config: usesLlmClassifier(params.classifierType) ? params.classifierLlmConfig : undefined,
|
||||
embedding_model: params.semanticMatchingEnabled ? params.embeddingModel : undefined,
|
||||
},
|
||||
availability,
|
||||
);
|
||||
if (
|
||||
params.classifierType === "jev" &&
|
||||
params.decisionDeploymentName &&
|
||||
!resolveAvailableModel(params.decisionDeploymentName, availability)
|
||||
) {
|
||||
return `Model(s) no longer available: ${params.decisionDeploymentName}`;
|
||||
}
|
||||
const references = {
|
||||
tiers: params.tiers,
|
||||
default_model: params.defaultModel,
|
||||
classifier_llm_config: usesLlmClassifier(params.classifierType) ? params.classifierLlmConfig : undefined,
|
||||
embedding_model: params.semanticMatchingEnabled ? params.embeddingModel : undefined,
|
||||
};
|
||||
const missing = getMissingModels(references, availability);
|
||||
return missing.length > 0 ? `Model(s) no longer available: ${missing.join(", ")}` : null;
|
||||
};
|
||||
|
||||
|
|
|
|||
84
ui/litellm-dashboard/src/lib/decisionModels.test.ts
Normal file
84
ui/litellm-dashboard/src/lib/decisionModels.test.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { EMPTY_DECISION_CATALOG, buildDecisionCatalog, isDecisionMode, isDecisionSelection } from "./decisionModels";
|
||||
|
||||
const COST_MAP = {
|
||||
"typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" },
|
||||
"typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" },
|
||||
"gpt-6-luna": {
|
||||
litellm_provider: "openai",
|
||||
mode: "chat",
|
||||
supported_endpoints: ["/v1/chat/completions", "/v1/decisions"],
|
||||
},
|
||||
"gpt-5.5": { litellm_provider: "openai", mode: "chat", supported_endpoints: ["/v1/chat/completions"] },
|
||||
"perplexity/pplx-decider-v1-27b": { litellm_provider: "perplexity", mode: "evaluation" },
|
||||
"perplexity/sonar-pro": { litellm_provider: "perplexity", mode: "chat" },
|
||||
"systemone-only": { litellm_provider: "somevendor", supported_endpoints: ["/v1/systemone"] },
|
||||
sample_spec: { max_tokens: "set to max_output_tokens if provider specifies it" },
|
||||
};
|
||||
|
||||
describe("isDecisionMode", () => {
|
||||
it("is true only for the evaluation mode", () => {
|
||||
expect(isDecisionMode("evaluation")).toBe(true);
|
||||
expect(isDecisionMode("chat")).toBe(false);
|
||||
expect(isDecisionMode(null)).toBe(false);
|
||||
expect(isDecisionMode(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDecisionCatalog", () => {
|
||||
const catalog = buildDecisionCatalog(COST_MAP);
|
||||
|
||||
it("collects evaluation models and models that list a decision endpoint", () => {
|
||||
expect([...catalog.models].sort()).toEqual(
|
||||
[
|
||||
"gpt-6-luna",
|
||||
"perplexity/pplx-decider-v1-27b",
|
||||
"systemone-only",
|
||||
"typesafe/jev-latest",
|
||||
"typesafe/jev-preview",
|
||||
].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
it("marks a provider decision-only when every one of its models is a decision model", () => {
|
||||
expect(catalog.providers.get("typesafe")?.decisionOnly).toBe(true);
|
||||
expect(catalog.providers.get("somevendor")?.decisionOnly).toBe(true);
|
||||
expect(catalog.providers.get("perplexity")?.decisionOnly).toBe(false);
|
||||
expect(catalog.providers.get("openai")?.decisionOnly).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves providers without decision models out", () => {
|
||||
expect(catalog.providers.has("anthropic")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns an empty catalog for a missing or malformed cost map", () => {
|
||||
expect(buildDecisionCatalog(undefined)).toEqual(EMPTY_DECISION_CATALOG);
|
||||
expect(buildDecisionCatalog("not a map")).toEqual(EMPTY_DECISION_CATALOG);
|
||||
expect(buildDecisionCatalog({ broken: { mode: "evaluation" } })).toEqual(EMPTY_DECISION_CATALOG);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isDecisionSelection", () => {
|
||||
const catalog = buildDecisionCatalog(COST_MAP);
|
||||
|
||||
it("is true for a decision-only provider before any model is picked", () => {
|
||||
expect(isDecisionSelection(catalog, "typesafe", [])).toBe(true);
|
||||
});
|
||||
|
||||
it("waits for a decision model on a provider that also serves chat models", () => {
|
||||
expect(isDecisionSelection(catalog, "perplexity", [])).toBe(false);
|
||||
expect(isDecisionSelection(catalog, "perplexity", ["perplexity/sonar-pro"])).toBe(false);
|
||||
expect(isDecisionSelection(catalog, "perplexity", ["perplexity/sonar-pro", "perplexity/pplx-decider-v1-27b"])).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("is true for a chat-mode model that lists a decision endpoint", () => {
|
||||
expect(isDecisionSelection(catalog, "openai", ["gpt-6-luna"])).toBe(true);
|
||||
expect(isDecisionSelection(catalog, "openai", ["gpt-5.5"])).toBe(false);
|
||||
});
|
||||
|
||||
it("is false with no provider and no models", () => {
|
||||
expect(isDecisionSelection(catalog, undefined, [])).toBe(false);
|
||||
});
|
||||
});
|
||||
63
ui/litellm-dashboard/src/lib/decisionModels.ts
Normal file
63
ui/litellm-dashboard/src/lib/decisionModels.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { z } from "zod";
|
||||
|
||||
export const DECISIONS_DOCS_URL = "https://docs.litellm.ai/docs/decisions";
|
||||
export const DECISIONS_PROVIDERS_DOCS_URL = `${DECISIONS_DOCS_URL}#supported-providers`;
|
||||
export const SYSTEM_ONE_PLAYGROUND_ROUTE = "playground?tab=system-one";
|
||||
|
||||
const DECISION_ENDPOINTS: ReadonlySet<string> = new Set(["/v1/decisions", "/v1/systemone"]);
|
||||
|
||||
const costMapEntrySchema = z.object({
|
||||
litellm_provider: z.string(),
|
||||
mode: z.string().nullish(),
|
||||
supported_endpoints: z.array(z.string()).nullish(),
|
||||
});
|
||||
|
||||
type CostMapEntry = z.infer<typeof costMapEntrySchema>;
|
||||
|
||||
interface ProviderDecisionModels {
|
||||
readonly decisionOnly: boolean;
|
||||
}
|
||||
|
||||
export interface DecisionCatalog {
|
||||
readonly models: ReadonlySet<string>;
|
||||
readonly providers: ReadonlyMap<string, ProviderDecisionModels>;
|
||||
}
|
||||
|
||||
export const EMPTY_DECISION_CATALOG: DecisionCatalog = { models: new Set(), providers: new Map() };
|
||||
|
||||
export const isDecisionMode = (mode: string | null | undefined): boolean => mode === "evaluation";
|
||||
|
||||
const isDecisionEntry = (entry: CostMapEntry): boolean =>
|
||||
isDecisionMode(entry.mode) || (entry.supported_endpoints ?? []).some((endpoint) => DECISION_ENDPOINTS.has(endpoint));
|
||||
|
||||
const parsedEntries = (costMap: unknown): ReadonlyArray<readonly [string, CostMapEntry]> => {
|
||||
const record = z.record(z.string(), z.unknown()).safeParse(costMap);
|
||||
if (!record.success) {
|
||||
return [];
|
||||
}
|
||||
return Object.entries(record.data).flatMap(([key, value]) => {
|
||||
const entry = costMapEntrySchema.safeParse(value);
|
||||
return entry.success ? [[key, entry.data] as const] : [];
|
||||
});
|
||||
};
|
||||
|
||||
export function buildDecisionCatalog(costMap: unknown): DecisionCatalog {
|
||||
const entries = parsedEntries(costMap);
|
||||
const decisionEntries = entries.filter(([, entry]) => isDecisionEntry(entry));
|
||||
const decisionProviders = new Set(decisionEntries.map(([, entry]) => entry.litellm_provider));
|
||||
const providers = new Map(
|
||||
[...decisionProviders].map((provider) => {
|
||||
const providerEntries = entries.filter(([, entry]) => entry.litellm_provider === provider);
|
||||
return [provider, { decisionOnly: providerEntries.every(([, entry]) => isDecisionEntry(entry)) }] as const;
|
||||
}),
|
||||
);
|
||||
return { models: new Set(decisionEntries.map(([key]) => key)), providers };
|
||||
}
|
||||
|
||||
export const isDecisionSelection = (
|
||||
catalog: DecisionCatalog,
|
||||
litellmProvider: string | undefined,
|
||||
selectedModels: readonly string[],
|
||||
): boolean =>
|
||||
(litellmProvider !== undefined && catalog.providers.get(litellmProvider)?.decisionOnly === true) ||
|
||||
selectedModels.some((model) => catalog.models.has(model));
|
||||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -31240,6 +31240,8 @@ export interface components {
|
|||
* @default true
|
||||
*/
|
||||
circuit_breaker_enabled: boolean;
|
||||
/** Deployment Name */
|
||||
deployment_name?: string | null;
|
||||
/**
|
||||
* Instructions
|
||||
* @description Replaces the built-in Jev question instructions
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue