mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
feat(auto-router): support classifier reasoning effort (#39372)
* feat(auto-router): support classifier reasoning effort * fix(auto-router): harden classifier reasoning effort * fix(ui): satisfy classifier config lint limits * refactor(auto-router): simplify classifier effort support * fix(auto-router): clear frontend-lint and type-discipline gates, trim LOC --------- Co-authored-by: Tin Chi Lo <tin@berri.ai>
This commit is contained in:
moe-berri
GitHub
parent
34d4f7f8ae
commit
4990f06acc
26 changed files with 774 additions and 68 deletions
|
|
@ -50,6 +50,7 @@ from litellm.constants import (
|
|||
DEFAULT_HEALTH_CHECK_INTERVAL,
|
||||
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
|
||||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
|
||||
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
|
||||
)
|
||||
|
|
@ -231,6 +232,7 @@ from litellm.types.router import (
|
|||
)
|
||||
from litellm.types.services import ServiceTypes
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
|
||||
CustomPricingLiteLLMParams,
|
||||
GenericBudgetConfigType,
|
||||
|
|
@ -2168,6 +2170,76 @@ class Router:
|
|||
verbose_router_logger.debug("Error occurred while printing deployment - %s", e)
|
||||
raise e
|
||||
|
||||
@staticmethod
|
||||
def _deployment_params_with_request_reasoning_override(
|
||||
deployment_params: Mapping[str, object], request_kwargs: Mapping[str, object]
|
||||
) -> dict[str, object]: # mutable-ok: litellm's request pipeline consumes a mutable kwargs mapping
|
||||
"""Return deployment params whose equivalent effort controls cannot outrank a request override.
|
||||
|
||||
Providers expose the same setting through several native carriers. A request-level
|
||||
``reasoning_effort`` is the portable override, so a deployment's ``thinking`` or nested
|
||||
``*.effort`` must not remain beside it and either win or trigger a conflicting-params 400.
|
||||
Every changed mapping is copied so the Router's shared deployment config stays immutable.
|
||||
"""
|
||||
sanitized: Final = dict(deployment_params) # mutable-ok: request-local copy protects shared Router state
|
||||
if request_kwargs.get("reasoning_effort") is None:
|
||||
return sanitized
|
||||
|
||||
sanitized.pop("thinking", None)
|
||||
Router._pop_effort_from_nested_carrier(sanitized, "output_config")
|
||||
Router._pop_effort_from_nested_carrier(sanitized, "reasoning")
|
||||
|
||||
extra_body: Final = sanitized.get("extra_body")
|
||||
if isinstance(extra_body, Mapping):
|
||||
sanitized_extra_body: Final = dict(extra_body) # mutable-ok: request-local nested copy
|
||||
sanitized_extra_body.pop("reasoning_effort", None)
|
||||
sanitized_extra_body.pop("thinking", None)
|
||||
Router._pop_effort_from_nested_carrier(sanitized_extra_body, "output_config")
|
||||
Router._pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning")
|
||||
if sanitized_extra_body:
|
||||
sanitized["extra_body"] = sanitized_extra_body
|
||||
else:
|
||||
sanitized.pop("extra_body", None)
|
||||
return sanitized
|
||||
|
||||
@staticmethod
|
||||
def _is_classifier_internal_call(kwargs: Mapping[str, object]) -> bool:
|
||||
metadata: Final = kwargs.get("metadata")
|
||||
litellm_metadata: Final = kwargs.get("litellm_metadata")
|
||||
return any(
|
||||
isinstance(candidate, Mapping)
|
||||
and candidate.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == AUTOROUTER_CLASSIFIER_CALL_ORIGIN
|
||||
for candidate in (metadata, litellm_metadata)
|
||||
)
|
||||
|
||||
def _drop_unsupported_classifier_reasoning_effort(
|
||||
self,
|
||||
deployment: DeploymentTypedDict,
|
||||
model: str,
|
||||
kwargs: dict[str, object], # mutable-ok: fallback must update the active request and its log body together
|
||||
) -> None:
|
||||
"""Let a classifier fallback without reasoning support remain a usable fallback.
|
||||
|
||||
The dashboard only offers explicitly advertised levels, but an existing config can outlive
|
||||
a model change and fallbacks can target a different group. Unknown capability fails open;
|
||||
only a provider that explicitly rejects the parameter has it removed.
|
||||
"""
|
||||
if kwargs.get("reasoning_effort") is None or not self._is_classifier_internal_call(kwargs):
|
||||
return
|
||||
if self._deployment_accepts_param(deployment, model, "reasoning_effort"):
|
||||
return
|
||||
verbose_router_logger.warning(
|
||||
"litellm.router.py: dropping classifier reasoning_effort for model=%s because the selected deployment does not support it",
|
||||
model,
|
||||
)
|
||||
kwargs.pop("reasoning_effort", None)
|
||||
proxy_server_request: Final = kwargs.get("proxy_server_request")
|
||||
if not isinstance(proxy_server_request, dict):
|
||||
return
|
||||
body: Final = proxy_server_request.get("body")
|
||||
if isinstance(body, dict):
|
||||
body.pop("reasoning_effort", None)
|
||||
|
||||
### COMPLETION, EMBEDDING, IMG GENERATION FUNCTIONS
|
||||
|
||||
def completion(self, model: str, messages: list[dict[str, str]], **kwargs) -> ModelResponse | CustomStreamWrapper:
|
||||
|
|
@ -2203,9 +2275,16 @@ class Router:
|
|||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
self._drop_unsupported_classifier_reasoning_effort(
|
||||
deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment
|
||||
model=model,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
# Check for silent model experiment
|
||||
# Make a local copy of litellm_params to avoid mutating the Router's state
|
||||
litellm_params: Final = deployment["litellm_params"].copy()
|
||||
litellm_params: Final = self._deployment_params_with_request_reasoning_override(
|
||||
deployment["litellm_params"], kwargs
|
||||
)
|
||||
silent_model: Final = litellm_params.pop("silent_model", None)
|
||||
|
||||
if silent_model is not None:
|
||||
|
|
@ -3216,6 +3295,11 @@ class Router:
|
|||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
self._drop_unsupported_classifier_reasoning_effort(
|
||||
deployment=cast(DeploymentTypedDict, deployment), # cast-ok: selection returns a router deployment
|
||||
model=model,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
_timeout_debug_deployment_dict = deployment
|
||||
end_time: Final = time.time()
|
||||
|
|
@ -3237,7 +3321,9 @@ class Router:
|
|||
|
||||
# Check for silent model experiment
|
||||
# Make a local copy of litellm_params to avoid mutating the Router's state
|
||||
litellm_params: Final = deployment["litellm_params"].copy()
|
||||
litellm_params: Final = self._deployment_params_with_request_reasoning_override(
|
||||
deployment["litellm_params"], kwargs
|
||||
)
|
||||
silent_model: Final = litellm_params.pop("silent_model", None)
|
||||
|
||||
if silent_model is not None:
|
||||
|
|
@ -10255,6 +10341,8 @@ class Router:
|
|||
total_itpm: int | None = None
|
||||
total_otpm: int | None = None
|
||||
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
|
||||
reasoning_efforts_initialized = False
|
||||
reasoning_efforts_unknown = False
|
||||
model_list: Final = self.get_model_list(model_name=model_group)
|
||||
if model_list is None:
|
||||
return None
|
||||
|
|
@ -10441,10 +10529,23 @@ class Router:
|
|||
if model_info.get("rpm", None) is not None and _deployment_rpm is None:
|
||||
_deployment_rpm = model_info.get("rpm")
|
||||
|
||||
model_group_info.supported_reasoning_efforts = intersect_supported_reasoning_efforts(
|
||||
model_group_info.supported_reasoning_efforts,
|
||||
resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=deployment_is_mapped),
|
||||
deployment_reasoning_efforts = (
|
||||
resolve_supported_reasoning_efforts( # rebind-ok: recalculated per deployment
|
||||
model_info, deployment_is_mapped=deployment_is_mapped
|
||||
)
|
||||
)
|
||||
if deployment_reasoning_efforts is None:
|
||||
reasoning_efforts_unknown = True
|
||||
model_group_info.supported_reasoning_efforts = None
|
||||
elif not reasoning_efforts_initialized:
|
||||
reasoning_efforts_initialized = True
|
||||
if not reasoning_efforts_unknown:
|
||||
model_group_info.supported_reasoning_efforts = deployment_reasoning_efforts
|
||||
elif not reasoning_efforts_unknown:
|
||||
model_group_info.supported_reasoning_efforts = intersect_supported_reasoning_efforts(
|
||||
model_group_info.supported_reasoning_efforts,
|
||||
deployment_reasoning_efforts,
|
||||
)
|
||||
|
||||
if _deployment_tpm is not None:
|
||||
if total_tpm is None:
|
||||
|
|
@ -12308,10 +12409,12 @@ class Router:
|
|||
@staticmethod
|
||||
def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None:
|
||||
nested: Final = request_kwargs.get(carrier)
|
||||
if not isinstance(nested, dict):
|
||||
if not isinstance(nested, Mapping):
|
||||
return
|
||||
nested.pop("effort", None)
|
||||
if not nested:
|
||||
sanitized: Final = {key: value for key, value in nested.items() if key != "effort"}
|
||||
if sanitized:
|
||||
request_kwargs[carrier] = sanitized # rebind-ok: copy-on-write, so a shared nested carrier is never edited
|
||||
else:
|
||||
request_kwargs.pop(carrier, None)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -255,7 +255,8 @@ model_list:
|
|||
classifier_type: heuristic_first
|
||||
heuristic_first_max_tier: SIMPLE
|
||||
classifier_llm_config:
|
||||
model: gpt-4o-mini
|
||||
model: gpt-5-mini
|
||||
reasoning_effort: low
|
||||
tiers:
|
||||
SIMPLE: gpt-4o-mini
|
||||
MEDIUM: gpt-4o
|
||||
|
|
@ -263,6 +264,10 @@ model_list:
|
|||
REASONING: o1-preview
|
||||
```
|
||||
|
||||
`classifier_llm_config.reasoning_effort` applies only to the internal classifier call. Omit it to
|
||||
keep the classifier deployment or provider default, or set a supported value such as `none` or
|
||||
`low` to override that call.
|
||||
|
||||
A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
|
||||
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
|
||||
one signal. Everything else goes to the classifier, which then decides as it normally would.
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from pydantic import BaseModel, create_model
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import (
|
||||
EMPTY_MAPPING,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
RETURN_RAW_MODEL_NAME_METADATA_KEY,
|
||||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
)
|
||||
|
|
@ -42,6 +43,7 @@ from litellm.router_strategy.complexity_router.tier_predictor import (
|
|||
TierSuccessPredictor,
|
||||
resolve_tier_artifact,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
ModelResponse,
|
||||
|
|
@ -1668,20 +1670,27 @@ class ComplexityRouter(CustomLogger):
|
|||
)
|
||||
|
||||
request_metadata = (request_kwargs or {}).get("litellm_metadata") or (request_kwargs or {}).get("metadata")
|
||||
metadata: Final = forwarded_internal_call_metadata(request_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN)
|
||||
metadata: Final = { # mutable-ok: SDK metadata kwarg is enriched by the request pipeline
|
||||
**forwarded_internal_call_metadata(request_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN),
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
}
|
||||
turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs)
|
||||
|
||||
messages_for_call: Final = [
|
||||
messages_for_call: Final[list[AllMessageValues]] = [ # mutable-ok: SDK request payload list is built once
|
||||
{"role": "system", "content": classifier_system_prompt},
|
||||
{"role": "user", "content": user_payload},
|
||||
]
|
||||
response_format: Final = classifier_response_format
|
||||
classifier_call_params: Mapping[str, str] = EMPTY_MAPPING
|
||||
if llm_config.reasoning_effort is not None:
|
||||
classifier_call_params = MappingProxyType({"reasoning_effort": llm_config.reasoning_effort})
|
||||
|
||||
proxy_server_request: Final = {
|
||||
"body": {
|
||||
"model": llm_config.model,
|
||||
"messages": messages_for_call,
|
||||
"response_format": response_format,
|
||||
**classifier_call_params,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1693,6 +1702,7 @@ class ComplexityRouter(CustomLogger):
|
|||
metadata=metadata,
|
||||
proxy_server_request=proxy_server_request,
|
||||
turn_off_message_logging=turn_off_message_logging,
|
||||
**classifier_call_params,
|
||||
**_parent_session_kwargs(request_kwargs),
|
||||
)
|
||||
content: Final = response.choices[0].message.content
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Annotated, Final, Literal
|
|||
|
||||
from pydantic import BaseModel, ConfigDict, Field, SkipValidation, field_serializer, field_validator, model_validator
|
||||
|
||||
from litellm.types.llms.openai import REASONING_EFFORT
|
||||
from litellm.types.router import AdaptiveRouterWeights, ClassifierPlugin, RoutingPlugin
|
||||
|
||||
from .tier_predictor import TrainedTierArtifact
|
||||
|
|
@ -432,6 +433,13 @@ class ClassifierLLMConfig(BaseModel):
|
|||
model: str = Field(
|
||||
description="Model name (from the router's model_list) to call for classification",
|
||||
)
|
||||
reasoning_effort: REASONING_EFFORT | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Reasoning effort override for classifier calls. Leave unset to use "
|
||||
"the classifier deployment or provider default."
|
||||
),
|
||||
)
|
||||
timeout_ms: int = Field(
|
||||
default=3000,
|
||||
description="Timeout budget for the classification call, in milliseconds",
|
||||
|
|
|
|||
|
|
@ -1558,6 +1558,14 @@ class TestLLMClassifierConfig:
|
|||
assert config.classifier_type == "heuristic"
|
||||
assert config.classifier_llm_config is None
|
||||
|
||||
@pytest.mark.parametrize("reasoning_effort", ["", "ultra"])
|
||||
def test_classifier_reasoning_effort_rejects_unsupported_values(self, reasoning_effort):
|
||||
with pytest.raises(ValidationError):
|
||||
ComplexityRouterConfig(
|
||||
classifier_type="llm",
|
||||
classifier_llm_config={"model": "haiku-classifier", "reasoning_effort": reasoning_effort},
|
||||
)
|
||||
|
||||
|
||||
CUSTOM_TIER_LABELS: Dict[str, str] = {
|
||||
"SIMPLE": "Cheap",
|
||||
|
|
@ -1871,6 +1879,19 @@ class TestLLMClassifier:
|
|||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
assert call_kwargs["metadata"] == {**request_metadata, "internal_call_origin": "autorouter_classifier"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_stamps_internal_origin_without_caller_metadata(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
):
|
||||
"""Fallback handling must still recognize the classifier when an SDK caller supplied no metadata."""
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}'))
|
||||
|
||||
await llm_complexity_router.aclassify("hi")
|
||||
|
||||
assert mock_router_instance.acompletion.call_args.kwargs["metadata"] == {
|
||||
"internal_call_origin": "autorouter_classifier"
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_kwargs",
|
||||
|
|
@ -1948,6 +1969,33 @@ class TestLLMClassifier:
|
|||
"REASONING",
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("reasoning_effort", [None, "none", "low"], ids=["omitted", "none", "low"])
|
||||
async def test_classifier_reasoning_effort_reaches_only_classifier_call(
|
||||
self, mock_router_instance, llm_classifier_config, reasoning_effort
|
||||
):
|
||||
classifier_llm_config = {
|
||||
**llm_classifier_config["classifier_llm_config"],
|
||||
**({"reasoning_effort": reasoning_effort} if reasoning_effort is not None else {}),
|
||||
}
|
||||
router = ComplexityRouter(
|
||||
model_name="test-complexity-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**llm_classifier_config, "classifier_llm_config": classifier_llm_config},
|
||||
)
|
||||
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
|
||||
|
||||
await router.aclassify("explain quantum tunneling in depth")
|
||||
|
||||
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
|
||||
body = call_kwargs["proxy_server_request"]["body"]
|
||||
if reasoning_effort is None:
|
||||
assert "reasoning_effort" not in call_kwargs
|
||||
assert "reasoning_effort" not in body
|
||||
else:
|
||||
assert call_kwargs["reasoning_effort"] == reasoning_effort
|
||||
assert body["reasoning_effort"] == reasoning_effort
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclassify_propagates_top_level_turn_off_message_logging(
|
||||
self, llm_complexity_router, mock_router_instance
|
||||
|
|
@ -8735,9 +8783,10 @@ class TestClassificationRubrics:
|
|||
[
|
||||
{"model": "haiku-classifier", "system_prompt": "Grade the data sensitivity of the request."},
|
||||
{"model": "haiku-classifier", "classification_rubric": "chat"},
|
||||
{"model": "haiku-classifier", "reasoning_effort": "low"},
|
||||
{"model": "haiku-classifier"},
|
||||
],
|
||||
ids=["custom-prompt", "chat-preset", "neither"],
|
||||
ids=["custom-prompt", "chat-preset", "reasoning-effort", "neither"],
|
||||
)
|
||||
def test_config_survives_a_dump_and_rebuild(self, classifier_llm_config):
|
||||
"""/auto_router/test_routing dumps this config and hands the dict straight back to
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.router import (
|
|||
_anthropic_stream_should_drop_pre_content_ping,
|
||||
_is_retriable_anthropic_status,
|
||||
)
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
|
||||
|
||||
def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
|
||||
|
|
@ -9783,11 +9784,12 @@ def test_model_group_info_intersects_supported_reasoning_efforts():
|
|||
assert result.supported_reasoning_efforts == ("minimal", "low", "medium", "high")
|
||||
|
||||
|
||||
def test_model_group_info_reasoning_efforts_ignore_a_deployment_off_the_map():
|
||||
def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_off_the_map():
|
||||
"""The router fills every ModelInfo key, so a deployment absent from the model map arrives with
|
||||
supports_reasoning None rather than with the key missing. Its synthesized entry carries no mode,
|
||||
which is what separates it from a mapped non-reasoning model, and nothing being known about it is
|
||||
no reason to drop the levels the rest of the group agrees on."""
|
||||
no evidence that the unknown deployment accepts levels its mapped sibling supports. The group
|
||||
therefore reports unknown instead of advertising a value routing might send to either one."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -9821,7 +9823,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_deployment_off_the_map():
|
|||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high", "max")
|
||||
assert result.supported_reasoning_efforts is None
|
||||
|
||||
|
||||
|
||||
|
|
@ -9958,11 +9960,11 @@ def test_model_group_info_survives_a_junk_typed_operator_effort_value():
|
|||
assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high")
|
||||
|
||||
|
||||
def test_model_group_info_reasoning_efforts_ignore_a_mode_the_operator_declared():
|
||||
def test_model_group_info_reasoning_efforts_are_unknown_for_an_operator_declared_mode():
|
||||
"""A deployment is registered in the cost map under its own id with whatever model_info the
|
||||
operator wrote, so a mode they set themselves reads back exactly like one the map supplied. Only
|
||||
a mode the map supplied marks the deployment as known, or an off-map deployment carrying any
|
||||
mode empties the group it sits in."""
|
||||
a mode the map supplied marks the deployment as known. An off-map deployment carrying an
|
||||
operator mode remains unknown and must keep the whole group's level support unknown."""
|
||||
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
|
||||
|
||||
mapped_model = "openai/gpt-5.6-sol"
|
||||
|
|
@ -9993,7 +9995,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_mode_the_operator_declared(
|
|||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.supported_reasoning_efforts == expected
|
||||
assert result.supported_reasoning_efforts is None
|
||||
|
||||
|
||||
class TestAddDeploymentApiBaseProviderResolution:
|
||||
|
|
@ -12252,6 +12254,120 @@ class TestTierParamsTheTargetAccepts:
|
|||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
|
||||
class TestRequestReasoningEffortOverride:
|
||||
def test_drop_effort_from_nested_carrier_preserves_other_nested_values(self):
|
||||
params: dict[str, object] = {"output_config": {"effort": "high", "format": "json"}}
|
||||
|
||||
litellm.Router._pop_effort_from_nested_carrier(params, "output_config")
|
||||
|
||||
assert params == {"output_config": {"format": "json"}}
|
||||
|
||||
@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
|
||||
def test_is_classifier_internal_call_recognizes_both_metadata_carriers(self, metadata_key):
|
||||
kwargs = {metadata_key: {"internal_call_origin": "autorouter_classifier"}}
|
||||
|
||||
assert litellm.Router._is_classifier_internal_call(kwargs) is True
|
||||
assert litellm.Router._is_classifier_internal_call({metadata_key: {}}) is False
|
||||
|
||||
def test_removes_every_deployment_native_effort_carrier_without_mutating_shared_config(self):
|
||||
extra_body: dict[str, object] = {
|
||||
"reasoning_effort": "high",
|
||||
"thinking": {"type": "enabled"},
|
||||
"output_config": {"effort": "high", "format": "json"},
|
||||
"reasoning": {"effort": "high", "summary": "detailed"},
|
||||
"provider_option": True,
|
||||
}
|
||||
deployment_params: dict[str, object] = {
|
||||
"model": "bedrock/converse/anthropic.claude-3-7-sonnet",
|
||||
"thinking": {"type": "enabled", "budget_tokens": 2048},
|
||||
"output_config": {"effort": "high", "format": {"type": "json_schema"}},
|
||||
"reasoning": {"effort": "high", "summary": "auto"},
|
||||
"extra_body": extra_body,
|
||||
}
|
||||
|
||||
sanitized = litellm.Router._deployment_params_with_request_reasoning_override(
|
||||
deployment_params, {"reasoning_effort": "low"}
|
||||
)
|
||||
|
||||
assert sanitized == {
|
||||
"model": "bedrock/converse/anthropic.claude-3-7-sonnet",
|
||||
"output_config": {"format": {"type": "json_schema"}},
|
||||
"reasoning": {"summary": "auto"},
|
||||
"extra_body": {
|
||||
"output_config": {"format": "json"},
|
||||
"reasoning": {"summary": "detailed"},
|
||||
"provider_option": True,
|
||||
},
|
||||
}
|
||||
assert deployment_params["thinking"] == {"type": "enabled", "budget_tokens": 2048}
|
||||
assert deployment_params["output_config"] == {"effort": "high", "format": {"type": "json_schema"}}
|
||||
assert extra_body["reasoning_effort"] == "high"
|
||||
|
||||
@pytest.mark.parametrize("request_kwargs", [{}, {"reasoning_effort": None}])
|
||||
def test_omitted_override_preserves_deployment_defaults(self, request_kwargs):
|
||||
deployment_params = {
|
||||
"model": "deepseek/deepseek-reasoner",
|
||||
"thinking": {"type": "enabled"},
|
||||
"output_config": {"effort": "high"},
|
||||
}
|
||||
|
||||
assert (
|
||||
litellm.Router._deployment_params_with_request_reasoning_override(deployment_params, request_kwargs)
|
||||
== deployment_params
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_280_concurrent_overrides_never_mutate_or_leak_through_shared_deployment_params(self):
|
||||
deployment_params = {
|
||||
"model": "fireworks_ai/accounts/fireworks/models/kimi-k2-thinking",
|
||||
"thinking": {"type": "enabled"},
|
||||
"output_config": {"effort": "high", "format": "json"},
|
||||
"extra_body": {"reasoning_effort": "high", "tenant": "shared"},
|
||||
}
|
||||
efforts = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(
|
||||
asyncio.to_thread(
|
||||
litellm.Router._deployment_params_with_request_reasoning_override,
|
||||
deployment_params,
|
||||
{"reasoning_effort": efforts[index % len(efforts)]},
|
||||
)
|
||||
for index in range(280)
|
||||
)
|
||||
)
|
||||
|
||||
assert all("thinking" not in result for result in results)
|
||||
assert all(result["output_config"] == {"format": "json"} for result in results)
|
||||
assert all(result["extra_body"] == {"tenant": "shared"} for result in results)
|
||||
assert deployment_params["thinking"] == {"type": "enabled"}
|
||||
assert deployment_params["output_config"] == {"effort": "high", "format": "json"}
|
||||
assert deployment_params["extra_body"] == {"reasoning_effort": "high", "tenant": "shared"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("metadata", "should_drop"),
|
||||
[({"internal_call_origin": "autorouter_classifier"}, True), ({}, False)],
|
||||
ids=["classifier", "ordinary-request"],
|
||||
)
|
||||
def test_only_classifier_calls_drop_effort_for_an_unsupported_fallback(self, metadata, should_drop):
|
||||
router = litellm.Router(model_list=[])
|
||||
body: dict[str, object] = {"model": "classifier", "reasoning_effort": "low"}
|
||||
kwargs: dict[str, object] = {
|
||||
"reasoning_effort": "low",
|
||||
"metadata": metadata,
|
||||
"proxy_server_request": {"body": body},
|
||||
}
|
||||
deployment: DeploymentTypedDict = {
|
||||
"model_name": "fallback",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini"},
|
||||
}
|
||||
|
||||
router._drop_unsupported_classifier_reasoning_effort(deployment, "fallback", kwargs)
|
||||
|
||||
assert ("reasoning_effort" not in kwargs) is should_drop
|
||||
assert ("reasoning_effort" not in body) is should_drop
|
||||
|
||||
|
||||
class TestPreRoutingTierDrivesFallbacks:
|
||||
"""#38832: a complexity/auto router picks a tier behind the router name, but fallback
|
||||
lookup stayed on the router name, so the tier's configured chain never ran and a
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import ClassifierPromptEditor from "./ClassifierPromptEditor";
|
|||
import CustomTierPromptEditor from "./CustomTierPromptEditor";
|
||||
import { RestrictedSection, restrictedBy } from "./TierRestrictions";
|
||||
import HeuristicScoringConfig from "./HeuristicScoringConfig";
|
||||
import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect";
|
||||
import type { ReasoningEffort } from "./complexity_router_tiers";
|
||||
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
|
||||
import {
|
||||
ClassificationFrequency,
|
||||
|
|
@ -154,6 +156,7 @@ interface ClassificationMethodConfigProps {
|
|||
value: ComplexityRouterConfigValue;
|
||||
onChange: (value: ComplexityRouterConfigValue) => void;
|
||||
modelOptions: { value: string; label: string }[];
|
||||
effortOptionsByModel: Record<string, string[] | null | undefined>;
|
||||
customTechnicalKeywords?: string[];
|
||||
onCustomTechnicalKeywordsChange?: (keywords: string[]) => void;
|
||||
showValidationErrors?: boolean;
|
||||
|
|
@ -236,6 +239,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
value,
|
||||
onChange,
|
||||
modelOptions,
|
||||
effortOptionsByModel,
|
||||
customTechnicalKeywords,
|
||||
onCustomTechnicalKeywordsChange,
|
||||
showValidationErrors = false,
|
||||
|
|
@ -251,6 +255,9 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
|
||||
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
|
||||
const classificationRubric = value.classifier_llm_config?.classification_rubric ?? DEFAULT_CLASSIFICATION_RUBRIC;
|
||||
const classifierModel = value.classifier_llm_config?.model ?? "";
|
||||
const classifierReasoningEffort = value.classifier_llm_config?.reasoning_effort;
|
||||
const explicitlySupportedClassifierEfforts = effortOptionsByModel[classifierModel];
|
||||
|
||||
const handleClassifierTypeChange = (classifierType: ClassifierType) => {
|
||||
const nextValue: ComplexityRouterConfigValue = {
|
||||
|
|
@ -299,16 +306,33 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
};
|
||||
|
||||
const handleClassifierModelChange = (model: string) => {
|
||||
if (model === value.classifier_llm_config?.model) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config ?? {
|
||||
model: "",
|
||||
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
};
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config: {
|
||||
...value.classifier_llm_config,
|
||||
...classifierLlmConfig,
|
||||
model,
|
||||
timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierReasoningEffortChange = (reasoningEffort: ReasoningEffort | undefined) => {
|
||||
if (!value.classifier_llm_config) return;
|
||||
const { reasoning_effort: _reasoningEffort, ...classifierLlmConfig } = value.classifier_llm_config;
|
||||
onChange({
|
||||
...value,
|
||||
classifier_llm_config:
|
||||
reasoningEffort === undefined
|
||||
? classifierLlmConfig
|
||||
: { ...classifierLlmConfig, reasoning_effort: reasoningEffort },
|
||||
});
|
||||
};
|
||||
|
||||
const handleClassifierTimeoutChange = (timeoutMs: number) => {
|
||||
onChange({
|
||||
...value,
|
||||
|
|
@ -493,9 +517,16 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
|
|||
emptyText="No models found"
|
||||
allowClear={false}
|
||||
className={classifierModelMissing ? "border-destructive" : undefined}
|
||||
aria-label="Classifier Model"
|
||||
/>
|
||||
{classifierModelMissing && <span className="text-xs text-destructive">A classifier model is required</span>}
|
||||
</div>
|
||||
<ClassifierReasoningEffortSelect
|
||||
model={classifierModel}
|
||||
value={classifierReasoningEffort}
|
||||
explicitlySupported={explicitlySupportedClassifierEfforts}
|
||||
onChange={handleClassifierReasoningEffortChange}
|
||||
/>
|
||||
<div>
|
||||
<Label htmlFor={CLASSIFIER_TIMEOUT_ID} className="block mb-1 font-semibold">
|
||||
Timeout (ms)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
import { Info } from "lucide-react";
|
||||
import { SimpleTooltip } from "@/components/ui/tooltip";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import type { ReasoningEffort } from "./complexity_router_tiers";
|
||||
|
||||
const PROVIDER_DEFAULT = "__classifier_provider_default__";
|
||||
|
||||
type EffortStatus = "supported" | "unsupported" | "unverified" | undefined;
|
||||
|
||||
const effortStatusFor = (
|
||||
effort: string | undefined,
|
||||
explicitlySupported: string[] | null | undefined,
|
||||
): EffortStatus => {
|
||||
if (effort === undefined) return undefined;
|
||||
if (!Array.isArray(explicitlySupported)) return "unverified";
|
||||
return explicitlySupported.includes(effort) ? "supported" : "unsupported";
|
||||
};
|
||||
|
||||
interface ClassifierReasoningEffortSelectProps {
|
||||
model: string;
|
||||
value: ReasoningEffort | undefined;
|
||||
explicitlySupported: string[] | null | undefined;
|
||||
onChange: (value: ReasoningEffort | undefined) => void;
|
||||
}
|
||||
|
||||
const ClassifierReasoningEffortSelect = ({
|
||||
model,
|
||||
value,
|
||||
explicitlySupported,
|
||||
onChange,
|
||||
}: ClassifierReasoningEffortSelectProps) => {
|
||||
const status = effortStatusFor(value, explicitlySupported);
|
||||
const options = Array.from(new Set([...(explicitlySupported ?? []), ...(value ? [value] : [])]));
|
||||
|
||||
if (!model || options.length === 0) return null;
|
||||
|
||||
const optionLabel = (effort: string): string =>
|
||||
effort === value && status !== "supported" ? `${effort} (${status})` : effort;
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<strong className="font-semibold">Reasoning Effort</strong>
|
||||
<SimpleTooltip content="Sent only to the classifier call. Default leaves the classifier deployment or provider setting unchanged.">
|
||||
<Info className="size-4 text-muted-foreground" />
|
||||
</SimpleTooltip>
|
||||
</div>
|
||||
<Select
|
||||
items={[
|
||||
{ value: PROVIDER_DEFAULT, label: "Default" },
|
||||
...options.map((effort) => ({ value: effort, label: optionLabel(effort) })),
|
||||
]}
|
||||
value={value ?? PROVIDER_DEFAULT}
|
||||
onValueChange={(effort: string | null) =>
|
||||
effort && onChange(effort === PROVIDER_DEFAULT ? undefined : (effort as ReasoningEffort))
|
||||
}
|
||||
>
|
||||
<SelectTrigger aria-label={`Reasoning effort for classifier model ${model}`} className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem>
|
||||
{options.map((effort) => (
|
||||
<SelectItem key={effort} value={effort}>
|
||||
{optionLabel(effort)}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
{status === "unverified" && (
|
||||
<p className="mt-1 text-xs text-amber-700 dark:text-amber-400">
|
||||
This saved effort cannot be verified for the selected model. Choose Default unless you have confirmed provider
|
||||
support.
|
||||
</p>
|
||||
)}
|
||||
{status === "unsupported" && (
|
||||
<p className="mt-1 text-xs text-destructive">
|
||||
This saved effort is not supported by every deployment in the selected model group. Choose Default or a
|
||||
supported value before saving.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ClassifierReasoningEffortSelect;
|
||||
|
|
@ -1124,6 +1124,109 @@ describe("ComplexityRouterConfig per-model reasoning effort", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("ComplexityRouterConfig classifier reasoning effort", () => {
|
||||
const llmValue: ComplexityRouterConfigValue = {
|
||||
...defaultValue,
|
||||
classifier_type: "llm",
|
||||
classifier_llm_config: { model: "gpt-4", timeout_ms: 3000 },
|
||||
};
|
||||
|
||||
const renderClassifier = (value: ComplexityRouterConfigValue = llmValue, onChange = vi.fn()) => {
|
||||
renderWithProviders(<ComplexityRouterConfig {...baseProps} value={value} onChange={onChange} />);
|
||||
fireEvent.click(screen.getByText("Advanced: Classification Method"));
|
||||
return onChange;
|
||||
};
|
||||
|
||||
it("defaults to the classifier provider setting and offers only supported efforts", async () => {
|
||||
renderClassifier();
|
||||
const user = userEvent.setup();
|
||||
const select = screen.getByRole("combobox", { name: "Reasoning effort for classifier model gpt-4" });
|
||||
expect(select).toHaveTextContent("Default");
|
||||
await user.click(select);
|
||||
expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([
|
||||
"Default",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
]);
|
||||
});
|
||||
|
||||
it("stores an explicit effort on the classifier config", async () => {
|
||||
const onChange = renderClassifier();
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Reasoning effort for classifier model gpt-4" }));
|
||||
await user.click(await screen.findByRole("option", { name: "high" }));
|
||||
expect(onChange).toHaveBeenCalledWith({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" },
|
||||
});
|
||||
});
|
||||
|
||||
it("removes the effort override when Default is selected", async () => {
|
||||
const onChange = renderClassifier({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" },
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Reasoning effort for classifier model gpt-4" }));
|
||||
await user.click(await screen.findByRole("option", { name: "Default" }));
|
||||
expect(onChange).toHaveBeenCalledWith(llmValue);
|
||||
});
|
||||
|
||||
it("clears the old effort when the classifier model changes", async () => {
|
||||
const onChange = renderClassifier({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" },
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier Model" }));
|
||||
await user.click(await screen.findByRole("option", { name: "gpt-3.5-turbo" }));
|
||||
expect(onChange).toHaveBeenCalledWith({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 },
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["click", "enter"] as const)(
|
||||
"keeps the effort when the selected model is confirmed by %s",
|
||||
async (action) => {
|
||||
const onChange = renderClassifier({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model: "gpt-4", timeout_ms: 3000, reasoning_effort: "high" },
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
await user.click(screen.getByRole("combobox", { name: "Classifier Model" }));
|
||||
if (action === "click") await user.click(await screen.findByRole("option", { name: "gpt-4" }));
|
||||
else await user.keyboard("{Enter}");
|
||||
expect(onChange).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["gpt-4", "max", "max (unsupported)", /not supported by every deployment/],
|
||||
["claude-3-opus", "low", "low (unverified)", /cannot be verified/],
|
||||
])("keeps a saved exceptional value visible for %s", (model, effort, label, warning) => {
|
||||
renderClassifier({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model, timeout_ms: 3000, reasoning_effort: effort },
|
||||
});
|
||||
expect(screen.getByRole("combobox", { name: `Reasoning effort for classifier model ${model}` })).toHaveTextContent(
|
||||
label,
|
||||
);
|
||||
expect(screen.getByText(warning)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each(["claude-3-opus", "gpt-3.5-turbo"])("hides the effort control when %s has no advertised options", (model) => {
|
||||
renderClassifier({
|
||||
...llmValue,
|
||||
classifier_llm_config: { model, timeout_ms: 3000 },
|
||||
});
|
||||
expect(
|
||||
screen.queryByRole("combobox", { name: `Reasoning effort for classifier model ${model}` }),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("ComplexityRouterConfig reasoning effort gating", () => {
|
||||
it("offers no effort select for a model group without reasoning support", () => {
|
||||
renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
|
||||
|
|
@ -1232,12 +1335,13 @@ describe("ComplexityRouterConfig custom technical keywords", () => {
|
|||
});
|
||||
|
||||
it("hides the keywords when the scorer never runs, so they cannot imply an effect they have none", () => {
|
||||
openClassificationPanel({
|
||||
const llmWithDefaultFallback = {
|
||||
...defaultValue,
|
||||
classifier_type: "llm",
|
||||
classifier_type: "llm" as const,
|
||||
classifier_llm_config: llmConfig,
|
||||
classifier_fallback: "default_model",
|
||||
});
|
||||
classifier_fallback: "default_model" as const,
|
||||
};
|
||||
openClassificationPanel(llmWithDefaultFallback);
|
||||
expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -36,10 +36,11 @@ import ContextWindowEscalationConfig from "./ContextWindowEscalationConfig";
|
|||
import { Restricted, restrictedBy } from "./TierRestrictions";
|
||||
import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
|
||||
import {
|
||||
REASONING_EFFORT_OPTIONS,
|
||||
ReasoningEffort,
|
||||
TierModelParamsByTier,
|
||||
classifierEffortOptionsForModels,
|
||||
setTierModelReasoningEffort,
|
||||
tierEffortOptionsForModels,
|
||||
tierRowLabel,
|
||||
} from "./complexity_router_tiers";
|
||||
import TierModelEffortRows from "./TierModelEffortRows";
|
||||
|
|
@ -124,6 +125,7 @@ export const CLASSIFICATION_RUBRIC_KEYS = Object.keys(CLASSIFICATION_RUBRIC_DESC
|
|||
export interface ClassifierLLMConfig {
|
||||
model: string;
|
||||
timeout_ms: number;
|
||||
reasoning_effort?: ReasoningEffort;
|
||||
classification_rubric?: ClassificationRubric;
|
||||
system_prompt?: string;
|
||||
}
|
||||
|
|
@ -632,14 +634,8 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
const removeTierRow = (id: string) => dispatch({ kind: "remove", id });
|
||||
const exitToBuiltInTiers = () => dispatch({ kind: "restore" });
|
||||
|
||||
// An absent list means the proxy does not send the field yet, so every level is offered as before.
|
||||
// An empty list is the group's own answer that its deployments share no level, and is left empty.
|
||||
const effortOptionsByModel: Record<string, string[]> = Object.fromEntries(
|
||||
modelInfo.map((model) => [
|
||||
model.model_group,
|
||||
model.supported_reasoning_efforts ?? (model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : []),
|
||||
]),
|
||||
);
|
||||
const tierEffortOptionsByModel = tierEffortOptionsForModels(modelInfo);
|
||||
const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo);
|
||||
|
||||
// Embedding models can't serve a chat-completion role, so they're excluded here.
|
||||
const modelOptions = modelInfo
|
||||
|
|
@ -746,7 +742,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
<TierModelEffortRows
|
||||
tierLabel={label}
|
||||
models={row.models}
|
||||
effortOptionsByModel={effortOptionsByModel}
|
||||
effortOptionsByModel={tierEffortOptionsByModel}
|
||||
paramsByModel={row.params}
|
||||
onEffortChange={(model, effort) => handleTierModelEffortChange(row.id, model, effort)}
|
||||
/>
|
||||
|
|
@ -818,6 +814,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
|
|||
value={value}
|
||||
onChange={onChange}
|
||||
modelOptions={modelOptions}
|
||||
effortOptionsByModel={classifierEffortOptionsByModel}
|
||||
customTechnicalKeywords={customTechnicalKeywords}
|
||||
onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange}
|
||||
showValidationErrors={showValidationErrors}
|
||||
|
|
|
|||
|
|
@ -155,7 +155,11 @@ describe("HeuristicScoringConfig", () => {
|
|||
});
|
||||
|
||||
describe("ClassificationMethodConfig scorer gating", () => {
|
||||
const props = { onChange: vi.fn(), modelOptions: [{ value: "gpt-4o-mini", label: "gpt-4o-mini" }] };
|
||||
const props = {
|
||||
onChange: vi.fn(),
|
||||
modelOptions: [{ value: "gpt-4o-mini", label: "gpt-4o-mini" }],
|
||||
effortOptionsByModel: {},
|
||||
};
|
||||
const withClassifier = (type: ClassifierType, fallback?: ClassifierFallback): ComplexityRouterConfigValue => ({
|
||||
...BASE,
|
||||
classifier_type: type,
|
||||
|
|
|
|||
|
|
@ -19,11 +19,12 @@ import { all_admin_roles } from "@/utils/roles";
|
|||
import { type ModelWriteScope } from "@/utils/modelPermissions";
|
||||
import TeamDropdown from "../common_components/team_dropdown";
|
||||
import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
||||
import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
|
||||
import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import ComplexityRouterConfig, {
|
||||
ComplexityRouterConfigValue,
|
||||
effectiveClassifierType,
|
||||
usesLlmClassifier,
|
||||
DEFAULT_ADAPTIVE_WEIGHTS,
|
||||
DEFAULT_SESSION_AFFINITY,
|
||||
DEFAULT_DEPLOYMENT_AFFINITY,
|
||||
|
|
@ -37,6 +38,7 @@ import {
|
|||
buildComplexityRouterConfig,
|
||||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getClassifierReasoningEffortError,
|
||||
getMissingTiersError,
|
||||
getPlanModeTierError,
|
||||
getSemanticConfigError,
|
||||
|
|
@ -120,14 +122,21 @@ export const getSubmitBlockedReason = (
|
|||
config: ComplexityRouterConfigValue,
|
||||
keywordTierRules: KeywordTierRule[],
|
||||
referencedModelsParams: Parameters<typeof getReferencedModelsError>[0],
|
||||
availability: ModelAvailability,
|
||||
): string | null =>
|
||||
(config.custom_tier_set ? getCustomTierRowsError(config.custom_tier_set) : getTierLabelsError(config.tier_labels)) ??
|
||||
getMissingTiersError(activeTierRows(config)) ??
|
||||
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
|
||||
getClassifierModelError(config) ??
|
||||
getReferencedModelsError(referencedModelsParams, availability);
|
||||
...capabilities: [availability: ModelAvailability, modelInfo?: readonly ModelGroup[]]
|
||||
): string | null => {
|
||||
const [availability, modelInfo = []] = capabilities;
|
||||
return (
|
||||
(config.custom_tier_set
|
||||
? getCustomTierRowsError(config.custom_tier_set)
|
||||
: getTierLabelsError(config.tier_labels)) ??
|
||||
getMissingTiersError(activeTierRows(config)) ??
|
||||
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
|
||||
getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
|
||||
getClassifierModelError(config) ??
|
||||
getClassifierReasoningEffortError(config, modelInfo) ??
|
||||
getReferencedModelsError(referencedModelsParams, availability)
|
||||
);
|
||||
};
|
||||
|
||||
const autoRouterSchema = (requiresTeamScope: boolean) =>
|
||||
z.object({
|
||||
|
|
@ -338,6 +347,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
);
|
||||
|
||||
const complexityRouterConfigParams: BuildComplexityRouterConfigParams = {
|
||||
|
|
@ -390,6 +400,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
keywordTierRules,
|
||||
referencedModelsParams,
|
||||
groupsOnlyAvailability,
|
||||
modelInfo,
|
||||
) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
|
||||
if (blockedReason) {
|
||||
setShowValidationErrors(true);
|
||||
|
|
@ -462,6 +473,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
semanticMatchingEnabled,
|
||||
embeddingModel,
|
||||
defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model),
|
||||
classifier: usesLlmClassifier(effectiveClassifierType(complexityRouterConfig))
|
||||
? {
|
||||
model: complexityRouterConfig.classifier_llm_config?.model ?? "",
|
||||
reasoningEffort: complexityRouterConfig.classifier_llm_config?.reasoning_effort,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
const targets = buildAutoRouterTestTargets(testTargetParams);
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ const targets: AutoRouterTestTarget[] = [
|
|||
{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" },
|
||||
{ labels: ["MEDIUM", "COMPLEX"], modelGroup: "claude-sonnet-4", mode: "chat" },
|
||||
{ labels: ["Embedding"], modelGroup: "voyage-3-5", mode: "embedding" },
|
||||
{ labels: ["Classifier"], modelGroup: "gpt-5-mini", mode: "chat", requestParams: { reasoning_effort: "low" } },
|
||||
];
|
||||
|
||||
describe("AutoRouterConnectionTest", () => {
|
||||
|
|
@ -30,11 +31,12 @@ describe("AutoRouterConnectionTest", () => {
|
|||
|
||||
renderWithProviders(<AutoRouterConnectionTest accessToken="sk-test" targets={targets} />);
|
||||
|
||||
await waitFor(() => expect(mock).toHaveBeenCalledTimes(3));
|
||||
await waitFor(() => expect(mock).toHaveBeenCalledTimes(4));
|
||||
|
||||
expect(mock).toHaveBeenCalledWith("sk-test", "gpt-4o-mini", "chat");
|
||||
expect(mock).toHaveBeenCalledWith("sk-test", "claude-sonnet-4", "chat");
|
||||
expect(mock).toHaveBeenCalledWith("sk-test", "voyage-3-5", "embedding");
|
||||
expect(mock).toHaveBeenCalledWith("sk-test", "gpt-5-mini", "chat", { reasoning_effort: "low" });
|
||||
});
|
||||
|
||||
it("shows a success indicator per target when the routing probe passes", async () => {
|
||||
|
|
@ -43,7 +45,7 @@ describe("AutoRouterConnectionTest", () => {
|
|||
|
||||
renderWithProviders(<AutoRouterConnectionTest accessToken="sk-test" targets={targets} />);
|
||||
|
||||
await waitFor(() => expect(screen.getAllByTestId("test-status-success")).toHaveLength(3));
|
||||
await waitFor(() => expect(screen.getAllByTestId("test-status-success")).toHaveLength(4));
|
||||
expect(screen.queryByTestId("test-status-error")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("MEDIUM, COMPLEX")).toBeInTheDocument();
|
||||
});
|
||||
|
|
@ -63,7 +65,7 @@ describe("AutoRouterConnectionTest", () => {
|
|||
expect(await screen.findByTestId("test-error-message")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("test-error-message")).toHaveTextContent("invalid api key");
|
||||
expect(screen.getByTestId("test-error-message")).not.toHaveTextContent("litellm.AuthenticationError");
|
||||
expect(screen.getAllByTestId("test-status-success")).toHaveLength(2);
|
||||
expect(screen.getAllByTestId("test-status-success")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("renders a non-litellm error string verbatim", async () => {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
const run = async () => {
|
||||
await Promise.all(
|
||||
targets.map(async (target, index) => {
|
||||
const result = await testModelGroupConnection(accessToken, target.modelGroup, target.mode);
|
||||
const result = target.requestParams
|
||||
? await testModelGroupConnection(accessToken, target.modelGroup, target.mode, target.requestParams)
|
||||
: await testModelGroupConnection(accessToken, target.modelGroup, target.mode);
|
||||
if (cancelled) return;
|
||||
const cleaned: TargetResult =
|
||||
result.status === "error" ? { status: "error", error: cleanErrorMessage(result.error) } : result;
|
||||
|
|
@ -56,14 +58,14 @@ const AutoRouterConnectionTest: React.FC<AutoRouterConnectionTestProps> = ({
|
|||
return (
|
||||
<div className="space-y-3">
|
||||
<p className="mb-2 text-sm text-muted-foreground">
|
||||
Each configured tier routes to a saved model group. Test Connection sends a minimal request through the proxy to
|
||||
each one, exactly as the auto router would.
|
||||
Test Connection sends a minimal request to every configured tier, classifier, default, and embedding model. The
|
||||
classifier probe includes its reasoning effort override.
|
||||
</p>
|
||||
{targets.map((target, index) => {
|
||||
const result = results[index] ?? { status: "pending" };
|
||||
return (
|
||||
<div
|
||||
key={`${target.modelGroup}-${target.mode}`}
|
||||
key={`${target.labels.join("-")}-${target.modelGroup}-${target.mode}`}
|
||||
data-testid="auto-router-test-row"
|
||||
className="flex items-start gap-3 rounded-lg border p-3"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -126,4 +126,32 @@ describe("buildAutoRouterTestTargets", () => {
|
|||
});
|
||||
expect(targets).toEqual([{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" }]);
|
||||
});
|
||||
|
||||
it("adds a distinct classifier probe with its reasoning effort", () => {
|
||||
const targets = buildAutoRouterTestTargets({
|
||||
tiers: tierEntries(["gpt-5-mini"]),
|
||||
semanticMatchingEnabled: false,
|
||||
embeddingModel: undefined,
|
||||
classifier: { model: "gpt-5-mini", reasoningEffort: "low" },
|
||||
});
|
||||
expect(targets).toEqual([
|
||||
{ labels: ["SIMPLE"], modelGroup: "gpt-5-mini", mode: "chat" },
|
||||
{
|
||||
labels: ["Classifier"],
|
||||
modelGroup: "gpt-5-mini",
|
||||
mode: "chat",
|
||||
requestParams: { reasoning_effort: "low" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits an empty classifier and omits params when the classifier uses provider defaults", () => {
|
||||
const base = { tiers: tierEntries([]), semanticMatchingEnabled: false, embeddingModel: undefined };
|
||||
const emptyClassifier = { ...base, classifier: { model: " " } };
|
||||
const providerDefaultClassifier = { ...base, classifier: { model: "gpt-5-mini" } };
|
||||
expect(buildAutoRouterTestTargets(emptyClassifier)).toEqual([]);
|
||||
expect(buildAutoRouterTestTargets(providerDefaultClassifier)).toEqual([
|
||||
{ labels: ["Classifier"], modelGroup: "gpt-5-mini", mode: "chat" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ export interface AutoRouterTestTarget {
|
|||
labels: string[];
|
||||
modelGroup: string;
|
||||
mode: AutoRouterTestMode;
|
||||
requestParams?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface BuildAutoRouterTestTargetsParams {
|
||||
|
|
@ -14,6 +15,7 @@ export interface BuildAutoRouterTestTargetsParams {
|
|||
/** The resolved default model - see resolveComplexityDefaultModel. A live fallback destination,
|
||||
* so it is probed even when no tier lists it. */
|
||||
defaultModel?: string;
|
||||
classifier?: { model: string; reasoningEffort?: string };
|
||||
}
|
||||
|
||||
export const buildAutoRouterTestTargets = ({
|
||||
|
|
@ -21,6 +23,7 @@ export const buildAutoRouterTestTargets = ({
|
|||
semanticMatchingEnabled,
|
||||
embeddingModel,
|
||||
defaultModel,
|
||||
classifier,
|
||||
}: BuildAutoRouterTestTargetsParams): AutoRouterTestTarget[] => {
|
||||
const tieredByModel = tiers.reduce<Record<string, string[]>>((acc, [tier, models]) => {
|
||||
return models.reduce((tierAcc, rawModel) => {
|
||||
|
|
@ -50,5 +53,17 @@ export const buildAutoRouterTestTargets = ({
|
|||
? [{ labels: ["Embedding"], modelGroup: embeddingModel.trim(), mode: "embedding" as const }]
|
||||
: [];
|
||||
|
||||
return [...tierTargets, ...embeddingTarget];
|
||||
const classifierModel = classifier?.model.trim();
|
||||
const classifierTarget: AutoRouterTestTarget[] = classifierModel
|
||||
? [
|
||||
{
|
||||
labels: ["Classifier"],
|
||||
modelGroup: classifierModel,
|
||||
mode: "chat",
|
||||
...(classifier?.reasoningEffort && { requestParams: { reasoning_effort: classifier.reasoningEffort } }),
|
||||
},
|
||||
]
|
||||
: [];
|
||||
|
||||
return [...tierTargets, ...embeddingTarget, ...classifierTarget];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
normalizeClassifierLlmConfig,
|
||||
getKeywordTierRulesError,
|
||||
getClassifierModelError,
|
||||
getClassifierReasoningEffortError,
|
||||
getMissingTiersError,
|
||||
hydrateCustomTierSet,
|
||||
getSemanticConfigError,
|
||||
|
|
@ -506,6 +507,19 @@ describe("classifier prompt and fallback", () => {
|
|||
expect(buildComplexityRouterConfig(llmParams)).not.toHaveProperty("classifier_fallback");
|
||||
});
|
||||
|
||||
it("keeps an explicit classifier reasoning effort", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...llmParams,
|
||||
classifierLlmConfig: { model: "haiku-classifier", timeout_ms: 400, reasoning_effort: "low" },
|
||||
});
|
||||
expect(config.classifier_llm_config?.reasoning_effort).toBe("low");
|
||||
});
|
||||
|
||||
it("omits classifier reasoning effort when the provider default is selected", () => {
|
||||
const config = buildComplexityRouterConfig(llmParams);
|
||||
expect(config.classifier_llm_config).not.toHaveProperty("reasoning_effort");
|
||||
});
|
||||
|
||||
it("sends the chat preset the operator picked", () => {
|
||||
const config = buildComplexityRouterConfig({
|
||||
...llmParams,
|
||||
|
|
@ -540,12 +554,16 @@ describe("classifier prompt and fallback", () => {
|
|||
});
|
||||
|
||||
it("normalizeClassifierLlmConfig leaves a real prompt untouched and strips an empty one", () => {
|
||||
expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, system_prompt: "x" })).toEqual({
|
||||
const customPromptConfig = { model: "m", timeout_ms: 1, reasoning_effort: "none" as const, system_prompt: "x" };
|
||||
const emptyPromptConfig = { model: "m", timeout_ms: 1, system_prompt: "" };
|
||||
const expectedCustomPromptConfig = {
|
||||
model: "m",
|
||||
timeout_ms: 1,
|
||||
reasoning_effort: "none",
|
||||
system_prompt: "x",
|
||||
});
|
||||
expect(normalizeClassifierLlmConfig({ model: "m", timeout_ms: 1, system_prompt: "" })).toEqual({
|
||||
};
|
||||
expect(normalizeClassifierLlmConfig(customPromptConfig)).toEqual(expectedCustomPromptConfig);
|
||||
expect(normalizeClassifierLlmConfig(emptyPromptConfig)).toEqual({
|
||||
model: "m",
|
||||
timeout_ms: 1,
|
||||
});
|
||||
|
|
@ -767,6 +785,26 @@ describe("getClassifierModelError", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("getClassifierReasoningEffortError", () => {
|
||||
const classifier = {
|
||||
classifier_type: "llm" as const,
|
||||
classifier_llm_config: { model: "classifier", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
};
|
||||
|
||||
it.each([
|
||||
[["low", "medium"], null],
|
||||
[["medium", "high"], "low reasoning effort is not supported"],
|
||||
[null, null],
|
||||
[undefined, null],
|
||||
])("validates capability levels %o", (supportedReasoningEfforts, expectedError) => {
|
||||
const error = getClassifierReasoningEffortError(classifier, [
|
||||
{ model_group: "classifier", supported_reasoning_efforts: supportedReasoningEfforts },
|
||||
]);
|
||||
if (expectedError) expect(error).toContain(expectedError);
|
||||
else expect(error).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getKeywordTierRulesError orphaned tiers", () => {
|
||||
const rows = activeTierRows({ tiers });
|
||||
|
||||
|
|
@ -944,11 +982,16 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
|
|||
classifierLlmConfig: {
|
||||
model: "gpt-4o-mini",
|
||||
timeout_ms: 3000,
|
||||
reasoning_effort: "low",
|
||||
system_prompt: "replace the whole rubric",
|
||||
classification_rubric: "agentic",
|
||||
},
|
||||
});
|
||||
expect(payload.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 });
|
||||
expect(payload.classifier_llm_config).toEqual({
|
||||
model: "gpt-4o-mini",
|
||||
timeout_ms: 3000,
|
||||
reasoning_effort: "low",
|
||||
});
|
||||
});
|
||||
|
||||
it.each(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { KeywordTierRule } from "./KeywordTierRules";
|
||||
import type { ModelGroup } from "../llm_calls/fetch_models";
|
||||
import {
|
||||
type CustomTierSet,
|
||||
type TierRow,
|
||||
|
|
@ -55,12 +56,18 @@ import {
|
|||
export const normalizeClassifierLlmConfig = ({
|
||||
model,
|
||||
timeout_ms,
|
||||
reasoning_effort,
|
||||
classification_rubric,
|
||||
system_prompt,
|
||||
}: ClassifierLLMConfig): ClassifierLLMConfig =>
|
||||
system_prompt?.trim()
|
||||
? { model, timeout_ms, system_prompt }
|
||||
: { model, timeout_ms, ...(classification_rubric && { classification_rubric }) };
|
||||
? { model, timeout_ms, ...(reasoning_effort && { reasoning_effort }), system_prompt }
|
||||
: {
|
||||
model,
|
||||
timeout_ms,
|
||||
...(reasoning_effort && { reasoning_effort }),
|
||||
...(classification_rubric && { classification_rubric }),
|
||||
};
|
||||
|
||||
interface ScorerKnobInputs {
|
||||
classifierType: ClassifierType;
|
||||
|
|
@ -268,6 +275,20 @@ export const getClassifierModelError = (
|
|||
: "Please select a classifier model, or switch back to Heuristic";
|
||||
};
|
||||
|
||||
export const getClassifierReasoningEffortError = (
|
||||
config: Pick<ComplexityRouterConfigValue, "custom_tier_set" | "classifier_type" | "classifier_llm_config">,
|
||||
modelInfo: readonly ModelGroup[],
|
||||
): string | null => {
|
||||
if (!usesLlmClassifier(effectiveClassifierType(config))) return null;
|
||||
const classifierConfig = config.classifier_llm_config;
|
||||
if (!classifierConfig?.model || !classifierConfig.reasoning_effort) return null;
|
||||
const supported = modelInfo.find(
|
||||
(model) => model.model_group === classifierConfig.model,
|
||||
)?.supported_reasoning_efforts;
|
||||
if (!Array.isArray(supported) || supported.includes(classifierConfig.reasoning_effort)) return null;
|
||||
return `${classifierConfig.reasoning_effort} reasoning effort is not supported by every deployment in ${classifierConfig.model}. Choose Default or a supported value.`;
|
||||
};
|
||||
|
||||
export const getSemanticConfigError = ({
|
||||
semanticMatchingEnabled,
|
||||
embeddingModel,
|
||||
|
|
@ -295,11 +316,15 @@ export const customTierWireFields = (
|
|||
tier_definitions: tierDefinitionsFromRows(rows),
|
||||
...(fallback && { fallback_tier: activeTierName(fallback) }),
|
||||
classifier_type: "llm",
|
||||
// Rebuilt from the two fields an edited tier set allows. The backend rejects system_prompt and
|
||||
// Rebuilt from the fields an edited tier set allows. The backend rejects system_prompt and
|
||||
// classification_rubric beside tier_definitions, and both live inside this object rather than at
|
||||
// the top level the omit list covers. The opening instructions ride classification_prompt below.
|
||||
...(classifierLlmConfig && {
|
||||
classifier_llm_config: { model: classifierLlmConfig.model, timeout_ms: classifierLlmConfig.timeout_ms },
|
||||
classifier_llm_config: {
|
||||
model: classifierLlmConfig.model,
|
||||
timeout_ms: classifierLlmConfig.timeout_ms,
|
||||
...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }),
|
||||
},
|
||||
}),
|
||||
session_affinity: false,
|
||||
...(classificationPrompt?.trim() && { classification_prompt: classificationPrompt.trim() }),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { ComplexityTier } from "./KeywordTierRules";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { TIER_ORDER } from "./tier_rows";
|
||||
|
||||
export type TierModelParams = Record<string, unknown>;
|
||||
|
|
@ -18,6 +19,23 @@ export const REASONING_EFFORT_OPTIONS = ["none", "minimal", "low", "medium", "hi
|
|||
*/
|
||||
export type ReasoningEffort = (typeof REASONING_EFFORT_OPTIONS)[number] | (string & {});
|
||||
|
||||
export const tierEffortOptionsForModels = (modelInfo: ModelGroup[]): Record<string, string[]> =>
|
||||
Object.fromEntries(
|
||||
modelInfo.map((model) => [
|
||||
model.model_group,
|
||||
model.supported_reasoning_efforts ?? (model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : []),
|
||||
]),
|
||||
);
|
||||
|
||||
/**
|
||||
* Stricter than the tier variant on purpose: classifier overrides are new in this release, so an
|
||||
* unknown capability list stays unknown instead of inventing provider levels.
|
||||
*/
|
||||
export const classifierEffortOptionsForModels = (
|
||||
modelInfo: ModelGroup[],
|
||||
): Record<string, string[] | null | undefined> =>
|
||||
Object.fromEntries(modelInfo.map((model) => [model.model_group, model.supported_reasoning_efforts]));
|
||||
|
||||
const asRecord = (raw: unknown): Record<string, unknown> | undefined =>
|
||||
typeof raw === "object" && raw !== null && !Array.isArray(raw) ? (raw as Record<string, unknown>) : undefined;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ const storedCustomConfig = (overrides: Record<string, unknown> = {}) => ({
|
|||
],
|
||||
fallback_tier: "CASUAL",
|
||||
classifier_type: "llm",
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
|
|||
const STORED_LLM = {
|
||||
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "llm",
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
classifier_context_window_size: 5,
|
||||
classifier_context_per_turn_chars: 300,
|
||||
};
|
||||
|
|
@ -522,7 +522,7 @@ describe("managed keys survive an untouched open-and-save", () => {
|
|||
tier_labels: { SIMPLE: "Cheap" },
|
||||
classifier_type: "heuristic_first",
|
||||
heuristic_first_max_tier: "SIMPLE",
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
|
||||
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
|
||||
classifier_context_window_size: 5,
|
||||
classifier_context_budget_chars: 4000,
|
||||
classifier_context_include_assistant_turns: true,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import {
|
|||
type BuildComplexityRouterConfigParams,
|
||||
buildComplexityRouterConfig,
|
||||
getClassifierModelError,
|
||||
getClassifierReasoningEffortError,
|
||||
getKeywordTierRulesError,
|
||||
getMissingTiersError,
|
||||
getSemanticConfigError,
|
||||
|
|
@ -560,6 +561,12 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
|
|||
toast.fromError(classifierError);
|
||||
return;
|
||||
}
|
||||
const classifierEffortError = getClassifierReasoningEffortError(complexityRouterConfig, modelInfo);
|
||||
if (classifierEffortError) {
|
||||
setShowValidationErrors(true);
|
||||
toast.fromError(classifierEffortError);
|
||||
return;
|
||||
}
|
||||
// Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a
|
||||
// keyword rule with no keyword, and semantic_keyword_matching without an embedding model
|
||||
// or keyword rules (complexity_router/config.py), so without these a save fails as a raw
|
||||
|
|
|
|||
|
|
@ -52,6 +52,24 @@ describe("fetchAvailableModels", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("preserves absent, unknown, empty, and explicit effort capability states", async () => {
|
||||
modelHubCallMock.mockResolvedValue({
|
||||
data: [
|
||||
{ model_group: "absent", supports_reasoning: true },
|
||||
{ model_group: "unknown", supports_reasoning: true, supported_reasoning_efforts: null },
|
||||
{ model_group: "empty", supports_reasoning: true, supported_reasoning_efforts: [] },
|
||||
{ model_group: "known", supports_reasoning: true, supported_reasoning_efforts: ["low"] },
|
||||
],
|
||||
});
|
||||
|
||||
expect(await fetchAvailableModels("token")).toEqual([
|
||||
{ model_group: "absent", supports_reasoning: true },
|
||||
{ model_group: "empty", supports_reasoning: true, supported_reasoning_efforts: [] },
|
||||
{ model_group: "known", supports_reasoning: true, supported_reasoning_efforts: ["low"] },
|
||||
{ model_group: "unknown", supports_reasoning: true, supported_reasoning_efforts: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["an error payload in place of the list", { data: { error: "no access" } }],
|
||||
["a missing data key", {}],
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ export interface ModelGroup {
|
|||
model_group: string;
|
||||
mode?: string;
|
||||
supports_reasoning?: boolean;
|
||||
supported_reasoning_efforts?: string[];
|
||||
supported_reasoning_efforts?: string[] | null;
|
||||
}
|
||||
|
||||
interface AvailableModel {
|
||||
|
|
@ -25,7 +25,9 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
|
|||
model_group: groupName,
|
||||
...(item.mode && { mode: item.mode }),
|
||||
...(item.supports_reasoning === true && { supports_reasoning: true }),
|
||||
...(item.supported_reasoning_efforts && { supported_reasoning_efforts: item.supported_reasoning_efforts }),
|
||||
...(item.supported_reasoning_efforts !== undefined && {
|
||||
supported_reasoning_efforts: item.supported_reasoning_efforts,
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -617,6 +617,15 @@ describe("buildModelGroupTestRequest", () => {
|
|||
expect(path).toBe("/v1/embeddings");
|
||||
expect(body).toEqual({ model: "text-embedding-3-small", input: "test from litellm" });
|
||||
});
|
||||
|
||||
it("adds classifier request parameters to a chat probe", () => {
|
||||
const { body } = Networking.buildModelGroupTestRequest("gpt-5-mini", "chat", { reasoning_effort: "low" });
|
||||
expect(body).toEqual({
|
||||
model: "gpt-5-mini",
|
||||
messages: [{ role: "user", content: "test from litellm" }],
|
||||
reasoning_effort: "low",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("testMCPToolsListRequest auth headers", () => {
|
||||
|
|
|
|||
|
|
@ -2381,20 +2381,22 @@ export type ModelGroupConnectionResult = { status: "success" } | { status: "erro
|
|||
export const buildModelGroupTestRequest = (
|
||||
modelGroup: string,
|
||||
mode: "chat" | "embedding",
|
||||
requestParams: Record<string, unknown> = {},
|
||||
): { path: string; body: Record<string, unknown> } =>
|
||||
mode === "embedding"
|
||||
? { path: "/v1/embeddings", body: { model: modelGroup, input: "test from litellm" } }
|
||||
: {
|
||||
path: "/v1/chat/completions",
|
||||
body: { model: modelGroup, messages: [{ role: "user", content: "test from litellm" }] },
|
||||
body: { ...requestParams, model: modelGroup, messages: [{ role: "user", content: "test from litellm" }] },
|
||||
};
|
||||
|
||||
export const testModelGroupConnection = async (
|
||||
accessToken: string,
|
||||
modelGroup: string,
|
||||
mode: "chat" | "embedding",
|
||||
requestParams?: Record<string, unknown>,
|
||||
): Promise<ModelGroupConnectionResult> => {
|
||||
const { path, body } = buildModelGroupTestRequest(modelGroup, mode);
|
||||
const { path, body } = buildModelGroupTestRequest(modelGroup, mode, requestParams);
|
||||
try {
|
||||
await apiClient.post(path, { accessToken, body });
|
||||
return { status: "success" };
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25211,6 +25211,11 @@ export interface components {
|
|||
* @description Model name (from the router's model_list) to call for classification
|
||||
*/
|
||||
model: string;
|
||||
/**
|
||||
* Reasoning Effort
|
||||
* @description Reasoning effort override for classifier calls. Leave unset to use the classifier deployment or provider default.
|
||||
*/
|
||||
reasoning_effort?: ("none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max") | null;
|
||||
/**
|
||||
* System Prompt
|
||||
* @description Replaces the built-in complexity rubric as the classifier's entire system role. When set, neither the default rubric nor the context-window closing line is appended, so the prompt owns the whole taxonomy and the tier names SIMPLE/MEDIUM/COMPLEX/REASONING become whatever buckets it defines: a prompt that classifies data sensitivity routes on that instead of on difficulty. Two consequences of full replacement. The default rubric's closing paragraph is the classifier's prompt-injection defense, telling it that the caller's quoted system prompt and prior turns are material to judge and never instructions; a replacement that omits it lets a caller ask for a tier and get it. And the heuristic fallback still scores complexity, so a router on some other taxonomy wants classifier_fallback='default_model'. Leave unset for the built-in rubric. Only applies when classifier_type is 'llm'.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue