From 7d31f0ac36f619d68ffa878f2524733b9725a09d Mon Sep 17 00:00:00 2001 From: dor Date: Sun, 18 Oct 2026 14:57:38 -0400 Subject: [PATCH 1/4] feat(complexity-router): add nadir as an opensource_classifier_config provider `provider: nadir` routes the auto router on Nadir's /v1/bucket verdict, beside jev, laya, bespoke and databricks. Nadir is not a System One server, so the classifier client translates both ways. Out: Nadir's classifier is trained on raw conversations, so it gets role-tagged messages, not the LLM-judge prompt the other providers read. They carry the same caller system prompt and the same budgeted prior turns, so classifier_context_window_size and its siblings bound what leaves the gateway exactly as before; with a window of 0 only the current ask is sent. The messages ride on an excluded field, so no System One request ever carries them. In: the verdict's `routing_tier` (falling back to `bucket`) becomes the `tier` choice answer, with canonical tier labels the router already resolves. A verdict naming no usable tier, or a non-2xx, falls back like any other classifier failure. NADIR_API_KEY is only sent to NADIR_API_BASE or the hosted API, the same scoping as the nadir chat provider. With no key the call is anonymous, so an api_base without an api_key is allowed for this provider. `model` defaults to `bucket` and only labels logs; `instructions` is rejected because Nadir classifies with its own model. Co-Authored-By: Claude Opus 5.5 --- litellm/llms/nadir/common_utils.py | 87 ++++++++++ .../auto_router_permissions.py | 2 +- .../complexity_router/complexity_router.py | 66 +++++-- .../complexity_router/config.py | 22 ++- .../complexity_router/jev_classifier.py | 21 ++- tests/unit/llms/nadir/test_common_utils.py | 93 ++++++++++ .../complexity_router/test_jev_classifier.py | 163 ++++++++++++++++++ 7 files changed, 436 insertions(+), 18 deletions(-) create mode 100644 litellm/llms/nadir/common_utils.py create mode 100644 tests/unit/llms/nadir/test_common_utils.py diff --git a/litellm/llms/nadir/common_utils.py b/litellm/llms/nadir/common_utils.py new file mode 100644 index 00000000000..4361daeae57 --- /dev/null +++ b/litellm/llms/nadir/common_utils.py @@ -0,0 +1,87 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import ConfigDict, Field, TypeAdapter + +from litellm.constants import NADIR_DEFAULT_API_BASE +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.base import LiteLLMBaseModel + +NADIR_CLASSIFIER_MODEL: Final = "bucket" +_BUCKET_PATH: Final = "/v1/bucket" +_REQUEST_SOURCE: Final = "litellm" + +NadirTier: TypeAlias = Literal["simple", "medium", "complex"] +_Probability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] + + +class _NadirBucketVerdict(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + + routing_tier: NadirTier | None = None + bucket: NadirTier | None = None + confidence: _Probability | None = None + probabilities: Mapping[str, _Probability] = Field(default_factory=lambda: MappingProxyType({})) + + +_VERDICT_ADAPTER: Final = TypeAdapter(_NadirBucketVerdict) + + +@dataclass(frozen=True, slots=True) +class NadirClassifierConnection: + api_base: str + api_key: str | None = field(repr=False) + + +def nadir_classifier_connection(api_base: str | None, api_key: str | None) -> NadirClassifierConnection: + """Resolve where the classifier calls go, with the chat provider's key scoping. + + ``NADIR_API_KEY`` is only sent to ``NADIR_API_BASE`` or the hosted API. With no key at all the + call is anonymous, which Nadir serves on a per-IP rate-limited tier. + """ + default_base: Final = get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE + trusted_base: Final = api_base is None or api_base.rstrip("/") == default_base.rstrip("/") + return NadirClassifierConnection( + api_base=(api_base or default_base).rstrip("/"), + api_key=api_key or (get_secret_str("NADIR_API_KEY") if trusted_base else None), + ) + + +def nadir_bucket_url(api_base: str) -> str: + # Nadir documents its base with the /v1 suffix for OpenAI-compatible clients. + return f"{api_base.rstrip('/').removesuffix('/v1')}{_BUCKET_PATH}" + + +def nadir_bucket_request(state: str, messages: Sequence[Mapping[str, str]] | None = None) -> Mapping[str, object]: + """Nadir's classifier reads the conversation itself, so it gets the messages rather than a judge prompt. + + Plain dicts, because httpx serializes ``json=`` with json.dumps, which rejects MappingProxyType. + """ + if messages: + return {"messages": [dict(message) for message in messages], "source": _REQUEST_SOURCE} + return {"prompt": state, "source": _REQUEST_SOURCE} + + +def nadir_classifier_response(body: Mapping[str, object], requested_model: str) -> Mapping[str, object]: + """Translate a ``/v1/bucket`` verdict into the System One ``tier`` choice answer the router reads. + + ``routing_tier`` is the tier Nadir itself routes on (the bucket after its confidence and gate + adjustments), so it wins over ``bucket``. Labels are the canonical tier names, which the router + resolves whatever display labels it uses. + """ + verdict: Final = _VERDICT_ADAPTER.validate_python(body) + tier: Final = verdict.routing_tier or verdict.bucket + if tier is None: + raise ValueError("Nadir response names neither routing_tier nor bucket") + confidence: Final = verdict.confidence if verdict.confidence is not None else verdict.probabilities.get(tier, 0.0) + answer: Final = MappingProxyType( + { + "type": "choice", + "choice": tier.upper(), + "probabilities": MappingProxyType({name.upper(): p for name, p in verdict.probabilities.items()}), + "confidence": confidence, + } + ) + return MappingProxyType({"model": requested_model, "answers": MappingProxyType({"tier": answer})}) diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index a4428eb3a85..719b99ca260 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -77,7 +77,7 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid") deployment_name: str | None = None - provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" + provider: Literal["jev", "laya", "bespoke", "databricks", "nadir"] = "jev" model: str api_key: None = None api_base: None = None diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..e7b412d2d40 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -61,6 +61,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG +from litellm.llms.nadir.common_utils import nadir_classifier_connection from litellm.router_strategy.adaptive_router.classifier import classify_prompt from litellm.router_strategy.complexity_router.context_compaction import compaction_pending from litellm.router_strategy.complexity_router.tier_predictor import ( @@ -1364,6 +1365,14 @@ class ComplexityRouter(CustomLogger): http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint), provider="databricks", ) + if config.provider == "nadir": + nadir: Final = nadir_classifier_connection(config.api_base, config.api_key) + return HttpJevClassifierClient( + api_key=nadir.api_key, + api_base=nadir.api_base, + http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint), + provider="nadir", + ) api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY") if not api_key: raise ValueError( @@ -2318,6 +2327,9 @@ class ComplexityRouter(CustomLogger): model=config.model, instructions=config.instructions or DEFAULT_JEV_INSTRUCTIONS, criteria=criteria, + messages=self._classifier_context_messages(prompt, system_prompt, request_kwargs, messages) + if config.provider == "nadir" + else None, ) try: response: Final = await asyncio.wait_for(client.evaluate(request, timeout_s, request_kwargs), timeout_s) @@ -2565,6 +2577,46 @@ class ComplexityRouter(CustomLogger): else system_prompt ) + def _classifier_prior_turns( + self, + prompt: str, + request_kwargs: Mapping[str, object] | None, + messages: Sequence[Mapping[str, object]] | None, + ) -> tuple[tuple[str, str], ...]: + if not messages or self.config.classifier_context_window_size <= 0: + return () + return _extract_prior_turns( + messages, + current_ask=prompt, + window_size=self.config.classifier_context_window_size, + budget_chars=self.config.classifier_context_budget_chars, + per_turn_chars=self.config.classifier_context_per_turn_chars, + include_assistant=self.config.classifier_context_include_assistant_turns, + marker_pairs=self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING), + ) + + def _classifier_context_messages( + self, + prompt: str, + system_prompt: str | None, + request_kwargs: Mapping[str, object] | None, + messages: Sequence[Mapping[str, object]] | None, + ) -> tuple[Mapping[str, str], ...]: + """The classifier payload's context as chat messages, for a classifier trained on raw conversations. + + Same caller constraints and same budgeted prior turns as the text payload, so the context window + settings bound what leaves the gateway exactly as they do there, minus the LLM-judge framing. + """ + caller_prompt: Final = self._classifier_caller_constraints(system_prompt, request_kwargs) + return ( + *((MappingProxyType({"role": "system", "content": caller_prompt}),) if caller_prompt else ()), + *( + MappingProxyType({"role": role, "content": text}) + for role, text in self._classifier_prior_turns(prompt, request_kwargs, messages) + ), + MappingProxyType({"role": "user", "content": prompt}), + ) + def _classifier_context_payload( self, prompt: str, @@ -2577,19 +2629,7 @@ class ComplexityRouter(CustomLogger): include_assistant: Final = self.config.classifier_context_include_assistant_turns marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING) context_enabled: Final = bool(messages) and self.config.classifier_context_window_size > 0 - prior_turns: Final = ( - _extract_prior_turns( - messages, - current_ask=prompt, - window_size=self.config.classifier_context_window_size, - budget_chars=self.config.classifier_context_budget_chars, - per_turn_chars=self.config.classifier_context_per_turn_chars, - include_assistant=include_assistant, - marker_pairs=marker_pairs, - ) - if context_enabled - else () - ) + prior_turns: Final = self._classifier_prior_turns(prompt, request_kwargs, messages) has_prior_conversation: Final = ( context_enabled and len(tuple(islice(_iter_context_turns_newest_first(messages or (), include_assistant, marker_pairs), 2))) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4a2c56daf00..c038e9e39da 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -717,7 +717,7 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True) deployment_name: str | None = Field(default=None, min_length=1) - provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" + provider: Literal["jev", "laya", "bespoke", "databricks", "nadir"] = "jev" model: str = "jev-latest" api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers") api_base: str | None = Field( @@ -737,6 +737,18 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): def _normalize_provider_alias(cls, value: object) -> object: return "jev" if value == "typesafe" else value + @model_validator(mode="before") + @classmethod + def _default_nadir_model(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + fields: Final = TypeAdapter(Mapping[str, object]).validate_python(data) + if fields.get("provider") != "nadir" or "model" in fields: + return fields + from litellm.llms.nadir.common_utils import NADIR_CLASSIFIER_MODEL + + return {**fields, "model": NADIR_CLASSIFIER_MODEL} + @field_validator("instructions") @classmethod def _reject_blank_instructions(cls, value: str | None) -> str | None: @@ -780,6 +792,14 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): f"{DATABRICKS_AI_DECIDE_MODEL!r} or a serving endpoint name, e.g. databricks-openjev-qwen35-4b" ) _ = validate_databricks_decisions_model(self.model) + if self.provider == "nadir": + if self.instructions is not None: + raise ValueError( + "opensource_classifier_config.instructions has no effect for provider 'nadir', " + "which classifies with its own model" + ) + # nadir_classifier_connection keeps NADIR_API_KEY on NADIR_API_BASE; a keyless call is anonymous. + return self if self.api_base is not None and self.api_key is None: raise ValueError( "opensource_classifier_config.api_base requires opensource_classifier_config.api_key: " diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 87bd031490a..f06e235a967 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG, is_ai_decide_model from litellm.llms.laya.common_utils import laya_response_model +from litellm.llms.nadir.common_utils import nadir_bucket_request, nadir_bucket_url, nadir_classifier_response from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS from litellm.types.decisions import DecisionsResponse @@ -31,7 +32,7 @@ if TYPE_CHECKING: from litellm.router import Router JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] -ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"] +ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks", "nadir"] DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS _WITHOUT_MODEL: Final = MappingProxyType({"model": True}) @@ -50,6 +51,8 @@ class JevSystemOneRequest(LiteLLMBaseModel): state: str model: str questions: Mapping[str, JevChoiceQuestion] + # Native input for a classifier that reads the conversation itself (nadir); never sent to a System One server. + messages: tuple[Mapping[str, str], ...] | None = Field(default=None, exclude=True) class JevChoiceAnswer(LiteLLMBaseModel): @@ -182,7 +185,7 @@ class HttpJevClassifierClient: ) response: Final = await self._http_client.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler has a dynamic post signature self._request_url(request.model), - json=request.model_dump(mode="json", exclude=self._excluded_fields(request.model)), + json=dict(self._request_body(request)), headers=MappingProxyType({**authorization, "Content-Type": "application/json"}), # pyright: ignore[reportArgumentType] # HTTP headers are not mutated by AsyncHTTPHandler timeout=timeout_s, ) @@ -195,12 +198,19 @@ class HttpJevClassifierClient: verbose_router_logger.warning("JEV response logging failed (%s)", type(exc).__name__) return TypeAdapter(JevSystemOneResponse).validate_python(normalized_body) + def _request_body(self, request: JevSystemOneRequest) -> Mapping[str, object]: + if self._provider == "nadir": + return nadir_bucket_request(request.state, request.messages) + return request.model_dump(mode="json", exclude=self._excluded_fields(request.model)) + def _excluded_fields(self, model: str) -> Mapping[str, bool] | None: return _WITHOUT_MODEL if self._provider == "databricks" and is_ai_decide_model(model) else None def _request_url(self, model: str) -> str: if self._provider == "databricks": return DATABRICKS_DECISIONS_CONFIG.get_complete_url(self._api_base, model) + if self._provider == "nadir": + return nadir_bucket_url(self._api_base) return f"{self._api_base}/v1/systemone" def _normalized_body(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]: @@ -209,6 +219,8 @@ class HttpJevClassifierClient: return MappingProxyType({**body, "model": laya_response_model(body, requested_model)}) case "databricks": return DATABRICKS_DECISIONS_CONFIG.classifier_response(body, requested_model) + case "nadir": + return nadir_classifier_response(body, requested_model) case "typesafe" | "bespoke": return body @@ -310,10 +322,13 @@ def build_jev_request( model: str, instructions: str, criteria: Mapping[str, str], + messages: tuple[Mapping[str, str], ...] | None = None, ) -> JevSystemOneRequest: state: Final = prompt if system_prompt is None else f"System prompt:\n{system_prompt}\n\nRequest:\n{prompt}" question: Final = JevChoiceQuestion(instructions=instructions, criteria=criteria) - return JevSystemOneRequest(state=state, model=model, questions=MappingProxyType({"tier": question})) + return JevSystemOneRequest( + state=state, model=model, questions=MappingProxyType({"tier": question}), messages=messages + ) def jev_classifier_cost( diff --git a/tests/unit/llms/nadir/test_common_utils.py b/tests/unit/llms/nadir/test_common_utils.py new file mode 100644 index 00000000000..dc8326838a3 --- /dev/null +++ b/tests/unit/llms/nadir/test_common_utils.py @@ -0,0 +1,93 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.llms.nadir.common_utils import ( + nadir_bucket_request, + nadir_bucket_url, + nadir_classifier_connection, + nadir_classifier_response, +) + + +def test_routing_tier_wins_over_bucket_and_labels_are_canonical() -> None: + translated: Final = nadir_classifier_response( + { + "bucket": "medium", + "routing_tier": "complex", + "confidence": 0.72, + "probabilities": {"simple": 0.1, "medium": 0.6, "complex": 0.3}, + "selected_model": None, + "decode": "complex_gate", + }, + "bucket", + ) + assert translated["model"] == "bucket" + assert dict(translated["answers"]["tier"]) == { + "type": "choice", + "choice": "COMPLEX", + "probabilities": {"SIMPLE": 0.1, "MEDIUM": 0.6, "COMPLEX": 0.3}, + "confidence": 0.72, + } + + +def test_bucket_is_used_without_routing_tier_and_confidence_falls_back_to_its_probability() -> None: + answer: Final = nadir_classifier_response( + {"bucket": "simple", "confidence": None, "probabilities": {"simple": 0.8, "medium": 0.2}}, "bucket" + )["answers"]["tier"] + assert (answer["choice"], answer["confidence"]) == ("SIMPLE", 0.8) + assert nadir_classifier_response({"routing_tier": "medium"}, "bucket")["answers"]["tier"]["confidence"] == 0.0 + + +@pytest.mark.parametrize( + "body", + [ + {}, + {"routing_tier": "reasoning"}, + {"routing_tier": "simple", "confidence": 1.5}, + {"routing_tier": "simple", "probabilities": {"simple": -0.1}}, + ], +) +def test_a_verdict_the_router_cannot_use_is_rejected(body: dict[str, object]) -> None: + with pytest.raises((ValueError, ValidationError)): + _ = nadir_classifier_response(body, "bucket") + + +@pytest.mark.parametrize( + "base", + ["https://api.getnadir.com/v1", "https://api.getnadir.com/v1/", "https://api.getnadir.com", "https://n.test/"], +) +def test_bucket_url_accepts_the_documented_base_with_or_without_v1(base: str) -> None: + assert nadir_bucket_url(base).endswith("/v1/bucket") + assert "/v1/v1/" not in nadir_bucket_url(base) + + +def test_bucket_request_prefers_the_conversation_and_falls_back_to_the_state() -> None: + messages: Final = ({"role": "system", "content": "Be brief"}, {"role": "user", "content": "hi"}) + assert dict(nadir_bucket_request("judge prompt", messages)) == {"messages": list(messages), "source": "litellm"} + assert dict(nadir_bucket_request("hi")) == {"prompt": "hi", "source": "litellm"} + + +def test_environment_key_only_goes_to_the_environment_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.setenv("NADIR_API_KEY", "ndr-env") + + hosted: Final = nadir_classifier_connection(None, None) + assert (hosted.api_base, hosted.api_key) == ("https://api.getnadir.com/v1", "ndr-env") + assert nadir_classifier_connection("https://api.getnadir.com/v1/", None).api_key == "ndr-env" + + elsewhere: Final = nadir_classifier_connection("https://collector.invalid", None) + assert (elsewhere.api_base, elsewhere.api_key) == ("https://collector.invalid", None) + assert nadir_classifier_connection("https://nadir.internal", "ndr-own").api_key == "ndr-own" + assert "ndr-env" not in repr(hosted) + + monkeypatch.setenv("NADIR_API_BASE", "https://nadir.internal/v1") + assert nadir_classifier_connection(None, None).api_base == "https://nadir.internal/v1" + assert nadir_classifier_connection("https://api.getnadir.com/v1", None).api_key is None + + +def test_no_key_anywhere_is_an_anonymous_call(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + assert nadir_classifier_connection(None, None).api_key is None diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 10a8f7929d9..1fc35031d78 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -459,6 +459,7 @@ def test_jev_config_requires_classifier_config() -> None: ("bespoke", "nimble-latest", "bespoke"), ("databricks", "databricks-openjev-qwen35-4b", "databricks"), ("databricks", "ai_decide", "databricks"), + ("nadir", "bucket", "nadir"), ], ) def test_classifier_aliases_load_and_serialize_one_canonical_config( @@ -743,6 +744,168 @@ def test_databricks_api_base_without_its_own_key_is_rejected_so_the_workspace_to assert (paired.api_base, paired.api_key) == ("https://workspace.invalid", "dapi-own") +_NADIR_VERDICT: Final = { + "bucket": "medium", + "routing_tier": "simple", + "confidence": 0.81, + "probabilities": {"simple": 0.81, "medium": 0.15, "complex": 0.04}, + "selected_model": None, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("connection", "url", "authorization"), + [ + ({}, "https://api.getnadir.com/v1/bucket", "Bearer ndr-env"), + ({"api_key": "ndr-own"}, "https://api.getnadir.com/v1/bucket", "Bearer ndr-own"), + ( + {"api_base": "https://nadir.internal/v1", "api_key": "ndr-own"}, + "https://nadir.internal/v1/bucket", + "Bearer ndr-own", + ), + ({"api_base": "https://nadir.internal"}, "https://nadir.internal/v1/bucket", None), + ], +) +async def test_nadir_routes_on_the_bucket_verdict_and_keeps_the_environment_key_home( + monkeypatch: pytest.MonkeyPatch, connection: Mapping[str, str], url: str, authorization: str | None +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key") + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.setenv("NADIR_API_KEY", "ndr-env") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _UsageRecorder("nadir/bucket") + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir", **connection}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(url).respond(200, json=_NADIR_VERDICT) + outcome: Final = await router.aclassify("What is two plus two?") + await GLOBAL_LOGGING_WORKER.flush() + + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("nadir", "bucket") + assert (outcome.jev_verdict.label, outcome.jev_verdict.confidence) == ("SIMPLE", 0.81) + assert outcome.classifier_cost is None + assert "nadir-classifier:SIMPLE" in outcome.signals + sent: Final = route.calls.last.request + assert sent.headers.get("authorization") == authorization + assert json.loads(sent.content) == { + "messages": [{"role": "user", "content": "What is two plus two?"}], + "source": "litellm", + } + assert len(recorder.calls) == 1 + assert recorder.calls[0]["response_cost"] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + [ + httpx.Response(200, json={"bucket": None, "routing_tier": None}), + httpx.Response(200, json={**_NADIR_VERDICT, "routing_tier": "reasoning"}), + httpx.Response(403, json={"detail": "anonymous_disabled"}), + ], +) +async def test_nadir_verdict_the_router_cannot_use_falls_back( + monkeypatch: pytest.MonkeyPatch, response: httpx.Response +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir"}, + "tiers": {"SIMPLE": "cheap", "COMPLEX": "strong"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as upstream: + _ = upstream.post("https://api.getnadir.com/v1/bucket").mock(return_value=response) + outcome: Final = await router.aclassify("What is two plus two?") + + assert outcome.cause == "heuristic_scorer" + assert outcome.jev_verdict is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("window", "expected_prior"), + [(None, [{"role": "user", "content": "Refactor the tokenizer into its own module"}]), (0, [])], +) +async def test_nadir_gets_the_budgeted_conversation_without_the_judge_framing( + monkeypatch: pytest.MonkeyPatch, window: int | None, expected_prior: list[dict[str, str]] +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir"}, + "tiers": {"SIMPLE": "cheap", "COMPLEX": "strong"}, + **({"classifier_context_window_size": window} if window is not None else {}), + }, + derive_savings_baseline=False, + ) + conversation: Final = [ + {"role": "system", "content": "You are a senior Python reviewer"}, + {"role": "user", "content": "Refactor the tokenizer into its own module"}, + {"role": "assistant", "content": "Done, moved it to tokenizer.py"}, + {"role": "user", "content": "now add tests"}, + ] + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://api.getnadir.com/v1/bucket").respond(200, json=_NADIR_VERDICT) + _ = await router.aclassify( + "now add tests", system_prompt="You are a senior Python reviewer", messages=conversation + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"] == [ + {"role": "system", "content": "You are a senior Python reviewer"}, + *expected_prior, + {"role": "user", "content": "now add tests"}, + ] + assert "Classify this message" not in route.calls.last.request.content.decode() + + +def test_nadir_messages_never_reach_a_system_one_server() -> None: + request: Final = build_jev_request( + "Classify this message:\nhi", + None, + "jev-latest", + "Pick a tier", + {"SIMPLE": "easy"}, + messages=({"role": "user", "content": "hi"},), + ) + assert "messages" not in request.model_dump(mode="json") + + +def test_nadir_defaults_its_model_and_allows_a_keyless_base() -> None: + assert JevClassifierConfig.model_validate({"provider": "nadir"}).model == "bucket" + keyless: Final = JevClassifierConfig.model_validate({"provider": "nadir", "api_base": "https://nadir.internal"}) + assert (keyless.api_base, keyless.api_key) == ("https://nadir.internal", None) + + +def test_nadir_rejects_instructions_it_would_ignore() -> None: + with pytest.raises(ValueError, match="instructions has no effect for provider 'nadir'"): + JevClassifierConfig.model_validate({"provider": "nadir", "instructions": "Pick a tier"}) + + @pytest.mark.parametrize( ("probabilities", "confidence"), [ From 78c9a8bc317fd78d648f1076d052e185bac0ff57 Mon Sep 17 00:00:00 2001 From: dor Date: Sun, 18 Oct 2026 14:57:39 -0400 Subject: [PATCH 2/4] feat(ui): offer Nadir as an auto router decision model Add Auto Router -> Decisions Model now lists Nadir beside Jev, Laya, Bespoke Nimble and Databricks. Picking it starts the classifier on the `bucket` label, hides the Classifier Model field (the label only names log rows) and the Classifier Instructions field, and drops any instructions carried over from another provider on save, since the gateway rejects instructions for Nadir. The editor tells the admin to set NADIR_API_KEY on the gateway, or NADIR_API_BASE for a self-hosted Nadir. schema.d.ts gains "nadir" in the opensource_classifier_config provider enum, matching the backend Literal (the one line `npm run gen:api` changes for this diff). Co-Authored-By: Claude Opus 5.5 --- .../add_model/AutoRouterClassifierTabs.tsx | 2 + .../add_model/ClassifierTypeRadios.tsx | 4 +- .../JevClassifierConfig.integration.test.tsx | 22 ++++ .../add_model/JevClassifierConfig.tsx | 110 ++++++++++-------- ...d_auto_router_routing_test_request.test.ts | 2 + .../build_complexity_router_config.test.ts | 6 +- .../add_model/jev_classifier_config.test.ts | 19 ++- .../add_model/jev_classifier_config.ts | 18 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 9 files changed, 126 insertions(+), 59 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 509329bb948..0b32995f707 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -233,6 +233,7 @@ const AutoRouterClassifierTabs: React.FC = ({ { value: "laya", label: "Laya" }, { value: "bespoke", label: "Bespoke Nimble" }, { value: "databricks", label: "Databricks" }, + { value: "nadir", label: "Nadir" }, ]} value={normalizeJevClassifierConfig(value.jev_classifier_config).provider} onValueChange={changeProvider} @@ -245,6 +246,7 @@ const AutoRouterClassifierTabs: React.FC = ({ Laya Bespoke Nimble Databricks + Nadir diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx index 6ed46e2a27d..ce98b25f36f 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx @@ -55,7 +55,9 @@ const ClassifierTypeRadios: React.FC = ({ value, clas OSS Classifier{" "} - uses Jev, Laya, Bespoke, or Databricks to decide the tier + + uses Jev, Laya, Bespoke, Databricks, or Nadir to decide the tier + diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index f829a2f2e41..32f56128987 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -164,6 +164,28 @@ describe("JEV classifier editor", () => { }, ); + it("hides the model and instructions for Nadir and saves it without instructions", async () => { + renderWithProviders(
); + fireEvent.click(screen.getByRole("radio", { name: /^Decisions Model$/ })); + fireEvent.change(screen.getByLabelText("Classifier Instructions"), { target: { value: "Pick carefully" } }); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Nadir"); + expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); + expect(screen.queryByLabelText("Classifier Instructions")).not.toBeInTheDocument(); + expect(screen.getByText(/Set NADIR_API_KEY on the gateway/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); + expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveTextContent("Nadir"); + fireEvent.click(screen.getByRole("button", { name: "Probe current config" })); + expect(testAutoRouterRouting).toHaveBeenLastCalledWith( + "token", + expect.objectContaining({ + complexity_router_config: expect.objectContaining({ + classifier_type: "oss_classifier", + opensource_classifier_config: { provider: "nadir", model: "bucket", timeout_ms: 3000 }, + }), + }), + ); + }); + it("allows licensed instructions and can restore built-in instructions", () => { const authorized = useAuthorized(); vi.mocked(useAuthorized).mockReturnValue({ ...authorized, premiumUser: true }); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 4b3cfa21f9d..f09384ca2b8 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -7,7 +7,11 @@ import { Textarea } from "@/components/ui/textarea"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { defaultJevClassifierConfig, fixedClassifierModels } from "./jev_classifier_config"; +import { + acceptsClassifierInstructions, + defaultJevClassifierConfig, + fixedClassifierModels, +} from "./jev_classifier_config"; const providerDescriptions = { jev: "Uses TypeSafe System One Choice evaluation with your configured tiers", @@ -16,6 +20,8 @@ const providerDescriptions = { "Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.", databricks: "Uses the Databricks ai_decide AI Function, or a decision serving endpoint you name, with your configured tiers. Set DATABRICKS_API_BASE to https:///serving-endpoints and DATABRICKS_API_KEY on the gateway.", + nadir: + "Uses Nadir's tier decision with your configured tiers. Set NADIR_API_KEY on the gateway; without it, calls use Nadir's anonymous, rate-limited tier. Set NADIR_API_BASE for a self-hosted Nadir.", }; export default function JevClassifierConfig({ @@ -38,34 +44,36 @@ export default function JevClassifierConfig({ {section !== "advanced" && !config.deployment_name && ( <>

{providerDescriptions[config.provider ?? "jev"]}

-
- - {models ? ( - - ) : ( - update({ model: event.target.value })} - /> - )} -
+ {config.provider !== "nadir" && ( +
+ + {models ? ( + + ) : ( + update({ model: event.target.value })} + /> + )} +
+ )} )} {section !== "model" && ( @@ -90,27 +98,29 @@ export default function JevClassifierConfig({ }) } /> -
- - -