diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..34feca88c58 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1730,6 +1730,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.nadir.decisions.transformation import ( + NadirDecisionsConfig as NadirDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e0cb7c680bc..c93985bb5b3 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -172,6 +172,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "NadirDecisionsConfig", "OpenAIDecisionsConfig", "AzureAIDecisionsConfig", "NvidiaNimRerankConfig", @@ -748,6 +749,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "NadirDecisionsConfig": (".llms.nadir.decisions.transformation", "NadirDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), "NvidiaNimRerankConfig": ( diff --git a/litellm/llms/nadir/decisions/transformation.py b/litellm/llms/nadir/decisions/transformation.py new file mode 100644 index 00000000000..3c485069fec --- /dev/null +++ b/litellm/llms/nadir/decisions/transformation.py @@ -0,0 +1,205 @@ +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError, model_validator +from pydantic_core import to_jsonable_python +from typing_extensions import assert_never + +from litellm.constants import NADIR_DEFAULT_API_BASE +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + parse_systemone_response, + systemone_keys, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import ( + DecisionsIRChoiceQuestion, + DecisionsIRMessages, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRState, + DecisionsJSON, + OpenAIDecisionContentPart, + OpenAIDecisionInputText, + UnsupportedDecisionsRequest, +) +from litellm.types.llms.base import LiteLLMBaseModel + +NADIR_CLASSIFIER_MODEL: Final = "bucket" +_REQUEST_SOURCE: Final = "litellm" + +NadirTier: TypeAlias = Literal["simple", "medium", "complex"] +_TIERS: Final[tuple[NadirTier, ...]] = ("simple", "medium", "complex") +_CANONICAL_CHOICES: Final[Mapping[NadirTier, str]] = MappingProxyType({tier: tier.upper() for tier in _TIERS}) +_Probability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] +_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_UNSUPPORTED_QUESTIONS: Final = UnsupportedDecisionsRequest( + reason="Nadir answers exactly one choice question whose choices include simple, medium and complex" +) + + +class _NadirBucketVerdict(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, allow_inf_nan=False) + + routing_tier: NadirTier + confidence: _Probability | None = None + probabilities: Mapping[str, _Probability] = Field(default_factory=lambda: MappingProxyType({})) + + @model_validator(mode="before") + @classmethod + def _routing_tier_falls_back_to_bucket(cls, data: object) -> object: + """``routing_tier`` is the tier Nadir itself routes on (the bucket after its confidence and gate + adjustments), so it wins; an older server that sends only ``bucket`` still answers.""" + try: + fields: Final = _MAPPING_ADAPTER.validate_python(data) + except ValidationError: + return data + return {**fields, "routing_tier": fields.get("routing_tier") or fields.get("bucket")} + + +_VERDICT_ADAPTER: Final = TypeAdapter(_NadirBucketVerdict) + + +class _ConversationMessage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + role: Literal["system", "user", "assistant"] + content: str + + +class _ConversationState(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + messages: tuple[_ConversationMessage, ...] = Field(min_length=1) + + +_CONVERSATION_ADAPTER: Final = TypeAdapter(_ConversationState) + + +def _part_text(content: str | Sequence[OpenAIDecisionContentPart]) -> str: + if isinstance(content, str): + return content + return "\n".join(part.text for part in content if isinstance(part, OpenAIDecisionInputText)) + + +def _state_body(state: DecisionsJSON) -> Mapping[str, object]: + if isinstance(state, str): + return {"prompt": state} + try: + conversation: Final = _CONVERSATION_ADAPTER.validate_python(state) + except ValidationError: + return {"prompt": json.dumps(to_jsonable_python(state))} + return {"messages": [message.model_dump() for message in conversation.messages]} + + +def _input_body(decision_input: DecisionsIRState | DecisionsIRMessages) -> Mapping[str, object]: + match decision_input: + case DecisionsIRMessages(messages=messages): + return { + "messages": [{"role": message.role, "content": _part_text(message.content)} for message in messages] + } + case DecisionsIRState(state=state): + return _state_body(state) + case _: + assert_never(decision_input) + + +def nadir_bucket_body(decision_input: DecisionsIRState | DecisionsIRMessages) -> Mapping[str, object]: + """The ``/v1/bucket`` body for a decision input. + + Nadir's classifier is trained on raw conversations, so a state shaped ``{"messages": [{role, content}]}`` + goes out as those messages, as do OpenAI-format input messages; any other state is the prompt. Plain + dicts and lists, because httpx serializes ``json=`` with json.dumps, which rejects MappingProxyType. + """ + return {**_input_body(decision_input), "source": _REQUEST_SOURCE} + + +def _tier_choices(request: DecisionsIRRequest) -> Mapping[NadirTier, str] | None: + """The question's own choice key for each Nadir tier, matched case-insensitively, when the request is + one choice question naming all three tiers.""" + match request.questions: + case (DecisionsIRChoiceQuestion(choices=choices),): + by_name: Final = {option.value.lower(): option.value for option in choices if isinstance(option.value, str)} + by_tier: Final[Mapping[NadirTier, str]] = {tier: by_name[tier] for tier in _TIERS if tier in by_name} + return MappingProxyType(by_tier) if len(by_tier) == len(_TIERS) else None + case _: + return None + + +def _choice_answer(verdict: _NadirBucketVerdict, choices: Mapping[NadirTier, str]) -> Mapping[str, object]: + tier: Final = verdict.routing_tier + return { + "type": "choice", + "choice": choices[tier], + "probabilities": {choices[name]: verdict.probabilities.get(name, 0.0) for name in _TIERS}, + "confidence": verdict.confidence if verdict.confidence is not None else verdict.probabilities.get(tier, 0.0), + } + + +class NadirDecisionsConfig(BaseDecisionsConfig): + """Nadir as a Decisions provider: ``/v1/bucket`` answers a single simple/medium/complex choice question. + + Nadir is a tier classifier, not a general judge, so any other question set is rejected before a call. + ``NADIR_API_KEY`` is only sent to ``NADIR_API_BASE`` or the hosted API, the same scoping as the ``nadir`` + chat provider; with no key the call is anonymous, which Nadir serves on a per-IP rate-limited tier. + """ + + path = "/v1/bucket" + api_key_env = ("NADIR_API_KEY",) + api_base_env = ("NADIR_API_BASE",) + api_key_required = False + health_check_questions = MappingProxyType( + { + "tier": MappingProxyType( + { + "type": "choice", + "instructions": "Which tier should serve this request?", + "criteria": {tier: None for tier in _TIERS}, + } + ) + } + ) + + def get_default_api_base(self) -> str: + return NADIR_DEFAULT_API_BASE + + def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + default_base: Final = get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE + trusted_base: Final = api_base is None or api_base.rstrip("/") == default_base.rstrip("/") + return ( + (api_base or default_base).rstrip("/"), + api_key or (get_secret_str("NADIR_API_KEY") if trusted_base else None), + ) + + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if _tier_choices(request) is None: + return _UNSUPPORTED_QUESTIONS + return nadir_bucket_body(request.input) + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + verdict: Final = _VERDICT_ADAPTER.validate_python(payload) + choices: Final = _tier_choices(request) + answers: Final = ( + {} + if choices is None + else {key: _choice_answer(verdict, choices) for key in systemone_keys(request.questions)} + ) + return parse_systemone_response({"answers": answers}, request) + + def classifier_response(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]: + """The verdict as the System One ``tier`` answer the auto router reads, on canonical tier labels the + router resolves whatever display labels it uses.""" + verdict: Final = _VERDICT_ADAPTER.validate_python(body) + return MappingProxyType( + {"model": requested_model, "answers": {"tier": _choice_answer(verdict, _CANONICAL_CHOICES)}} + ) + + +NADIR_DECISIONS_CONFIG: Final[NadirDecisionsConfig] = NadirDecisionsConfig() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb9bfa6ffe4..ca4165d726e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72192,6 +72192,16 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "nadir/bucket": { + "input_cost_per_token": 0.0, + "litellm_provider": "nadir", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://getnadir.com/docs/decision-api/", + "metadata": { + "notes": "Nadir's simple, medium or complex tier decision from POST /v1/bucket, which Nadir does not bill" + } + }, "strands_decider/strands-decider-2B-hobson-v19": { "input_cost_per_token": 0.0, "litellm_provider": "strands_decider", diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index a4428eb3a85..719b99ca260 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -77,7 +77,7 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid") deployment_name: str | None = None - provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" + provider: Literal["jev", "laya", "bespoke", "databricks", "nadir"] = "jev" model: str api_key: None = None api_base: None = None diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..c2eb692bd34 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -61,6 +61,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG +from litellm.llms.nadir.decisions.transformation import NADIR_DECISIONS_CONFIG from litellm.router_strategy.adaptive_router.classifier import classify_prompt from litellm.router_strategy.complexity_router.context_compaction import compaction_pending from litellm.router_strategy.complexity_router.tier_predictor import ( @@ -887,6 +888,22 @@ def _truncate(text: str, limit: int) -> str: return f"{text[:head_chars]}{_TRUNCATION_MARKER}{text[len(text) - tail_chars :]}" +_DEPLOYMENT_PARAMS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_NO_REQUEST_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _is_nadir_deployment(deployment: Mapping[str, object]) -> bool: + try: + params: Final = _DEPLOYMENT_PARAMS.validate_python(deployment.get("litellm_params")) + except ValidationError: + return False + provider: Final = params.get("custom_llm_provider") + if isinstance(provider, str) and provider: + return provider == "nadir" + model: Final = params.get("model") + return isinstance(model, str) and model.split("/", 1)[0] == "nadir" + + def _iter_context_turns_newest_first( messages: Sequence[Mapping[str, object]], include_assistant: bool, @@ -1364,6 +1381,14 @@ class ComplexityRouter(CustomLogger): http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint), provider="databricks", ) + if config.provider == "nadir": + nadir_base, nadir_key = NADIR_DECISIONS_CONFIG.resolve_connection(config.api_base, config.api_key) + return HttpJevClassifierClient( + api_key=nadir_key, + api_base=nadir_base, + http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint), + provider="nadir", + ) api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY") if not api_key: raise ValueError( @@ -2318,6 +2343,9 @@ class ComplexityRouter(CustomLogger): model=config.model, instructions=config.instructions or DEFAULT_JEV_INSTRUCTIONS, criteria=criteria, + messages=self._classifier_context_messages(prompt, system_prompt, request_kwargs, messages) + if self._classifier_reads_conversation(config) + else None, ) try: response: Final = await asyncio.wait_for(client.evaluate(request, timeout_s, request_kwargs), timeout_s) @@ -2565,6 +2593,54 @@ class ComplexityRouter(CustomLogger): else system_prompt ) + def _classifier_reads_conversation(self, config: OpenSourceClassifierConfig) -> bool: + """Nadir classifies the conversation itself, whether it is named as the provider or serves the + decision deployment, so it gets role-tagged messages in place of the LLM-judge payload.""" + if config.deployment_name is None: + return config.provider == "nadir" + deployments: Final = self._group_deployments(config.deployment_name) + return bool(deployments) and all(_is_nadir_deployment(deployment) for deployment in deployments) + + def _classifier_prior_turns( + self, + prompt: str, + request_kwargs: Mapping[str, object] | None, + messages: Sequence[Mapping[str, object]] | None, + ) -> tuple[tuple[str, str], ...]: + if not messages or self.config.classifier_context_window_size <= 0: + return () + return _extract_prior_turns( + messages, + current_ask=prompt, + window_size=self.config.classifier_context_window_size, + budget_chars=self.config.classifier_context_budget_chars, + per_turn_chars=self.config.classifier_context_per_turn_chars, + include_assistant=self.config.classifier_context_include_assistant_turns, + marker_pairs=self._reminder_markers_for_request(request_kwargs or _NO_REQUEST_KWARGS), + ) + + def _classifier_context_messages( + self, + prompt: str, + system_prompt: str | None, + request_kwargs: Mapping[str, object] | None, + messages: Sequence[Mapping[str, object]] | None, + ) -> tuple[Mapping[str, str], ...]: + """The classifier payload's context as chat messages, for a classifier trained on raw conversations. + + Same caller constraints and same budgeted prior turns as the text payload, so the context window + settings bound what leaves the gateway exactly as they do there, minus the LLM-judge framing. + """ + caller_prompt: Final = self._classifier_caller_constraints(system_prompt, request_kwargs) + return ( + *((MappingProxyType({"role": "system", "content": caller_prompt}),) if caller_prompt else ()), + *( + MappingProxyType({"role": role, "content": text}) + for role, text in self._classifier_prior_turns(prompt, request_kwargs, messages) + ), + MappingProxyType({"role": "user", "content": prompt}), + ) + def _classifier_context_payload( self, prompt: str, @@ -2577,19 +2653,7 @@ class ComplexityRouter(CustomLogger): include_assistant: Final = self.config.classifier_context_include_assistant_turns marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING) context_enabled: Final = bool(messages) and self.config.classifier_context_window_size > 0 - prior_turns: Final = ( - _extract_prior_turns( - messages, - current_ask=prompt, - window_size=self.config.classifier_context_window_size, - budget_chars=self.config.classifier_context_budget_chars, - per_turn_chars=self.config.classifier_context_per_turn_chars, - include_assistant=include_assistant, - marker_pairs=marker_pairs, - ) - if context_enabled - else () - ) + prior_turns: Final = self._classifier_prior_turns(prompt, request_kwargs, messages) has_prior_conversation: Final = ( context_enabled and len(tuple(islice(_iter_context_turns_newest_first(messages or (), include_assistant, marker_pairs), 2))) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4a2c56daf00..0a2694c4365 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -717,7 +717,7 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True) deployment_name: str | None = Field(default=None, min_length=1) - provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev" + provider: Literal["jev", "laya", "bespoke", "databricks", "nadir"] = "jev" model: str = "jev-latest" api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers") api_base: str | None = Field( @@ -737,6 +737,18 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): def _normalize_provider_alias(cls, value: object) -> object: return "jev" if value == "typesafe" else value + @model_validator(mode="before") + @classmethod + def _default_nadir_model(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + fields: Final = TypeAdapter(Mapping[str, object]).validate_python(data) + if fields.get("provider") != "nadir" or "model" in fields: + return fields + from litellm.llms.nadir.decisions.transformation import NADIR_CLASSIFIER_MODEL + + return {**fields, "model": NADIR_CLASSIFIER_MODEL} + @field_validator("instructions") @classmethod def _reject_blank_instructions(cls, value: str | None) -> str | None: @@ -780,6 +792,14 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): f"{DATABRICKS_AI_DECIDE_MODEL!r} or a serving endpoint name, e.g. databricks-openjev-qwen35-4b" ) _ = validate_databricks_decisions_model(self.model) + if self.provider == "nadir": + if self.instructions is not None: + raise ValueError( + "opensource_classifier_config.instructions has no effect for provider 'nadir', " + "which classifies with its own model" + ) + # NADIR_DECISIONS_CONFIG keeps NADIR_API_KEY on NADIR_API_BASE; a keyless call is anonymous. + return self if self.api_base is not None and self.api_key is None: raise ValueError( "opensource_classifier_config.api_base requires opensource_classifier_config.api_key: " diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 87bd031490a..0f6448f2cf5 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -21,9 +21,10 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG, is_ai_decide_model from litellm.llms.laya.common_utils import laya_response_model +from litellm.llms.nadir.decisions.transformation import NADIR_DECISIONS_CONFIG, nadir_bucket_body from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS -from litellm.types.decisions import DecisionsResponse +from litellm.types.decisions import DecisionsIRState, DecisionsJSON, DecisionsResponse from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN @@ -31,7 +32,7 @@ if TYPE_CHECKING: from litellm.router import Router JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] -ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"] +ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks", "nadir"] DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS _WITHOUT_MODEL: Final = MappingProxyType({"model": True}) @@ -50,6 +51,12 @@ class JevSystemOneRequest(LiteLLMBaseModel): state: str model: str questions: Mapping[str, JevChoiceQuestion] + # The conversation, for a classifier that reads it itself (nadir); excluded from the System One body. + messages: tuple[Mapping[str, str], ...] | None = Field(default=None, exclude=True) + + def decision_state(self) -> DecisionsJSON: + """The state a decision call carries: the conversation when the classifier reads it, else the payload.""" + return self.state if self.messages is None else {"messages": [dict(message) for message in self.messages]} class JevChoiceAnswer(LiteLLMBaseModel): @@ -92,7 +99,7 @@ class _DecisionsRouter(Protocol): self, *, model: str, - state: str, + state: DecisionsJSON, questions: Mapping[str, object], timeout: float, num_retries: int, @@ -127,14 +134,15 @@ class RouterJevClassifierClient: ), INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN, } + state: Final = request.decision_state() payload: Final = { - "state": request.state, + "state": state, "questions": {key: value.model_dump() for key, value in request.questions.items()}, } response: Final = DecisionsResponse.model_validate( await self._router.adecisions( model=self._deployment_name, - state=request.state, + state=state, questions={key: value.model_dump() for key, value in request.questions.items()}, timeout=timeout_s, num_retries=0, @@ -182,7 +190,7 @@ class HttpJevClassifierClient: ) response: Final = await self._http_client.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler has a dynamic post signature self._request_url(request.model), - json=request.model_dump(mode="json", exclude=self._excluded_fields(request.model)), + json=dict(self._request_body(request)), headers=MappingProxyType({**authorization, "Content-Type": "application/json"}), # pyright: ignore[reportArgumentType] # HTTP headers are not mutated by AsyncHTTPHandler timeout=timeout_s, ) @@ -195,12 +203,19 @@ class HttpJevClassifierClient: verbose_router_logger.warning("JEV response logging failed (%s)", type(exc).__name__) return TypeAdapter(JevSystemOneResponse).validate_python(normalized_body) + def _request_body(self, request: JevSystemOneRequest) -> Mapping[str, object]: + if self._provider == "nadir": + return nadir_bucket_body(DecisionsIRState(state=request.decision_state())) + return request.model_dump(mode="json", exclude=self._excluded_fields(request.model)) + def _excluded_fields(self, model: str) -> Mapping[str, bool] | None: return _WITHOUT_MODEL if self._provider == "databricks" and is_ai_decide_model(model) else None def _request_url(self, model: str) -> str: if self._provider == "databricks": return DATABRICKS_DECISIONS_CONFIG.get_complete_url(self._api_base, model) + if self._provider == "nadir": + return NADIR_DECISIONS_CONFIG.get_complete_url(self._api_base, model) return f"{self._api_base}/v1/systemone" def _normalized_body(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]: @@ -209,6 +224,8 @@ class HttpJevClassifierClient: return MappingProxyType({**body, "model": laya_response_model(body, requested_model)}) case "databricks": return DATABRICKS_DECISIONS_CONFIG.classifier_response(body, requested_model) + case "nadir": + return NADIR_DECISIONS_CONFIG.classifier_response(body, requested_model) case "typesafe" | "bespoke": return body @@ -310,10 +327,13 @@ def build_jev_request( model: str, instructions: str, criteria: Mapping[str, str], + messages: tuple[Mapping[str, str], ...] | None = None, ) -> JevSystemOneRequest: state: Final = prompt if system_prompt is None else f"System prompt:\n{system_prompt}\n\nRequest:\n{prompt}" question: Final = JevChoiceQuestion(instructions=instructions, criteria=criteria) - return JevSystemOneRequest(state=state, model=model, questions=MappingProxyType({"tier": question})) + return JevSystemOneRequest( + state=state, model=model, questions=MappingProxyType({"tier": question}), messages=messages + ) def jev_classifier_cost( diff --git a/litellm/utils.py b/litellm/utils.py index 7de7ce6188d..90dd8bd3b1e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9638,6 +9638,8 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() + if provider == LlmProviders.NADIR: + return litellm.NadirDecisionsConfig() if provider == LlmProviders.OPENAI: return litellm.OpenAIDecisionsConfig() if provider == LlmProviders.AZURE_AI: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb9bfa6ffe4..ca4165d726e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72192,6 +72192,16 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "nadir/bucket": { + "input_cost_per_token": 0.0, + "litellm_provider": "nadir", + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://getnadir.com/docs/decision-api/", + "metadata": { + "notes": "Nadir's simple, medium or complex tier decision from POST /v1/bucket, which Nadir does not bill" + } + }, "strands_decider/strands-decider-2B-hobson-v19": { "input_cost_per_token": 0.0, "litellm_provider": "strands_decider", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 84e7a970195..c2de1f5ed99 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -584,6 +584,45 @@ async def test_oss_classifier_env_key_goes_only_to_its_env_api_base( assert "authorization" not in other_upstream.calls[0].request.headers +@pytest.mark.asyncio +async def test_nadir_answers_a_tier_question_from_its_bucket_verdict_and_keeps_its_key_home( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.setenv("NADIR_API_KEY", "ndr-env") + verdict: Final = {"routing_tier": "complex", "confidence": 0.9, "probabilities": {"simple": 0.02, "complex": 0.9}} + hosted: Final = respx_mock.post("https://api.getnadir.com/v1/bucket").respond(json=verdict) + elsewhere: Final = respx_mock.post("https://collector.example/v1/bucket").respond(json=verdict) + questions: Final = {"tier": {"type": "choice", "criteria": {"Simple": None, "Medium": None, "Complex": None}}} + + response: Final = await litellm.asystemone( + model="nadir/bucket", state="Design a consensus protocol", questions=questions + ) + _ = await litellm.asystemone( + model="nadir/bucket", state="hi", questions=questions, api_base="https://collector.example/v1" + ) + + assert response.answers["tier"] == ChoiceAnswer( + type="choice", choice="Complex", confidence=0.9, probabilities={"Simple": 0.02, "Medium": 0.0, "Complex": 0.9} + ) + assert json.loads(hosted.calls.last.request.content) == { + "prompt": "Design a consensus protocol", + "source": "litellm", + } + assert hosted.calls.last.request.headers["authorization"] == "Bearer ndr-env" + assert "authorization" not in elsewhere.calls.last.request.headers + + +@pytest.mark.asyncio +async def test_nadir_refuses_a_question_it_cannot_answer_before_any_call(respx_mock: respx.MockRouter) -> None: + upstream: Final = respx_mock.post("https://api.getnadir.com/v1/bucket") + with pytest.raises(litellm.BadRequestError, match="cannot serve this request"): + _ = await litellm.asystemone( + model="nadir/bucket", state="review", questions={"is_defect": {"type": "noul", "instructions": "Defect?"}} + ) + assert not upstream.called + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "api_base", "error"), @@ -620,7 +659,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": ("reply", "detail"), ( pytest.param(_PPLX_USAGE, "DecisionsResponse answers: Field required", id="usage"), - pytest.param({"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text"), + pytest.param( + {"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text" + ), pytest.param({"error": "Request timed out"}, "DecisionsResponse answers: Field required", id="timeout_text"), pytest.param( {**_PPLX_USAGE, "answers": {"rate_limit": None}}, @@ -632,7 +673,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": "DecisionsResponse answers.is_defect", id="answer_type_timeout_text", ), - pytest.param(["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object"), + pytest.param( + ["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object" + ), ), ) async def test_a_malformed_upstream_reply_is_a_server_error_whatever_its_text( @@ -3729,7 +3772,12 @@ async def test_a_returned_model_registered_without_prices_is_billed_at_the_reque monkeypatch.setitem( litellm.model_cost, "typesafe/jev-test-alias", - {"litellm_provider": "typesafe", "mode": "evaluation", "input_cost_per_token": 0.001, "output_cost_per_token": 0}, + { + "litellm_provider": "typesafe", + "mode": "evaluation", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0, + }, ) monkeypatch.setitem(litellm.model_cost, "typesafe/jev-test-unpriced", unpriced_row) respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json={**_RESPONSE, "model": "jev-test-unpriced"}) diff --git a/tests/unit/llms/nadir/decisions/__init__.py b/tests/unit/llms/nadir/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/nadir/decisions/test_transformation.py b/tests/unit/llms/nadir/decisions/test_transformation.py new file mode 100644 index 00000000000..0dd8401da59 --- /dev/null +++ b/tests/unit/llms/nadir/decisions/test_transformation.py @@ -0,0 +1,185 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.llms.base_llm.decisions.transformation import systemone_request_to_ir +from litellm.llms.nadir.decisions.transformation import NADIR_DECISIONS_CONFIG, nadir_bucket_body +from litellm.types.decisions import ( + DecisionsIRChoiceAnswer, + DecisionsIRChoiceProbability, + DecisionsIRMessages, + DecisionsIRRequest, + DecisionsIRState, + DecisionsRequestBody, + OpenAIDecisionInputMessage, + UnsupportedDecisionsRequest, +) + +_TIER_QUESTION: Final = { + "tier": { + "type": "choice", + "instructions": "Which tier?", + "criteria": {"SIMPLE": "facts", "MEDIUM": "edits", "COMPLEX": "design", "REASONING": "proofs"}, + } +} +_VERDICT: Final = { + "bucket": "medium", + "routing_tier": "complex", + "confidence": 0.72, + "probabilities": {"simple": 0.1, "medium": 0.6, "complex": 0.3}, + "selected_model": None, + "decode": "complex_gate", +} + + +def _request(state: object = "What is 2+2?", questions: object = _TIER_QUESTION) -> DecisionsIRRequest: + return systemone_request_to_ir(DecisionsRequestBody.model_validate({"state": state, "questions": questions})) + + +def test_routing_tier_wins_over_bucket_and_answers_on_the_question_s_own_choice_keys() -> None: + parsed: Final = NADIR_DECISIONS_CONFIG.parse_response(_VERDICT, _request()) + assert parsed.answers == ( + DecisionsIRChoiceAnswer( + choice="COMPLEX", + confidence=0.72, + probabilities=( + DecisionsIRChoiceProbability(value="SIMPLE", probability=0.1), + DecisionsIRChoiceProbability(value="MEDIUM", probability=0.6), + DecisionsIRChoiceProbability(value="COMPLEX", probability=0.3), + ), + ), + ) + + +def test_bucket_is_used_without_routing_tier_and_confidence_falls_back_to_its_probability() -> None: + lowercase: Final = _request( + questions={"t": {"type": "choice", "criteria": {"simple": None, "medium": None, "complex": None}}} + ) + answer: Final = NADIR_DECISIONS_CONFIG.parse_response( + {"bucket": "simple", "confidence": None, "probabilities": {"simple": 0.8, "medium": 0.2}}, lowercase + ).answers[0] + assert isinstance(answer, DecisionsIRChoiceAnswer) + assert (answer.choice, answer.confidence) == ("simple", 0.8) + + +def test_the_router_s_tier_answer_uses_canonical_labels() -> None: + translated: Final = NADIR_DECISIONS_CONFIG.classifier_response(_VERDICT, "bucket") + assert translated["model"] == "bucket" + assert translated["answers"] == { + "tier": { + "type": "choice", + "choice": "COMPLEX", + "probabilities": {"SIMPLE": 0.1, "MEDIUM": 0.6, "COMPLEX": 0.3}, + "confidence": 0.72, + } + } + assert ( + NADIR_DECISIONS_CONFIG.classifier_response({"routing_tier": "medium"}, "bucket")["answers"]["tier"][ + "confidence" + ] + == 0.0 + ) + + +@pytest.mark.parametrize( + "body", + [ + {}, + {"routing_tier": "reasoning"}, + {"routing_tier": "simple", "confidence": 1.5}, + {"routing_tier": "simple", "probabilities": {"simple": -0.1}}, + ], +) +def test_a_verdict_the_router_cannot_use_is_rejected(body: dict[str, object]) -> None: + with pytest.raises(ValidationError): + _ = NADIR_DECISIONS_CONFIG.classifier_response(body, "bucket") + with pytest.raises(ValidationError): + _ = NADIR_DECISIONS_CONFIG.parse_response(body, _request()) + + +@pytest.mark.parametrize( + "questions", + [ + {"q": {"type": "noul", "instructions": "Is this hard?"}}, + {"q": {"type": "choice", "criteria": {"simple": None, "complex": None}}}, + {"q": {"type": "choice", "criteria": {"easy": None, "hard": None, "harder": None}}}, + {**_TIER_QUESTION, "second": {"type": "choice", "criteria": {"simple": None, "medium": None, "complex": None}}}, + ], +) +def test_anything_but_one_tier_choice_question_is_refused_before_a_call(questions: object) -> None: + refused: Final = NADIR_DECISIONS_CONFIG.transform_decisions_request( + "bucket", _request(questions=questions), "nadir" + ) + assert isinstance(refused, UnsupportedDecisionsRequest) + assert "simple, medium and complex" in refused.reason + + +def test_the_health_check_asks_a_question_nadir_can_answer() -> None: + request: Final = _request(questions=NADIR_DECISIONS_CONFIG.health_check_questions) + assert not isinstance( + NADIR_DECISIONS_CONFIG.transform_decisions_request("bucket", request, "nadir"), UnsupportedDecisionsRequest + ) + + +@pytest.mark.parametrize( + ("decision_input", "expected"), + [ + (DecisionsIRState(state="hi"), {"prompt": "hi"}), + ( + DecisionsIRState( + state={"messages": [{"role": "system", "content": "Be brief"}, {"role": "user", "content": "hi"}]} + ), + {"messages": [{"role": "system", "content": "Be brief"}, {"role": "user", "content": "hi"}]}, + ), + (DecisionsIRState(state={"messages": []}), {"prompt": '{"messages": []}'}), + (DecisionsIRState(state={"ticket": 7}), {"prompt": '{"ticket": 7}'}), + ( + DecisionsIRMessages( + messages=( + OpenAIDecisionInputMessage(content="first"), + OpenAIDecisionInputMessage.model_validate( + {"content": [{"type": "input_text", "text": "a"}, {"type": "input_text", "text": "b"}]} + ), + ) + ), + {"messages": [{"role": "user", "content": "first"}, {"role": "user", "content": "a\nb"}]}, + ), + ], +) +def test_the_bucket_body_carries_the_conversation_when_there_is_one( + decision_input: DecisionsIRState | DecisionsIRMessages, expected: dict[str, object] +) -> None: + assert nadir_bucket_body(decision_input) == {**expected, "source": "litellm"} + + +@pytest.mark.parametrize( + "base", + ["https://api.getnadir.com/v1", "https://api.getnadir.com/v1/", "https://api.getnadir.com", "https://n.test/"], +) +def test_bucket_url_accepts_the_documented_base_with_or_without_v1(base: str) -> None: + url: Final = NADIR_DECISIONS_CONFIG.get_complete_url(base, "bucket") + assert url.endswith("/v1/bucket") + assert "/v1/v1/" not in url + + +def test_environment_key_only_goes_to_the_environment_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.setenv("NADIR_API_KEY", "ndr-env") + connect: Final = NADIR_DECISIONS_CONFIG.resolve_connection + + assert connect(None, None) == ("https://api.getnadir.com/v1", "ndr-env") + assert connect("https://api.getnadir.com/v1/", None) == ("https://api.getnadir.com/v1", "ndr-env") + assert connect("https://collector.invalid", None) == ("https://collector.invalid", None) + assert connect("https://nadir.internal", "ndr-own") == ("https://nadir.internal", "ndr-own") + + monkeypatch.setenv("NADIR_API_BASE", "https://nadir.internal/v1") + assert connect(None, None) == ("https://nadir.internal/v1", "ndr-env") + assert connect("https://api.getnadir.com/v1", None) == ("https://api.getnadir.com/v1", None) + + +def test_no_key_anywhere_is_an_anonymous_call(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + assert NADIR_DECISIONS_CONFIG.resolve_connection(None, None) == ("https://api.getnadir.com/v1", None) + assert not NADIR_DECISIONS_CONFIG.api_key_required diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 10a8f7929d9..ea7951e0480 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -459,6 +459,7 @@ def test_jev_config_requires_classifier_config() -> None: ("bespoke", "nimble-latest", "bespoke"), ("databricks", "databricks-openjev-qwen35-4b", "databricks"), ("databricks", "ai_decide", "databricks"), + ("nadir", "bucket", "nadir"), ], ) def test_classifier_aliases_load_and_serialize_one_canonical_config( @@ -743,6 +744,213 @@ def test_databricks_api_base_without_its_own_key_is_rejected_so_the_workspace_to assert (paired.api_base, paired.api_key) == ("https://workspace.invalid", "dapi-own") +_NADIR_VERDICT: Final = { + "bucket": "medium", + "routing_tier": "simple", + "confidence": 0.81, + "probabilities": {"simple": 0.81, "medium": 0.15, "complex": 0.04}, + "selected_model": None, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("connection", "url", "authorization"), + [ + ({}, "https://api.getnadir.com/v1/bucket", "Bearer ndr-env"), + ({"api_key": "ndr-own"}, "https://api.getnadir.com/v1/bucket", "Bearer ndr-own"), + ( + {"api_base": "https://nadir.internal/v1", "api_key": "ndr-own"}, + "https://nadir.internal/v1/bucket", + "Bearer ndr-own", + ), + ({"api_base": "https://nadir.internal"}, "https://nadir.internal/v1/bucket", None), + ], +) +async def test_nadir_routes_on_the_bucket_verdict_and_keeps_the_environment_key_home( + monkeypatch: pytest.MonkeyPatch, connection: Mapping[str, str], url: str, authorization: str | None +) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key") + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.setenv("NADIR_API_KEY", "ndr-env") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _UsageRecorder("nadir/bucket") + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir", **connection}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(url).respond(200, json=_NADIR_VERDICT) + outcome: Final = await router.aclassify("What is two plus two?") + await GLOBAL_LOGGING_WORKER.flush() + + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("nadir", "bucket") + assert (outcome.jev_verdict.label, outcome.jev_verdict.confidence) == ("SIMPLE", 0.81) + assert outcome.classifier_cost is None + assert "nadir-classifier:SIMPLE" in outcome.signals + sent: Final = route.calls.last.request + assert sent.headers.get("authorization") == authorization + assert json.loads(sent.content) == { + "messages": [{"role": "user", "content": "What is two plus two?"}], + "source": "litellm", + } + assert len(recorder.calls) == 1 + assert recorder.calls[0]["response_cost"] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + [ + httpx.Response(200, json={"bucket": None, "routing_tier": None}), + httpx.Response(200, json={**_NADIR_VERDICT, "routing_tier": "reasoning"}), + httpx.Response(403, json={"detail": "anonymous_disabled"}), + ], +) +async def test_nadir_verdict_the_router_cannot_use_falls_back( + monkeypatch: pytest.MonkeyPatch, response: httpx.Response +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir"}, + "tiers": {"SIMPLE": "cheap", "COMPLEX": "strong"}, + }, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=True) as upstream: + _ = upstream.post("https://api.getnadir.com/v1/bucket").mock(return_value=response) + outcome: Final = await router.aclassify("What is two plus two?") + + assert outcome.cause == "heuristic_scorer" + assert outcome.jev_verdict is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("window", "expected_prior"), + [(None, [{"role": "user", "content": "Refactor the tokenizer into its own module"}]), (0, [])], +) +async def test_nadir_gets_the_budgeted_conversation_without_the_judge_framing( + monkeypatch: pytest.MonkeyPatch, window: int | None, expected_prior: list[dict[str, str]] +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = ComplexityRouter( + "nadir-route", + litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"provider": "nadir"}, + "tiers": {"SIMPLE": "cheap", "COMPLEX": "strong"}, + **({"classifier_context_window_size": window} if window is not None else {}), + }, + derive_savings_baseline=False, + ) + conversation: Final = [ + {"role": "system", "content": "You are a senior Python reviewer"}, + {"role": "user", "content": "Refactor the tokenizer into its own module"}, + {"role": "assistant", "content": "Done, moved it to tokenizer.py"}, + {"role": "user", "content": "now add tests"}, + ] + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://api.getnadir.com/v1/bucket").respond(200, json=_NADIR_VERDICT) + _ = await router.aclassify( + "now add tests", system_prompt="You are a senior Python reviewer", messages=conversation + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"] == [ + {"role": "system", "content": "You are a senior Python reviewer"}, + *expected_prior, + {"role": "user", "content": "now add tests"}, + ] + assert "Classify this message" not in route.calls.last.request.content.decode() + + +@pytest.mark.asyncio +async def test_a_nadir_decision_deployment_gets_the_conversation_and_routes_on_its_verdict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("NADIR_API_BASE", raising=False) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + {"model_name": "nadir-decider", "litellm_params": {"model": "nadir/bucket", "api_key": "ndr-deployment"}} + ], + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "decision-router", + router, + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"deployment_name": "nadir-decider"}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + ) + conversation: Final = [ + {"role": "user", "content": "Refactor the tokenizer into its own module"}, + {"role": "assistant", "content": "Done, moved it to tokenizer.py"}, + {"role": "user", "content": "now add tests"}, + ] + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://api.getnadir.com/v1/bucket").respond(200, json=_NADIR_VERDICT) + outcome: Final = await classifier.aclassify("now add tests", messages=conversation) + + assert json.loads(route.calls.last.request.content) == { + "messages": [ + {"role": "user", "content": "Refactor the tokenizer into its own module"}, + {"role": "user", "content": "now add tests"}, + ], + "source": "litellm", + } + assert route.calls.last.request.headers["authorization"] == "Bearer ndr-deployment" + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert (outcome.jev_verdict.label, outcome.jev_verdict.provider) == ("SIMPLE", "nadir") + + +def test_nadir_messages_never_reach_a_system_one_server() -> None: + request: Final = build_jev_request( + "Classify this message:\nhi", + None, + "jev-latest", + "Pick a tier", + {"SIMPLE": "easy"}, + messages=({"role": "user", "content": "hi"},), + ) + assert "messages" not in request.model_dump(mode="json") + + +def test_nadir_defaults_its_model_and_allows_a_keyless_base() -> None: + assert JevClassifierConfig.model_validate({"provider": "nadir"}).model == "bucket" + keyless: Final = JevClassifierConfig.model_validate({"provider": "nadir", "api_base": "https://nadir.internal"}) + assert (keyless.api_base, keyless.api_key) == ("https://nadir.internal", None) + + +def test_nadir_rejects_instructions_it_would_ignore() -> None: + with pytest.raises(ValueError, match="instructions has no effect for provider 'nadir'"): + JevClassifierConfig.model_validate({"provider": "nadir", "instructions": "Pick a tier"}) + + @pytest.mark.parametrize( ("probabilities", "confidence"), [ @@ -890,6 +1098,7 @@ async def test_deployed_decision_classifier_uses_registered_connection( assert route.call_count == 1 assert route.calls[0].request.headers["authorization"] == "Bearer deployment-key" assert json.loads(route.calls[0].request.content)["model"] == model + assert "Classify this message" in json.loads(route.calls[0].request.content)["state"] assert (outcome.cause == "jev_classifier") is (status_code == 200) if status_code == 200: assert outcome.jev_verdict is not None diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 509329bb948..0b32995f707 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -233,6 +233,7 @@ const AutoRouterClassifierTabs: React.FC = ({ { value: "laya", label: "Laya" }, { value: "bespoke", label: "Bespoke Nimble" }, { value: "databricks", label: "Databricks" }, + { value: "nadir", label: "Nadir" }, ]} value={normalizeJevClassifierConfig(value.jev_classifier_config).provider} onValueChange={changeProvider} @@ -245,6 +246,7 @@ const AutoRouterClassifierTabs: React.FC = ({ Laya Bespoke Nimble Databricks + Nadir diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx index 6ed46e2a27d..ce98b25f36f 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierTypeRadios.tsx @@ -55,7 +55,9 @@ const ClassifierTypeRadios: React.FC = ({ value, clas OSS Classifier{" "} - uses Jev, Laya, Bespoke, or Databricks to decide the tier + + uses Jev, Laya, Bespoke, Databricks, or Nadir to decide the tier + diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index f829a2f2e41..32f56128987 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -164,6 +164,28 @@ describe("JEV classifier editor", () => { }, ); + it("hides the model and instructions for Nadir and saves it without instructions", async () => { + renderWithProviders(
); + fireEvent.click(screen.getByRole("radio", { name: /^Decisions Model$/ })); + fireEvent.change(screen.getByLabelText("Classifier Instructions"), { target: { value: "Pick carefully" } }); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Nadir"); + expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); + expect(screen.queryByLabelText("Classifier Instructions")).not.toBeInTheDocument(); + expect(screen.getByText(/Set NADIR_API_KEY on the gateway/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); + expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveTextContent("Nadir"); + fireEvent.click(screen.getByRole("button", { name: "Probe current config" })); + expect(testAutoRouterRouting).toHaveBeenLastCalledWith( + "token", + expect.objectContaining({ + complexity_router_config: expect.objectContaining({ + classifier_type: "oss_classifier", + opensource_classifier_config: { provider: "nadir", model: "bucket", timeout_ms: 3000 }, + }), + }), + ); + }); + it("allows licensed instructions and can restore built-in instructions", () => { const authorized = useAuthorized(); vi.mocked(useAuthorized).mockReturnValue({ ...authorized, premiumUser: true }); diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 4b3cfa21f9d..f09384ca2b8 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -7,7 +7,11 @@ import { Textarea } from "@/components/ui/textarea"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { defaultJevClassifierConfig, fixedClassifierModels } from "./jev_classifier_config"; +import { + acceptsClassifierInstructions, + defaultJevClassifierConfig, + fixedClassifierModels, +} from "./jev_classifier_config"; const providerDescriptions = { jev: "Uses TypeSafe System One Choice evaluation with your configured tiers", @@ -16,6 +20,8 @@ const providerDescriptions = { "Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.", databricks: "Uses the Databricks ai_decide AI Function, or a decision serving endpoint you name, with your configured tiers. Set DATABRICKS_API_BASE to https:///serving-endpoints and DATABRICKS_API_KEY on the gateway.", + nadir: + "Uses Nadir's tier decision with your configured tiers. Set NADIR_API_KEY on the gateway; without it, calls use Nadir's anonymous, rate-limited tier. Set NADIR_API_BASE for a self-hosted Nadir.", }; export default function JevClassifierConfig({ @@ -38,34 +44,36 @@ export default function JevClassifierConfig({ {section !== "advanced" && !config.deployment_name && ( <>

{providerDescriptions[config.provider ?? "jev"]}

-
- - {models ? ( - - ) : ( - update({ model: event.target.value })} - /> - )} -
+ {config.provider !== "nadir" && ( +
+ + {models ? ( + + ) : ( + update({ model: event.target.value })} + /> + )} +
+ )} )} {section !== "model" && ( @@ -90,27 +98,29 @@ export default function JevClassifierConfig({ }) } /> -
- - -