diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index aeedbf702eb..d9f1b7b2262 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -275,9 +275,9 @@ your provider keys, fallbacks, and spend tracking all stay on this proxy. #### Nadir `litellm.router_strategy.complexity_router.nadir_classifier.nadir_classifier` is a bundled -plugin that asks [Nadir](https://getnadir.com)'s `/v1/bucket` endpoint, a trained -complexity classifier rather than an LLM call, which of `simple` / `medium` / `complex` a -request is. Those map to the SIMPLE / MEDIUM / COMPLEX tiers: +plugin that asks [Nadir](https://getnadir.com)'s `/v1/bucket` endpoint which of `simple` / +`medium` / `complex` a request is. The endpoint grades the request without generating an +answer, and the three buckets map to the SIMPLE / MEDIUM / COMPLEX tiers: ```yaml model_list: @@ -289,32 +289,41 @@ model_list: classifier_plugin: litellm.router_strategy.complexity_router.nadir_classifier.nadir_classifier classifier_fallback: heuristic tiers: - SIMPLE: gpt-4o-mini - MEDIUM: gpt-4o - COMPLEX: o1-preview + SIMPLE: gpt-5-nano + MEDIUM: gpt-5-mini + COMPLEX: gpt-5 ``` -`NADIR_API_KEY` attributes decisions to an account and lifts the anonymous rate limit; -without it the endpoint still answers, burst-limited per IP and stored nowhere. -`NADIR_API_BASE` points at a self-hosted or on-prem Nadir instead of the hosted API. +`NADIR_API_KEY` attributes decisions to a Nadir account and lifts the anonymous rate limit. +That account's request log then keeps the user text of each classified request, or only a +hash of it when the account has prompt storage turned off. Without a key the endpoint still +answers and keeps no prompt, but its per-IP rate limit is sized for trying the plugin out, +and requests over it route on `classifier_fallback`. -A router whose tiers are renamed with `tier_labels`, or defined with `tier_definitions`, -constructs the classifier with the matching names instead of using the module instance: +`NADIR_API_BASE` points at a self-hosted or on-prem Nadir instead of the hosted API. +`NADIR_API_KEY` is only sent to that base, or to the hosted API when it is unset. + +Renaming the tiers with `tier_labels` needs no change here: the router still accepts the +default tier names the plugin returns. A router built on `tier_definitions` names its own +tiers, so point `classifier_plugin` at an instance of your own that maps Nadir's buckets +onto them: ```python +# my_classifiers.py, named in the config as my_classifiers.nadir from litellm.router_strategy.complexity_router.nadir_classifier import NadirComplexityClassifier -classifier = NadirComplexityClassifier(tier_map={"simple": "Cheap", "medium": "Standard", "complex": "Premium"}) +nadir = NadirComplexityClassifier(tier_map={"simple": "cheap", "medium": "standard", "complex": "premium"}) ``` Two properties to weigh against the local scorers. The classification is a network call, so it costs a round trip per request where `heuristic` and `heuristic_v2` cost under a millisecond (a laptop against the hosted API measured about 160ms warm, and the first call in a fresh process also pays connection setup; anything that overruns -`classifier_plugin_timeout_ms` routes on the fallback instead of waiting). And the messages are sent to the configured Nadir host, which is a third party -unless that host is yours, the same disclosure `classifier_type: llm` carries when the -classifier model is a hosted one. A REASONING tier is never returned, since Nadir grades -three buckets; keyword rules and the reasoning override still place requests there. +`classifier_plugin_timeout_ms` routes on the fallback instead of waiting). And the messages +are sent to the configured Nadir host, which is a third party unless that host is yours, the +same disclosure `classifier_type: llm` carries when the classifier model is a hosted one. A +REASONING tier is never returned, since Nadir grades three buckets; keyword rules and the +reasoning override still place requests there. ### Renaming the tiers diff --git a/litellm/router_strategy/complexity_router/nadir_classifier.py b/litellm/router_strategy/complexity_router/nadir_classifier.py index 183a8200e1b..d7465bf6bc5 100644 --- a/litellm/router_strategy/complexity_router/nadir_classifier.py +++ b/litellm/router_strategy/complexity_router/nadir_classifier.py @@ -4,13 +4,13 @@ Nadir as the Complexity Router's classifier. A decision-only integration: the tier comes from a call to Nadir's ``/v1/bucket`` endpoint, and everything else stays here. The tier's model pool, the provider call, the operator's own provider keys, fallbacks, spend tracking and the response path are -untouched, so Nadir sees the messages it is asked to classify and never the completion. +untouched, so Nadir sees the messages it is asked to classify and never the response. -``/v1/bucket`` runs a trained complexity classifier (a fine-tuned encoder, not an LLM -call) and answers ``simple`` / ``medium`` / ``complex``, which map to the router's -default SIMPLE / MEDIUM / COMPLEX tiers. A router whose tiers are renamed with -``tier_labels``, or defined with ``tier_definitions``, passes its own names as -``tier_map``. +``/v1/bucket`` grades the request without generating an answer and replies ``simple`` / +``medium`` / ``complex``, which map to the router's default SIMPLE / MEDIUM / COMPLEX +tiers. Renaming the tiers with ``tier_labels`` needs nothing here, since the router still +accepts the default names. A router built on ``tier_definitions`` names its own tiers, so +it constructs its own instance with a ``tier_map`` onto them. Configure it in the proxy by pointing ``classifier_plugin`` at the module-level instance:: @@ -23,17 +23,24 @@ instance:: classifier_type: custom classifier_plugin: litellm.router_strategy.complexity_router.nadir_classifier.nadir_classifier tiers: - SIMPLE: gpt-4o-mini - MEDIUM: gpt-4o - COMPLEX: o1-preview + SIMPLE: gpt-5-nano + MEDIUM: gpt-5-mini + COMPLEX: gpt-5 -``NADIR_API_KEY`` attributes the decision to an account and lifts the anonymous rate -limit; without it the endpoint still answers, burst-limited per IP and stored nowhere. -``NADIR_API_BASE`` overrides the host for a self-hosted or on-prem deployment. +``NADIR_API_KEY`` attributes decisions to a Nadir account and lifts the anonymous rate +limit. That account's request log then keeps the user text of each classified request, or +only a hash of it when the account has prompt storage turned off. Without a key the +endpoint still answers and keeps no prompt, but its per-IP rate limit is sized for trying +the plugin out, and requests over it route on ``classifier_fallback``. + +``NADIR_API_BASE`` points at a self-hosted or on-prem Nadir. ``NADIR_API_KEY`` is only +ever sent to that base, or to the hosted API when it is unset: an ``api_base`` passed in +code brings its own ``api_key``. Every failure mode is the router's existing one: this returns None to decline, and a -network error, a timeout past ``classifier_plugin_timeout_ms`` or an unknown bucket all -hand the request to ``classifier_fallback``. Nothing here can fail a completion. +network error, an error status, a timeout past ``classifier_plugin_timeout_ms`` or an +unknown bucket all hand the request to ``classifier_fallback``. Nothing here can fail a +completion. Privacy note for operators: the messages are sent to the configured Nadir host, which is a third party unless that host is your own. That is the same disclosure a remote LLM @@ -42,17 +49,16 @@ classifier carries, and it is the reason the local heuristic scorers exist. from __future__ import annotations -import os from collections.abc import Mapping from types import MappingProxyType from typing import Final -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.constants import NADIR_DEFAULT_API_BASE +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.router import RoutingContext -DEFAULT_NADIR_API_BASE: Final = "https://api.getnadir.com" - DEFAULT_TIER_MAP: Final[Mapping[str, str]] = MappingProxyType( {"simple": "SIMPLE", "medium": "MEDIUM", "complex": "COMPLEX"} ) @@ -72,14 +78,23 @@ def _bucket_url(api_base: str) -> str: return f"{base}{_API_PATH}" +def _configured_bucket_url() -> str: + """The endpoint the environment names: ``NADIR_API_BASE``, else the hosted API.""" + return _bucket_url(get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE) + + class NadirComplexityClassifier: """Classifier plugin that asks Nadir's decision API which tier a request belongs to. Args: api_base: Nadir host. Defaults to ``NADIR_API_BASE``, then to the hosted API. - api_key: Nadir API key. Defaults to ``NADIR_API_KEY``; anonymous when unset. + api_key: Nadir API key. Defaults to ``NADIR_API_KEY`` only when the request goes to the + host the environment names, so the environment key never follows an ``api_base`` + set in code; anonymous otherwise. tier_map: Nadir bucket name -> the tier name this router routes on. Defaults to the router's built-in tier names. + client: HTTP client the request goes out on. Defaults to litellm's shared async client. + Leave it unset on an instance a proxy config names, since the proxy deepcopies it. """ def __init__( @@ -87,9 +102,11 @@ class NadirComplexityClassifier: api_base: str | None = None, api_key: str | None = None, tier_map: Mapping[str, str] | None = None, + client: AsyncHTTPHandler | None = None, ) -> None: self._api_base: Final = api_base self._api_key: Final = api_key + self._client: Final = client # A plain dict, not a MappingProxyType: the proxy deepcopies a deployment's # litellm_params, this instance travels inside them, and a mappingproxy cannot be # deepcopied. Router construction would fail before the first request. @@ -97,32 +114,31 @@ class NadirComplexityClassifier: DEFAULT_TIER_MAP if tier_map is None else tier_map ) - def _headers(self) -> dict[str, str]: - api_key: Final = self._api_key or os.getenv("NADIR_API_KEY") + def _headers(self, url: str) -> dict[str, str]: + env_key: Final = get_secret_str("NADIR_API_KEY") if url == _configured_bucket_url() else None + api_key: Final = self._api_key or env_key return {"X-API-Key": api_key} if api_key else {} # mutable-ok: request headers, handed straight to httpx async def classify(self, context: RoutingContext) -> str | None: """Return the tier Nadir places this request in, or None to decline. Declines rather than guesses on anything the endpoint cannot grade: a request with no - messages, and a bucket name outside ``tier_map`` (which is what a renamed tier set looks - like before ``tier_map`` is configured). Both leave the decision to ``classifier_fallback``, - where a local scorer still routes the request. + messages, and a bucket name outside ``tier_map`` (which is what a ``tier_definitions`` + router looks like before ``tier_map`` is configured). Both leave the decision to + ``classifier_fallback``, where a local scorer still routes the request. """ messages: Final = context.structured_messages or context.raw_messages if not messages: return None - api_base: Final = self._api_base or os.getenv("NADIR_API_BASE") or DEFAULT_NADIR_API_BASE - client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.ComplexityClassifier) + url: Final = _bucket_url(self._api_base) if self._api_base else _configured_bucket_url() + client: Final = self._client or get_async_httpx_client(llm_provider=httpxSpecialProvider.ComplexityClassifier) body: Final = {"messages": list(messages), "source": "litellm"} # mutable-ok: one request body - response: Final = await client.post(url=_bucket_url(api_base), json=body, headers=self._headers()) - payload: Final = response.json() - if not isinstance(payload, Mapping): - return None - bucket: Final = payload.get("bucket") - if not isinstance(bucket, str): - return None - return self._tier_map.get(bucket.strip().lower()) + response: Final = await client.post(url=url, json=body, headers=self._headers(url)) + match response.json(): + case {"bucket": str(bucket)}: + return self._tier_map.get(bucket.strip().lower()) + case _: + return None nadir_classifier: Final = NadirComplexityClassifier() diff --git a/tests/unit/router_strategy/complexity_router/test_nadir_classifier.py b/tests/unit/router_strategy/complexity_router/test_nadir_classifier.py index d65a625304c..2af0824d4f4 100644 --- a/tests/unit/router_strategy/complexity_router/test_nadir_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_nadir_classifier.py @@ -6,13 +6,14 @@ import pytest from litellm.router_strategy.complexity_router import ComplexityRouter from litellm.router_strategy.complexity_router.nadir_classifier import ( - DEFAULT_NADIR_API_BASE, NadirComplexityClassifier, _bucket_url, nadir_classifier, ) from litellm.types.router import ClassifierPlugin, RoutingContext +HOSTED_BUCKET_URL = "https://api.getnadir.com/v1/bucket" + class _FakeResponse: def __init__(self, payload): @@ -37,19 +38,6 @@ class _FakeClient: return _FakeResponse(self._payload) -@pytest.fixture -def fake_client(monkeypatch): - def _install(payload=None, error=None): - client = _FakeClient(payload=payload, error=error) - monkeypatch.setattr( - "litellm.router_strategy.complexity_router.nadir_classifier.get_async_httpx_client", - lambda llm_provider: client, - ) - return client - - return _install - - def _context(messages=None): resolved = [{"role": "user", "content": "refactor the retry loop"}] if messages is None else messages return RoutingContext( @@ -73,7 +61,7 @@ class TestBucketURL: ) def test_v1_is_never_doubled(self, api_base): """Nadir's docs advertise the base URL with /v1, so operators paste it with and without.""" - assert _bucket_url(api_base) == "https://api.getnadir.com/v1/bucket" + assert _bucket_url(api_base) == HOSTED_BUCKET_URL def test_self_hosted_host_with_a_path_prefix_is_preserved(self): assert _bucket_url("https://gateway.internal/nadir") == "https://gateway.internal/nadir/v1/bucket" @@ -84,60 +72,92 @@ class TestClassify: @pytest.mark.parametrize( ("bucket", "expected_tier"), [("simple", "SIMPLE"), ("medium", "MEDIUM"), ("complex", "COMPLEX")] ) - async def test_each_bucket_maps_to_its_default_tier(self, fake_client, bucket, expected_tier): - client = fake_client(payload={"bucket": bucket, "confidence": 0.91}) - assert await NadirComplexityClassifier().classify(_context()) == expected_tier - assert client.calls[0]["url"] == f"{DEFAULT_NADIR_API_BASE}/v1/bucket" + async def test_each_bucket_maps_to_its_default_tier(self, bucket, expected_tier): + client = _FakeClient(payload={"bucket": bucket, "confidence": 0.91}) + assert await NadirComplexityClassifier(client=client).classify(_context()) == expected_tier + assert client.calls[0]["url"] == HOSTED_BUCKET_URL assert client.calls[0]["json"]["messages"] == [{"role": "user", "content": "refactor the retry loop"}] assert client.calls[0]["json"]["source"] == "litellm" @pytest.mark.asyncio - async def test_tier_map_override_serves_renamed_tiers(self, fake_client): - """tier_labels/tier_definitions rename the tiers; the plugin must answer in the router's names.""" - fake_client(payload={"bucket": "complex"}) - classifier = NadirComplexityClassifier(tier_map={"simple": "cheap", "medium": "mid", "complex": "deep"}) + async def test_tier_map_override_serves_custom_tier_names(self): + """A tier_definitions router names its own tiers; tier_map must answer in those names.""" + classifier = NadirComplexityClassifier( + tier_map={"simple": "cheap", "medium": "mid", "complex": "deep"}, + client=_FakeClient(payload={"bucket": "complex"}), + ) assert await classifier.classify(_context()) == "deep" @pytest.mark.asyncio - async def test_api_key_is_sent_when_configured(self, fake_client): - client = fake_client(payload={"bucket": "simple"}) - await NadirComplexityClassifier(api_key="ndr_test").classify(_context()) + async def test_api_key_is_sent_when_configured(self): + client = _FakeClient(payload={"bucket": "simple"}) + await NadirComplexityClassifier(api_key="ndr_test", client=client).classify(_context()) assert client.calls[0]["headers"] == {"X-API-Key": "ndr_test"} @pytest.mark.asyncio - async def test_env_supplies_key_and_base(self, fake_client, monkeypatch): - client = fake_client(payload={"bucket": "simple"}) + async def test_env_supplies_key_and_base(self, monkeypatch): + client = _FakeClient(payload={"bucket": "simple"}) monkeypatch.setenv("NADIR_API_KEY", "ndr_from_env") monkeypatch.setenv("NADIR_API_BASE", "https://nadir.internal") - await NadirComplexityClassifier().classify(_context()) + await NadirComplexityClassifier(client=client).classify(_context()) assert client.calls[0]["headers"] == {"X-API-Key": "ndr_from_env"} assert client.calls[0]["url"] == "https://nadir.internal/v1/bucket" @pytest.mark.asyncio - async def test_anonymous_when_no_key_is_configured(self, fake_client, monkeypatch): - client = fake_client(payload={"bucket": "simple"}) - monkeypatch.delenv("NADIR_API_KEY", raising=False) - await NadirComplexityClassifier().classify(_context()) + async def test_env_key_never_follows_a_base_set_in_code(self, monkeypatch): + """The environment names where NADIR_API_KEY may go; an api_base set in code brings its own key.""" + client = _FakeClient(payload={"bucket": "simple"}) + monkeypatch.setenv("NADIR_API_KEY", "ndr_from_env") + monkeypatch.delenv("NADIR_API_BASE", raising=False) + await NadirComplexityClassifier(api_base="https://elsewhere.example", client=client).classify(_context()) + assert client.calls[0]["url"] == "https://elsewhere.example/v1/bucket" assert client.calls[0]["headers"] == {} @pytest.mark.asyncio - async def test_no_messages_declines_without_a_call(self, fake_client): + async def test_env_key_goes_to_the_hosted_api_named_in_code(self, monkeypatch): + """Naming the hosted base in code, with or without /v1, is still the base the environment trusts.""" + client = _FakeClient(payload={"bucket": "simple"}) + monkeypatch.setenv("NADIR_API_KEY", "ndr_from_env") + monkeypatch.delenv("NADIR_API_BASE", raising=False) + await NadirComplexityClassifier(api_base="https://api.getnadir.com/v1", client=client).classify(_context()) + assert client.calls[0]["headers"] == {"X-API-Key": "ndr_from_env"} + + @pytest.mark.asyncio + async def test_a_base_set_in_code_sends_its_own_key(self, monkeypatch): + client = _FakeClient(payload={"bucket": "simple"}) + monkeypatch.setenv("NADIR_API_KEY", "ndr_from_env") + classifier = NadirComplexityClassifier( + api_base="https://nadir.onprem.internal", api_key="ndr_own", client=client + ) + await classifier.classify(_context()) + assert client.calls[0]["headers"] == {"X-API-Key": "ndr_own"} + + @pytest.mark.asyncio + async def test_anonymous_when_no_key_is_configured(self, monkeypatch): + client = _FakeClient(payload={"bucket": "simple"}) + monkeypatch.delenv("NADIR_API_KEY", raising=False) + await NadirComplexityClassifier(client=client).classify(_context()) + assert client.calls[0]["headers"] == {} + + @pytest.mark.asyncio + async def test_no_messages_declines_without_a_call(self): """An empty request is a 400 from the endpoint; decline locally instead of spending the trip.""" - client = fake_client(payload={"bucket": "simple"}) - assert await NadirComplexityClassifier().classify(_context(messages=[])) is None + client = _FakeClient(payload={"bucket": "simple"}) + assert await NadirComplexityClassifier(client=client).classify(_context(messages=[])) is None assert client.calls == [] @pytest.mark.asyncio - @pytest.mark.parametrize("payload", [{"bucket": "reasoning"}, {"bucket": None}, {}, ["not", "a", "mapping"]]) - async def test_unusable_verdicts_decline(self, fake_client, payload): + @pytest.mark.parametrize( + "payload", [{"bucket": "reasoning"}, {"bucket": None}, {"bucket": 3}, {}, ["not", "a", "mapping"]] + ) + async def test_unusable_verdicts_decline(self, payload): """Anything outside tier_map is declined, so classifier_fallback decides rather than this guessing.""" - fake_client(payload=payload) - assert await NadirComplexityClassifier().classify(_context()) is None + assert await NadirComplexityClassifier(client=_FakeClient(payload=payload)).classify(_context()) is None @pytest.mark.asyncio - async def test_bucket_name_is_matched_case_insensitively(self, fake_client): - fake_client(payload={"bucket": " Complex "}) - assert await NadirComplexityClassifier().classify(_context()) == "COMPLEX" + async def test_bucket_name_is_matched_case_insensitively(self): + classifier = NadirComplexityClassifier(client=_FakeClient(payload={"bucket": " Complex "})) + assert await classifier.classify(_context()) == "COMPLEX" def test_module_instance_satisfies_the_plugin_protocol(self): """The proxy resolves the dotted path to this instance and interface-checks it at startup.""" @@ -159,22 +179,22 @@ class TestDeploymentConfigSurvivesRouterInit: class TestThroughTheComplexityRouter: """The plugin as the router actually drives it: verdict in, tier out, failures fall back.""" - def _router(self, mock_router_instance=None, **overrides): + def _router(self, client, **overrides): return ComplexityRouter( model_name="test-nadir-router", - litellm_router_instance=mock_router_instance, + litellm_router_instance=None, complexity_router_config={ "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o", "COMPLEX": "o1-preview"}, "classifier_type": "custom", - "classifier_plugin": NadirComplexityClassifier(), + "classifier_plugin": NadirComplexityClassifier(client=client), **overrides, }, ) @pytest.mark.asyncio - async def test_verdict_becomes_the_routed_tier(self, fake_client): - fake_client(payload={"bucket": "complex"}) - outcome = await self._router().aclassify( + async def test_verdict_becomes_the_routed_tier(self): + router = self._router(_FakeClient(payload={"bucket": "complex"})) + outcome = await router.aclassify( prompt="port the scheduler to the new executor", raw_messages=[{"role": "user", "content": "port the scheduler to the new executor"}], ) @@ -182,9 +202,20 @@ class TestThroughTheComplexityRouter: assert outcome.cause == "classifier_plugin" @pytest.mark.asyncio - async def test_network_failure_falls_back_to_the_local_scorer(self, fake_client): + async def test_renamed_tier_labels_need_no_tier_map(self): + """tier_labels only renames what the dashboard shows; the router still accepts the default names.""" + router = self._router( + _FakeClient(payload={"bucket": "complex"}), + tier_labels={"SIMPLE": "Cheap", "MEDIUM": "Standard", "COMPLEX": "Premium"}, + ) + outcome = await router.aclassify(prompt="hi", raw_messages=[{"role": "user", "content": "hi"}]) + assert outcome.tier.value == "COMPLEX" + assert outcome.cause == "classifier_plugin" + + @pytest.mark.asyncio + async def test_network_failure_falls_back_to_the_local_scorer(self): """A Nadir outage must never fail a completion: the heuristic scorer still places the request.""" - fake_client(error=ConnectionError("nadir unreachable")) - outcome = await self._router().aclassify(prompt="hi", raw_messages=[{"role": "user", "content": "hi"}]) + router = self._router(_FakeClient(error=ConnectionError("nadir unreachable"))) + outcome = await router.aclassify(prompt="hi", raw_messages=[{"role": "user", "content": "hi"}]) assert outcome.cause != "classifier_plugin" assert outcome.tier is not None