mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
feat(providers): add Nadir intelligent-router provider (nadir/auto) (#33227)
* feat(dd_span_tagger): emit litellm_user_email span tag for JWT-authenticated requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(dd_span_tagger): use dotted litellm.user_email tag for consistency Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(model_hub): surface model_info.description in model group info and Model Hub UI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(router): aggregate model group description without in-place mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(health): skip background health check DB writes when the latest-row read fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(providers): add Nadir intelligent-router provider (nadir/auto) Nadir (https://getnadir.com) is an OpenAI-compatible intelligent router. A single virtual model, nadir/auto, is classified server-side and routed to the cheapest model that clears the quality bar. The response reports the routed model in the model field, so LiteLLM cost tracking prices the real underlying model. - litellm/llms/nadir/chat/transformation.py: NadirConfig(OpenAIGPTConfig) - register nadir across enum, provider lists, get_llm_provider, __init__, lazy imports, utils, get_supported_openai_params - add https://api.getnadir.com/v1 to openai_compatible_endpoints so base_url only usage reverse-maps to the provider - provider_endpoints_support.json entry (chat_completions only) - docs page + unit tests (11 passing) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(nadir): scope credentials to the trusted base and validate reported cost NADIR_API_KEY only loads for the https default base, the SDK no longer falls back to litellm.api_key, the reported cost is validated before it reaches the spend log, and streams price from the routed model. --------- Co-authored-by: milan <milan@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: ryan-crabbe-berri <ryan@berri.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
020e5dee9b
commit
7e05581f93
12 changed files with 440 additions and 1 deletions
|
|
@ -657,6 +657,7 @@ azure_anthropic_models: Set = set()
|
|||
azure_text_models: Set = set()
|
||||
anyscale_models: Set = set()
|
||||
cerebras_models: Set = set()
|
||||
nadir_models: Set = set() # mutable-ok: provider registry, filled from model_cost at import like every sibling provider
|
||||
galadriel_models: Set = set()
|
||||
nvidia_nim_models: Set = set()
|
||||
nvidia_riva_models: Set = set()
|
||||
|
|
@ -893,6 +894,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
|
|||
anyscale_models.add(key)
|
||||
elif value.get("litellm_provider") == "cerebras":
|
||||
cerebras_models.add(key)
|
||||
elif value.get("litellm_provider") == "nadir":
|
||||
nadir_models.add(key)
|
||||
elif value.get("litellm_provider") == "galadriel":
|
||||
galadriel_models.add(key)
|
||||
elif value.get("litellm_provider") == "nvidia_nim":
|
||||
|
|
@ -1083,6 +1086,7 @@ model_list = list(
|
|||
| azure_anthropic_models
|
||||
| anyscale_models
|
||||
| cerebras_models
|
||||
| nadir_models
|
||||
| galadriel_models
|
||||
| nvidia_nim_models
|
||||
| nvidia_riva_models
|
||||
|
|
@ -1191,6 +1195,7 @@ def _build_models_by_provider() -> dict:
|
|||
"azure_text": azure_text_models,
|
||||
"anyscale": anyscale_models,
|
||||
"cerebras": cerebras_models,
|
||||
"nadir": nadir_models,
|
||||
"galadriel": galadriel_models,
|
||||
"nvidia_nim": nvidia_nim_models,
|
||||
"nvidia_riva": nvidia_riva_models,
|
||||
|
|
@ -1994,6 +1999,7 @@ if TYPE_CHECKING:
|
|||
FeatherlessAIConfig as FeatherlessAIConfig,
|
||||
)
|
||||
from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig
|
||||
from .llms.nadir.chat.transformation import NadirConfig as NadirConfig
|
||||
from .llms.baseten.chat import BasetenConfig as BasetenConfig
|
||||
from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig
|
||||
from .llms.sambanova.embedding.transformation import (
|
||||
|
|
|
|||
|
|
@ -264,6 +264,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"NvidiaNimEmbeddingConfig",
|
||||
"FeatherlessAIConfig",
|
||||
"CerebrasConfig",
|
||||
"NadirConfig",
|
||||
"BasetenConfig",
|
||||
"SambanovaConfig",
|
||||
"SambaNovaEmbeddingConfig",
|
||||
|
|
@ -1061,6 +1062,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"FeatherlessAIConfig",
|
||||
),
|
||||
"CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"),
|
||||
"NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"),
|
||||
"BasetenConfig": (".llms.baseten.chat", "BasetenConfig"),
|
||||
"SambanovaConfig": (".llms.sambanova.chat", "SambanovaConfig"),
|
||||
"SambaNovaEmbeddingConfig": (
|
||||
|
|
|
|||
|
|
@ -330,6 +330,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float(
|
|||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
||||
DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
|
||||
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
|
||||
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
|
||||
|
||||
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
|
||||
|
|
@ -711,6 +712,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"gigachat",
|
||||
"nvidia_nim",
|
||||
"cerebras",
|
||||
"nadir",
|
||||
"baseten",
|
||||
"ai21_chat",
|
||||
"volcengine",
|
||||
|
|
@ -904,6 +906,7 @@ openai_compatible_endpoints: Final[list] = [
|
|||
"codestral.mistral.ai/v1/fim/completions",
|
||||
"api.groq.com/openai/v1",
|
||||
"https://integrate.api.nvidia.com/v1",
|
||||
NADIR_DEFAULT_API_BASE,
|
||||
"api.deepseek.com/v1",
|
||||
"api.together.ai/v1",
|
||||
"api.together.xyz/v1",
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ from typing import Final, cast
|
|||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
||||
from litellm.constants import (
|
||||
NADIR_DEFAULT_API_BASE,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
REPLICATE_MODEL_NAME_WITH_ID_LENGTH,
|
||||
)
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
match_routing_generalization,
|
||||
)
|
||||
|
|
@ -277,6 +281,11 @@ def get_llm_provider(
|
|||
elif endpoint == "https://api.cerebras.ai/v1":
|
||||
custom_llm_provider = "cerebras"
|
||||
dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
|
||||
elif endpoint == NADIR_DEFAULT_API_BASE:
|
||||
custom_llm_provider = "nadir" # rebind-ok: mirrors sibling endpoint branches
|
||||
dynamic_api_key = (
|
||||
get_secret_str("NADIR_API_KEY") if api_base.lower().startswith("https://") else None
|
||||
)
|
||||
elif endpoint == "https://inference.baseten.co/v1":
|
||||
custom_llm_provider = "baseten"
|
||||
dynamic_api_key = get_secret_str("BASETEN_API_KEY")
|
||||
|
|
@ -649,6 +658,13 @@ def _get_openai_compatible_provider_info(
|
|||
elif custom_llm_provider == "cerebras":
|
||||
api_base = api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
|
||||
elif custom_llm_provider == "nadir":
|
||||
default_nadir_base: Final = get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE
|
||||
caller_base: Final = api_base
|
||||
api_base = api_base or default_nadir_base # rebind-ok: mirrors sibling provider branches
|
||||
trusted_base: Final = caller_base is None or caller_base.rstrip("/") == default_nadir_base.rstrip("/")
|
||||
env_key: Final = get_secret_str("NADIR_API_KEY") if trusted_base else None
|
||||
dynamic_api_key = api_key or env_key # rebind-ok: mirrors sibling provider branches
|
||||
elif custom_llm_provider == "baseten":
|
||||
# Use BasetenConfig to determine the appropriate API base URL
|
||||
if api_base is None:
|
||||
|
|
|
|||
|
|
@ -91,6 +91,8 @@ def get_supported_openai_params(
|
|||
return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
|
||||
elif custom_llm_provider == "cerebras":
|
||||
return litellm.CerebrasConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "nadir":
|
||||
return litellm.NadirConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "baseten":
|
||||
return litellm.BasetenConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "xai":
|
||||
|
|
|
|||
68
litellm/llms/nadir/chat/transformation.py
Normal file
68
litellm/llms/nadir/chat/transformation.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import math
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
_SUPPORTED_OPENAI_PARAMS: Final = (
|
||||
"extra_headers",
|
||||
"frequency_penalty",
|
||||
"max_retries",
|
||||
"max_tokens",
|
||||
"presence_penalty",
|
||||
"response_format",
|
||||
"stream",
|
||||
"temperature",
|
||||
"top_p",
|
||||
)
|
||||
|
||||
|
||||
def _reported_cost_usd(raw_response: httpx.Response) -> float | None:
|
||||
try:
|
||||
cost: Final = raw_response.json()["nadir_metadata"]["cost"]["total_cost_usd"]
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
if isinstance(cost, bool) or not isinstance(cost, (int, float)):
|
||||
return None
|
||||
if not math.isfinite(cost) or cost < 0:
|
||||
return None
|
||||
return float(cost)
|
||||
|
||||
|
||||
class NadirConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: return type fixed by the base interface
|
||||
return list(_SUPPORTED_OPENAI_PARAMS) # mutable-ok: the base interface returns a list
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: object,
|
||||
request_data: dict, # mutable-ok: signature fixed by the base interface
|
||||
messages: list[AllMessageValues], # mutable-ok: signature fixed by the base interface
|
||||
optional_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
litellm_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
encoding: object,
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
transformed: Final = super().transform_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
request_data=request_data,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
set_response_cost_in_hidden_params(transformed, _reported_cost_usd(raw_response))
|
||||
return transformed
|
||||
|
|
@ -64,6 +64,7 @@ from litellm.constants import (
|
|||
AZURE_OPENAI_AUDIO_PROVIDERS,
|
||||
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
|
||||
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
|
||||
NADIR_DEFAULT_API_BASE,
|
||||
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS,
|
||||
)
|
||||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
|
|
@ -3494,6 +3495,33 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch
|
|||
return response
|
||||
|
||||
|
||||
def _complete_nadir(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
|
||||
api_base: Final = ctx.api_base or litellm.api_base or get_secret_str("NADIR_API_BASE") or NADIR_DEFAULT_API_BASE
|
||||
api_key: Final = ctx.api_key
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=ctx.model,
|
||||
stream=ctx.stream,
|
||||
messages=ctx.messages,
|
||||
acompletion=ctx.acompletion,
|
||||
api_base=api_base,
|
||||
model_response=ctx.model_response,
|
||||
optional_params=ctx.optional_params,
|
||||
litellm_params=ctx.litellm_params,
|
||||
shared_session=ctx.shared_session,
|
||||
custom_llm_provider="nadir",
|
||||
timeout=ctx.timeout,
|
||||
headers=ctx.headers or litellm.headers,
|
||||
encoding=_get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=ctx.logging,
|
||||
client=ctx.client,
|
||||
)
|
||||
ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def _complete_vercel_ai_gateway(
|
||||
ctx: _CompletionDispatchContext,
|
||||
) -> _CompletionDispatchResult:
|
||||
|
|
@ -5923,6 +5951,8 @@ def completion(
|
|||
response = _complete_datarobot(_dispatch_ctx)
|
||||
elif custom_llm_provider == "openrouter":
|
||||
response = _complete_openrouter(_dispatch_ctx)
|
||||
elif custom_llm_provider == "nadir":
|
||||
response = _complete_nadir(_dispatch_ctx) # rebind-ok: mirrors sibling provider branches
|
||||
elif custom_llm_provider == "vercel_ai_gateway":
|
||||
response = _complete_vercel_ai_gateway(_dispatch_ctx)
|
||||
elif custom_llm_provider == "palm":
|
||||
|
|
|
|||
|
|
@ -2639,6 +2639,24 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "Nadir",
|
||||
"provider_display_name": "Nadir",
|
||||
"litellm_provider": "nadir",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "nadir/auto"
|
||||
},
|
||||
{
|
||||
"provider": "Oracle",
|
||||
"provider_display_name": "Oracle Cloud Infrastructure (OCI)",
|
||||
|
|
|
|||
|
|
@ -4016,6 +4016,7 @@ class LlmProviders(str, Enum):
|
|||
NVIDIA_RIVA = "nvidia_riva"
|
||||
SONIOX = "soniox"
|
||||
CEREBRAS = "cerebras"
|
||||
NADIR = "nadir"
|
||||
AI21_CHAT = "ai21_chat"
|
||||
VOLCENGINE = "volcengine"
|
||||
CODESTRAL = "codestral"
|
||||
|
|
|
|||
|
|
@ -4828,6 +4828,13 @@ def get_optional_params(
|
|||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "nadir":
|
||||
optional_params = litellm.NadirConfig().map_openai_params( # rebind-ok: same optional_params rebinding every sibling provider branch does
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "xai":
|
||||
optional_params = litellm.XAIChatConfig().map_openai_params(
|
||||
model=model,
|
||||
|
|
@ -5690,6 +5697,8 @@ def _check_provider_match(model_info: dict, custom_llm_provider: str | None) ->
|
|||
elif custom_llm_provider == "github":
|
||||
# Allow github/<model> aliases to reuse existing provider metadata.
|
||||
return True
|
||||
elif custom_llm_provider == "nadir":
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
|
@ -6815,6 +6824,11 @@ def validate_environment(
|
|||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("CEREBRAS_API_KEY")
|
||||
elif custom_llm_provider == "nadir":
|
||||
if "NADIR_API_KEY" in os.environ:
|
||||
keys_in_environment = True # rebind-ok: same flag rebinding every sibling provider branch does
|
||||
else:
|
||||
missing_keys.append("NADIR_API_KEY")
|
||||
elif custom_llm_provider == "baseten":
|
||||
if "BASETEN_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
|
|
@ -8473,6 +8487,7 @@ class ProviderConfigManager:
|
|||
LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
|
||||
LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
|
||||
LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
|
||||
LlmProviders.NADIR: (lambda: litellm.NadirConfig(), False),
|
||||
LlmProviders.VERCEL_AI_GATEWAY: (
|
||||
lambda: litellm.VercelAIGatewayConfig(),
|
||||
False,
|
||||
|
|
|
|||
|
|
@ -457,6 +457,24 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"nadir": {
|
||||
"display_name": "Nadir (`nadir`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/nadir",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": false,
|
||||
"responses": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": false,
|
||||
"interactions": false
|
||||
}
|
||||
},
|
||||
"cerebras": {
|
||||
"display_name": "Cerebras (`cerebras`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/cerebras",
|
||||
|
|
|
|||
260
tests/test_litellm/llms/nadir/test_nadir.py
Normal file
260
tests/test_litellm/llms/nadir/test_nadir.py
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
import json
|
||||
import math
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
NADIR_BASE = "https://api.getnadir.com/v1"
|
||||
COST_HEADER = "llm_provider-x-litellm-response-cost"
|
||||
|
||||
|
||||
def _transform(payload):
|
||||
raw = httpx.Response(
|
||||
200,
|
||||
content=json.dumps(payload).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
request=httpx.Request("POST", f"{NADIR_BASE}/chat/completions"),
|
||||
)
|
||||
return litellm.NadirConfig().transform_response(
|
||||
model="auto",
|
||||
raw_response=raw,
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=MagicMock(),
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
|
||||
def _payload(**extra):
|
||||
return {
|
||||
"id": "req-1",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "claude-haiku-4-5",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _cost(response, provider):
|
||||
return litellm.completion_cost(completion_response=response, custom_llm_provider=provider)
|
||||
|
||||
|
||||
def _logged_cost(response):
|
||||
return litellm.response_cost_calculator(
|
||||
response_object=response,
|
||||
model="auto",
|
||||
custom_llm_provider="nadir",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
)
|
||||
|
||||
|
||||
class TestNadirProviderResolution:
|
||||
def test_model_prefix_resolves_to_nadir(self):
|
||||
model, provider, _, _ = get_llm_provider(model="nadir/auto", api_key="sk-test")
|
||||
assert (model, provider) == ("auto", "nadir")
|
||||
|
||||
def test_default_api_base(self):
|
||||
_, _, _, api_base = get_llm_provider(model="nadir/auto", api_key="sk-test")
|
||||
assert api_base == NADIR_BASE
|
||||
|
||||
def test_api_base_override(self):
|
||||
_, _, _, api_base = get_llm_provider(
|
||||
model="nadir/auto",
|
||||
api_key="sk-test",
|
||||
api_base="https://gateway.internal/v1",
|
||||
)
|
||||
assert api_base == "https://gateway.internal/v1"
|
||||
|
||||
def test_endpoint_reverse_maps_to_nadir_with_the_env_key(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, provider, dynamic_api_key, _ = get_llm_provider(model="auto", api_base=NADIR_BASE)
|
||||
assert (provider, dynamic_api_key) == ("nadir", "sk-server-secret")
|
||||
|
||||
def test_plaintext_endpoint_never_loads_the_env_key(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, provider, dynamic_api_key, _ = get_llm_provider(model="auto", api_base="http://api.getnadir.com/v1")
|
||||
assert provider == "nadir"
|
||||
assert dynamic_api_key is None
|
||||
|
||||
|
||||
class TestNadirCredentialScoping:
|
||||
def test_env_key_used_for_default_endpoint(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, _, dynamic_api_key, _ = get_llm_provider(model="nadir/auto")
|
||||
assert dynamic_api_key == "sk-server-secret"
|
||||
|
||||
def test_env_key_used_when_base_matches_default(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, _, dynamic_api_key, _ = get_llm_provider(model="nadir/auto", api_base=f"{NADIR_BASE}/")
|
||||
assert dynamic_api_key == "sk-server-secret"
|
||||
|
||||
def test_env_key_not_leaked_to_custom_base(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, _, dynamic_api_key, _ = get_llm_provider(model="nadir/auto", api_base="https://attacker.example/v1")
|
||||
assert dynamic_api_key is None
|
||||
|
||||
def test_caller_key_used_for_custom_base(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
_, _, dynamic_api_key, _ = get_llm_provider(
|
||||
model="nadir/auto",
|
||||
api_base="https://self-hosted.internal/v1",
|
||||
api_key="sk-caller-own",
|
||||
)
|
||||
assert dynamic_api_key == "sk-caller-own"
|
||||
|
||||
def test_env_key_used_for_operator_configured_base(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-server-secret")
|
||||
monkeypatch.setenv("NADIR_API_BASE", "https://nadir.mycorp.internal/v1")
|
||||
_, _, dynamic_api_key, _ = get_llm_provider(model="nadir/auto", api_base="https://nadir.mycorp.internal/v1")
|
||||
assert dynamic_api_key == "sk-server-secret"
|
||||
|
||||
|
||||
class TestNadirParamMapping:
|
||||
def test_supported_params_are_mapped(self):
|
||||
params = litellm.get_optional_params(
|
||||
model="auto",
|
||||
custom_llm_provider="nadir",
|
||||
temperature=0.5,
|
||||
max_tokens=64,
|
||||
)
|
||||
assert params["temperature"] == 0.5
|
||||
assert params["max_tokens"] == 64
|
||||
|
||||
def test_streaming_is_advertised_and_tools_are_not(self):
|
||||
params = litellm.get_supported_openai_params(model="auto", custom_llm_provider="nadir")
|
||||
assert "stream" in params
|
||||
assert "tools" not in params
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
{"tools": [{"type": "function", "function": {"name": "f", "parameters": {}}}]},
|
||||
{"stop": ["\n"]},
|
||||
{"seed": 7},
|
||||
{"n": 2},
|
||||
],
|
||||
)
|
||||
def test_params_nadir_would_silently_drop_are_rejected(self, unsupported):
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
litellm.get_optional_params(model="auto", custom_llm_provider="nadir", **unsupported)
|
||||
|
||||
def test_unsupported_params_are_dropped_when_asked(self):
|
||||
params = litellm.get_optional_params(
|
||||
model="auto",
|
||||
custom_llm_provider="nadir",
|
||||
drop_params=True,
|
||||
seed=7,
|
||||
temperature=0.2,
|
||||
)
|
||||
assert "seed" not in params
|
||||
assert params["temperature"] == 0.2
|
||||
|
||||
|
||||
class TestNadirEnvValidation:
|
||||
def test_validate_environment_detects_key(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-live-xyz")
|
||||
result = litellm.validate_environment(model="nadir/auto")
|
||||
assert result["keys_in_environment"] is True
|
||||
|
||||
def test_validate_environment_flags_missing_key(self, monkeypatch):
|
||||
monkeypatch.delenv("NADIR_API_KEY", raising=False)
|
||||
result = litellm.validate_environment(model="nadir/auto")
|
||||
assert "NADIR_API_KEY" in result["missing_keys"]
|
||||
|
||||
|
||||
class TestNadirCostAttribution:
|
||||
def test_reported_cost_wins_over_model_pricing(self):
|
||||
res = _transform(_payload(nadir_metadata={"cost": {"total_cost_usd": 0.00123}}))
|
||||
assert _logged_cost(res) == pytest.approx(0.00123)
|
||||
assert _logged_cost(res) != _cost(res, "anthropic")
|
||||
|
||||
def test_routed_model_is_preserved(self):
|
||||
res = _transform(_payload(nadir_metadata={"cost": {"total_cost_usd": 0.001}}))
|
||||
assert res.model == "claude-haiku-4-5"
|
||||
|
||||
def test_missing_cost_prices_the_routed_model_from_its_own_entry(self):
|
||||
res = _transform(_payload())
|
||||
assert res.choices[0].message.content == "hi"
|
||||
assert COST_HEADER not in res._hidden_params.get("additional_headers", {})
|
||||
assert _cost(res, "nadir") == _logged_cost(res) == _cost(res, "anthropic") > 0
|
||||
|
||||
def test_streamed_routed_model_prices_from_its_own_entry(self):
|
||||
res = ModelResponse(
|
||||
model="gemini-3.5-flash-lite",
|
||||
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
|
||||
)
|
||||
assert _cost(res, "nadir") == _cost(res, "gemini") > 0
|
||||
|
||||
@pytest.mark.parametrize("bad", [-0.001, math.nan, math.inf, -math.inf, True, "0.001", None])
|
||||
def test_invalid_reported_cost_falls_back_to_model_pricing(self, bad):
|
||||
res = _transform(_payload(nadir_metadata={"cost": {"total_cost_usd": bad}}))
|
||||
assert COST_HEADER not in res._hidden_params.get("additional_headers", {})
|
||||
assert _cost(res, "nadir") == _cost(res, "anthropic") > 0
|
||||
|
||||
@pytest.mark.parametrize("metadata", ["oops", {"cost": "free"}, {"cost": None}, {}])
|
||||
def test_malformed_metadata_falls_back_to_model_pricing(self, metadata):
|
||||
res = _transform(_payload(nadir_metadata=metadata))
|
||||
assert COST_HEADER not in res._hidden_params.get("additional_headers", {})
|
||||
assert _cost(res, "nadir") == _cost(res, "anthropic") > 0
|
||||
|
||||
|
||||
class TestNadirCompletionDispatch:
|
||||
def _call(self, **kwargs):
|
||||
captured = {}
|
||||
|
||||
def fake_completion(**call_kwargs):
|
||||
captured.update(call_kwargs)
|
||||
return ModelResponse()
|
||||
|
||||
with patch( # test-quality-ok: these tests assert the dispatch wiring itself (nadir must reach base_llm_http_handler, and which credentials it is handed); faking HTTP would not observe that
|
||||
"litellm.main.base_llm_http_handler.completion", side_effect=fake_completion
|
||||
):
|
||||
litellm.completion(
|
||||
model="nadir/auto",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
**kwargs,
|
||||
)
|
||||
return captured
|
||||
|
||||
def test_routes_through_the_http_handler_as_nadir(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-env")
|
||||
captured = self._call()
|
||||
assert captured["custom_llm_provider"] == "nadir"
|
||||
assert captured["api_base"] == NADIR_BASE
|
||||
|
||||
def test_env_key_is_used_for_the_default_endpoint(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-env")
|
||||
assert self._call()["api_key"] == "sk-env"
|
||||
|
||||
def test_caller_key_wins(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-env")
|
||||
assert self._call(api_key="sk-caller")["api_key"] == "sk-caller"
|
||||
|
||||
def test_env_key_is_not_forwarded_to_a_caller_supplied_host(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-env")
|
||||
captured = self._call(api_base="https://attacker.example/v1")
|
||||
assert captured["api_key"] != "sk-env"
|
||||
assert captured["api_base"] == "https://attacker.example/v1"
|
||||
|
||||
def test_global_key_is_not_forwarded_to_a_caller_supplied_host(self, monkeypatch):
|
||||
monkeypatch.delenv("NADIR_API_KEY", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_key", "sk-global")
|
||||
captured = self._call(api_base="https://attacker.example/v1")
|
||||
assert captured["api_key"] != "sk-global"
|
||||
|
||||
def test_custom_api_base_is_honoured(self, monkeypatch):
|
||||
monkeypatch.setenv("NADIR_API_KEY", "sk-env")
|
||||
captured = self._call(api_base="https://nadir.internal/v1", api_key="sk-own")
|
||||
assert captured["api_base"] == "https://nadir.internal/v1"
|
||||
assert captured["api_key"] == "sk-own"
|
||||
Loading…
Add table
Reference in a new issue