From 6d0d0bb68babd2202c61692eb61f0eb947643f45 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 16 Oct 2026 20:07:50 -0500 Subject: [PATCH] feat: add Nace Drex decisions provider (#46523) * feat: add Nace Drex decisions provider * fix: expose Nace in dashboard provider metadata * ci: include Nace tests in the provider shard * fix: complete Nace provider catalog and preserve OpenRouter tests * test: mark Nace test directories as packages * fix: share Drex score parsing across decision providers --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- README.md | 1 + litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/decisions/call.py | 7 +- .../llms/base_llm/decisions/transformation.py | 18 ++- litellm/llms/nace/decisions/transformation.py | 9 ++ ...odel_prices_and_context_window_backup.json | 32 +++++ .../provider_create_fields.json | 28 ++++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 32 +++++ provider_endpoints_support.json | 7 + .../providers/test_decisions_wire.py | 9 +- tests/unit/decisions/test_main.py | 135 ++++++++++++++++++ .../test_base_decisions_transformation.py | 69 +++++++++ .../public_endpoints/test_public_endpoints.py | 15 +- .../components/provider_info_helpers.test.tsx | 11 ++ .../src/components/provider_info_helpers.tsx | 3 + 18 files changed, 371 insertions(+), 13 deletions(-) create mode 100644 litellm/llms/nace/decisions/transformation.py diff --git a/README.md b/README.md index 3bd0fd54a65..77f7e3baa4d 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | | | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | | +| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | | | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | | | [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | | | [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index df52808643d..cdcf9d1d80c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1694,6 +1694,9 @@ if TYPE_CHECKING: from .llms.perplexity.decisions.transformation import ( PerplexityDecisionsConfig as PerplexityDecisionsConfig, ) + from .llms.nace.decisions.transformation import ( + NaceDecisionsConfig as NaceDecisionsConfig, + ) from .llms.typesafe.decisions.transformation import ( TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index d47d72ef7a8..8abc24d1417 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMRerankConfig", "PerplexityDecisionsConfig", "TypeSafeDecisionsConfig", + "NaceDecisionsConfig", "OpenRouterDecisionsConfig", "FireworksAIDecisionsConfig", "CloudflareDecisionsConfig", @@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "HostedVLLMRerankConfig", ), "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 040d80e1ac0..4943b22e1fb 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -57,7 +57,7 @@ from litellm.types.decisions import ( UnsupportedDecisionsRequest, ) from litellm.types.utils import LlmProviders, ProviderSpecificHeader -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, get_model_cost_key SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]] OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]] @@ -667,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci { "model": f"{call.custom_llm_provider}/{call.model}", "custom_llm_provider": call.custom_llm_provider, - "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + "provider_response_model": ( + get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") + or f"{call.custom_llm_provider}/{call.model}" + ), **({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}), } ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index df7b7645840..bbec66c6765 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe ) +_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...]) + + +def _normalize_answer(value: object) -> Mapping[str, object]: + answer: Final = _MAPPING.validate_python(value) + legend: Final = answer.get("legend") + if answer.get("type") != "score" or not isinstance(legend, list): + return answer + labels: Final = _LABELS.validate_python(legend) + return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}} + + def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: - return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request) + response: Final = _MAPPING.validate_python(payload) + answers: Final = _MAPPING.validate_python(response.get("answers")) + normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}} + return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request) def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None: diff --git a/litellm/llms/nace/decisions/transformation.py b/litellm/llms/nace/decisions/transformation.py new file mode 100644 index 00000000000..c96177bb7d5 --- /dev/null +++ b/litellm/llms/nace/decisions/transformation.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class NaceDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("NACE_API_KEY",) + api_base_env = ("NACE_API_BASE", "NACE_BASE_URL") + + def get_default_api_base(self) -> str | None: + return "https://console.nace.ai" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c3efaf168ab..1851e818ec7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71184,6 +71184,38 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "nace/drex-v1.0": { + "input_cost_per_token": 4e-08, + "litellm_provider": "nace", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-v1.5": { + "input_cost_per_token": 5e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-v2.0": { + "input_cost_per_token": 6e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-latest": { + "input_cost_per_token": 6e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 4a6a88f9a86..cb9e193fa48 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2417,6 +2417,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "Nace", + "provider_display_name": "Nace", + "litellm_provider": "nace", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://console.nace.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "nace/drex-latest" + }, { "provider": "NEBIUS", "provider_display_name": "Nebius", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 054a0a1c034..c9925580190 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4269,6 +4269,7 @@ class LlmProviders(str, Enum): DEEPINFRA = "deepinfra" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" + NACE = "nace" STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" diff --git a/litellm/utils.py b/litellm/utils.py index 3b43c570466..04c1e628288 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9103,6 +9103,8 @@ class ProviderConfigManager: return litellm.PerplexityDecisionsConfig() if provider == LlmProviders.TYPESAFE: return litellm.TypeSafeDecisionsConfig() + if provider == LlmProviders.NACE: + return litellm.NaceDecisionsConfig() if provider == LlmProviders.OPENROUTER: return litellm.OpenRouterDecisionsConfig() if provider == LlmProviders.FIREWORKS_AI: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c3efaf168ab..1851e818ec7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71184,6 +71184,38 @@ "notes": "Self-hosted decision model; infrastructure costs are paid separately" } }, + "nace/drex-v1.0": { + "input_cost_per_token": 4e-08, + "litellm_provider": "nace", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-v1.5": { + "input_cost_per_token": 5e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-v2.0": { + "input_cost_per_token": 6e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, + "nace/drex-latest": { + "input_cost_per_token": 6e-08, + "litellm_provider": "nace", + "max_input_tokens": 131072, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://console.nace.ai/docs/drex/reference/models" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0325fe1f796..ad38b064903 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2625,6 +2625,13 @@ "image_variations": true } }, + "nace": { + "display_name": "Nace (`nace`)", + "url": "https://console.nace.ai/docs/drex", + "endpoints": { + "systemone": true + } + }, "typesafe": { "display_name": "TypeSafe (`typesafe`)", "url": "https://docs.typesafe.ai/models", diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 87cd3105cc7..44dc98c9d01 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -83,6 +83,7 @@ _PROVIDERS: Final = ( "perplexity/pplx-decider-v1-27b", ), _Provider("typesafe", "typesafe/jev-1.13.0", "/v1/systemone", "jev-1.13.0", _API_KEY, False, "typesafe/jev-1.13.0"), + _Provider("nace", "nace/drex-v2.0", "/v1/systemone", "drex-v2.0", _API_KEY, False, "nace/drex-v2.0"), _Provider( "openrouter", "openrouter/typesafe/jev-1.13", @@ -135,7 +136,7 @@ _PROVIDERS: Final = ( ), ) _PERPLEXITY: Final = _PROVIDERS[0] -_OPENROUTER: Final = _PROVIDERS[2] +_OPENROUTER: Final = next(provider for provider in _PROVIDERS if provider.name == "openrouter") _OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" _UNSUPPORTED_PROVIDER_MODEL: Final = "anthropic/claude-opus-5-5" _CONNECTION_ERROR: Final = "litellm.APIConnectionError" @@ -581,14 +582,16 @@ async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gatew api_base=dead_api_base, api_key=_API_KEY, ) - with pytest.raises(litellm.APIConnectionError): + with pytest.raises(litellm.APIConnectionError) as decisions_error: litellm.decisions( model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY ) - with pytest.raises(litellm.APIConnectionError): + assert decisions_error.value.llm_provider == "openrouter" + with pytest.raises(litellm.APIConnectionError) as systemone_error: await litellm.asystemone( model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY ) + assert systemone_error.value.llm_provider == "openrouter" def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_every_supported_provider( diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 8b1a536d8e3..46b768e4924 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -120,6 +120,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( "pplx-decider-v1-27b", ), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), + ("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"), ( "openrouter", "openrouter/typesafe/jev-1.13", @@ -3345,3 +3346,137 @@ def test_dropping_every_question_answers_400_before_http(respx_mock: respx.MockR assert caught.value.status_code == 400 assert not route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize("openai_format", (False, True)) +@pytest.mark.parametrize( + ("provider", "model", "url", "key_env"), + ( + ("nace", "drex-test-alias", "https://console.nace.ai/v1/systemone", "NACE_API_KEY"), + ("openrouter", "nace/drex-test-alias", "https://openrouter.ai/api/alpha/decisions", "OPENROUTER_API_KEY"), + ( + "fireworks_ai", + "accounts/test/models/drex-test-alias", + "https://api.fireworks.ai/inference/v1/systemone", + "FIREWORKS_API_KEY", + ), + ), +) +async def test_decision_hosts_preserve_drex_model_routing_credentials_and_returned_model_pricing( + provider: str, + model: str, + url: str, + key_env: str, + openai_format: bool, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv(key_env, "host-test-key") + monkeypatch.delenv("NACE_API_BASE", raising=False) + monkeypatch.delenv("NACE_BASE_URL", raising=False) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + monkeypatch.delenv("FIREWORKS_API_BASE", raising=False) + monkeypatch.setitem( + litellm.model_cost, + f"{provider}/{model}", + { + "litellm_provider": provider, + "mode": "evaluation", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0, + }, + ) + monkeypatch.setitem( + litellm.model_cost, + f"{provider}/drex-test-resolved", + { + "litellm_provider": provider, + "mode": "evaluation", + "input_cost_per_token": 0.002, + "output_cost_per_token": 0, + }, + ) + payload: Final = {**_RESPONSE, "model": "drex-test-resolved", "request_id": "request-test"} + upstream: Final = respx_mock.post(url).respond(json=payload) + if openai_format: + response: Final = await litellm.adecisions( + model=f"{provider}/{model}", + input="damaged order", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + ) + assert isinstance(response, OpenAIDecisionResponse) + assert response.answers[0].model_dump() == {"type": "predicate", "name": "is_defect", "probability": 0.9} + else: + systemone_response: Final = await litellm.asystemone( + model=f"{provider}/{model}", + state="damaged order", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + assert systemone_response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + assert litellm.completion_cost(completion_response=systemone_response, call_type="asystemone") == pytest.approx( + _INPUT_TOKENS * 0.002 + ) + assert upstream.call_count == 1 + assert upstream.calls[0].request.headers["authorization"] == "Bearer host-test-key" + assert json.loads(upstream.calls[0].request.content) == { + "model": model, + "state": "damaged order", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + + +@pytest.mark.parametrize("base_env", ("NACE_API_BASE", "NACE_BASE_URL")) +def test_nace_sync_call_uses_environment_base_and_explicit_credentials( + base_env: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv(base_env, "https://nace.example/v1/") + monkeypatch.setenv("NACE_API_KEY", "environment-key") + upstream: Final = respx_mock.post("https://nace.example/v1/systemone").respond(json=_RESPONSE) + response: Final = litellm.systemone( + model="nace/drex-v2.0", + state={"message": "damaged order"}, + api_key="explicit-key", + questions={"is_defect": {"type": "noul", "criteria": {"true": "damaged", "false": "intact"}}}, + ) + assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + assert upstream.calls[0].request.headers["authorization"] == "Bearer explicit-key" + assert json.loads(upstream.calls[0].request.content) == { + "model": "drex-v2.0", + "state": {"message": "damaged order"}, + "questions": {"is_defect": {"type": "noul", "criteria": {"true": "damaged", "false": "intact"}}}, + } + + +@pytest.mark.parametrize( + ("status", "exception"), + ( + (401, litellm.AuthenticationError), + (402, litellm.PaymentRequiredError), + (422, litellm.BadRequestError), + (429, litellm.RateLimitError), + (529, litellm.APIError), + ), +) +def test_nace_upstream_failures_preserve_status_and_message( + status: int, + exception: type[Exception], + respx_mock: respx.MockRouter, +) -> None: + upstream: Final = respx_mock.post("https://console.nace.ai/v1/systemone").respond( + status, + json={"error": {"message": "decision rejected"}, "request_id": "request-test"}, + headers={"retry-after": "1"}, + ) + with pytest.raises(exception, match="decision rejected") as error: + litellm.systemone( + model="nace/drex-v2.0", + state="text", + api_key="test-key", + questions={"a": {"type": "noul", "instructions": "Is it urgent?"}}, + num_retries=0, + ) + assert getattr(error.value, "status_code") == status + assert upstream.call_count == 1 diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index d0219ea2963..924b192a00e 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -1,9 +1,12 @@ +import json from collections.abc import Mapping from typing import Final +import httpx import pytest from pydantic import TypeAdapter, ValidationError +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( ir_to_systemone_request, ir_to_systemone_response, @@ -15,10 +18,13 @@ from litellm.types.decisions import ( MAX_DECISION_QUESTIONS, DecisionsIRRefusal, DecisionsIRRequest, + DecisionsIRScoreAnswer, DecisionsRequestBody, OpenAIDecisionRequestBody, UnsupportedDecisionsRequest, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager _SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) _OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) @@ -264,3 +270,66 @@ def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None: response: Final = ir_to_systemone_response(parse_systemone_response(payload, ir), ir) assert response.model_dump(mode="json") == payload + + +@pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe")) +@pytest.mark.parametrize("legend", (["low", "high"], {"0": "low", "1": "high"})) +def test_score_legends_preserve_labels_probabilities_and_metadata(provider: str, legend: object) -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("drex-v2.0", LlmProviders(provider)) + assert config is not None + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate( + { + "state": {"text": "The order arrived damaged"}, + "questions": {"severity": {"type": "score", "criteria": ["low", "high"]}}, + } + ) + ) + payload: Final = { + "model": "drex-v2.0", + "answers": { + "severity": { + "type": "score", + "score": 0.75, + "confidence": 0.8, + "legend": legend, + "probabilities": {"0": 0.25, "1": 0.75}, + } + }, + "usage": {"input_tokens": 20, "output_tokens": 30}, + "request_id": "test-request", + "evaluation_time_ms": 8.5, + } + original: Final = json.dumps(payload) + response: Final = config.parse_response(payload, request) + answer: Final = response.answers[0] + assert isinstance(answer, DecisionsIRScoreAnswer) + assert (answer.score, answer.confidence) == (0.75, 0.8) + assert tuple((item.value, item.label, item.probability) for item in answer.probabilities) == ( + (0, "low", 0.25), + (1, "high", 0.75), + ) + assert response.model == payload["model"] + assert (response.usage.input_tokens, response.usage.output_tokens) == (20, 30) + assert response.extra == {"request_id": "test-request", "evaluation_time_ms": 8.5} + assert json.dumps(payload) == original + + +@pytest.mark.parametrize( + "payload", ([], {"answers": []}, {"answers": {"a": None}}, {"answers": {"a": {"type": "score", "legend": [1]}}}) +) +@pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe")) +def test_invalid_provider_response_is_an_api_error(payload: object, provider: str) -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("drex-v2.0", LlmProviders(provider)) + assert config is not None + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate( + { + "state": "text", + "questions": {"a": {"type": "score", "criteria": ["low", "high"]}}, + } + ) + ) + with pytest.raises(BaseLLMException, match="unexpected response") as error: + config.transform_decisions_response("drex-v2.0", provider, httpx.Response(200, json=payload), request) + assert error.value.status_code == 500 diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 9e8910e44de..d5bacd9fd3c 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -437,15 +437,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: return entry -def test_typesafe_provider_fields(): - typesafe: Final = _decisions_provider_entry("TypeSafe") +@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE))) +def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None: + entry: Final = _decisions_provider_entry(display_name) - assert typesafe.provider_display_name == "TypeSafe" - assert typesafe.litellm_provider == LlmProviders.TYPESAFE.value - assert typesafe.default_model_placeholder is not None - assert typesafe.default_model_placeholder.startswith("typesafe/") + assert entry.provider_display_name == display_name + assert entry.litellm_provider == provider.value + assert entry.default_model_placeholder is not None + assert entry.default_model_placeholder.startswith(f"{provider.value}/") - fields_by_key: Final = {f.key: f for f in typesafe.credential_fields} + fields_by_key: Final = {f.key: f for f in entry.credential_fields} assert fields_by_key["api_key"].required is True assert fields_by_key["api_key"].field_type == "password" diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 4d0d590521f..0ebd2cdff17 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -214,6 +214,7 @@ describe("provider_info_helpers", () => { Providers.LEMONADE, Providers.LLAMAFILE, Providers.MARITALK, + Providers.Nace, Providers.NLP_CLOUD, Providers.NSCALE, Providers.OVHCLOUD, @@ -517,6 +518,16 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("lists Nace decision models and resolves the provider name and model placeholder", () => { + const modelMap = { + "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + }; + expect(getProviderModels(Providers.Nace, modelMap)).toEqual(["nace/drex-latest"]); + expect(getProviderLogoAndName("nace").displayName).toBe("Nace"); + expect(getPlaceholder("Nace")).toBe("nace/drex-latest"); + }); + it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { const modelMap = { "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 3925118a881..5a163bce3b6 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -181,6 +181,7 @@ export enum Providers { TOPAZ = "Topaz", Triton = "Triton", TypeSafe = "TypeSafe", + Nace = "Nace", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -307,6 +308,7 @@ export const provider_map: Record = { TOPAZ: "topaz", Triton: "triton", TypeSafe: "typesafe", + Nace: "nace", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -485,6 +487,7 @@ const providerPlaceholderMap: Partial> = { [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19", [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", + [Providers.Nace]: "nace/drex-latest", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/",