diff --git a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py index 9cda0ea4a7c..6b636594e27 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py @@ -148,7 +148,10 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N ({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"), ({"provider": "databricks", "model": "ai_decide", "api_base": "https://collector.invalid"}, "opensource_classifier_config"), ({"provider": "databricks", "model": "ai_decide", "api_key": "dapi-member"}, "api_key"), - ({"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, "opensource_classifier_config"), + ( + {"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, + "opensource_classifier_config", + ), ({"provider": "nace", "model": "drex-max", "api_key": "sk-member"}, "api_key"), ], ) diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 62caa376f11..144e107f52d 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -1495,11 +1495,8 @@ async def test_nace_routes_through_its_console_with_its_own_key_and_accounts_the @pytest.mark.asyncio @pytest.mark.parametrize("provider", ["nace", "typesafe"]) async def test_only_the_nace_classifier_sends_the_client_header(provider: ClassifierProvider) -> None: - seen: Final[list[httpx.Headers]] = [] - - def respond(http_request: httpx.Request) -> httpx.Response: - seen.append(http_request.headers) - return httpx.Response( + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://classifier.test/v1/systemone").respond( 200, json={ "model": "drex-max", @@ -1513,12 +1510,12 @@ async def test_only_the_nace_classifier_sends_the_client_header(provider: Classi }, }, ) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(upstream.async_handler)) + client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider) + await client.evaluate( + build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0 + ) - handler: Final = AsyncHTTPHandler() - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) - client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider) - await client.evaluate( - build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0 + assert route.calls.last.request.headers.get("x-nace-client") == ( + f"litellm/{litellm_version}" if provider == "nace" else None ) - - assert seen[0].get("x-nace-client") == (f"litellm/{litellm_version}" if provider == "nace" else None) diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index e4a2beb3a86..9d2a5e6e7d6 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -911,3 +911,45 @@ def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblin def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True + + +def _nace_rows(path: Path) -> Mapping[str, Mapping[str, JsonValue]]: + prices: Final = TypeAdapter(dict[str, dict[str, JsonValue]]).validate_json(path.read_bytes()) + return {key: row for key, row in prices.items() if key.startswith("nace/")} + + +def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens(local_model_cost_map: None) -> None: + """Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed + (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" + for model in ("nace/drex-max", "nace/drex-max-v1.1"): + row = litellm.model_cost[model] + assert row["input_cost_per_token"] == 3e-08 + assert row["output_cost_per_token"] == 0.0 + assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144) + assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08 + + +def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it(local_model_cost_map: None) -> None: + """drex-v1.0 is retired and answered by drex-max at drex-max's price and limits + (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" + drex_v1: Final = litellm.model_cost["nace/drex-v1.0"] + drex_max: Final = litellm.model_cost["nace/drex-max"] + + assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"] + assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"] + + +def test_drex_max_versions_are_the_nace_models_that_read_images(local_model_cost_map: None) -> None: + """Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex + models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31).""" + vision_rows: Final = tuple(key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")) + + assert litellm.supports_vision("nace/drex-max") is True + assert litellm.supports_vision("nace/drex-max-v1.1") is True + assert litellm.supports_vision("nace/drex-v1.0") is True + assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"] + + +def test_nace_backup_rows_match_the_main_catalog() -> None: + assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys() + assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH) diff --git a/tests/unit/test_nace_model_metadata.py b/tests/unit/test_nace_model_metadata.py deleted file mode 100644 index e61a69c645f..00000000000 --- a/tests/unit/test_nace_model_metadata.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -from pathlib import Path -from typing import Final - -import pytest - -import litellm - -pytestmark = pytest.mark.usefixtures("local_model_cost_map") - -REPO_ROOT: Final = Path(__file__).resolve().parents[2] -PRICES_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json" -BACKUP_PRICES_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" - - -def _nace_rows(path: Path) -> dict[str, object]: - prices: Final = json.loads(path.read_text()) - return {key: row for key, row in prices.items() if key.startswith("nace/")} - - -def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens() -> None: - """Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed - (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" - for model in ("nace/drex-max", "nace/drex-max-v1.1"): - row = litellm.model_cost[model] - assert row["input_cost_per_token"] == 3e-08 - assert row["output_cost_per_token"] == 0.0 - assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144) - assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08 - - -def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it() -> None: - """drex-v1.0 is retired and answered by drex-max at drex-max's price and limits - (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" - drex_v1: Final = litellm.model_cost["nace/drex-v1.0"] - drex_max: Final = litellm.model_cost["nace/drex-max"] - - assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"] - assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"] - - -def test_drex_max_versions_are_the_nace_models_that_read_images() -> None: - """Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex - models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31).""" - vision_rows: Final = [key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")] - - assert litellm.supports_vision("nace/drex-max") is True - assert litellm.supports_vision("nace/drex-max-v1.1") is True - assert litellm.supports_vision("nace/drex-v1.0") is True - assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"] - - -def test_nace_backup_rows_match_the_main_catalog() -> None: - assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys() - assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH) diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index 3c5d555cff8..08b0599d36d 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -1,6 +1,7 @@ import React, { useState } from "react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; -import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; +import { chooseSelectOption, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import { selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; @@ -54,6 +55,25 @@ function Form({ } describe("Auto-router classifier selection", () => { + it("selects Nace with its default model and preserves classifier settings", async () => { + const onChange = vi.fn(); + const jevConfig = { provider: "jev", model: "jev-latest", timeout_ms: 4200 } as const; + renderWithProviders( + + Existing settings + , + ); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Nace"); + expect(onChange).toHaveBeenCalledWith({ + ...initial, + classifier_type: "jev", + jev_classifier_config: { ...jevConfig, provider: "nace", model: "drex-max" }, + }); + }); + it.each([false, true])("applies the v2 allowance to chain selection, owned: %s", (owned) => { const onChange = vi.fn(); renderWithProviders( diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 509329bb948..088a5faefe3 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -233,6 +233,7 @@ const AutoRouterClassifierTabs: React.FC = ({ { value: "laya", label: "Laya" }, { value: "bespoke", label: "Bespoke Nimble" }, { value: "databricks", label: "Databricks" }, + { value: "nace", label: "Nace" }, ]} value={normalizeJevClassifierConfig(value.jev_classifier_config).provider} onValueChange={changeProvider} @@ -245,6 +246,7 @@ const AutoRouterClassifierTabs: React.FC = ({ Laya Bespoke Nimble Databricks + Nace diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index f829a2f2e41..67b72785025 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -102,10 +102,11 @@ describe("JEV classifier editor", () => { ["laya", "Laya", "multilingual"], ["bespoke", "Bespoke Nimble", "bespokelabs/Bespoke-Nimble-9B"], ["databricks", "Databricks", "databricks-openjev-qwen35-4b"], + ["nace", "Nace", "drex-max"], ] as const)( "preserves %s, custom tiers and context through save, reload and probe", async (provider, label, model) => { - const typesTheModel = provider === "jev" || provider === "databricks"; + const typesTheModel = provider === "jev" || provider === "databricks" || provider === "nace"; renderWithProviders(
); expect(screen.getByLabelText("Judge model")).toBeInTheDocument(); expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); @@ -126,6 +127,10 @@ describe("JEV classifier editor", () => { await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), label); if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest"); if (provider === "databricks") expect(screen.getByLabelText("Classifier Model")).toHaveValue("ai_decide"); + if (provider === "nace") { + expect(screen.getByLabelText("Classifier Model")).toHaveValue("drex-max"); + expect(screen.getByText(/Uses Nace/)).toHaveTextContent("NACE_API_KEY"); + } if (typesTheModel) { fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: model } }); } else { diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 4b3cfa21f9d..c425d38a138 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -14,6 +14,7 @@ const providerDescriptions = { laya: "Uses Laya with your configured tiers. Set LAYA_API_BASE on the gateway to connect your Laya server.", bespoke: "Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.", + nace: "Uses Nace Drex with your configured tiers. Set NACE_API_KEY on the gateway.", databricks: "Uses the Databricks ai_decide AI Function, or a decision serving endpoint you name, with your configured tiers. Set DATABRICKS_API_BASE to https:///serving-endpoints and DATABRICKS_API_KEY on the gateway.", };