From 9d220fe4fdf643fd994e5b47d7e4bb54007faee8 Mon Sep 17 00:00:00 2001 From: Saruar Date: Fri, 6 Nov 2026 14:37:01 +0500 Subject: [PATCH] fix(nace): list Nace in the classifier UI and follow the test conventions The dashboard's classifier config widened its provider type to nace without a description entry, so the type check failed, and the Auto Router's inline provider picker did not offer Nace. Both now include it, with tests The header test reads the request respx captured instead of appending to a list, and the drex-max catalog tests move into the existing catalog test file --- .../test_auto_router_permissions.py | 5 +- .../complexity_router/test_jev_classifier.py | 21 +++---- .../test_model_prices_and_context_window.py | 42 ++++++++++++++ tests/unit/test_nace_model_metadata.py | 55 ------------------- ...oRouterClassifierTabs.integration.test.tsx | 22 +++++++- .../add_model/AutoRouterClassifierTabs.tsx | 2 + .../JevClassifierConfig.integration.test.tsx | 7 ++- .../add_model/JevClassifierConfig.tsx | 1 + 8 files changed, 85 insertions(+), 70 deletions(-) delete mode 100644 tests/unit/test_nace_model_metadata.py diff --git a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py index 9cda0ea4a7c..6b636594e27 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py @@ -148,7 +148,10 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N ({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"), ({"provider": "databricks", "model": "ai_decide", "api_base": "https://collector.invalid"}, "opensource_classifier_config"), ({"provider": "databricks", "model": "ai_decide", "api_key": "dapi-member"}, "api_key"), - ({"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, "opensource_classifier_config"), + ( + {"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, + "opensource_classifier_config", + ), ({"provider": "nace", "model": "drex-max", "api_key": "sk-member"}, "api_key"), ], ) diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 62caa376f11..144e107f52d 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -1495,11 +1495,8 @@ async def test_nace_routes_through_its_console_with_its_own_key_and_accounts_the @pytest.mark.asyncio @pytest.mark.parametrize("provider", ["nace", "typesafe"]) async def test_only_the_nace_classifier_sends_the_client_header(provider: ClassifierProvider) -> None: - seen: Final[list[httpx.Headers]] = [] - - def respond(http_request: httpx.Request) -> httpx.Response: - seen.append(http_request.headers) - return httpx.Response( + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://classifier.test/v1/systemone").respond( 200, json={ "model": "drex-max", @@ -1513,12 +1510,12 @@ async def test_only_the_nace_classifier_sends_the_client_header(provider: Classi }, }, ) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(upstream.async_handler)) + client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider) + await client.evaluate( + build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0 + ) - handler: Final = AsyncHTTPHandler() - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) - client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider) - await client.evaluate( - build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0 + assert route.calls.last.request.headers.get("x-nace-client") == ( + f"litellm/{litellm_version}" if provider == "nace" else None ) - - assert seen[0].get("x-nace-client") == (f"litellm/{litellm_version}" if provider == "nace" else None) diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index e4a2beb3a86..9d2a5e6e7d6 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -911,3 +911,45 @@ def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblin def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True + + +def _nace_rows(path: Path) -> Mapping[str, Mapping[str, JsonValue]]: + prices: Final = TypeAdapter(dict[str, dict[str, JsonValue]]).validate_json(path.read_bytes()) + return {key: row for key, row in prices.items() if key.startswith("nace/")} + + +def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens(local_model_cost_map: None) -> None: + """Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed + (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" + for model in ("nace/drex-max", "nace/drex-max-v1.1"): + row = litellm.model_cost[model] + assert row["input_cost_per_token"] == 3e-08 + assert row["output_cost_per_token"] == 0.0 + assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144) + assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08 + + +def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it(local_model_cost_map: None) -> None: + """drex-v1.0 is retired and answered by drex-max at drex-max's price and limits + (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" + drex_v1: Final = litellm.model_cost["nace/drex-v1.0"] + drex_max: Final = litellm.model_cost["nace/drex-max"] + + assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"] + assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"] + + +def test_drex_max_versions_are_the_nace_models_that_read_images(local_model_cost_map: None) -> None: + """Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex + models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31).""" + vision_rows: Final = tuple(key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")) + + assert litellm.supports_vision("nace/drex-max") is True + assert litellm.supports_vision("nace/drex-max-v1.1") is True + assert litellm.supports_vision("nace/drex-v1.0") is True + assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"] + + +def test_nace_backup_rows_match_the_main_catalog() -> None: + assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys() + assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH) diff --git a/tests/unit/test_nace_model_metadata.py b/tests/unit/test_nace_model_metadata.py deleted file mode 100644 index e61a69c645f..00000000000 --- a/tests/unit/test_nace_model_metadata.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -from pathlib import Path -from typing import Final - -import pytest - -import litellm - -pytestmark = pytest.mark.usefixtures("local_model_cost_map") - -REPO_ROOT: Final = Path(__file__).resolve().parents[2] -PRICES_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json" -BACKUP_PRICES_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" - - -def _nace_rows(path: Path) -> dict[str, object]: - prices: Final = json.loads(path.read_text()) - return {key: row for key, row in prices.items() if key.startswith("nace/")} - - -def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens() -> None: - """Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed - (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" - for model in ("nace/drex-max", "nace/drex-max-v1.1"): - row = litellm.model_cost[model] - assert row["input_cost_per_token"] == 3e-08 - assert row["output_cost_per_token"] == 0.0 - assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144) - assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08 - - -def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it() -> None: - """drex-v1.0 is retired and answered by drex-max at drex-max's price and limits - (console.nace.ai/docs/drex/reference/models, read 2026-10-30).""" - drex_v1: Final = litellm.model_cost["nace/drex-v1.0"] - drex_max: Final = litellm.model_cost["nace/drex-max"] - - assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"] - assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"] - - -def test_drex_max_versions_are_the_nace_models_that_read_images() -> None: - """Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex - models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31).""" - vision_rows: Final = [key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")] - - assert litellm.supports_vision("nace/drex-max") is True - assert litellm.supports_vision("nace/drex-max-v1.1") is True - assert litellm.supports_vision("nace/drex-v1.0") is True - assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"] - - -def test_nace_backup_rows_match_the_main_catalog() -> None: - assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys() - assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH) diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index 3c5d555cff8..08b0599d36d 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -1,6 +1,7 @@ import React, { useState } from "react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; -import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; +import { chooseSelectOption, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import { selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; @@ -54,6 +55,25 @@ function Form({ } describe("Auto-router classifier selection", () => { + it("selects Nace with its default model and preserves classifier settings", async () => { + const onChange = vi.fn(); + const jevConfig = { provider: "jev", model: "jev-latest", timeout_ms: 4200 } as const; + renderWithProviders( + + Existing settings + , + ); + await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Nace"); + expect(onChange).toHaveBeenCalledWith({ + ...initial, + classifier_type: "jev", + jev_classifier_config: { ...jevConfig, provider: "nace", model: "drex-max" }, + }); + }); + it.each([false, true])("applies the v2 allowance to chain selection, owned: %s", (owned) => { const onChange = vi.fn(); renderWithProviders( diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 509329bb948..088a5faefe3 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -233,6 +233,7 @@ const AutoRouterClassifierTabs: React.FC = ({ { value: "laya", label: "Laya" }, { value: "bespoke", label: "Bespoke Nimble" }, { value: "databricks", label: "Databricks" }, + { value: "nace", label: "Nace" }, ]} value={normalizeJevClassifierConfig(value.jev_classifier_config).provider} onValueChange={changeProvider} @@ -245,6 +246,7 @@ const AutoRouterClassifierTabs: React.FC = ({ Laya Bespoke Nimble Databricks + Nace diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index f829a2f2e41..67b72785025 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -102,10 +102,11 @@ describe("JEV classifier editor", () => { ["laya", "Laya", "multilingual"], ["bespoke", "Bespoke Nimble", "bespokelabs/Bespoke-Nimble-9B"], ["databricks", "Databricks", "databricks-openjev-qwen35-4b"], + ["nace", "Nace", "drex-max"], ] as const)( "preserves %s, custom tiers and context through save, reload and probe", async (provider, label, model) => { - const typesTheModel = provider === "jev" || provider === "databricks"; + const typesTheModel = provider === "jev" || provider === "databricks" || provider === "nace"; renderWithProviders(
); expect(screen.getByLabelText("Judge model")).toBeInTheDocument(); expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); @@ -126,6 +127,10 @@ describe("JEV classifier editor", () => { await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), label); if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest"); if (provider === "databricks") expect(screen.getByLabelText("Classifier Model")).toHaveValue("ai_decide"); + if (provider === "nace") { + expect(screen.getByLabelText("Classifier Model")).toHaveValue("drex-max"); + expect(screen.getByText(/Uses Nace/)).toHaveTextContent("NACE_API_KEY"); + } if (typesTheModel) { fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: model } }); } else { diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 4b3cfa21f9d..c425d38a138 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -14,6 +14,7 @@ const providerDescriptions = { laya: "Uses Laya with your configured tiers. Set LAYA_API_BASE on the gateway to connect your Laya server.", bespoke: "Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.", + nace: "Uses Nace Drex with your configured tiers. Set NACE_API_KEY on the gateway.", databricks: "Uses the Databricks ai_decide AI Function, or a decision serving endpoint you name, with your configured tiers. Set DATABRICKS_API_BASE to https:///serving-endpoints and DATABRICKS_API_KEY on the gateway.", };