scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(nace): list Nace in the classifier UI and follow the test conventions

Browse source 
The dashboard's classifier config widened its provider type to nace without a
description entry, so the type check failed, and the Auto Router's inline
provider picker did not offer Nace. Both now include it, with tests

The header test reads the request respx captured instead of appending to a
list, and the drex-max catalog tests move into the existing catalog test file
This commit is contained in:
Saruar 2026-11-06 14:37:01 +05:00
parent 89fe81a799
commit 9d220fe4fd
8 changed files with 85 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
tests/unit/proxy/management_helpers/test_auto_router_permissions.py
View file

@ -148,7 +148,10 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N
({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"),
({"provider": "databricks", "model": "ai_decide", "api_base": "https://collector.invalid"}, "opensource_classifier_config"),
({"provider": "databricks", "model": "ai_decide", "api_key": "dapi-member"}, "api_key"),
({"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, "opensource_classifier_config"),
(
{"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"},
"opensource_classifier_config",
),
({"provider": "nace", "model": "drex-max", "api_key": "sk-member"}, "api_key"),
],
)

21
tests/unit/router_strategy/complexity_router/test_jev_classifier.py
View file

@ -1495,11 +1495,8 @@ async def test_nace_routes_through_its_console_with_its_own_key_and_accounts_the
@pytest.mark.asyncio
@pytest.mark.parametrize("provider", ["nace", "typesafe"])
async def test_only_the_nace_classifier_sends_the_client_header(provider: ClassifierProvider) -> None:
seen: Final[list[httpx.Headers]] = []
def respond(http_request: httpx.Request) -> httpx.Response:
seen.append(http_request.headers)
return httpx.Response(
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://classifier.test/v1/systemone").respond(
200,
json={
"model": "drex-max",
@ -1513,12 +1510,12 @@ async def test_only_the_nace_classifier_sends_the_client_header(provider: Classi
},
},
)
handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(upstream.async_handler))
client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider)
await client.evaluate(
build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0
)
handler: Final = AsyncHTTPHandler()
handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond))
client: Final = HttpJevClassifierClient("secret", "https://classifier.test", handler, provider)
await client.evaluate(
build_jev_request("Hello", None, "drex-max", DEFAULT_JEV_INSTRUCTIONS, {"SIMPLE": "facts"}), 1.0
assert route.calls.last.request.headers.get("x-nace-client") == (
f"litellm/{litellm_version}" if provider == "nace" else None
)
assert seen[0].get("x-nace-client") == (f"litellm/{litellm_version}" if provider == "nace" else None)

42
tests/unit/test_model_prices_and_context_window.py
View file

@ -911,3 +911,45 @@ def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblin
def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None:
assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True
assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True
def _nace_rows(path: Path) -> Mapping[str, Mapping[str, JsonValue]]:
prices: Final = TypeAdapter(dict[str, dict[str, JsonValue]]).validate_json(path.read_bytes())
return {key: row for key, row in prices.items() if key.startswith("nace/")}
def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens(local_model_cost_map: None) -> None:
"""Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
for model in ("nace/drex-max", "nace/drex-max-v1.1"):
row = litellm.model_cost[model]
assert row["input_cost_per_token"] == 3e-08
assert row["output_cost_per_token"] == 0.0
assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144)
assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08
def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it(local_model_cost_map: None) -> None:
"""drex-v1.0 is retired and answered by drex-max at drex-max's price and limits
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
drex_v1: Final = litellm.model_cost["nace/drex-v1.0"]
drex_max: Final = litellm.model_cost["nace/drex-max"]
assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"]
assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"]
def test_drex_max_versions_are_the_nace_models_that_read_images(local_model_cost_map: None) -> None:
"""Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex
models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31)."""
vision_rows: Final = tuple(key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision"))
assert litellm.supports_vision("nace/drex-max") is True
assert litellm.supports_vision("nace/drex-max-v1.1") is True
assert litellm.supports_vision("nace/drex-v1.0") is True
assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"]
def test_nace_backup_rows_match_the_main_catalog() -> None:
assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys()
assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH)

55
tests/unit/test_nace_model_metadata.py
View file

@ -1,55 +0,0 @@
import json
from pathlib import Path
from typing import Final
import pytest
import litellm
pytestmark = pytest.mark.usefixtures("local_model_cost_map")
REPO_ROOT: Final = Path(__file__).resolve().parents[2]
PRICES_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json"
BACKUP_PRICES_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
def _nace_rows(path: Path) -> dict[str, object]:
prices: Final = json.loads(path.read_text())
return {key: row for key, row in prices.items() if key.startswith("nace/")}
def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens() -> None:
"""Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
for model in ("nace/drex-max", "nace/drex-max-v1.1"):
row = litellm.model_cost[model]
assert row["input_cost_per_token"] == 3e-08
assert row["output_cost_per_token"] == 0.0
assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144)
assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08
def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it() -> None:
"""drex-v1.0 is retired and answered by drex-max at drex-max's price and limits
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
drex_v1: Final = litellm.model_cost["nace/drex-v1.0"]
drex_max: Final = litellm.model_cost["nace/drex-max"]
assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"]
assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"]
def test_drex_max_versions_are_the_nace_models_that_read_images() -> None:
"""Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex
models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31)."""
vision_rows: Final = [key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")]
assert litellm.supports_vision("nace/drex-max") is True
assert litellm.supports_vision("nace/drex-max-v1.1") is True
assert litellm.supports_vision("nace/drex-v1.0") is True
assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"]
def test_nace_backup_rows_match_the_main_catalog() -> None:
assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys()
assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH)

22
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx
View file

@ -1,6 +1,7 @@
import React, { useState } from "react";
import userEvent from "@testing-library/user-event";
import { describe, expect, it, vi } from "vitest";
import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils";
import { chooseSelectOption, fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils";
import { selectAutoRouterOption } from "../../../tests/autoRouterSetup";
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
import ClassificationMethodConfig from "./ClassificationMethodConfig";
@ -54,6 +55,25 @@ function Form({
}
describe("Auto-router classifier selection", () => {
it("selects Nace with its default model and preserves classifier settings", async () => {
const onChange = vi.fn();
const jevConfig = { provider: "jev", model: "jev-latest", timeout_ms: 4200 } as const;
renderWithProviders(
<AutoRouterClassifierTabs
value={{ ...initial, classifier_type: "jev", jev_classifier_config: jevConfig }}
onChange={onChange}
>
Existing settings
</AutoRouterClassifierTabs>,
);
await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), "Nace");
expect(onChange).toHaveBeenCalledWith({
...initial,
classifier_type: "jev",
jev_classifier_config: { ...jevConfig, provider: "nace", model: "drex-max" },
});
});
it.each([false, true])("applies the v2 allowance to chain selection, owned: %s", (owned) => {
const onChange = vi.fn();
renderWithProviders(

2
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
View file

@ -233,6 +233,7 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({
{ value: "laya", label: "Laya" },
{ value: "bespoke", label: "Bespoke Nimble" },
{ value: "databricks", label: "Databricks" },
{ value: "nace", label: "Nace" },
]}
value={normalizeJevClassifierConfig(value.jev_classifier_config).provider}
onValueChange={changeProvider}
@ -245,6 +246,7 @@ const AutoRouterClassifierTabs: React.FC<AutoRouterClassifierTabsProps> = ({
<SelectItem value="laya">Laya</SelectItem>
<SelectItem value="bespoke">Bespoke Nimble</SelectItem>
<SelectItem value="databricks">Databricks</SelectItem>
<SelectItem value="nace">Nace</SelectItem>
</SelectContent>
</Select>
</div>

7
ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx
View file

@ -102,10 +102,11 @@ describe("JEV classifier editor", () => {
["laya", "Laya", "multilingual"],
["bespoke", "Bespoke Nimble", "bespokelabs/Bespoke-Nimble-9B"],
["databricks", "Databricks", "databricks-openjev-qwen35-4b"],
["nace", "Nace", "drex-max"],
] as const)(
"preserves %s, custom tiers and context through save, reload and probe",
async (provider, label, model) => {
const typesTheModel = provider === "jev" || provider === "databricks";
const typesTheModel = provider === "jev" || provider === "databricks" || provider === "nace";
renderWithProviders(<Form />);
expect(screen.getByLabelText("Judge model")).toBeInTheDocument();
expect(screen.getByText("Reasoning Effort")).toBeInTheDocument();
@ -126,6 +127,10 @@ describe("JEV classifier editor", () => {
await chooseSelectOption(userEvent, screen.getByRole("combobox", { name: "Decision model" }), label);
if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest");
if (provider === "databricks") expect(screen.getByLabelText("Classifier Model")).toHaveValue("ai_decide");
if (provider === "nace") {
expect(screen.getByLabelText("Classifier Model")).toHaveValue("drex-max");
expect(screen.getByText(/Uses Nace/)).toHaveTextContent("NACE_API_KEY");
}
if (typesTheModel) {
fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: model } });
} else {

1
ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx
View file

@ -14,6 +14,7 @@ const providerDescriptions = {
laya: "Uses Laya with your configured tiers. Set LAYA_API_BASE on the gateway to connect your Laya server.",
bespoke:
"Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.",
nace: "Uses Nace Drex with your configured tiers. Set NACE_API_KEY on the gateway.",
databricks:
"Uses the Databricks ai_decide AI Function, or a decision serving endpoint you name, with your configured tiers. Set DATABRICKS_API_BASE to https://<workspace-host>/serving-endpoints and DATABRICKS_API_KEY on the gateway.",
};