mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-07 07:58:09 +00:00
feat(nace): add drex-max to the catalog, image support, and the complexity router
Nace released drex-max, the Drex decision model that also reads images, and the existing nace provider could not use it: the catalog had no row, so cost tracking and the dashboard did not know the model, and image input was refused before the request left the gateway. Add the nace/drex-max catalog row to both cost map files with supports_vision and the per-question token limit, let the nace decisions provider forward images as the top-level images array Nace reads (inline images as data URLs, remote URLs fetched and inlined, at most 8 per request), and accept provider nace in the Auto Router classifier config so a tier can be chosen by drex-max with NACE_API_KEY. Also: resolve os.environ/ references for the classifier's nace api_key and api_base once, and refuse an unset one instead of falling back to NACE_API_KEY; price drex-max at $0.03 per 1M input tokens and add drex-max-v1.1; mark the retired drex-v1.0, now served by drex-max, as vision-capable; and accept provider nace in the dashboard's classifier config (default drex-max), the generated API types and the member auto-router permission check.
This commit is contained in:
parent
b71491c6c9
commit
3336461492
14 changed files with 440 additions and 15 deletions
|
|
@ -1,9 +1,47 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
from litellm.types.decisions import (
|
||||
DecisionsBase64Image,
|
||||
DecisionsImage,
|
||||
DecisionsIRRequest,
|
||||
DecisionsIRState,
|
||||
UnsupportedDecisionsRequest,
|
||||
)
|
||||
|
||||
|
||||
def _data_url(image: DecisionsImage) -> DecisionsImage:
|
||||
if isinstance(image, DecisionsBase64Image):
|
||||
return f"data:{image.content_type};base64,{image.base64}"
|
||||
return image
|
||||
|
||||
|
||||
def _with_images_as_data_urls(request: DecisionsIRRequest) -> DecisionsIRRequest:
|
||||
if not isinstance(request.input, DecisionsIRState):
|
||||
return request
|
||||
return replace(request, input=replace(request.input, images=tuple(map(_data_url, request.input.images))))
|
||||
|
||||
|
||||
class NaceDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("NACE_API_KEY",)
|
||||
api_base_env = ("NACE_API_BASE", "NACE_BASE_URL")
|
||||
supports_images = True
|
||||
max_images = 8
|
||||
image_media_types = frozenset({"image/png", "image/jpeg", "image/jpg", "image/webp"})
|
||||
inlines_remote_images = True
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://console.nace.ai"
|
||||
|
||||
def transform_decisions_request(
|
||||
self,
|
||||
model: str,
|
||||
request: DecisionsIRRequest,
|
||||
custom_llm_provider: str,
|
||||
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
|
||||
return super().transform_decisions_request(model, _with_images_as_data_urls(request), custom_llm_provider)
|
||||
|
||||
|
||||
NACE_DECISIONS_CONFIG: Final = NaceDecisionsConfig()
|
||||
|
|
|
|||
|
|
@ -72513,12 +72513,13 @@
|
|||
}
|
||||
},
|
||||
"nace/drex-v1.0": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 32768,
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models"
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models",
|
||||
"supports_vision": true
|
||||
},
|
||||
"nace/drex-v1.5": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
|
|
@ -72544,6 +72545,24 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models"
|
||||
},
|
||||
"nace/drex-max": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/guides/drex-max",
|
||||
"supports_vision": true
|
||||
},
|
||||
"nace/drex-max-v1.1": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/guides/drex-max",
|
||||
"supports_vision": true
|
||||
},
|
||||
"typesafe/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel):
|
|||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
deployment_name: str | None = None
|
||||
provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev"
|
||||
provider: Literal["jev", "laya", "bespoke", "databricks", "nace"] = "jev"
|
||||
model: str
|
||||
api_key: None = None
|
||||
api_base: None = None
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
|
|||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG
|
||||
from litellm.llms.nace.decisions.transformation import NACE_DECISIONS_CONFIG
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
|
|
@ -1432,6 +1433,29 @@ class ComplexityRouter(CustomLogger):
|
|||
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
|
||||
provider="databricks",
|
||||
)
|
||||
if config.provider == "nace":
|
||||
resolved: Final = {
|
||||
name: get_secret_str(value) if value is not None and value.startswith("os.environ/") else value
|
||||
for name, value in (("api_base", config.api_base), ("api_key", config.api_key))
|
||||
}
|
||||
for name, value in (("api_base", config.api_base), ("api_key", config.api_key)):
|
||||
if value is not None and value.startswith("os.environ/") and not (resolved[name] or "").strip():
|
||||
return _JevConnectionError(
|
||||
f"opensource_classifier_config.{name} references {value!r}, which is not set"
|
||||
)
|
||||
nace_api_base, nace_api_key = NACE_DECISIONS_CONFIG.resolve_connection(
|
||||
resolved["api_base"], resolved["api_key"]
|
||||
)
|
||||
if nace_api_base is None or not nace_api_key or not nace_api_key.strip():
|
||||
return _JevConnectionError(
|
||||
"opensource_classifier_config.api_key or NACE_API_KEY is required for provider 'nace'"
|
||||
)
|
||||
return HttpJevClassifierClient(
|
||||
api_key=nace_api_key,
|
||||
api_base=nace_api_base,
|
||||
http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint),
|
||||
provider="nace",
|
||||
)
|
||||
api_key: Final = (
|
||||
get_secret_str(config.api_key)
|
||||
if config.api_key is not None and config.api_key.startswith("os.environ/")
|
||||
|
|
|
|||
|
|
@ -714,6 +714,7 @@ _ENVIRONMENT_KEY_SCOPE: Final = MappingProxyType(
|
|||
{
|
||||
"jev": "TYPESAFE_API_KEY is only sent to TYPESAFE_API_BASE or https://api.typesafe.ai",
|
||||
"databricks": "DATABRICKS_API_KEY or DATABRICKS_TOKEN is only sent to DATABRICKS_API_BASE",
|
||||
"nace": "NACE_API_KEY is only sent to NACE_API_BASE, NACE_BASE_URL or https://console.nace.ai",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -722,7 +723,7 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel):
|
|||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
deployment_name: str | None = Field(default=None, min_length=1)
|
||||
provider: Literal["jev", "laya", "bespoke", "databricks"] = "jev"
|
||||
provider: Literal["jev", "laya", "bespoke", "databricks", "nace"] = "jev"
|
||||
model: str = "jev-latest"
|
||||
api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers")
|
||||
api_base: str | None = Field(
|
||||
|
|
@ -785,6 +786,10 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel):
|
|||
f"{DATABRICKS_AI_DECIDE_MODEL!r} or a serving endpoint name, e.g. databricks-openjev-qwen35-4b"
|
||||
)
|
||||
_ = validate_databricks_decisions_model(self.model)
|
||||
if self.provider == "nace" and "model" not in self.model_fields_set:
|
||||
raise ValueError(
|
||||
"opensource_classifier_config.model is required for provider 'nace', e.g. drex-max or drex-latest"
|
||||
)
|
||||
if self.api_base is not None and self.api_key is None:
|
||||
raise ValueError(
|
||||
"opensource_classifier_config.api_base requires opensource_classifier_config.api_key: "
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.databricks.decisions.transformation import DATABRICKS_DECISIONS_CONFIG, is_ai_decide_model
|
||||
from litellm.llms.laya.common_utils import laya_response_model
|
||||
from litellm.llms.nace.decisions.transformation import NACE_DECISIONS_CONFIG
|
||||
from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthroughLoggingHandler
|
||||
from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS
|
||||
from litellm.types.decisions import DecisionsResponse
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)]
|
||||
ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"]
|
||||
ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks", "nace"]
|
||||
DEFAULT_JEV_INSTRUCTIONS: Final = _DEFAULT_JEV_INSTRUCTIONS
|
||||
_WITHOUT_MODEL: Final = MappingProxyType({"model": True})
|
||||
|
||||
|
|
@ -201,6 +202,8 @@ class HttpJevClassifierClient:
|
|||
def _request_url(self, model: str) -> str:
|
||||
if self._provider == "databricks":
|
||||
return DATABRICKS_DECISIONS_CONFIG.get_complete_url(self._api_base, model)
|
||||
if self._provider == "nace":
|
||||
return NACE_DECISIONS_CONFIG.get_complete_url(self._api_base, model)
|
||||
return f"{self._api_base}/v1/systemone"
|
||||
|
||||
def _normalized_body(self, body: Mapping[str, object], requested_model: str) -> Mapping[str, object]:
|
||||
|
|
@ -209,7 +212,7 @@ class HttpJevClassifierClient:
|
|||
return MappingProxyType({**body, "model": laya_response_model(body, requested_model)})
|
||||
case "databricks":
|
||||
return DATABRICKS_DECISIONS_CONFIG.classifier_response(body, requested_model)
|
||||
case "typesafe" | "bespoke":
|
||||
case "typesafe" | "bespoke" | "nace":
|
||||
return body
|
||||
|
||||
def _log_response(
|
||||
|
|
|
|||
|
|
@ -72513,12 +72513,13 @@
|
|||
}
|
||||
},
|
||||
"nace/drex-v1.0": {
|
||||
"input_cost_per_token": 4e-08,
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 32768,
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models"
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models",
|
||||
"supports_vision": true
|
||||
},
|
||||
"nace/drex-v1.5": {
|
||||
"input_cost_per_token": 5e-08,
|
||||
|
|
@ -72544,6 +72545,24 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/reference/models"
|
||||
},
|
||||
"nace/drex-max": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/guides/drex-max",
|
||||
"supports_vision": true
|
||||
},
|
||||
"nace/drex-max-v1.1": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
"litellm_provider": "nace",
|
||||
"max_input_tokens": 262144,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://console.nace.ai/docs/drex/guides/drex-max",
|
||||
"supports_vision": true
|
||||
},
|
||||
"typesafe/jev-1.13.0": {
|
||||
"input_cost_per_token": 4.2e-08,
|
||||
"litellm_provider": "typesafe",
|
||||
|
|
|
|||
|
|
@ -4340,3 +4340,94 @@ def test_hosted_vllm_systemone_maps_image_rejection_to_bad_request(
|
|||
|
||||
assert caught.value.status_code == 400
|
||||
assert route.call_count == 1
|
||||
|
||||
|
||||
_NACE_SYSTEMONE_URL: Final = "https://console.nace.ai/v1/systemone"
|
||||
_DREX_MAX_RESPONSE: Final[Mapping[str, object]] = {**_RESPONSE, "model": "drex-max"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def nace_account(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("NACE_API_KEY", "nace-key")
|
||||
monkeypatch.delenv("NACE_API_BASE", raising=False)
|
||||
monkeypatch.delenv("NACE_BASE_URL", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("nace_account")
|
||||
async def test_input_images_reach_nace_drex_max_as_a_top_level_images_array(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_NACE_SYSTEMONE_URL).respond(json=_DREX_MAX_RESPONSE)
|
||||
|
||||
response: Final = await litellm.adecisions(
|
||||
model="nace/drex-max",
|
||||
input=(
|
||||
*_screen_messages(_PNG_DATA_URL),
|
||||
{"role": "user", "content": [{"type": "input_image", "image_url": "data:image/webp;base64,BB=="}]},
|
||||
{"role": "user", "content": "Order 1234."},
|
||||
),
|
||||
questions=_PREDICATE_QUESTIONS,
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": "drex-max",
|
||||
"state": f"{_SCREEN_TEXT}\n\nOrder 1234.",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL, "data:image/webp;base64,BB=="],
|
||||
}
|
||||
assert isinstance(response, OpenAIDecisionResponse)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("nace_account")
|
||||
@pytest.mark.parametrize("nace_model", ["drex-max", "drex-max-v1.1"])
|
||||
def test_systemone_images_reach_nace_as_data_urls(nace_model: str, respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(_NACE_SYSTEMONE_URL).respond(json={**_DREX_MAX_RESPONSE, "model": nace_model})
|
||||
|
||||
response: Final = litellm.systemone(
|
||||
model=f"nace/{nace_model}",
|
||||
state="Inspect Image 1 and Image 2.",
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL, {"content_type": "image/webp", "base64": "BB=="}],
|
||||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {
|
||||
"model": nace_model,
|
||||
"state": "Inspect Image 1 and Image 2.",
|
||||
"questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}},
|
||||
"images": [_PNG_DATA_URL, "data:image/webp;base64,BB=="],
|
||||
}
|
||||
assert isinstance(response, DecisionsResponse)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("nace_account")
|
||||
@pytest.mark.parametrize(("api", "request_kwargs"), _image_requests(_REMOTE_IMAGE_URL), ids=_PREDICATE_REQUEST_IDS)
|
||||
def test_remote_images_are_fetched_and_inlined_for_nace(
|
||||
api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
png: Final = b"\x89PNG\r\n\x1a\n"
|
||||
image_route: Final = respx_mock.get(_REMOTE_IMAGE_URL).respond(content=png, headers={"content-type": "image/png"})
|
||||
route: Final = respx_mock.post(_NACE_SYSTEMONE_URL).respond(json=_DREX_MAX_RESPONSE)
|
||||
|
||||
getattr(litellm, api)(model="nace/drex-max", **request_kwargs)
|
||||
|
||||
assert image_route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["images"] == [
|
||||
f"data:image/png;base64,{base64.b64encode(png).decode()}"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("nace_account")
|
||||
def test_nace_refuses_a_ninth_image_before_http(respx_mock: respx.MockRouter) -> None:
|
||||
"""Drex Max takes up to 8 images per request (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-26)."""
|
||||
route: Final = respx_mock.post(_NACE_SYSTEMONE_URL).respond(json=_DREX_MAX_RESPONSE)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="at most 8 images per request, got 9") as caught:
|
||||
litellm.systemone(
|
||||
model="nace/drex-max",
|
||||
state="Inspect the images.",
|
||||
questions=_SYSTEMONE_PREDICATE,
|
||||
images=[_PNG_DATA_URL] * 9,
|
||||
)
|
||||
|
||||
assert caught.value.status_code == 400
|
||||
assert not route.called
|
||||
|
|
|
|||
|
|
@ -148,6 +148,8 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N
|
|||
({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"),
|
||||
({"provider": "databricks", "model": "ai_decide", "api_base": "https://collector.invalid"}, "opensource_classifier_config"),
|
||||
({"provider": "databricks", "model": "ai_decide", "api_key": "dapi-member"}, "api_key"),
|
||||
({"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}, "opensource_classifier_config"),
|
||||
({"provider": "nace", "model": "drex-max", "api_key": "sk-member"}, "api_key"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
|
|
@ -174,6 +176,7 @@ def test_members_cannot_move_the_jev_classifier_off_the_proxys_typesafe_account(
|
|||
("bespoke", "nimble-latest"),
|
||||
("databricks", "ai_decide"),
|
||||
("databricks", "my-openjev"),
|
||||
("nace", "drex-max"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
|
|
|
|||
|
|
@ -459,6 +459,8 @@ def test_jev_config_requires_classifier_config() -> None:
|
|||
("bespoke", "nimble-latest", "bespoke"),
|
||||
("databricks", "databricks-openjev-qwen35-4b", "databricks"),
|
||||
("databricks", "ai_decide", "databricks"),
|
||||
("nace", "drex-max", "nace"),
|
||||
("nace", "drex-max-v1.1", "nace"),
|
||||
],
|
||||
)
|
||||
def test_classifier_aliases_load_and_serialize_one_canonical_config(
|
||||
|
|
@ -770,7 +772,9 @@ async def test_jev_connection_resolves_only_explicit_environment_references(
|
|||
expected_key: Final = (
|
||||
"default-key"
|
||||
if connection == "defaults"
|
||||
else "resolved-key" if connection in ("key-reference", "references") else "literal-key"
|
||||
else "resolved-key"
|
||||
if connection in ("key-reference", "references")
|
||||
else "literal-key"
|
||||
)
|
||||
assert outcome.cause == "jev_classifier"
|
||||
assert route.calls[0].request.headers["authorization"] == f"Bearer {expected_key}"
|
||||
|
|
@ -1055,3 +1059,152 @@ def test_switching_to_deployed_classifier_does_not_copy_legacy_credentials() ->
|
|||
)
|
||||
assert write.effective is not None
|
||||
assert write.effective["opensource_classifier_config"] == {"deployment_name": "registered"}
|
||||
|
||||
|
||||
def test_nace_requires_a_model_because_the_default_names_a_typesafe_model() -> None:
|
||||
with pytest.raises(ValueError, match="model is required for provider 'nace'"):
|
||||
JevClassifierConfig.model_validate({"provider": "nace"})
|
||||
assert JevClassifierConfig.model_validate({"provider": "nace", "model": "drex-max"}).model == "drex-max"
|
||||
|
||||
|
||||
def test_nace_api_base_without_its_own_key_is_rejected_so_the_environment_key_stays_home() -> None:
|
||||
with pytest.raises(ValueError, match="NACE_API_KEY is only sent to NACE_API_BASE"):
|
||||
JevClassifierConfig.model_validate(
|
||||
{"provider": "nace", "model": "drex-max", "api_base": "https://collector.invalid"}
|
||||
)
|
||||
paired: Final = JevClassifierConfig.model_validate(
|
||||
{"provider": "nace", "model": "drex-max", "api_base": "https://eu.nace.invalid", "api_key": "sk-own"}
|
||||
)
|
||||
assert (paired.api_base, paired.api_key) == ("https://eu.nace.invalid", "sk-own")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "unresolved"),
|
||||
[
|
||||
({"api_base": "https://collector.invalid", "api_key": "os.environ/MISSING_KEY"}, "api_key"),
|
||||
({"api_base": "https://collector.invalid", "api_key": "os.environ/EMPTY_KEY"}, "api_key"),
|
||||
({"api_base": "os.environ/MISSING_BASE", "api_key": "sk-own"}, "api_base"),
|
||||
],
|
||||
)
|
||||
def test_nace_unresolved_reference_fails_without_falling_back_to_the_environment_key(
|
||||
monkeypatch: pytest.MonkeyPatch, configured: Mapping[str, str], unresolved: str, legacy: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("NACE_API_KEY", "nace-env-key")
|
||||
monkeypatch.setenv("EMPTY_KEY", " ")
|
||||
monkeypatch.delenv("MISSING_KEY", raising=False)
|
||||
monkeypatch.delenv("MISSING_BASE", raising=False)
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
with pytest.raises(ValueError, match=unresolved):
|
||||
ComplexityRouter(
|
||||
"nace-route",
|
||||
litellm.Router(model_list=[]),
|
||||
{
|
||||
"classifier_type": "jev" if legacy else "oss_classifier",
|
||||
"jev_classifier_config" if legacy else "opensource_classifier_config": {
|
||||
"provider": "nace",
|
||||
"model": "drex-max",
|
||||
**configured,
|
||||
},
|
||||
"tiers": {"SIMPLE": "cheap"},
|
||||
},
|
||||
derive_savings_baseline=False,
|
||||
)
|
||||
assert upstream.calls.call_count == 0
|
||||
|
||||
|
||||
def test_nace_without_a_key_fails_at_client_build(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("NACE_API_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="NACE_API_KEY"):
|
||||
ComplexityRouter(
|
||||
"nace-route",
|
||||
litellm.Router(model_list=[]),
|
||||
{
|
||||
"classifier_type": "oss_classifier",
|
||||
"opensource_classifier_config": {"provider": "nace", "model": "drex-max"},
|
||||
"tiers": {"SIMPLE": "cheap"},
|
||||
},
|
||||
derive_savings_baseline=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "env_base", "server", "expected_authorization"),
|
||||
[
|
||||
({}, None, "https://console.nace.ai", "Bearer nace-env-key"),
|
||||
({}, "https://nace.test/v1/", "https://nace.test", "Bearer nace-env-key"),
|
||||
(
|
||||
{"api_base": "https://eu.nace.test", "api_key": "nace-configured"},
|
||||
None,
|
||||
"https://eu.nace.test",
|
||||
"Bearer nace-configured",
|
||||
),
|
||||
(
|
||||
{"api_base": "os.environ/NACE_TEST_BASE", "api_key": "os.environ/NACE_TEST_KEY"},
|
||||
None,
|
||||
"https://ref.nace.test",
|
||||
"Bearer nace-referenced",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
async def test_nace_routes_through_its_console_with_its_own_key_and_accounts_the_model(
|
||||
local_model_cost_map: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: Mapping[str, str],
|
||||
env_base: str | None,
|
||||
server: str,
|
||||
expected_authorization: str,
|
||||
legacy: bool,
|
||||
) -> None:
|
||||
monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key")
|
||||
monkeypatch.setenv("NACE_API_KEY", "nace-env-key")
|
||||
monkeypatch.setenv("NACE_TEST_KEY", "nace-referenced")
|
||||
monkeypatch.setenv("NACE_TEST_BASE", "https://ref.nace.test")
|
||||
monkeypatch.delenv("NACE_BASE_URL", raising=False)
|
||||
if env_base is None:
|
||||
monkeypatch.delenv("NACE_API_BASE", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("NACE_API_BASE", env_base)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setitem(litellm.model_cost, "nace/drex-max", {"input_cost_per_token": 0.01})
|
||||
recorder: Final = _UsageRecorder("nace/drex-max")
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
|
||||
router: Final = ComplexityRouter(
|
||||
"nace-route",
|
||||
litellm.Router(model_list=[]),
|
||||
{
|
||||
"classifier_type": "jev" if legacy else "oss_classifier",
|
||||
"jev_classifier_config" if legacy else "opensource_classifier_config": {
|
||||
"provider": "nace",
|
||||
"model": "drex-max",
|
||||
**configured,
|
||||
},
|
||||
"tiers": {"SIMPLE": "cheap"},
|
||||
},
|
||||
derive_savings_baseline=False,
|
||||
)
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post(f"{server}/v1/systemone").respond(
|
||||
200,
|
||||
json={
|
||||
"model": "drex-max",
|
||||
"answers": {"tier": _answer().model_dump()},
|
||||
"usage": {"input_tokens": 31, "output_tokens": 0},
|
||||
"evaluation_time_ms": 12.5,
|
||||
"request_id": "req_test",
|
||||
},
|
||||
)
|
||||
outcome: Final = await router.aclassify("choose a tier")
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
assert outcome.cause == "jev_classifier"
|
||||
assert outcome.jev_verdict is not None
|
||||
assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("nace", "drex-max")
|
||||
assert outcome.classifier_cost == pytest.approx(0.31)
|
||||
sent: Final = route.calls.last.request
|
||||
assert sent.headers.get("authorization") == expected_authorization
|
||||
assert json.loads(sent.content)["model"] == "drex-max"
|
||||
assert len(recorder.calls) == 1
|
||||
assert recorder.calls[0]["response_cost"] == pytest.approx(0.31)
|
||||
|
|
|
|||
55
tests/unit/test_nace_model_metadata.py
Normal file
55
tests/unit/test_nace_model_metadata.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
PRICES_PATH: Final = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PRICES_PATH: Final = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
|
||||
def _nace_rows(path: Path) -> dict[str, object]:
|
||||
prices: Final = json.loads(path.read_text())
|
||||
return {key: row for key, row in prices.items() if key.startswith("nace/")}
|
||||
|
||||
|
||||
def test_drex_max_versions_are_priced_at_three_cents_per_million_input_tokens() -> None:
|
||||
"""Both Drex Max versions cost $0.03 per 1M input tokens, half of drex-v2.0's $0.06; output is not billed
|
||||
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
|
||||
for model in ("nace/drex-max", "nace/drex-max-v1.1"):
|
||||
row = litellm.model_cost[model]
|
||||
assert row["input_cost_per_token"] == 3e-08
|
||||
assert row["output_cost_per_token"] == 0.0
|
||||
assert (row["litellm_provider"], row["mode"], row["max_input_tokens"]) == ("nace", "evaluation", 262144)
|
||||
assert litellm.model_cost["nace/drex-v2.0"]["input_cost_per_token"] == 6e-08
|
||||
|
||||
|
||||
def test_drex_v1_0_is_priced_as_the_drex_max_that_now_serves_it() -> None:
|
||||
"""drex-v1.0 is retired and answered by drex-max at drex-max's price and limits
|
||||
(console.nace.ai/docs/drex/reference/models, read 2026-10-30)."""
|
||||
drex_v1: Final = litellm.model_cost["nace/drex-v1.0"]
|
||||
drex_max: Final = litellm.model_cost["nace/drex-max"]
|
||||
|
||||
assert drex_v1["input_cost_per_token"] == drex_max["input_cost_per_token"]
|
||||
assert drex_v1["max_input_tokens"] == drex_max["max_input_tokens"]
|
||||
|
||||
|
||||
def test_drex_max_versions_are_the_nace_models_that_read_images() -> None:
|
||||
"""Only drex-max, drex-max-v1.1 and the retired drex-v1.0 (now served by drex-max) read images; the other Drex
|
||||
models answer 422 to a non-empty images array (console.nace.ai/docs/drex/guides/drex-max, read 2026-10-31)."""
|
||||
vision_rows: Final = [key for key, row in _nace_rows(PRICES_PATH).items() if row.get("supports_vision")]
|
||||
|
||||
assert litellm.supports_vision("nace/drex-max") is True
|
||||
assert litellm.supports_vision("nace/drex-max-v1.1") is True
|
||||
assert litellm.supports_vision("nace/drex-v1.0") is True
|
||||
assert sorted(vision_rows) == ["nace/drex-max", "nace/drex-max-v1.1", "nace/drex-v1.0"]
|
||||
|
||||
|
||||
def test_nace_backup_rows_match_the_main_catalog() -> None:
|
||||
assert {"nace/drex-max", "nace/drex-max-v1.1"} <= _nace_rows(PRICES_PATH).keys()
|
||||
assert _nace_rows(BACKUP_PRICES_PATH) == _nace_rows(PRICES_PATH)
|
||||
|
|
@ -16,6 +16,19 @@ describe("hydrateOssClassifier", () => {
|
|||
expect(hydrated).toEqual({ classifier_type: "jev", jev_classifier_config: stored });
|
||||
});
|
||||
|
||||
it("keeps a stored Nace classifier instead of falling back to Jev", () => {
|
||||
const stored = {
|
||||
provider: "nace",
|
||||
model: "drex-max",
|
||||
timeout_ms: 3000,
|
||||
};
|
||||
|
||||
const hydrated = hydrateOssClassifier({ classifier_type: "oss_classifier", opensource_classifier_config: stored });
|
||||
|
||||
expect(hydrated).toEqual({ classifier_type: "jev", jev_classifier_config: stored });
|
||||
expect(defaultJevClassifierConfig("nace")).toEqual(stored);
|
||||
});
|
||||
|
||||
it("falls back to Jev only when the stored classifier cannot be parsed", () => {
|
||||
const hydrated = hydrateOssClassifier({
|
||||
classifier_type: "oss_classifier",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ export const OSS_CLASSIFIER_MODELS = {
|
|||
bespoke: ["nimble-latest", "nimble", "bespokelabs/Bespoke-Nimble-9B"],
|
||||
} as const;
|
||||
|
||||
const OSS_CLASSIFIER_PROVIDERS = ["jev", "laya", "bespoke", "databricks"] as const;
|
||||
const OSS_CLASSIFIER_PROVIDERS = ["jev", "laya", "bespoke", "databricks", "nace"] as const;
|
||||
export type OssClassifierProvider = (typeof OSS_CLASSIFIER_PROVIDERS)[number];
|
||||
export const isOssClassifierProvider = (value: unknown): value is OssClassifierProvider =>
|
||||
OSS_CLASSIFIER_PROVIDERS.some((provider) => provider === value);
|
||||
|
|
@ -14,12 +14,14 @@ export const isOssClassifierProvider = (value: unknown): value is OssClassifierP
|
|||
const DEFAULT_TIMEOUT_MS = 3000;
|
||||
const SERVING_ENDPOINT_NAME = /^[A-Za-z0-9_-][A-Za-z0-9._-]*$/;
|
||||
const DATABRICKS_AI_DECIDE_MODEL = "ai_decide";
|
||||
const NACE_DEFAULT_MODEL = "drex-max";
|
||||
|
||||
export const fixedClassifierModels = (provider: OssClassifierProvider | undefined): readonly string[] | undefined =>
|
||||
provider === "laya" || provider === "bespoke" ? OSS_CLASSIFIER_MODELS[provider] : undefined;
|
||||
|
||||
const defaultClassifierModel = (provider: OssClassifierProvider | undefined): string =>
|
||||
fixedClassifierModels(provider)?.[0] ?? (provider === "databricks" ? DATABRICKS_AI_DECIDE_MODEL : "jev-latest");
|
||||
fixedClassifierModels(provider)?.[0] ??
|
||||
(provider === "databricks" ? DATABRICKS_AI_DECIDE_MODEL : provider === "nace" ? NACE_DEFAULT_MODEL : "jev-latest");
|
||||
|
||||
const isSupportedClassifierModel = (provider: OssClassifierProvider | undefined, model: string): boolean => {
|
||||
if (provider === "databricks") return SERVING_ENDPOINT_NAME.test(model);
|
||||
|
|
@ -31,7 +33,7 @@ const jevClassifierConfigFields = {
|
|||
deployment_name: z.preprocess((value) => value ?? undefined, z.string().trim().min(1).optional()),
|
||||
provider: z.preprocess(
|
||||
(value) => (value === "typesafe" ? "jev" : value),
|
||||
z.enum(["jev", "laya", "bespoke", "databricks"]).optional(),
|
||||
z.enum(["jev", "laya", "bespoke", "databricks", "nace"]).optional(),
|
||||
),
|
||||
model: z.string().trim().min(1).optional(),
|
||||
timeout_ms: z.number().int().positive().default(DEFAULT_TIMEOUT_MS),
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -40219,7 +40219,7 @@ export interface components {
|
|||
* @default jev
|
||||
* @enum {string}
|
||||
*/
|
||||
provider: "jev" | "laya" | "bespoke" | "databricks";
|
||||
provider: "jev" | "laya" | "bespoke" | "databricks" | "nace";
|
||||
/**
|
||||
* Timeout Ms
|
||||
* @default 3000
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue