fix(pass-through): apply eu_jev key limits, register eu_jev, price unlisted releases

Pre-call hooks and guardrails now see the body model as eu_jev/<release>, so
model_rpm_limit and model_tpm_limit set on eu_jev/eu-jeff-latest apply, and
upstream still gets the client's exact body. eu_jev is a registered decisions
provider, so access groups, config deployments, /v1/systemone and /v1/decisions
accept it, and the Admin UI can add it. A release without its own cost map row
is priced as eu-jeff-latest instead of $0
This commit is contained in:
mateo-berri 2026-10-21 06:11:09 -07:00
parent 3988b236d0
commit 4923892642
20 changed files with 170 additions and 35 deletions

View file

@ -354,6 +354,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -1730,6 +1730,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)

View file

@ -172,6 +172,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureAIDecisionsConfig",
"NvidiaNimRerankConfig",
@ -748,6 +749,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
"NvidiaNimRerankConfig": (

View file

@ -1,10 +0,0 @@
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
def eu_jev_registry_model(requested_model: str | None) -> str:
if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return requested_model

View file

View file

@ -0,0 +1,30 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
_EU_JEV_PREFIX: Final = "eu_jev/"
def eu_jev_registry_model(requested_model: str | None) -> str:
if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return requested_model
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
requested_model: Final = decision_body.get("model")
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
hook_model: Final = hook_body.get("model")
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
if hook_model == eu_jev_scoped_model(client_body):
return MappingProxyType(
{**without_model, **{key: value for key, value in client_body.items() if key == "model"}}
)
if isinstance(hook_model, str):
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
return hook_body

View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EuJevDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("EU_JEV_API_KEY",)
api_base_env = ("EU_JEV_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://eu-jev.com"

View file

@ -11,7 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.llms.eu_jev import eu_jev_registry_model
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
@ -91,8 +91,13 @@ class TypeSafePassthroughLoggingHandler:
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_models: Final = (
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
if custom_llm_provider == "eu_jev"
else (response_model, registry_request_model)
)
candidate_model_keys: Final = tuple(
f"{custom_llm_provider}/{model}" for model in (response_model, registry_request_model) if model is not None
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (

View file

@ -72270,10 +72270,7 @@
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
"source": "https://eu-jev.com/docs"
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
@ -72281,10 +72278,7 @@
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
"source": "https://eu-jev.com/docs"
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,

View file

@ -2162,7 +2162,7 @@ def _extract_model_candidates_from_request(
raise HTTPException(status_code=400, detail=str(exc)) from exc
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
if route.rstrip("/") == "/eu_jev/v1/systemone":
from litellm.llms.eu_jev import eu_jev_registry_model
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
if requested_model is not None and not isinstance(requested_model, str):

View file

@ -71,6 +71,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
from litellm.llms.oss_decision import validate_oss_request
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._lazy_features import lazy_owned_routes
@ -1228,6 +1229,13 @@ async def pass_through_request(
decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body)
checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request)
_parsed_body["model"] = f"{custom_llm_provider}/{checkpoint}"
eu_jev_client_body: Final = (
MappingProxyType(TypeAdapter(dict[str, object]).validate_python(_parsed_body))
if custom_llm_provider == "eu_jev"
else None
)
if eu_jev_client_body is not None:
_parsed_body["model"] = eu_jev_scoped_model(eu_jev_client_body)
### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ###
# Passthrough endpoints are opt-in only for guardrails
@ -1307,6 +1315,12 @@ async def pass_through_request(
)
_ = validate_oss_request(custom_llm_provider, decision_body)
_parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body)
if eu_jev_client_body is not None:
_parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage
eu_jev_upstream_body(
TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body
)
)
resolved_timeout: Final = resolve_pass_through_request_timeout(timeout)
async_client_obj: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.PassThroughEndpoint,

View file

@ -1497,6 +1497,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "EU_JEV",
"provider_display_name": "eu/jev",
"litellm_provider": "eu_jev",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://eu-jev.com",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "eu_jev/eu-jeff-latest"
},
{
"provider": "FalAI",
"provider_display_name": "Fal AI",

View file

@ -4432,6 +4432,7 @@ class LlmProviders(str, Enum):
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
BESPOKE = "bespoke"
EU_JEV = "eu_jev"
MISTRAL = "mistral"
MILVUS = "milvus"
GROQ = "groq"

View file

@ -9565,6 +9565,8 @@ class ProviderConfigManager:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider == LlmProviders.EU_JEV:
return litellm.EuJevDecisionsConfig()
if provider == LlmProviders.OPENAI:
return litellm.OpenAIDecisionsConfig()
if provider == LlmProviders.AZURE_AI:

View file

@ -72270,10 +72270,7 @@
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
"source": "https://eu-jev.com/docs"
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
@ -72281,10 +72278,7 @@
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
"source": "https://eu-jev.com/docs"
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,

View file

@ -1508,6 +1508,13 @@
"systemone": true
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"systemone": true
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -122,6 +122,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
),
("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"),
("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"),
("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"),
(
"openrouter",
"openrouter/typesafe/jev-1.13",

View file

@ -392,9 +392,11 @@ def test_eu_jev_decision_is_priced_from_its_registry_row():
# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release,
# which a future eu/jev release reports under a version the registry does not list yet.
@pytest.mark.parametrize("request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}])
def test_eu_jev_current_release_requests_are_priced_as_the_latest_alias(request_body: dict[str, object]) -> None:
# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it.
@pytest.mark.parametrize(
"request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}]
)
def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None:
unlisted_release: Final = "eu-jeff-99.0.0"
assert f"eu_jev/{unlisted_release}" not in litellm.model_cost
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(

View file

@ -8325,6 +8325,19 @@ class TestTypeSafePassthroughRoute:
)
class _RecordsPreCallModel(CustomLogger):
def __init__(self, switch_to: str | None = None) -> None:
super().__init__()
self.switch_to: Final = switch_to
self.models: Final[list[object]] = []
async def async_pre_call_hook(
self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str
) -> dict | None:
self.models.append(data.get("model"))
return None if self.switch_to is None else {**data, "model": self.switch_to}
class TestEuJevPassthroughRoute:
@pytest.fixture
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
@ -8351,6 +8364,38 @@ class TestEuJevPassthroughRoute:
assert sent.headers["authorization"] == "Bearer eu-jev-test-key"
assert json.loads(sent.content) == body
@pytest.mark.parametrize(
("body", "hook_model"),
[
({"state": "x"}, "eu_jev/eu-jeff-latest"),
({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"),
({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"),
],
)
def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str
) -> None:
recorder: Final = _RecordsPreCallModel()
monkeypatch.setattr(litellm, "callbacks", [recorder])
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json=body)
assert response.status_code == 200
assert recorder.models == [hook_model]
assert json.loads(route.calls.last.request.content) == body
def test_forwards_the_release_a_pre_call_hook_switched_to(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")])
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"})
assert response.status_code == 200
assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"}
def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond(

View file

@ -438,9 +438,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
return entry
@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE)))
def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None:
entry: Final = _decisions_provider_entry(display_name)
@pytest.mark.parametrize(
("provider_key", "display_name", "provider"),
(
("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE),
("Nace", "Nace", LlmProviders.NACE),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
),
)
def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None:
entry: Final = _decisions_provider_entry(provider_key)
assert entry.provider_display_name == display_name
assert entry.litellm_provider == provider.value