mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(pass-through): apply eu_jev key limits, register eu_jev, price unlisted releases
Pre-call hooks and guardrails now see the body model as eu_jev/<release>, so model_rpm_limit and model_tpm_limit set on eu_jev/eu-jeff-latest apply, and upstream still gets the client's exact body. eu_jev is a registered decisions provider, so access groups, config deployments, /v1/systemone and /v1/decisions accept it, and the Admin UI can add it. A release without its own cost map row is priced as eu-jeff-latest instead of $0
This commit is contained in:
parent
3988b236d0
commit
4923892642
20 changed files with 170 additions and 35 deletions
|
|
@ -354,6 +354,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|
||||
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
|
||||
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
|
||||
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
|
||||
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -1730,6 +1730,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bespoke.decisions.transformation import (
|
||||
BespokeDecisionsConfig as BespokeDecisionsConfig,
|
||||
)
|
||||
from .llms.eu_jev.decisions.transformation import (
|
||||
EuJevDecisionsConfig as EuJevDecisionsConfig,
|
||||
)
|
||||
from .llms.openai.decisions.transformation import (
|
||||
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -172,6 +172,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"HostedVLLMDecisionsConfig",
|
||||
"LayaDecisionsConfig",
|
||||
"BespokeDecisionsConfig",
|
||||
"EuJevDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
"AzureAIDecisionsConfig",
|
||||
"NvidiaNimRerankConfig",
|
||||
|
|
@ -748,6 +749,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
),
|
||||
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
|
||||
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
|
||||
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
|
||||
"NvidiaNimRerankConfig": (
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
from typing import Final
|
||||
|
||||
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
|
||||
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
|
||||
|
||||
|
||||
def eu_jev_registry_model(requested_model: str | None) -> str:
|
||||
if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES:
|
||||
return EU_JEV_LATEST_MODEL
|
||||
return requested_model
|
||||
0
litellm/llms/eu_jev/__init__.py
Normal file
0
litellm/llms/eu_jev/__init__.py
Normal file
30
litellm/llms/eu_jev/common_utils.py
Normal file
30
litellm/llms/eu_jev/common_utils.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
|
||||
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
|
||||
_EU_JEV_PREFIX: Final = "eu_jev/"
|
||||
|
||||
|
||||
def eu_jev_registry_model(requested_model: str | None) -> str:
|
||||
if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES:
|
||||
return EU_JEV_LATEST_MODEL
|
||||
return requested_model
|
||||
|
||||
|
||||
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
|
||||
requested_model: Final = decision_body.get("model")
|
||||
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
|
||||
|
||||
|
||||
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
hook_model: Final = hook_body.get("model")
|
||||
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
|
||||
if hook_model == eu_jev_scoped_model(client_body):
|
||||
return MappingProxyType(
|
||||
{**without_model, **{key: value for key, value in client_body.items() if key == "model"}}
|
||||
)
|
||||
if isinstance(hook_model, str):
|
||||
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
|
||||
return hook_body
|
||||
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
|
||||
class EuJevDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("EU_JEV_API_KEY",)
|
||||
api_base_env = ("EU_JEV_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://eu-jev.com"
|
||||
|
|
@ -11,7 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
|
||||
)
|
||||
from litellm.llms.eu_jev import eu_jev_registry_model
|
||||
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
|
||||
from litellm.llms.laya.common_utils import laya_response_model
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
|
||||
|
|
@ -91,8 +91,13 @@ class TypeSafePassthroughLoggingHandler:
|
|||
usage: Final = response.usage or _TypeSafeUsage()
|
||||
input_tokens: Final = usage.input_tokens
|
||||
output_tokens: Final = usage.output_tokens
|
||||
candidate_models: Final = (
|
||||
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
|
||||
if custom_llm_provider == "eu_jev"
|
||||
else (response_model, registry_request_model)
|
||||
)
|
||||
candidate_model_keys: Final = tuple(
|
||||
f"{custom_llm_provider}/{model}" for model in (response_model, registry_request_model) if model is not None
|
||||
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
response_cost: Final = (
|
||||
|
|
|
|||
|
|
@ -72270,10 +72270,7 @@
|
|||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
"source": "https://eu-jev.com/docs"
|
||||
},
|
||||
"eu_jev/eu-jeff-latest": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
|
|
@ -72281,10 +72278,7 @@
|
|||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
"source": "https://eu-jev.com/docs"
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
|
|
|
|||
|
|
@ -2162,7 +2162,7 @@ def _extract_model_candidates_from_request(
|
|||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
|
||||
if route.rstrip("/") == "/eu_jev/v1/systemone":
|
||||
from litellm.llms.eu_jev import eu_jev_registry_model
|
||||
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
|
||||
|
||||
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
|
||||
if requested_model is not None and not isinstance(requested_model, str):
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
|
|||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
|
||||
from litellm.llms.oss_decision import validate_oss_request
|
||||
from litellm.passthrough import BasePassthroughUtils
|
||||
from litellm.proxy._lazy_features import lazy_owned_routes
|
||||
|
|
@ -1228,6 +1229,13 @@ async def pass_through_request(
|
|||
decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body)
|
||||
checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request)
|
||||
_parsed_body["model"] = f"{custom_llm_provider}/{checkpoint}"
|
||||
eu_jev_client_body: Final = (
|
||||
MappingProxyType(TypeAdapter(dict[str, object]).validate_python(_parsed_body))
|
||||
if custom_llm_provider == "eu_jev"
|
||||
else None
|
||||
)
|
||||
if eu_jev_client_body is not None:
|
||||
_parsed_body["model"] = eu_jev_scoped_model(eu_jev_client_body)
|
||||
|
||||
### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ###
|
||||
# Passthrough endpoints are opt-in only for guardrails
|
||||
|
|
@ -1307,6 +1315,12 @@ async def pass_through_request(
|
|||
)
|
||||
_ = validate_oss_request(custom_llm_provider, decision_body)
|
||||
_parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body)
|
||||
if eu_jev_client_body is not None:
|
||||
_parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage
|
||||
eu_jev_upstream_body(
|
||||
TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body
|
||||
)
|
||||
)
|
||||
resolved_timeout: Final = resolve_pass_through_request_timeout(timeout)
|
||||
async_client_obj: Final = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.PassThroughEndpoint,
|
||||
|
|
|
|||
|
|
@ -1497,6 +1497,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "EU_JEV",
|
||||
"provider_display_name": "eu/jev",
|
||||
"litellm_provider": "eu_jev",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://eu-jev.com",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "eu_jev/eu-jeff-latest"
|
||||
},
|
||||
{
|
||||
"provider": "FalAI",
|
||||
"provider_display_name": "Fal AI",
|
||||
|
|
|
|||
|
|
@ -4432,6 +4432,7 @@ class LlmProviders(str, Enum):
|
|||
STRANDS_DECIDER = "strands_decider"
|
||||
LAYA = "laya"
|
||||
BESPOKE = "bespoke"
|
||||
EU_JEV = "eu_jev"
|
||||
MISTRAL = "mistral"
|
||||
MILVUS = "milvus"
|
||||
GROQ = "groq"
|
||||
|
|
|
|||
|
|
@ -9565,6 +9565,8 @@ class ProviderConfigManager:
|
|||
return litellm.LayaDecisionsConfig()
|
||||
if provider == LlmProviders.BESPOKE:
|
||||
return litellm.BespokeDecisionsConfig()
|
||||
if provider == LlmProviders.EU_JEV:
|
||||
return litellm.EuJevDecisionsConfig()
|
||||
if provider == LlmProviders.OPENAI:
|
||||
return litellm.OpenAIDecisionsConfig()
|
||||
if provider == LlmProviders.AZURE_AI:
|
||||
|
|
|
|||
|
|
@ -72270,10 +72270,7 @@
|
|||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
"source": "https://eu-jev.com/docs"
|
||||
},
|
||||
"eu_jev/eu-jeff-latest": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
|
|
@ -72281,10 +72278,7 @@
|
|||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
"source": "https://eu-jev.com/docs"
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
|
|
|
|||
|
|
@ -1508,6 +1508,13 @@
|
|||
"systemone": true
|
||||
}
|
||||
},
|
||||
"eu_jev": {
|
||||
"display_name": "eu/jev (`eu_jev`)",
|
||||
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
|
||||
"endpoints": {
|
||||
"systemone": true
|
||||
}
|
||||
},
|
||||
"laya": {
|
||||
"display_name": "Laya (`laya`)",
|
||||
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
|
|||
),
|
||||
("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"),
|
||||
("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"),
|
||||
("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"),
|
||||
(
|
||||
"openrouter",
|
||||
"openrouter/typesafe/jev-1.13",
|
||||
|
|
|
|||
|
|
@ -392,9 +392,11 @@ def test_eu_jev_decision_is_priced_from_its_registry_row():
|
|||
|
||||
|
||||
# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release,
|
||||
# which a future eu/jev release reports under a version the registry does not list yet.
|
||||
@pytest.mark.parametrize("request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}])
|
||||
def test_eu_jev_current_release_requests_are_priced_as_the_latest_alias(request_body: dict[str, object]) -> None:
|
||||
# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it.
|
||||
@pytest.mark.parametrize(
|
||||
"request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}]
|
||||
)
|
||||
def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None:
|
||||
unlisted_release: Final = "eu-jeff-99.0.0"
|
||||
assert f"eu_jev/{unlisted_release}" not in litellm.model_cost
|
||||
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
|
||||
|
|
|
|||
|
|
@ -8325,6 +8325,19 @@ class TestTypeSafePassthroughRoute:
|
|||
)
|
||||
|
||||
|
||||
class _RecordsPreCallModel(CustomLogger):
|
||||
def __init__(self, switch_to: str | None = None) -> None:
|
||||
super().__init__()
|
||||
self.switch_to: Final = switch_to
|
||||
self.models: Final[list[object]] = []
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str
|
||||
) -> dict | None:
|
||||
self.models.append(data.get("model"))
|
||||
return None if self.switch_to is None else {**data, "model": self.switch_to}
|
||||
|
||||
|
||||
class TestEuJevPassthroughRoute:
|
||||
@pytest.fixture
|
||||
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||
|
|
@ -8351,6 +8364,38 @@ class TestEuJevPassthroughRoute:
|
|||
assert sent.headers["authorization"] == "Bearer eu-jev-test-key"
|
||||
assert json.loads(sent.content) == body
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "hook_model"),
|
||||
[
|
||||
({"state": "x"}, "eu_jev/eu-jeff-latest"),
|
||||
({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"),
|
||||
({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"),
|
||||
],
|
||||
)
|
||||
def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body(
|
||||
self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str
|
||||
) -> None:
|
||||
recorder: Final = _RecordsPreCallModel()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
|
||||
response: Final = client.post("/eu_jev/v1/systemone", json=body)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert recorder.models == [hook_model]
|
||||
assert json.loads(route.calls.last.request.content) == body
|
||||
|
||||
def test_forwards_the_release_a_pre_call_hook_switched_to(
|
||||
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")])
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
|
||||
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"}
|
||||
|
||||
def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None:
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond(
|
||||
|
|
|
|||
|
|
@ -438,9 +438,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
|
|||
return entry
|
||||
|
||||
|
||||
@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE)))
|
||||
def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None:
|
||||
entry: Final = _decisions_provider_entry(display_name)
|
||||
@pytest.mark.parametrize(
|
||||
("provider_key", "display_name", "provider"),
|
||||
(
|
||||
("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE),
|
||||
("Nace", "Nace", LlmProviders.NACE),
|
||||
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
|
||||
),
|
||||
)
|
||||
def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None:
|
||||
entry: Final = _decisions_provider_entry(provider_key)
|
||||
|
||||
assert entry.provider_display_name == display_name
|
||||
assert entry.litellm_provider == provider.value
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue