From fe4ba0f5eeafe737a568ee81deddb3f2b7c8f371 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:48:05 +0200 Subject: [PATCH 01/15] feat(pass-through): add eu/jev decision API pass-through with spend tracking Adds a /eu_jev/{endpoint:path} pass-through route for eu/jev, an EU-hosted decision API that speaks the same POST /v1/systemone contract as TypeSafe. It reads EU_JEV_API_KEY and EU_JEV_API_BASE (defaults to https://jev.bevel.software), books spend through the shared decision logging handler, and prices eu_jev/eu-jeff-1.0.0 and eu_jev/eu-jeff-latest from the cost map --- gateway/routes/allowlist.py | 1 + ...odel_prices_and_context_window_backup.json | 22 ++ litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 222 ++++++++++++++++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 38 +++ .../pass_through_endpoints.py | 6 +- .../pass_through_endpoints/success_handler.py | 2 +- model_prices_and_context_window.json | 22 ++ ...st_typesafe_passthrough_logging_handler.py | 41 ++++ .../test_llm_pass_through_endpoints.py | 94 ++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 191 +++++++++++++++ 12 files changed, 637 insertions(+), 4 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cced49f788a..d365cbd1abf 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -107,6 +107,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/vllm/", "/mistral/", "/typesafe/", + "/eu_jev/", "/openrouter/", "/nvidia_nim/", "/groq/", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b744032fb6c..f71bb039417 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -70896,6 +70896,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index a909a7a993a..19c80691a87 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -230,6 +230,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/tinyfish/", "/transcribe", "/typesafe/", + "/eu_jev/", "/laya/", "/bespoke/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 20819a59a43..3cf5ca88f4c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -30372,6 +30372,228 @@ ] } }, + "/eu_jev/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/fal_ai/{endpoint}": { "delete": { "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d62c6b0d51f..36c0c86cf06 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -407,6 +407,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", + "/eu_jev", "/laya", "/bespoke", "/openrouter", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 464cfee270a..3d8b5f7f26e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -681,6 +681,44 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/eu_jev/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], + tags=["eu/jev Pass-through", "pass-through"], +) +async def eu_jev_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://jev.bevel.software" + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + eu_jev_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="eu_jev", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ + "Authorization": f"Bearer {eu_jev_api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( + await endpoint_func(request, fastapi_response, user_api_key_dict) + ) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b1c448b023b..5dcbece655d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -403,9 +403,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): @staticmethod def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) - if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( - "/v1/systemone" - ): + if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix( + "/" + ).endswith("/v1/systemone"): return EndpointType.DECISIONS if ( ("generateContent") in url diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..f0e0906d486 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -338,7 +338,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider in ("laya", "bespoke") + or custom_llm_provider in ("eu_jev", "laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b744032fb6c..f71bb039417 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -70896,6 +70896,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 79209a32f28..9958805d6aa 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -366,3 +366,44 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou assert normalized["standard_logging_response_object"] is None assert "combined_usage_object" not in normalized["kwargs"] assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" + + +def test_eu_jev_decision_is_priced_from_its_registry_row(): + model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://jev.bevel.software/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-latest"}, + custom_llm_provider="eu_jev", + ) + + assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" + assert response["kwargs"]["custom_llm_provider"] == "eu_jev" + assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"]) + assert response["kwargs"]["response_cost"] > 0 + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 + + +def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + request_body={"model": "eu-jeff-latest"}, + logging_obj=_logging_obj(), + url_route="https://jev.bevel.software/v1/systemone", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="eu_jev", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev" + assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 21170abaeb1..9b7c9e38dd9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + eu_jev_proxy_route, fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, @@ -8236,6 +8237,99 @@ class TestTypeSafePassthroughRoute: ) +class TestEuJevPassthroughRoute: + @staticmethod + def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = query_params or {} + request.json = AsyncMock(return_value=body) + return request + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: + body: Final = {"model": "eu-jeff-latest", "state": "x"} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response = client.post("/eu_jev/v1/systemone", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer eu-jev-test-key" + assert json.loads(sent.content) == body + + @pytest.mark.asyncio + async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + + async def fake_upstream(request, *_args): + target: Final = create_route.call_args.kwargs["target"] + upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params) + return Response(content=json.dumps({"upstream_query": parse_qs(upstream_url.query.decode())})) + + endpoint_func = AsyncMock(side_effect=fake_upstream) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + result = await eu_jev_proxy_route( + endpoint="v1/systemone", + request=self._request({"state": "x"}, {"trace": "yes"}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert json.loads(result.body) == {"upstream_query": {"trace": ["yes"]}} + endpoint_func.assert_awaited_once() + create_route.assert_called_once_with( + endpoint="v1/systemone", + target="https://eu-jev.example/base/v1/systemone", + custom_headers={ + "Authorization": "Bearer eu-jev-test-key", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + + @pytest.mark.asyncio + async def test_uses_the_hosted_api_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.delenv("EU_JEV_API_BASE", raising=False) + + endpoint_func = AsyncMock(return_value=Response()) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await eu_jev_proxy_route( + endpoint="v1/systemone", + request=self._request({"state": "x"}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == "https://jev.bevel.software/v1/systemone" + + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: @pytest.fixture diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f6eae1f98e..a2dc31c9140 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,6 +5333,42 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; + options?: never; + head?: never; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; + trace?: never; + }; "/fal_ai/{endpoint}": { parameters: { query?: never; @@ -58817,6 +58853,161 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; fal_ai_proxy_route_fal_ai__endpoint__get: { parameters: { query?: never; From d58095896593c6d1ee09ac3360720d070d6197af Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:53:21 +0200 Subject: [PATCH 02/15] fix(pass-through): default eu_jev to its canonical eu-jev.com origin --- litellm/model_prices_and_context_window_backup.json | 4 ++-- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 2 +- model_prices_and_context_window.json | 4 ++-- .../test_typesafe_passthrough_logging_handler.py | 4 ++-- .../pass_through_endpoints/test_llm_pass_through_endpoints.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f71bb039417..ff293456e8d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -70902,7 +70902,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] @@ -70913,7 +70913,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 3d8b5f7f26e..0d7d3f78b53 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -693,7 +693,7 @@ async def eu_jev_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" - base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://jev.bevel.software" + base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" base_url: Final = httpx.URL(base_target_url) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f71bb039417..ff293456e8d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -70902,7 +70902,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] @@ -70913,7 +70913,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 9958805d6aa..dd7b82070f1 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -374,7 +374,7 @@ def test_eu_jev_decision_is_priced_from_its_registry_row(): httpx_response=_response(), response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, logging_obj=_logging_obj(), - url_route="https://jev.bevel.software/v1/systemone", + url_route="https://eu-jev.com/v1/systemone", result='{"answers": {}}', start_time=datetime.now(), end_time=datetime.now(), @@ -397,7 +397,7 @@ def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, request_body={"model": "eu-jeff-latest"}, logging_obj=_logging_obj(), - url_route="https://jev.bevel.software/v1/systemone", + url_route="https://eu-jev.com/v1/systemone", result="{}", start_time=datetime.now(), end_time=datetime.now(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9b7c9e38dd9..434e139b9c2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8327,7 +8327,7 @@ class TestEuJevPassthroughRoute: user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), ) - assert create_route.call_args.kwargs["target"] == "https://jev.bevel.software/v1/systemone" + assert create_route.call_args.kwargs["target"] == "https://eu-jev.com/v1/systemone" @pytest.mark.parametrize("provider", ["laya", "bespoke"]) From 185378215cbe01237601a95cb485b93aa3d9e309 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:59:28 +0200 Subject: [PATCH 03/15] fix(pass-through): reject custom_body on the eu_jev route A key limited to other models could send an eu/jev request through custom_body, because auth checks the outer body while the shared pass-through forwards custom_body upstream. The route now answers 400 before anything is forwarded, the same way validate_oss_request guards the laya and bespoke routes --- .../llm_passthrough_endpoints.py | 2 ++ .../test_llm_pass_through_endpoints.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0d7d3f78b53..fa731c0e1fe 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -693,6 +693,8 @@ async def eu_jev_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + if "custom_body" in await read_request_body(request): + raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 434e139b9c2..c790f7f3b16 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8271,6 +8271,16 @@ class TestEuJevPassthroughRoute: assert sent.headers["authorization"] == "Bearer eu-jev-test-key" assert json.loads(sent.content) == body + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: + smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled}) + + assert response.status_code == 400 + assert "custom_body" in response.text + assert not route.called + @pytest.mark.asyncio async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") From 8c975c7d91b50f1085807508a289bb6c320d2ffb Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:59:44 +0200 Subject: [PATCH 04/15] fix(pass-through): give the eu_jev route request and 200 response schemas The OpenAPI docs gate flagged POST /eu_jev/{endpoint} for OA001 and OA003. The route now documents the eu/jev System One request body and its 200 answer, usage and meta shape, taken from the eu/jev OpenAPI document. The lazy OpenAPI snapshot and the dashboard API types are regenerated with python -m litellm.proxy._lazy_openapi_snapshot and npm run gen:api --- litellm/proxy/_lazy_openapi_snapshot.json | 1652 ++++++++++++++++- .../llm_passthrough_endpoints.py | 36 +- litellm/types/passthrough_endpoints/eu_jev.py | 100 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 694 ++++++- 4 files changed, 2456 insertions(+), 26 deletions(-) create mode 100644 litellm/types/passthrough_endpoints/eu_jev.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3cf5ca88f4c..2204448afd5 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -21808,6 +21808,39 @@ "title": "CachedTokensDetails", "type": "object" }, + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, "Click": { "additionalProperties": true, "description": "A click action.", @@ -22572,6 +22605,112 @@ "title": "ErrorResponse", "type": "object" }, + "EuJevDecisionResponse": { + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "description": "One answer per question, under the same names", + "title": "Answers", + "type": "object" + }, + "meta": { + "$ref": "#/components/schemas/EuJevMeta" + }, + "model": { + "description": "The release that answered, such as eu-jeff-1.0.0", + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/EuJevUsage" + } + }, + "required": [ + "model", + "answers", + "usage", + "meta" + ], + "title": "EuJevDecisionResponse", + "type": "object" + }, + "EuJevMeta": { + "properties": { + "cost_eur": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The exact charge in euros, on authenticated requests", + "title": "Cost Eur" + }, + "latency_ms": { + "description": "Server-side time for token quoting, credit reservation and inference", + "title": "Latency Ms", + "type": "integer" + }, + "mode": { + "const": "live", + "title": "Mode", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "mode", + "latency_ms" + ], + "title": "EuJevMeta", + "type": "object" + }, + "EuJevUsage": { + "properties": { + "input_tokens": { + "minimum": 0.0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "minimum": 0.0, + "title": "Output Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "EuJevUsage", + "type": "object" + }, "FileSearchTool": { "additionalProperties": true, "description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).", @@ -24211,6 +24350,26 @@ "title": "NamespaceTool", "type": "object" }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -27480,6 +27639,59 @@ "title": "Result", "type": "object" }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + }, "Screenshot": { "additionalProperties": true, "description": "A screenshot action.", @@ -30374,7 +30586,7 @@ }, "/eu_jev/{endpoint}": { "delete": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__delete", "parameters": [ { @@ -30387,11 +30599,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30418,7 +30914,7 @@ ] }, "get": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__get", "parameters": [ { @@ -30431,11 +30927,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30462,7 +31242,7 @@ ] }, "patch": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__patch", "parameters": [ { @@ -30475,11 +31255,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30506,7 +31570,7 @@ ] }, "post": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__post", "parameters": [ { @@ -30519,11 +31583,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30550,7 +31898,7 @@ ] }, "put": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__put", "parameters": [ { @@ -30563,11 +31911,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index fa731c0e1fe..fb10ed664a6 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import ( model_deployments_support_stream_options, open_sse_before_first_byte, ) +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -122,6 +123,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, ) from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool +from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, @@ -681,10 +683,34 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( + EuJevDecisionRequest, + { + "model": "eu-jeff-latest", + "state": "I was charged twice. Can I get a refund?", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"}, + }, + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) + + @router.api_route( "/eu_jev/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], tags=["eu/jev Pass-through", "pass-through"], + responses={200: {"model": EuJevDecisionResponse}}, + openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, ) async def eu_jev_proxy_route( endpoint: str, @@ -692,7 +718,15 @@ async def eu_jev_proxy_route( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: - """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + """ + Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + + `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + rejected with a 400. + + [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + """ if "custom_body" in await read_request_body(request): raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" diff --git a/litellm/types/passthrough_endpoints/eu_jev.py b/litellm/types/passthrough_endpoints/eu_jev.py new file mode 100644 index 00000000000..3b81d68deea --- /dev/null +++ b/litellm/types/passthrough_endpoints/eu_jev.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +EuJevEntry: TypeAlias = Annotated[ + DecisionsJSON | None, + Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"), +] + + +class EuJevChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: EuJevEntry + criteria: Annotated[ + Mapping[str, EuJevEntry], + Field( + min_length=2, + max_length=64, + description="Two to 64 options, option name to description. A null description uses the option name", + ), + ] + + model_config = ConfigDict(frozen=True) + + +class EuJevNoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: EuJevEntry + criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = ( + None + ) + + model_config = ConfigDict(frozen=True) + + +class EuJevScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: EuJevEntry + criteria: Annotated[ + Sequence[EuJevEntry], + Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevQuestion: TypeAlias = Annotated[ + EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion, + Field(discriminator="type"), +] + + +class EuJevDecisionRequest(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = ( + "eu-jeff-latest" + ) + state: Annotated[ + Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object], + Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"), + ] + questions: Annotated[ + Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion], + Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")] + + +class EuJevUsage(LiteLLMPydanticObjectBase): + input_tokens: Annotated[int, Field(ge=0)] + output_tokens: Annotated[int, Field(ge=0)] + + model_config = ConfigDict(frozen=True) + + +class EuJevMeta(LiteLLMPydanticObjectBase): + request_id: str + mode: Literal["live"] + latency_ms: Annotated[ + int, Field(description="Server-side time for token quoting, credit reservation and inference") + ] + cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None + + model_config = ConfigDict(frozen=True) + + +class EuJevDecisionResponse(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")] + answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")] + usage: EuJevUsage + meta: EuJevMeta + + model_config = ConfigDict(frozen=True) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a2dc31c9140..7649b2772cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5342,29 +5342,59 @@ export interface paths { }; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; options?: never; head?: never; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; trace?: never; @@ -32115,6 +32145,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -58862,7 +58936,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58870,7 +59062,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58893,7 +59085,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58901,7 +59211,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58924,7 +59234,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58932,7 +59360,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58955,7 +59383,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58963,7 +59509,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58986,7 +59532,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58994,7 +59658,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ From d3f8062e388b2a1dae214f5aba289036724db361 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:19:14 +0200 Subject: [PATCH 05/15] fix(pass-through): reject streaming on the eu_jev route A body with stream=true switched the route to streaming, which skips the decision spend tracking. Reject it with a 400, as validate_oss_request does for laya and bespoke. --- .../llm_passthrough_endpoints.py | 5 ++++- .../test_llm_pass_through_endpoints.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index fb10ed664a6..af4754c99e2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -727,8 +727,11 @@ async def eu_jev_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ - if "custom_body" in await read_request_body(request): + body: Final = await read_request_body(request) + if "custom_body" in body: raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index c790f7f3b16..f3ef1dcd033 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8281,6 +8281,17 @@ class TestEuJevPassthroughRoute: assert "custom_body" in response.text assert not route.called + def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post( + "/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True} + ) + + assert response.status_code == 400 + assert "Streaming" in response.text + assert not route.called + @pytest.mark.asyncio async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") From 27f804790494e3f25728dae84268a54aeaf3ab5f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:21:29 -0700 Subject: [PATCH 06/15] fix(pass-through): type the eu_jev request body like the other decision routes --- .../llm_passthrough_endpoints.py | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 855 ++++++++++++++++++ 2 files changed, 856 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 8dfffadc584..ecc9b46105c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -737,7 +737,7 @@ async def eu_jev_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ - body: Final = await read_request_body(request) + body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request)) if "custom_body" in body: raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") if body.get("stream"): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2b174a1141a..44f8932f212 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,6 +5333,72 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; + options?: never; + head?: never; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; + trace?: never; + }; "/extraction": { parameters: { query?: never; @@ -32233,6 +32299,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -59146,6 +59256,751 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; extraction_extraction_post: { parameters: { query?: never; From 3988b236d0a2565048542dffda2f3300aa740fbd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:24:09 -0700 Subject: [PATCH 07/15] fix(pass-through): check eu_jev decisions against the model eu/jev runs Serve only POST /eu_jev/v1/systemone, eu/jev's one endpoint, resolve an omitted model and the legacy names to eu-jeff-latest for key model checks and pricing, check scopes under eu_jev/ like Laya and Bespoke, and answer 503 when EU_JEV_API_KEY is unset --- litellm/llms/eu_jev.py | 10 + .../pass_through/typesafe_logging_handler.py | 8 +- litellm/proxy/_lazy_openapi_snapshot.json | 1339 +---------------- litellm/proxy/auth/auth_utils.py | 7 + .../llm_passthrough_endpoints.py | 25 +- tests/unit/proxy/auth/test_auth_utils.py | 27 + ...st_typesafe_passthrough_logging_handler.py | 48 + .../test_llm_pass_through_endpoints.py | 109 +- .../test_pass_through_endpoints.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 667 +------- 10 files changed, 166 insertions(+), 2075 deletions(-) create mode 100644 litellm/llms/eu_jev.py diff --git a/litellm/llms/eu_jev.py b/litellm/llms/eu_jev.py new file mode 100644 index 00000000000..315b26bbcea --- /dev/null +++ b/litellm/llms/eu_jev.py @@ -0,0 +1,10 @@ +from typing import Final + +EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" +_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) + + +def eu_jev_registry_model(requested_model: str | None) -> str: + if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES: + return EU_JEV_LATEST_MODEL + return requested_model diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 397ec408ae9..83511adbb4c 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature ) +from litellm.llms.eu_jev import eu_jev_registry_model from litellm.llms.laya.common_utils import laya_response_model from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage @@ -82,13 +83,16 @@ class TypeSafePassthroughLoggingHandler: response_model: Final = ( laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model ) - logged_model: Final = response_model or request_model or "unknown" + registry_request_model: Final = ( + eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model + ) + logged_model: Final = response_model or registry_request_model or "unknown" model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in (response_model, registry_request_model) if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3622bcc671b..b9005be094c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -31675,1005 +31675,10 @@ ] } }, - "/eu_jev/{endpoint}": { - "delete": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", - "operationId": "eu_jev_proxy_route_eu_jev__endpoint__delete", - "parameters": [ - { - "in": "path", - "name": "endpoint", - "required": true, - "schema": { - "title": "Endpoint", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "eu-jeff-latest", - "questions": { - "frustration": { - "criteria": [ - "calm", - "mildly annoyed", - "angry" - ], - "instructions": "How frustrated is the customer?", - "type": "score" - }, - "is_refund_request": { - "instructions": "Is the customer asking for a refund?", - "type": "noul" - }, - "team": { - "criteria": { - "billing": "Payments, invoices, and refunds", - "support": "Technical issues and bugs" - }, - "instructions": "Which team should handle this?", - "type": "choice" - } - }, - "state": "I was charged twice. Can I get a refund?" - }, - "schema": { - "properties": { - "model": { - "default": "eu-jeff-latest", - "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", - "title": "Model", - "type": "string" - }, - "questions": { - "additionalProperties": { - "discriminator": { - "propertyName": "type" - }, - "oneOf": [ - { - "properties": { - "criteria": { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "description": "Two to 64 options, option name to description. A null description uses the option name", - "maxProperties": 64, - "minProperties": 2, - "title": "Criteria", - "type": "object" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "choice", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevChoiceQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "anyOf": [ - { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "propertyNames": { - "enum": [ - "true", - "false" - ] - }, - "type": "object" - }, - { - "type": "null" - } - ], - "description": "Optional descriptions of what true and false mean", - "title": "Criteria" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "noul", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions" - ], - "title": "EuJevNoulQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "description": "Two to 10 level descriptions, ordered from low to high", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "maxItems": 10, - "minItems": 2, - "title": "Criteria", - "type": "array" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "score", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevScoreQuestion", - "type": "object" - } - ] - }, - "description": "One to eight questions, keyed by names of up to 80 characters", - "maxProperties": 8, - "minProperties": 1, - "propertyNames": { - "maxLength": 80, - "minLength": 1 - }, - "title": "Questions", - "type": "object" - }, - "state": { - "anyOf": [ - { - "maxLength": 100000, - "minLength": 1, - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - } - ], - "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", - "title": "State" - } - }, - "required": [ - "state", - "questions" - ], - "title": "EuJevDecisionRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EuJevDecisionResponse" - } - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Eu Jev Proxy Route", - "tags": [ - "llm_passthrough" - ] - }, - "get": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", - "operationId": "eu_jev_proxy_route_eu_jev__endpoint__get", - "parameters": [ - { - "in": "path", - "name": "endpoint", - "required": true, - "schema": { - "title": "Endpoint", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "eu-jeff-latest", - "questions": { - "frustration": { - "criteria": [ - "calm", - "mildly annoyed", - "angry" - ], - "instructions": "How frustrated is the customer?", - "type": "score" - }, - "is_refund_request": { - "instructions": "Is the customer asking for a refund?", - "type": "noul" - }, - "team": { - "criteria": { - "billing": "Payments, invoices, and refunds", - "support": "Technical issues and bugs" - }, - "instructions": "Which team should handle this?", - "type": "choice" - } - }, - "state": "I was charged twice. Can I get a refund?" - }, - "schema": { - "properties": { - "model": { - "default": "eu-jeff-latest", - "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", - "title": "Model", - "type": "string" - }, - "questions": { - "additionalProperties": { - "discriminator": { - "propertyName": "type" - }, - "oneOf": [ - { - "properties": { - "criteria": { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "description": "Two to 64 options, option name to description. A null description uses the option name", - "maxProperties": 64, - "minProperties": 2, - "title": "Criteria", - "type": "object" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "choice", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevChoiceQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "anyOf": [ - { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "propertyNames": { - "enum": [ - "true", - "false" - ] - }, - "type": "object" - }, - { - "type": "null" - } - ], - "description": "Optional descriptions of what true and false mean", - "title": "Criteria" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "noul", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions" - ], - "title": "EuJevNoulQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "description": "Two to 10 level descriptions, ordered from low to high", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "maxItems": 10, - "minItems": 2, - "title": "Criteria", - "type": "array" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "score", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevScoreQuestion", - "type": "object" - } - ] - }, - "description": "One to eight questions, keyed by names of up to 80 characters", - "maxProperties": 8, - "minProperties": 1, - "propertyNames": { - "maxLength": 80, - "minLength": 1 - }, - "title": "Questions", - "type": "object" - }, - "state": { - "anyOf": [ - { - "maxLength": 100000, - "minLength": 1, - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - } - ], - "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", - "title": "State" - } - }, - "required": [ - "state", - "questions" - ], - "title": "EuJevDecisionRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EuJevDecisionResponse" - } - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Eu Jev Proxy Route", - "tags": [ - "llm_passthrough" - ] - }, - "patch": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", - "operationId": "eu_jev_proxy_route_eu_jev__endpoint__patch", - "parameters": [ - { - "in": "path", - "name": "endpoint", - "required": true, - "schema": { - "title": "Endpoint", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "eu-jeff-latest", - "questions": { - "frustration": { - "criteria": [ - "calm", - "mildly annoyed", - "angry" - ], - "instructions": "How frustrated is the customer?", - "type": "score" - }, - "is_refund_request": { - "instructions": "Is the customer asking for a refund?", - "type": "noul" - }, - "team": { - "criteria": { - "billing": "Payments, invoices, and refunds", - "support": "Technical issues and bugs" - }, - "instructions": "Which team should handle this?", - "type": "choice" - } - }, - "state": "I was charged twice. Can I get a refund?" - }, - "schema": { - "properties": { - "model": { - "default": "eu-jeff-latest", - "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", - "title": "Model", - "type": "string" - }, - "questions": { - "additionalProperties": { - "discriminator": { - "propertyName": "type" - }, - "oneOf": [ - { - "properties": { - "criteria": { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "description": "Two to 64 options, option name to description. A null description uses the option name", - "maxProperties": 64, - "minProperties": 2, - "title": "Criteria", - "type": "object" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "choice", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevChoiceQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "anyOf": [ - { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "propertyNames": { - "enum": [ - "true", - "false" - ] - }, - "type": "object" - }, - { - "type": "null" - } - ], - "description": "Optional descriptions of what true and false mean", - "title": "Criteria" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "noul", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions" - ], - "title": "EuJevNoulQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "description": "Two to 10 level descriptions, ordered from low to high", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "maxItems": 10, - "minItems": 2, - "title": "Criteria", - "type": "array" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "score", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevScoreQuestion", - "type": "object" - } - ] - }, - "description": "One to eight questions, keyed by names of up to 80 characters", - "maxProperties": 8, - "minProperties": 1, - "propertyNames": { - "maxLength": 80, - "minLength": 1 - }, - "title": "Questions", - "type": "object" - }, - "state": { - "anyOf": [ - { - "maxLength": 100000, - "minLength": 1, - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - } - ], - "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", - "title": "State" - } - }, - "required": [ - "state", - "questions" - ], - "title": "EuJevDecisionRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EuJevDecisionResponse" - } - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Eu Jev Proxy Route", - "tags": [ - "llm_passthrough" - ] - }, + "/eu_jev/v1/systemone": { "post": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", - "operationId": "eu_jev_proxy_route_eu_jev__endpoint__post", - "parameters": [ - { - "in": "path", - "name": "endpoint", - "required": true, - "schema": { - "title": "Endpoint", - "type": "string" - } - } - ], + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post", "requestBody": { "content": { "application/json": { @@ -32966,344 +31971,6 @@ } }, "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Eu Jev Proxy Route", - "tags": [ - "llm_passthrough" - ] - }, - "put": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", - "operationId": "eu_jev_proxy_route_eu_jev__endpoint__put", - "parameters": [ - { - "in": "path", - "name": "endpoint", - "required": true, - "schema": { - "title": "Endpoint", - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "eu-jeff-latest", - "questions": { - "frustration": { - "criteria": [ - "calm", - "mildly annoyed", - "angry" - ], - "instructions": "How frustrated is the customer?", - "type": "score" - }, - "is_refund_request": { - "instructions": "Is the customer asking for a refund?", - "type": "noul" - }, - "team": { - "criteria": { - "billing": "Payments, invoices, and refunds", - "support": "Technical issues and bugs" - }, - "instructions": "Which team should handle this?", - "type": "choice" - } - }, - "state": "I was charged twice. Can I get a refund?" - }, - "schema": { - "properties": { - "model": { - "default": "eu-jeff-latest", - "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", - "title": "Model", - "type": "string" - }, - "questions": { - "additionalProperties": { - "discriminator": { - "propertyName": "type" - }, - "oneOf": [ - { - "properties": { - "criteria": { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "description": "Two to 64 options, option name to description. A null description uses the option name", - "maxProperties": 64, - "minProperties": 2, - "title": "Criteria", - "type": "object" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "choice", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevChoiceQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "anyOf": [ - { - "additionalProperties": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "propertyNames": { - "enum": [ - "true", - "false" - ] - }, - "type": "object" - }, - { - "type": "null" - } - ], - "description": "Optional descriptions of what true and false mean", - "title": "Criteria" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "noul", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions" - ], - "title": "EuJevNoulQuestion", - "type": "object" - }, - { - "properties": { - "criteria": { - "description": "Two to 10 level descriptions, ordered from low to high", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" - }, - "maxItems": 10, - "minItems": 2, - "title": "Criteria", - "type": "array" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - }, - { - "type": "null" - } - ], - "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", - "title": "Instructions" - }, - "type": { - "const": "score", - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "instructions", - "criteria" - ], - "title": "EuJevScoreQuestion", - "type": "object" - } - ] - }, - "description": "One to eight questions, keyed by names of up to 80 characters", - "maxProperties": 8, - "minProperties": 1, - "propertyNames": { - "maxLength": 80, - "minLength": 1 - }, - "title": "Questions", - "type": "object" - }, - "state": { - "anyOf": [ - { - "maxLength": 100000, - "minLength": 1, - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "items": {}, - "type": "array" - } - ], - "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", - "title": "State" - } - }, - "required": [ - "state", - "questions" - ], - "title": "EuJevDecisionRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EuJevDecisionResponse" - } - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" } }, "security": [ diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a186d84aa5..25021453ab8 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2161,6 +2161,13 @@ def _extract_model_candidates_from_request( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) + if route.rstrip("/") == "/eu_jev/v1/systemone": + from litellm.llms.eu_jev import eu_jev_registry_model + + requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") + if requested_model is not None and not isinstance(requested_model, str): + raise HTTPException(status_code=400, detail="eu_jev model must be a string") + return [f"eu_jev/{eu_jev_registry_model(requested_model)}"] if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload return _dedupe_model_candidates(prediction_models) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ecc9b46105c..7b259fe6d19 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -715,15 +715,13 @@ _EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( ) -@router.api_route( - "/eu_jev/{endpoint:path}", - methods=["GET", "POST", "PUT", "DELETE", "PATCH"], +@router.post( + "/eu_jev/v1/systemone", tags=["eu/jev Pass-through", "pass-through"], responses={200: {"model": EuJevDecisionResponse}}, openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, ) async def eu_jev_proxy_route( - endpoint: str, request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], @@ -732,8 +730,8 @@ async def eu_jev_proxy_route( Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - rejected with a 400. + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + `stream` are rejected with a 400. [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ @@ -742,19 +740,18 @@ async def eu_jev_proxy_route( raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") if body.get("stream"): raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") - base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" - encoded_endpoint: Final = httpx.URL(endpoint).path - normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" - base_url: Final = httpx.URL(base_target_url) - updated_url: Final = base_url.copy_with( - path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), - ) eu_jev_api_key: Final = passthrough_endpoint_router.get_credentials( custom_llm_provider="eu_jev", region_name=None, ) + if not eu_jev_api_key: + raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") + base_url: Final = httpx.URL(get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), + ) endpoint_func: Final = create_pass_through_route( - endpoint=endpoint, + endpoint="v1/systemone", target=str(updated_url), custom_headers={ "Authorization": f"Bearer {eu_jev_api_key}", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index ee1ae43fd3a..ed606036fe0 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1028,6 +1028,33 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr assert denied.value.status_code == 400 +# eu-jev.com/docs (read 2026-10-21): an omitted model means eu-jeff-latest, and the legacy names +# jeff-latest and jeff-1.0.0 answer with the current release. +@pytest.mark.parametrize( + ("request_data", "permission_model"), + [ + ({}, "eu_jev/eu-jeff-latest"), + ({"model": None}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ], +) +@pytest.mark.parametrize("suffix", ["", "/"]) +def test_eu_jev_decision_is_checked_against_the_model_eu_jev_runs( + request_data: dict[str, object], permission_model: str, suffix: str +) -> None: + assert get_model_from_request(request_data=request_data, route=f"/eu_jev/v1/systemone{suffix}") == permission_model + + +@pytest.mark.parametrize("model", [["eu-jeff-latest"], 7, {"name": "eu-jeff-latest"}]) +def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> None: + with pytest.raises(HTTPException) as denied: + get_model_from_request(request_data={"model": model}, route="/eu_jev/v1/systemone") + assert denied.value.status_code == 400 + + def test_laya_model_normalization_does_not_change_other_provider_routes() -> None: assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest" assert get_model_from_request(request_data={}, route="/laya/health") is None diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index dd7b82070f1..6374eed716e 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -391,6 +391,54 @@ def test_eu_jev_decision_is_priced_from_its_registry_row(): assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 +# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release, +# which a future eu/jev release reports under a version the registry does not list yet. +@pytest.mark.parametrize("request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}]) +def test_eu_jev_current_release_requests_are_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: + unlisted_release: Final = "eu-jeff-99.0.0" + assert f"eu_jev/{unlisted_release}" not in litellm.model_cost + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body, + custom_llm_provider="eu_jev", + ) + + latest_cost: Final = litellm.model_cost["eu_jev/eu-jeff-latest"]["input_cost_per_token"] + assert response["kwargs"]["model"] == f"eu_jev/{unlisted_release}" + assert latest_cost > 0 + assert response["kwargs"]["response_cost"] == pytest.approx(64 * latest_cost) + + +def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "eu_jev/eu-jeff-latest", + {**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0}, + ) + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-1.0.0"}, + custom_llm_provider="eu_jev", + ) + + pinned_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]["input_cost_per_token"] + assert response["kwargs"]["response_cost"] == pytest.approx(64 * pinned_cost) + + def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( httpx_response=_response(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9ef360a2c00..5df6e534f60 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8326,14 +8326,6 @@ class TestTypeSafePassthroughRoute: class TestEuJevPassthroughRoute: - @staticmethod - def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: - request = MagicMock(spec=Request) - request.method = "POST" - request.query_params = query_params or {} - request.json = AsyncMock(return_value=body) - return request - @pytest.fixture def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: from litellm.proxy.proxy_server import app @@ -8359,6 +8351,49 @@ class TestEuJevPassthroughRoute: assert sent.headers["authorization"] == "Bearer eu-jev-test-key" assert json.loads(sent.content) == body + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone?trace=yes", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_uses_the_hosted_api_when_base_is_unset(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("EU_JEV_API_BASE") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.com/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_answers_503_without_calling_upstream_when_the_key_is_unset( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("EU_JEV_API_KEY") + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 503 + assert "EU_JEV_API_KEY" in response.text + assert not route.called + + @pytest.mark.parametrize( + ("method", "path"), + [("GET", "/eu_jev/api/account/keys"), ("DELETE", "/eu_jev/v1/keys/abc"), ("GET", "/eu_jev/v1/systemone")], + ) + def test_forwards_nothing_but_a_decision(self, client: TestClient, method: str, path: str) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={}) + response: Final = client.request(method, path) + + assert response.status_code in (404, 405) + assert not route.called + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} with respx.mock(assert_all_called=False) as upstream: @@ -8380,64 +8415,6 @@ class TestEuJevPassthroughRoute: assert "Streaming" in response.text assert not route.called - @pytest.mark.asyncio - async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") - monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") - - async def fake_upstream(request, *_args): - target: Final = create_route.call_args.kwargs["target"] - upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params) - return Response(content=json.dumps({"upstream_query": parse_qs(upstream_url.query.decode())})) - - endpoint_func = AsyncMock(side_effect=fake_upstream) - create_route = Mock(return_value=endpoint_func) - monkeypatch.setattr( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", - create_route, - ) - - result = await eu_jev_proxy_route( - endpoint="v1/systemone", - request=self._request({"state": "x"}, {"trace": "yes"}), - fastapi_response=MagicMock(spec=Response), - user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), - ) - - assert json.loads(result.body) == {"upstream_query": {"trace": ["yes"]}} - endpoint_func.assert_awaited_once() - create_route.assert_called_once_with( - endpoint="v1/systemone", - target="https://eu-jev.example/base/v1/systemone", - custom_headers={ - "Authorization": "Bearer eu-jev-test-key", - "Content-Type": "application/json", - }, - custom_llm_provider="eu_jev", - is_streaming_request=False, - ) - - @pytest.mark.asyncio - async def test_uses_the_hosted_api_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") - monkeypatch.delenv("EU_JEV_API_BASE", raising=False) - - endpoint_func = AsyncMock(return_value=Response()) - create_route = Mock(return_value=endpoint_func) - monkeypatch.setattr( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", - create_route, - ) - - await eu_jev_proxy_route( - endpoint="v1/systemone", - request=self._request({"state": "x"}), - fastapi_response=MagicMock(spec=Response), - user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), - ) - - assert create_route.call_args.kwargs["target"] == "https://eu-jev.com/v1/systemone" - @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index f0baceecdaa..ffa30d11b52 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -82,6 +82,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES: Final = MappingProxyType( "/tinyfish/{endpoint:path}": frozenset({"GET", "POST"}), "/laya/v1/systemone": frozenset({"POST"}), "/bespoke/v1/systemone": frozenset({"POST"}), + "/eu_jev/v1/systemone": frozenset({"POST"}), } ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 44f8932f212..3d8c50be4ca 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,70 +5333,30 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; - "/eu_jev/{endpoint}": { + "/eu_jev/v1/systemone": { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; + get?: never; + put?: never; /** * Eu Jev Proxy Route * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. * * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - * rejected with a 400. + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + * `stream` are rejected with a 400. * * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ - get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; - /** - * Eu Jev Proxy Route - * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. - * - * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - * rejected with a 400. - * - * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) - */ - put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; - /** - * Eu Jev Proxy Route - * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. - * - * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - * rejected with a 400. - * - * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) - */ - post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; - /** - * Eu Jev Proxy Route - * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. - * - * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - * rejected with a 400. - * - * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) - */ - delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; + post: operations["eu_jev_proxy_route_eu_jev_v1_systemone_post"]; + delete?: never; options?: never; head?: never; - /** - * Eu Jev Proxy Route - * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. - * - * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) - * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is - * rejected with a 400. - * - * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) - */ - patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; + patch?: never; trace?: never; }; "/extraction": { @@ -59256,13 +59216,11 @@ export interface operations { }; }; }; - eu_jev_proxy_route_eu_jev__endpoint__get: { + eu_jev_proxy_route_eu_jev_v1_systemone_post: { parameters: { query?: never; header?: never; - path: { - endpoint: string; - }; + path?: never; cookie?: never; }; requestBody: { @@ -59394,611 +59352,6 @@ export interface operations { "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - eu_jev_proxy_route_eu_jev__endpoint__put: { - parameters: { - query?: never; - header?: never; - path: { - endpoint: string; - }; - cookie?: never; - }; - requestBody: { - content: { - /** - * @example { - * "model": "eu-jeff-latest", - * "questions": { - * "frustration": { - * "criteria": [ - * "calm", - * "mildly annoyed", - * "angry" - * ], - * "instructions": "How frustrated is the customer?", - * "type": "score" - * }, - * "is_refund_request": { - * "instructions": "Is the customer asking for a refund?", - * "type": "noul" - * }, - * "team": { - * "criteria": { - * "billing": "Payments, invoices, and refunds", - * "support": "Technical issues and bugs" - * }, - * "instructions": "Which team should handle this?", - * "type": "choice" - * } - * }, - * "state": "I was charged twice. Can I get a refund?" - * } - */ - "application/json": { - /** - * Model - * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it - * @default eu-jeff-latest - */ - model?: string; - /** - * Questions - * @description One to eight questions, keyed by names of up to 80 characters - */ - questions: { - [key: string]: { - /** - * Criteria - * @description Two to 64 options, option name to description. A null description uses the option name - */ - criteria: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - }; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "choice"; - } | { - /** - * Criteria - * @description Optional descriptions of what true and false mean - */ - criteria?: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - } | null; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "noul"; - } | { - /** - * Criteria - * @description Two to 10 level descriptions, ordered from low to high - */ - criteria: (string | { - [key: string]: unknown; - } | unknown[] | null)[]; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "score"; - }; - }; - /** - * State - * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array - */ - state: string | { - [key: string]: unknown; - } | unknown[]; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EuJevDecisionResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - eu_jev_proxy_route_eu_jev__endpoint__post: { - parameters: { - query?: never; - header?: never; - path: { - endpoint: string; - }; - cookie?: never; - }; - requestBody: { - content: { - /** - * @example { - * "model": "eu-jeff-latest", - * "questions": { - * "frustration": { - * "criteria": [ - * "calm", - * "mildly annoyed", - * "angry" - * ], - * "instructions": "How frustrated is the customer?", - * "type": "score" - * }, - * "is_refund_request": { - * "instructions": "Is the customer asking for a refund?", - * "type": "noul" - * }, - * "team": { - * "criteria": { - * "billing": "Payments, invoices, and refunds", - * "support": "Technical issues and bugs" - * }, - * "instructions": "Which team should handle this?", - * "type": "choice" - * } - * }, - * "state": "I was charged twice. Can I get a refund?" - * } - */ - "application/json": { - /** - * Model - * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it - * @default eu-jeff-latest - */ - model?: string; - /** - * Questions - * @description One to eight questions, keyed by names of up to 80 characters - */ - questions: { - [key: string]: { - /** - * Criteria - * @description Two to 64 options, option name to description. A null description uses the option name - */ - criteria: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - }; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "choice"; - } | { - /** - * Criteria - * @description Optional descriptions of what true and false mean - */ - criteria?: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - } | null; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "noul"; - } | { - /** - * Criteria - * @description Two to 10 level descriptions, ordered from low to high - */ - criteria: (string | { - [key: string]: unknown; - } | unknown[] | null)[]; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "score"; - }; - }; - /** - * State - * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array - */ - state: string | { - [key: string]: unknown; - } | unknown[]; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EuJevDecisionResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - eu_jev_proxy_route_eu_jev__endpoint__delete: { - parameters: { - query?: never; - header?: never; - path: { - endpoint: string; - }; - cookie?: never; - }; - requestBody: { - content: { - /** - * @example { - * "model": "eu-jeff-latest", - * "questions": { - * "frustration": { - * "criteria": [ - * "calm", - * "mildly annoyed", - * "angry" - * ], - * "instructions": "How frustrated is the customer?", - * "type": "score" - * }, - * "is_refund_request": { - * "instructions": "Is the customer asking for a refund?", - * "type": "noul" - * }, - * "team": { - * "criteria": { - * "billing": "Payments, invoices, and refunds", - * "support": "Technical issues and bugs" - * }, - * "instructions": "Which team should handle this?", - * "type": "choice" - * } - * }, - * "state": "I was charged twice. Can I get a refund?" - * } - */ - "application/json": { - /** - * Model - * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it - * @default eu-jeff-latest - */ - model?: string; - /** - * Questions - * @description One to eight questions, keyed by names of up to 80 characters - */ - questions: { - [key: string]: { - /** - * Criteria - * @description Two to 64 options, option name to description. A null description uses the option name - */ - criteria: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - }; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "choice"; - } | { - /** - * Criteria - * @description Optional descriptions of what true and false mean - */ - criteria?: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - } | null; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "noul"; - } | { - /** - * Criteria - * @description Two to 10 level descriptions, ordered from low to high - */ - criteria: (string | { - [key: string]: unknown; - } | unknown[] | null)[]; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "score"; - }; - }; - /** - * State - * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array - */ - state: string | { - [key: string]: unknown; - } | unknown[]; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EuJevDecisionResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; - }; - }; - eu_jev_proxy_route_eu_jev__endpoint__patch: { - parameters: { - query?: never; - header?: never; - path: { - endpoint: string; - }; - cookie?: never; - }; - requestBody: { - content: { - /** - * @example { - * "model": "eu-jeff-latest", - * "questions": { - * "frustration": { - * "criteria": [ - * "calm", - * "mildly annoyed", - * "angry" - * ], - * "instructions": "How frustrated is the customer?", - * "type": "score" - * }, - * "is_refund_request": { - * "instructions": "Is the customer asking for a refund?", - * "type": "noul" - * }, - * "team": { - * "criteria": { - * "billing": "Payments, invoices, and refunds", - * "support": "Technical issues and bugs" - * }, - * "instructions": "Which team should handle this?", - * "type": "choice" - * } - * }, - * "state": "I was charged twice. Can I get a refund?" - * } - */ - "application/json": { - /** - * Model - * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it - * @default eu-jeff-latest - */ - model?: string; - /** - * Questions - * @description One to eight questions, keyed by names of up to 80 characters - */ - questions: { - [key: string]: { - /** - * Criteria - * @description Two to 64 options, option name to description. A null description uses the option name - */ - criteria: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - }; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "choice"; - } | { - /** - * Criteria - * @description Optional descriptions of what true and false mean - */ - criteria?: { - [key: string]: string | { - [key: string]: unknown; - } | unknown[] | null; - } | null; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "noul"; - } | { - /** - * Criteria - * @description Two to 10 level descriptions, ordered from low to high - */ - criteria: (string | { - [key: string]: unknown; - } | unknown[] | null)[]; - /** - * Instructions - * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference - */ - instructions: string | { - [key: string]: unknown; - } | unknown[] | null; - /** - * Type - * @constant - */ - type: "score"; - }; - }; - /** - * State - * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array - */ - state: string | { - [key: string]: unknown; - } | unknown[]; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["EuJevDecisionResponse"]; - }; - }; - /** @description Validation Error */ - 422: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["HTTPValidationError"]; - }; - }; }; }; extraction_extraction_post: { From 49238926420be2bff24d41a00eecd96800d76715 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:11:09 -0700 Subject: [PATCH 08/15] fix(pass-through): apply eu_jev key limits, register eu_jev, price unlisted releases Pre-call hooks and guardrails now see the body model as eu_jev/, so model_rpm_limit and model_tpm_limit set on eu_jev/eu-jeff-latest apply, and upstream still gets the client's exact body. eu_jev is a registered decisions provider, so access groups, config deployments, /v1/systemone and /v1/decisions accept it, and the Admin UI can add it. A release without its own cost map row is priced as eu-jeff-latest instead of $0 --- README.md | 1 + litellm/__init__.py | 3 ++ litellm/_lazy_imports_registry.py | 2 + litellm/llms/eu_jev.py | 10 ----- litellm/llms/eu_jev/__init__.py | 0 litellm/llms/eu_jev/common_utils.py | 30 +++++++++++++ .../llms/eu_jev/decisions/transformation.py | 9 ++++ .../pass_through/typesafe_logging_handler.py | 9 +++- ...odel_prices_and_context_window_backup.json | 10 +---- litellm/proxy/auth/auth_utils.py | 2 +- .../pass_through_endpoints.py | 14 ++++++ .../provider_create_fields.json | 28 ++++++++++++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 10 +---- provider_endpoints_support.json | 7 +++ tests/unit/decisions/test_main.py | 1 + ...st_typesafe_passthrough_logging_handler.py | 8 ++-- .../test_llm_pass_through_endpoints.py | 45 +++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 13 ++++-- 20 files changed, 170 insertions(+), 35 deletions(-) delete mode 100644 litellm/llms/eu_jev.py create mode 100644 litellm/llms/eu_jev/__init__.py create mode 100644 litellm/llms/eu_jev/common_utils.py create mode 100644 litellm/llms/eu_jev/decisions/transformation.py diff --git a/README.md b/README.md index 5fd84b529dc..03456d07fe4 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | | | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | +| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | | | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | | | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..5bc998ce839 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1730,6 +1730,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.eu_jev.decisions.transformation import ( + EuJevDecisionsConfig as EuJevDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e0cb7c680bc..f1dee93deea 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -172,6 +172,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "EuJevDecisionsConfig", "OpenAIDecisionsConfig", "AzureAIDecisionsConfig", "NvidiaNimRerankConfig", @@ -748,6 +749,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), "NvidiaNimRerankConfig": ( diff --git a/litellm/llms/eu_jev.py b/litellm/llms/eu_jev.py deleted file mode 100644 index 315b26bbcea..00000000000 --- a/litellm/llms/eu_jev.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Final - -EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" -_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) - - -def eu_jev_registry_model(requested_model: str | None) -> str: - if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES: - return EU_JEV_LATEST_MODEL - return requested_model diff --git a/litellm/llms/eu_jev/__init__.py b/litellm/llms/eu_jev/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/eu_jev/common_utils.py b/litellm/llms/eu_jev/common_utils.py new file mode 100644 index 00000000000..0bc29ba2d73 --- /dev/null +++ b/litellm/llms/eu_jev/common_utils.py @@ -0,0 +1,30 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" +_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) +_EU_JEV_PREFIX: Final = "eu_jev/" + + +def eu_jev_registry_model(requested_model: str | None) -> str: + if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES: + return EU_JEV_LATEST_MODEL + return requested_model + + +def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str: + requested_model: Final = decision_body.get("model") + return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}" + + +def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]: + hook_model: Final = hook_body.get("model") + without_model: Final = {key: value for key, value in hook_body.items() if key != "model"} + if hook_model == eu_jev_scoped_model(client_body): + return MappingProxyType( + {**without_model, **{key: value for key, value in client_body.items() if key == "model"}} + ) + if isinstance(hook_model, str): + return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)}) + return hook_body diff --git a/litellm/llms/eu_jev/decisions/transformation.py b/litellm/llms/eu_jev/decisions/transformation.py new file mode 100644 index 00000000000..18ef391b040 --- /dev/null +++ b/litellm/llms/eu_jev/decisions/transformation.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class EuJevDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("EU_JEV_API_KEY",) + api_base_env = ("EU_JEV_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://eu-jev.com" diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 83511adbb4c..9eae57774c2 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -11,7 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature ) -from litellm.llms.eu_jev import eu_jev_registry_model +from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model from litellm.llms.laya.common_utils import laya_response_model from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage @@ -91,8 +91,13 @@ class TypeSafePassthroughLoggingHandler: usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens + candidate_models: Final = ( + (response_model, registry_request_model, EU_JEV_LATEST_MODEL) + if custom_llm_provider == "eu_jev" + else (response_model, registry_request_model) + ) candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, registry_request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 659ba9d6bbb..9a98fc7631e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72270,10 +72270,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs", - "supported_endpoints": [ - "/v1/systemone" - ] + "source": "https://eu-jev.com/docs" }, "eu_jev/eu-jeff-latest": { "input_cost_per_token": 4.15e-08, @@ -72281,10 +72278,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs", - "supported_endpoints": [ - "/v1/systemone" - ] + "source": "https://eu-jev.com/docs" }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 25021453ab8..83c56c511e2 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2162,7 +2162,7 @@ def _extract_model_candidates_from_request( raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) if route.rstrip("/") == "/eu_jev/v1/systemone": - from litellm.llms.eu_jev import eu_jev_registry_model + from litellm.llms.eu_jev.common_utils import eu_jev_registry_model requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") if requested_model is not None and not isinstance(requested_model, str): diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index eb398ec45ca..f8b751a852d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -71,6 +71,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body from litellm.llms.oss_decision import validate_oss_request from litellm.passthrough import BasePassthroughUtils from litellm.proxy._lazy_features import lazy_owned_routes @@ -1228,6 +1229,13 @@ async def pass_through_request( decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) _parsed_body["model"] = f"{custom_llm_provider}/{checkpoint}" + eu_jev_client_body: Final = ( + MappingProxyType(TypeAdapter(dict[str, object]).validate_python(_parsed_body)) + if custom_llm_provider == "eu_jev" + else None + ) + if eu_jev_client_body is not None: + _parsed_body["model"] = eu_jev_scoped_model(eu_jev_client_body) ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### # Passthrough endpoints are opt-in only for guardrails @@ -1307,6 +1315,12 @@ async def pass_through_request( ) _ = validate_oss_request(custom_llm_provider, decision_body) _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) + if eu_jev_client_body is not None: + _parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage + eu_jev_upstream_body( + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body + ) + ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 4bec42a7868..367d14ba655 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1497,6 +1497,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "EU_JEV", + "provider_display_name": "eu/jev", + "litellm_provider": "eu_jev", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://eu-jev.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "eu_jev/eu-jeff-latest" + }, { "provider": "FalAI", "provider_display_name": "Fal AI", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..6dcf7811f21 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4432,6 +4432,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + EU_JEV = "eu_jev" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 897138a940c..6dad3bb02a9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9565,6 +9565,8 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() + if provider == LlmProviders.EU_JEV: + return litellm.EuJevDecisionsConfig() if provider == LlmProviders.OPENAI: return litellm.OpenAIDecisionsConfig() if provider == LlmProviders.AZURE_AI: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 659ba9d6bbb..9a98fc7631e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72270,10 +72270,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs", - "supported_endpoints": [ - "/v1/systemone" - ] + "source": "https://eu-jev.com/docs" }, "eu_jev/eu-jeff-latest": { "input_cost_per_token": 4.15e-08, @@ -72281,10 +72278,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs", - "supported_endpoints": [ - "/v1/systemone" - ] + "source": "https://eu-jev.com/docs" }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d65da3f41fd..4bb5615cb9e 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1508,6 +1508,13 @@ "systemone": true } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "systemone": true + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 84e7a970195..eeb1cef29c9 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -122,6 +122,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), ("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"), + ("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"), ( "openrouter", "openrouter/typesafe/jev-1.13", diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 6374eed716e..a20d6eb50e0 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -392,9 +392,11 @@ def test_eu_jev_decision_is_priced_from_its_registry_row(): # eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release, -# which a future eu/jev release reports under a version the registry does not list yet. -@pytest.mark.parametrize("request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}]) -def test_eu_jev_current_release_requests_are_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: +# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it. +@pytest.mark.parametrize( + "request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}] +) +def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: unlisted_release: Final = "eu-jeff-99.0.0" assert f"eu_jev/{unlisted_release}" not in litellm.model_cost response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 5df6e534f60..9aae1a6cb43 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8325,6 +8325,19 @@ class TestTypeSafePassthroughRoute: ) +class _RecordsPreCallModel(CustomLogger): + def __init__(self, switch_to: str | None = None) -> None: + super().__init__() + self.switch_to: Final = switch_to + self.models: Final[list[object]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str + ) -> dict | None: + self.models.append(data.get("model")) + return None if self.switch_to is None else {**data, "model": self.switch_to} + + class TestEuJevPassthroughRoute: @pytest.fixture def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: @@ -8351,6 +8364,38 @@ class TestEuJevPassthroughRoute: assert sent.headers["authorization"] == "Bearer eu-jev-test-key" assert json.loads(sent.content) == body + @pytest.mark.parametrize( + ("body", "hook_model"), + [ + ({"state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"), + ], + ) + def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert response.status_code == 200 + assert recorder.models == [hook_model] + assert json.loads(route.calls.last.request.content) == body + + def test_forwards_the_release_a_pre_call_hook_switched_to( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: with respx.mock(assert_all_called=True) as upstream: route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 8f27365fcc1..358d298874b 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -438,9 +438,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: return entry -@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE))) -def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None: - entry: Final = _decisions_provider_entry(display_name) +@pytest.mark.parametrize( + ("provider_key", "display_name", "provider"), + ( + ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), + ("Nace", "Nace", LlmProviders.NACE), + ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), + ), +) +def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: + entry: Final = _decisions_provider_entry(provider_key) assert entry.provider_display_name == display_name assert entry.litellm_provider == provider.value From 5774d72b447187d82fa87b9b563ef4440c37dc13 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:04:17 -0700 Subject: [PATCH 09/15] fix(eu_jev): use the pass-through deployment's api_base and accept provider-prefixed models --- litellm/llms/eu_jev/common_utils.py | 8 +++- .../llm_passthrough_endpoints.py | 6 ++- .../passthrough_endpoint_router.py | 37 +++++++++++++---- tests/unit/proxy/auth/test_auth_utils.py | 2 + .../test_llm_pass_through_endpoints.py | 41 +++++++++++++++++++ .../test_passthrough_endpoint_router.py | 17 ++++++++ 6 files changed, 99 insertions(+), 12 deletions(-) diff --git a/litellm/llms/eu_jev/common_utils.py b/litellm/llms/eu_jev/common_utils.py index 0bc29ba2d73..beda1d371f9 100644 --- a/litellm/llms/eu_jev/common_utils.py +++ b/litellm/llms/eu_jev/common_utils.py @@ -8,9 +8,10 @@ _EU_JEV_PREFIX: Final = "eu_jev/" def eu_jev_registry_model(requested_model: str | None) -> str: - if requested_model is None or requested_model in _EU_JEV_CURRENT_RELEASE_NAMES: + release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None + if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES: return EU_JEV_LATEST_MODEL - return requested_model + return release def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str: @@ -22,6 +23,9 @@ def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[s hook_model: Final = hook_body.get("model") without_model: Final = {key: value for key, value in hook_body.items() if key != "model"} if hook_model == eu_jev_scoped_model(client_body): + client_model: Final = client_body.get("model") + if isinstance(client_model, str): + return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)}) return MappingProxyType( {**without_model, **{key: value for key, value in client_body.items() if key == "model"}} ) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 7b259fe6d19..988fe140460 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -746,7 +746,11 @@ async def eu_jev_proxy_route( ) if not eu_jev_api_key: raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") - base_url: Final = httpx.URL(get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + base_url: Final = httpx.URL( + passthrough_endpoint_router.get_deployment_api_base(custom_llm_provider="eu_jev", region_name=None) + or get_secret_str("EU_JEV_API_BASE") + or "https://eu-jev.com" + ) updated_url: Final = base_url.copy_with( path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), ) diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 270a1cc2f3b..4cd90829ad4 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,5 +1,6 @@ import json from collections.abc import Callable +from dataclasses import dataclass from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -23,6 +24,12 @@ if TYPE_CHECKING: from litellm.router import Router +@dataclass(frozen=True, slots=True) +class _DeploymentCredentials: + api_key: str + api_base: str | None + + def _get_proxy_llm_router() -> "Router | None": from litellm.proxy.proxy_server import llm_router @@ -74,12 +81,12 @@ class PassthroughEndpointRouter: custom_llm_provider: str, region_name: str | None, ) -> str | None: - deployment_api_key: Final = self._get_deployment_api_key( + deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, region_name=region_name, ) - if deployment_api_key is not None: - return _resolve_oidc_reference(deployment_api_key) + if deployment_credentials is not None: + return _resolve_oidc_reference(deployment_credentials.api_key) verbose_router_logger.debug( "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider ) @@ -88,21 +95,32 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) - def _get_deployment_api_key( + def get_deployment_api_base( self, custom_llm_provider: str, region_name: str | None, ) -> str | None: + deployment_credentials: Final = self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, + region_name=region_name, + ) + return deployment_credentials.api_base if deployment_credentials is not None else None + + def _get_deployment_credentials( + self, + custom_llm_provider: str, + region_name: str | None, + ) -> _DeploymentCredentials | None: llm_router: Final = self.llm_router_getter() if llm_router is None: return None deployments: Final = llm_router.get_model_list() or () return next( ( - api_key + credentials for deployment in deployments if ( - api_key := self._resolve_matching_deployment_api_key( + credentials := self._resolve_matching_deployment_credentials( litellm_params=deployment["litellm_params"], custom_llm_provider=custom_llm_provider, region_name=region_name, @@ -113,12 +131,12 @@ class PassthroughEndpointRouter: None, ) - def _resolve_matching_deployment_api_key( + def _resolve_matching_deployment_credentials( self, litellm_params: LiteLLMParamsTypedDict, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + ) -> _DeploymentCredentials | None: if litellm_params.get("use_in_pass_through") is not True: return None if self._get_deployment_provider(litellm_params) != custom_llm_provider: @@ -134,7 +152,8 @@ class PassthroughEndpointRouter: ) if deployment_region != region_name: return None - return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + return _DeploymentCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None: model: Final = litellm_params.get("model") diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index ed606036fe0..93fede4ae97 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1039,6 +1039,8 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr ({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"), ({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"), ({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/jeff-latest"}, "eu_jev/eu-jeff-latest"), ], ) @pytest.mark.parametrize("suffix", ["", "/"]) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9aae1a6cb43..376860dcbc7 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8396,6 +8396,47 @@ class TestEuJevPassthroughRoute: assert response.status_code == 200 assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + def test_a_provider_prefixed_model_is_limited_once_and_forwarded_without_the_prefix( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu_jev/eu-jeff-1.0.0", "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-1.0.0"] + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_pass_through_deployment_key_goes_to_that_deployment_base( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "eu-jev-friendly", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "eu-jev-deployment-key", + "api_base": "https://eu-jev-deployment.example/v2", + "use_in_pass_through": True, + }, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", deployment_router) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev-deployment.example/v2/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key" + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: with respx.mock(assert_all_called=True) as upstream: route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 9c50a6d4df6..0b8a36cd4b9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -60,6 +60,23 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) +def test_deployment_api_base_belongs_to_the_deployment_whose_key_is_used(): + llm_router = litellm.Router( + model_list=[ + _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), + _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), + ] + ) + passthrough_router = _passthrough_router(llm_router) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-deployment" + assert ( + passthrough_router.get_deployment_api_base(custom_llm_provider="openai", region_name=None) + == "https://keyed.example/v1" + ) + assert passthrough_router.get_deployment_api_base(custom_llm_provider="anthropic", region_name=None) is None + + def test_credential_rotation_is_reflected_without_deployment_update(): CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")]) llm_router = litellm.Router( From b0ebc1372f69012a1521315e4417ec70742d7043 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:05:27 -0700 Subject: [PATCH 10/15] fix(eu_jev): declare /v1/systemone on the eu/jev cost map entries so the generated endpoint table keeps it --- ...odel_prices_and_context_window_backup.json | 10 +++++-- .../provider_endpoints_support_backup.json | 26 +++++++++++++++++++ model_prices_and_context_window.json | 10 +++++-- provider_endpoints_support.json | 21 ++++++++++++++- 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 76d8617e942..77e928969d7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -69990,7 +69990,10 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs" + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] }, "eu_jev/eu-jeff-latest": { "input_cost_per_token": 4.15e-08, @@ -69998,7 +70001,10 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs" + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 9ba6ed342ea..230ead5fc68 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2215,6 +2215,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 76d8617e942..77e928969d7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -69990,7 +69990,10 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs" + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] }, "eu_jev/eu-jeff-latest": { "input_cost_per_token": 4.15e-08, @@ -69998,7 +70001,10 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://eu-jev.com/docs" + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0086f7daa7e..230ead5fc68 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2219,7 +2219,26 @@ "display_name": "eu/jev (`eu_jev`)", "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", "endpoints": { - "systemone": true + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false } }, "laya": { From 11fd82d4b9a8d432e607db0949cd5dabae2763db Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:24:58 -0700 Subject: [PATCH 11/15] fix(eu_jev): take pass-through credentials only from proxy-wide deployments The eu/jev route now ignores team-scoped deployments when it picks the upstream key and base, so one team's deployment can no longer redirect another team's calls, and a base ending in /v1 is not doubled --- .../llm_passthrough_endpoints.py | 15 +++---- .../passthrough_endpoint_router.py | 44 +++++++++++++------ .../test_llm_pass_through_endpoints.py | 40 +++++++++++++++++ .../test_passthrough_endpoint_router.py | 32 ++++++++++---- 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 319ce247a12..cfc67931feb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -740,16 +740,13 @@ async def eu_jev_proxy_route( raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") if body.get("stream"): raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") - eu_jev_api_key: Final = passthrough_endpoint_router.get_credentials( - custom_llm_provider="eu_jev", - region_name=None, - ) - if not eu_jev_api_key: + eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev") + if eu_jev_credentials is None: raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") base_url: Final = httpx.URL( - passthrough_endpoint_router.get_deployment_api_base(custom_llm_provider="eu_jev", region_name=None) - or get_secret_str("EU_JEV_API_BASE") - or "https://eu-jev.com" + (eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + .rstrip("/") + .removesuffix("/v1") ) updated_url: Final = base_url.copy_with( path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), @@ -758,7 +755,7 @@ async def eu_jev_proxy_route( endpoint="v1/systemone", target=str(updated_url), custom_headers={ - "Authorization": f"Bearer {eu_jev_api_key}", + "Authorization": f"Bearer {eu_jev_credentials.api_key}", "Content-Type": "application/json", }, custom_llm_provider="eu_jev", diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 4cd90829ad4..8c551a78929 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,5 +1,5 @@ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Final @@ -11,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.secret_managers.main import ( UnresolvableSecretReference, get_secret_str, @@ -25,7 +26,7 @@ if TYPE_CHECKING: @dataclass(frozen=True, slots=True) -class _DeploymentCredentials: +class _PassThroughCredentials: api_key: str api_base: str | None @@ -36,6 +37,11 @@ def _get_proxy_llm_router() -> "Router | None": return llm_router +def _is_team_deployment(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id")) + + def _get_str_value(values: dict[str, object] | None, key: str) -> str | None: value: Final = values.get(key) if values is not None else None return value if isinstance(value, str) else None @@ -84,6 +90,7 @@ class PassthroughEndpointRouter: deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, region_name=region_name, + include_team_deployments=True, ) if deployment_credentials is not None: return _resolve_oidc_reference(deployment_credentials.api_key) @@ -95,22 +102,32 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) - def get_deployment_api_base( - self, - custom_llm_provider: str, - region_name: str | None, - ) -> str | None: + def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None: deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, - region_name=region_name, + region_name=None, + include_team_deployments=False, + ) + api_key: Final = _resolve_oidc_reference( + deployment_credentials.api_key + if deployment_credentials is not None + else get_secret_str( + self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider) + ) + ) + if not api_key: + return None + return _PassThroughCredentials( + api_key=api_key, + api_base=deployment_credentials.api_base if deployment_credentials is not None else None, ) - return deployment_credentials.api_base if deployment_credentials is not None else None def _get_deployment_credentials( self, custom_llm_provider: str, region_name: str | None, - ) -> _DeploymentCredentials | None: + include_team_deployments: bool, + ) -> _PassThroughCredentials | None: llm_router: Final = self.llm_router_getter() if llm_router is None: return None @@ -119,7 +136,8 @@ class PassthroughEndpointRouter: ( credentials for deployment in deployments - if ( + if (include_team_deployments or not _is_team_deployment(deployment)) + and ( credentials := self._resolve_matching_deployment_credentials( litellm_params=deployment["litellm_params"], custom_llm_provider=custom_llm_provider, @@ -136,7 +154,7 @@ class PassthroughEndpointRouter: litellm_params: LiteLLMParamsTypedDict, custom_llm_provider: str, region_name: str | None, - ) -> _DeploymentCredentials | None: + ) -> _PassThroughCredentials | None: if litellm_params.get("use_in_pass_through") is not True: return None if self._get_deployment_provider(litellm_params) != custom_llm_provider: @@ -153,7 +171,7 @@ class PassthroughEndpointRouter: if deployment_region != region_name: return None api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") - return _DeploymentCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None + return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None: model: Final = litellm_params.get("model") diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 035d5e0bb6a..bc7cfce4feb 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8438,6 +8438,46 @@ class TestEuJevPassthroughRoute: assert response.status_code == 200 assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key" + def test_a_team_deployment_never_redirects_another_team_s_calls( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + team_router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-a-eu-jev", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "team-a-own-key", + "api_base": "https://team-a-server.example", + "use_in_pass_through": True, + }, + "model_info": {"team_id": "team-a"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", team_router) + with respx.mock(assert_all_called=False) as upstream: + team_server: Final = upstream.route(host="team-a-server.example").respond(200, json={"answers": {}}) + eu_jev: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "team b data"}) + + assert response.status_code == 200 + assert not team_server.called + assert eu_jev.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + + def test_a_base_ending_in_v1_is_not_doubled(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base/v1/") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: with respx.mock(assert_all_called=True) as upstream: route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 0b8a36cd4b9..81d0a574449 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -60,21 +60,37 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) -def test_deployment_api_base_belongs_to_the_deployment_whose_key_is_used(): +def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base(): llm_router = litellm.Router( model_list=[ _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), ] ) - passthrough_router = _passthrough_router(llm_router) + credentials = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") - assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-deployment" - assert ( - passthrough_router.get_deployment_api_base(custom_llm_provider="openai", region_name=None) - == "https://keyed.example/v1" - ) - assert passthrough_router.get_deployment_api_base(custom_llm_provider="anthropic", region_name=None) is None + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1") + + +def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env") + team_deployment = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + credentials = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None) + + +def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key(): + passthrough_router = _passthrough_router(litellm.Router(model_list=[])) + + assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None def test_credential_rotation_is_reflected_without_deployment_update(): From 3df76ba77847b6073e17d765ec4333c9aa9a9ed9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:22:17 -0700 Subject: [PATCH 12/15] fix(eu_jev): let a configured pass-through at /eu_jev/v1/systemone keep serving it Scope the mapped pass-through and gateway allowlist entries to the decision route instead of the whole /eu_jev prefix, so custom routes under /eu_jev keep their auth, metering, and gateway behavior from before this PR --- gateway/routes/allowlist.py | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 2 +- litellm/proxy/_types.py | 2 +- .../llm_passthrough_endpoints.py | 11 ++++- .../pass_through_endpoints.py | 16 ++++++- .../test_llm_pass_through_endpoints.py | 45 +++++++++++++++++++ .../test_passthrough_auth_default.py | 18 ++++++++ .../test_passthrough_endpoint_router.py | 29 ++++++++---- tests/unit/proxy/test_component_allowlists.py | 8 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 10 files changed, 119 insertions(+), 16 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 375fad27b7a..0cd3cf1d59d 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -109,7 +109,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/vllm/", "/mistral/", "/typesafe/", - "/eu_jev/", + "/eu_jev/v1/systemone", "/openrouter/", "/nvidia_nim/", "/groq/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b1993068f7f..671ec87be4a 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -31896,7 +31896,7 @@ }, "/eu_jev/v1/systemone": { "post": { - "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post", "requestBody": { "content": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e2dd7a7a80f..7c227e04b9e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -413,7 +413,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", - "/eu_jev", + "/eu_jev/v1/systemone", "/laya", "/bespoke", "/openrouter", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index cfc67931feb..6c086e99a13 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -111,6 +111,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, + YieldsToConfiguredPassThroughRoute, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -715,7 +716,10 @@ _EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( ) -@router.post( +eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute) + + +@eu_jev_router.post( "/eu_jev/v1/systemone", tags=["eu/jev Pass-through", "pass-through"], responses={200: {"model": EuJevDecisionResponse}}, @@ -731,7 +735,7 @@ async def eu_jev_proxy_route( `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and - `stream` are rejected with a 400. + `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ @@ -766,6 +770,9 @@ async def eu_jev_proxy_route( ) +router.include_router(eu_jev_router) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 73f4ada4f1f..aa06ca01c8b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -26,9 +26,11 @@ from fastapi import ( status, ) from fastapi.responses import StreamingResponse +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile -from starlette.routing import BaseRoute, Route +from starlette.routing import BaseRoute, Match, Route +from starlette.types import Scope from starlette.websockets import WebSocketState from websockets.asyncio.client import connect from websockets.exceptions import ( @@ -3497,6 +3499,18 @@ class InitPassThroughEndpointHelpers: return None +class YieldsToConfiguredPassThroughRoute(APIRoute): + def matches(self, scope: Scope) -> tuple[Match, Scope]: + match, child_scope = super().matches(scope) + if match is not Match.FULL: + return match, child_scope + request: Final = Request(scope) + configured: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( + route=request.url.path, method=request.method + ) + return (match, child_scope) if configured is None else (Match.NONE, {}) + + def _get_combined_pass_through_endpoints( pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint], config_pass_through_endpoints: list[dict], diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index bc7cfce4feb..126fdd6c931 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8542,6 +8542,51 @@ class TestEuJevPassthroughRoute: assert "Streaming" in response.text assert not route.called + def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from fastapi import FastAPI + + from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + monkeypatch.delenv("EU_JEV_API_KEY", raising=False) + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") + app: Final = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + saved_registry: Final = dict(_registered_pass_through_routes) + try: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + with respx.mock(assert_all_called=True) as upstream, TestClient(app) as client: + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py index 0e337195bb8..a683f064bae 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -26,6 +26,7 @@ import pytest from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from starlette.requests import Request from litellm.proxy._types import PassThroughGenericEndpoint, ProxyException @@ -137,3 +138,20 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation(): ) assert isinstance(result, UserAPIKeyAuth) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "expected_key"), + [("/eu_jev/v1/systemone", "sk-from-header"), ("/eu_jev/custom", "sk-from-bearer")], +) +async def test_only_the_eu_jev_decision_route_reads_the_litellm_user_api_key_header( + route: str, expected_key: str +) -> None: + request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]}) + + result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints( + request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer" + ) + + assert result == expected_key diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index 81d0a574449..cb50a4cbe89 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import asyncio import os import unittest +from typing import Final from unittest.mock import patch import pytest @@ -60,35 +61,45 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) -def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base(): - llm_router = litellm.Router( +def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base() -> None: + llm_router: Final = litellm.Router( model_list=[ _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), ] ) - credentials = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") + credentials: Final = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") assert credentials is not None assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1") -def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch): +def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env") - team_deployment = { + team_deployment: Final = { **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"), "model_info": {"team_id": "team-a"}, } - passthrough_router = _passthrough_router(litellm.Router(model_list=[team_deployment])) + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) - credentials = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") + credentials: Final = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") assert credentials is not None assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None) -def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key(): - passthrough_router = _passthrough_router(litellm.Router(model_list=[])) +def test_credentials_for_a_provider_route_still_include_a_team_deployment_key() -> None: + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-team-a" + + +def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key() -> None: + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[])) assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index eb058d01770..63a72d7b0f0 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -397,6 +397,14 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" +@pytest.mark.parametrize(("path", "served"), [("/eu_jev/v1/systemone", True), ("/eu_jev/custom", False)]) +def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None: + async def endpoint() -> None: + return None + + assert _is_gateway_route(Route(path, endpoint, methods=["POST"])) is served + + def test_every_app_mount_is_assigned_to_a_component(): """Every Mount on the proxy app must be consciously assigned to a component. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index cb74428d871..7ffeac0b006 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5372,7 +5372,7 @@ export interface paths { * * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and - * `stream` are rejected with a 400. + * `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. * * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ From e9660b04d5ae0307f3f69e81cd78903c5ee6630b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:37:51 -0700 Subject: [PATCH 13/15] fix(eu_jev): treat /eu_jev/v1/systemone as a provider route only while the built-in route serves it Drop the decision path from the mapped pass-through list and classify it with one check that asks the pass-through registry, so an operator's own pass-through at that path keeps its auth, metering, LLM-route status and litellm_user_api_key handling from before this PR. The built-in route now yields on a per-request root path and on any configured method, and the gateway keeps only the exact decision path --- gateway/routes/allowlist.py | 2 +- litellm/proxy/_types.py | 1 - litellm/proxy/auth/auth_utils.py | 4 +- litellm/proxy/auth/route_checks.py | 13 ++++- .../billable_request_metrics_middleware.py | 3 ++ .../pass_through_endpoints.py | 6 ++- tests/unit/proxy/auth/test_auth_utils.py | 5 ++ tests/unit/proxy/auth/test_route_checks.py | 27 ++++++++++ tests/unit/proxy/conftest.py | 27 ++++++++++ ...est_billable_request_metrics_middleware.py | 8 +++ .../test_llm_pass_through_endpoints.py | 52 +++++++++++++------ .../test_passthrough_auth_default.py | 11 ++-- tests/unit/proxy/test_component_allowlists.py | 10 +++- 13 files changed, 137 insertions(+), 32 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 0cd3cf1d59d..c2740ac1d41 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -109,7 +109,6 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/vllm/", "/mistral/", "/typesafe/", - "/eu_jev/v1/systemone", "/openrouter/", "/nvidia_nim/", "/groq/", @@ -143,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/test", "/debug/memory/summary", "/api/event_logging/batch", + "/eu_jev/v1/systemone", } ) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7c227e04b9e..dd91bb6eef3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -413,7 +413,6 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", - "/eu_jev/v1/systemone", "/laya", "/bespoke", "/openrouter", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 91a2512fbfa..a08a2ee0708 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import ( from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.proxy._types import * +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, @@ -883,7 +884,6 @@ def route_in_additonal_public_routes(current_route: str): public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"] ``` """ - from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.proxy_server import general_settings, premium_user try: @@ -2173,7 +2173,7 @@ def _extract_model_candidates_from_request( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) - if route.rstrip("/") == "/eu_jev/v1/systemone": + if RouteChecks.is_builtin_eu_jev_decision_route(route=route): from litellm.llms.eu_jev.common_utils import eu_jev_registry_model requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 74e4c4c6d11..0a20f414bb2 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend") _LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp")) +_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone" _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS)) # Writes that only touch the caller's own credential row (BYOK connections and @@ -503,7 +504,17 @@ class RouteChecks: for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value: if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"): return True - return False + return RouteChecks.is_builtin_eu_jev_decision_route(route=route) + + @staticmethod + def is_builtin_eu_jev_decision_route(route: str) -> bool: + if route.rstrip("/") != _EU_JEV_DECISION_ROUTE: + return False + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack + InitPassThroughEndpointHelpers, + ) + + return InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=route.rstrip("/")) is None @staticmethod def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool: diff --git a/litellm/proxy/middleware/billable_request_metrics_middleware.py b/litellm/proxy/middleware/billable_request_metrics_middleware.py index 96c3276efac..fd78b6b58da 100644 --- a/litellm/proxy/middleware/billable_request_metrics_middleware.py +++ b/litellm/proxy/middleware/billable_request_metrics_middleware.py @@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm._logging import verbose_proxy_logger from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.auth.route_checks import RouteChecks class BillableCategory(str, Enum): @@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None: suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None) if suffix_match is not None: return suffix_match + if RouteChecks.is_builtin_eu_jev_decision_route(route=path): + return path # Deep passthrough paths only: the bare prefix itself is not an inference call. return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index aa06ca01c8b..bbb4fb4846a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -91,6 +91,7 @@ from litellm.proxy.auth.auth_utils import ( get_request_route, request_dispatched_to_pass_through_endpoint, ) +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -3422,6 +3423,8 @@ class InitPassThroughEndpointHelpers: return True comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) + if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route): + return True # Fast path: check if any registered route key contains this path # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" @@ -3504,9 +3507,8 @@ class YieldsToConfiguredPassThroughRoute(APIRoute): match, child_scope = super().matches(scope) if match is not Match.FULL: return match, child_scope - request: Final = Request(scope) configured: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=request.url.path, method=request.method + route=get_request_route(Request(scope)) ) return (match, child_scope) if configured is None else (Match.NONE, {}) diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index eb00d4c95cf..c40042bc4c1 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1057,6 +1057,11 @@ def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> assert denied.value.status_code == 400 +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_keeps_the_requested_model() -> None: + assert get_model_from_request(request_data={"model": "jeff-latest"}, route="/eu_jev/v1/systemone") == "jeff-latest" + + def test_laya_model_normalization_does_not_change_other_provider_routes() -> None: assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest" assert get_model_from_request(request_data={}, route="/laya/health") is None diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index b2c67b4e009..2bcfe06c6d3 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -662,6 +662,33 @@ def test_virtual_key_llm_api_routes_allows_model_info(route): assert result is True +@pytest.mark.parametrize( + ("route", "is_llm_route"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone/extra", False), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/custom", False), + ], +) +def test_only_the_built_in_eu_jev_decision_path_is_an_llm_route(route: str, is_llm_route: bool) -> None: + assert RouteChecks.is_llm_api_route(route=route) is is_llm_route + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_route() -> None: + assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False + + +@pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"]) +def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route]) + + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/eu_jev/v1/systemone", valid_token=valid_token, request=_mock_request("POST") + ) + + @pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"]) def test_model_info_not_classified_as_llm_api(route): """Membership in `llm_api_routes` must not promote /model/info to an diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 58c9a5e4d76..8229701cb38 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -12,11 +12,16 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml +from fastapi import FastAPI from fastapi.testclient import TestClient from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, +) from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -237,6 +242,28 @@ def secret_vault_factory() -> type[FakeSecretVault]: return FakeSecretVault +@pytest.fixture +def configured_eu_jev_route() -> Iterator[None]: + saved_registry: Final = dict(_registered_pass_through_routes) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + try: + yield + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + @pytest.fixture def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py index 8c81a2c9b36..d2e961ad158 100644 --- a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py +++ b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py @@ -123,6 +123,7 @@ def test_is_pure_asgi_not_base_http_middleware(): ("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")), ("/transcribe", (BillableCategory.LLM, "/transcribe")), ("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")), + ("/eu_jev/v1/systemone", (BillableCategory.LLM, "/eu_jev/v1/systemone")), ("/mcp", (BillableCategory.MCP, "/mcp")), ("/mcp/", (BillableCategory.MCP, "/mcp")), ("/mcp/tools/list", (BillableCategory.MCP, "/mcp")), @@ -165,6 +166,8 @@ def test_classify_billable(path: str, expected: Tuple[BillableCategory, str]): "/langfuse/api/public/ingestion", # a bare provider prefix is not an inference call "/bedrock", + "/eu_jev/v1/systemone/extra", + "/eu_jev/v1/systemone2", "/v1/mcp", "/v1/mcp/tools", "/v1/mcp/server", @@ -184,6 +187,11 @@ def test_classify_non_billable_returns_none(path: str): assert classify_billable_request(path) is None +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_billed() -> None: + assert classify_billable_request("/eu_jev/v1/systemone") is None + + @pytest.mark.parametrize( "path", [ diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 126fdd6c931..af2910d54ff 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -15,7 +15,7 @@ from urllib.parse import parse_qs import httpx import pytest import respx -from fastapi import HTTPException, Request, Response +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.routing import APIRoute from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient @@ -8542,11 +8542,8 @@ class TestEuJevPassthroughRoute: assert "Streaming" in response.text assert not route.called - def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from fastapi import FastAPI - + @pytest.fixture + def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str]], FastAPI]]: from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, @@ -8559,12 +8556,14 @@ class TestEuJevPassthroughRoute: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") - app: Final = FastAPI() - reserve_lazy_slot(app, "llm_passthrough", features=features) - attach_lazy_features(app, features) - monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) - saved_registry: Final = dict(_registered_pass_through_routes) - try: + + def build(methods: list[str]) -> FastAPI: + app: Final = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + monkeypatch.setitem( + app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) InitPassThroughEndpointHelpers.add_exact_path_route( app=app, path="/eu_jev/v1/systemone", @@ -8575,18 +8574,39 @@ class TestEuJevPassthroughRoute: dependencies=None, cost_per_request=None, endpoint_id="operator-eu-jev", - methods=["POST"], + methods=methods, ) - with respx.mock(assert_all_called=True) as upstream, TestClient(app) as client: - route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) - response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + return app + + saved_registry: Final = dict(_registered_pass_through_routes) + try: + yield build finally: _registered_pass_through_routes.clear() _registered_pass_through_routes.update(saved_registry) + @pytest.mark.parametrize("root_path", ["", "/tenant-a"]) + def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( + self, operator_app: Callable[[list[str]], FastAPI], root_path: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"]), root_path=root_path) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + assert (response.status_code, response.json()) == (200, {"operator": True}) assert route.calls.last.request.headers["x-operator"] == "on" + def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused( + self, operator_app: Callable[[list[str]], FastAPI] + ) -> None: + with TestClient(operator_app(["GET"])) as client: + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 405 + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py index a683f064bae..7771ff0274e 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -141,17 +141,12 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation(): @pytest.mark.asyncio -@pytest.mark.parametrize( - ("route", "expected_key"), - [("/eu_jev/v1/systemone", "sk-from-header"), ("/eu_jev/custom", "sk-from-bearer")], -) -async def test_only_the_eu_jev_decision_route_reads_the_litellm_user_api_key_header( - route: str, expected_key: str -) -> None: +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/extra", "/eu_jev/v1/systemone2"]) +async def test_no_eu_jev_route_reads_the_litellm_user_api_key_header(route: str) -> None: request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]}) result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints( request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer" ) - assert result == expected_key + assert result == "sk-from-bearer" diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index 63a72d7b0f0..4c0b3ad913b 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -397,7 +397,15 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" -@pytest.mark.parametrize(("path", "served"), [("/eu_jev/v1/systemone", True), ("/eu_jev/custom", False)]) +@pytest.mark.parametrize( + ("path", "served"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/v1/systemone/{subpath:path}", False), + ("/eu_jev/custom", False), + ], +) def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None: async def endpoint() -> None: return None From 31e429267d599775d005dfaa396825a43a5ae740 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:37:06 -0700 Subject: [PATCH 14/15] fix(eu_jev): find a configured route at the decision path with or without a trailing slash, and stop a deleted one forwarding --- litellm/proxy/auth/route_checks.py | 4 +- .../pass_through_endpoints.py | 16 +++++-- tests/unit/proxy/auth/test_route_checks.py | 8 ++++ tests/unit/proxy/conftest.py | 19 +++++++-- .../test_llm_pass_through_endpoints.py | 42 +++++++++++++++---- 5 files changed, 75 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 0a20f414bb2..56be44600fd 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -514,7 +514,9 @@ class RouteChecks: InitPassThroughEndpointHelpers, ) - return InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=route.rstrip("/")) is None + return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=_EU_JEV_DECISION_ROUTE + ) @staticmethod def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool: diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index bbb4fb4846a..76425480341 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2230,7 +2230,9 @@ def create_pass_through_route( stream, ) = await _parse_request_data_by_content_type(request) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path): + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or ( + custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path) + ): raise HTTPException( status_code=404, detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", @@ -3501,16 +3503,24 @@ class InitPassThroughEndpointHelpers: return None + @staticmethod + def has_configured_route_ignoring_trailing_slash(route: str) -> bool: + bare_route: Final = route.rstrip("/") + return any( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None + for form in (bare_route, f"{bare_route}/") + ) + class YieldsToConfiguredPassThroughRoute(APIRoute): def matches(self, scope: Scope) -> tuple[Match, Scope]: match, child_scope = super().matches(scope) if match is not Match.FULL: return match, child_scope - configured: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( + configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( route=get_request_route(Request(scope)) ) - return (match, child_scope) if configured is None else (Match.NONE, {}) + return (Match.NONE, {}) if configured else (match, child_scope) def _get_combined_pass_through_endpoints( diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 2bcfe06c6d3..529f639999d 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -680,6 +680,14 @@ def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_rou assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False +@pytest.mark.usefixtures("configured_eu_jev_route_with_trailing_slash") +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/"]) +def test_a_pass_through_configured_at_the_eu_jev_decision_path_with_a_trailing_slash_is_not_an_llm_route( + route: str, +) -> None: + assert RouteChecks.is_llm_api_route(route=route) is False + + @pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"]) def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None: valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route]) diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 8229701cb38..d2d118df8f4 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -7,6 +7,7 @@ import os import tempfile import warnings from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -242,12 +243,12 @@ def secret_vault_factory() -> type[FakeSecretVault]: return FakeSecretVault -@pytest.fixture -def configured_eu_jev_route() -> Iterator[None]: +@contextmanager +def _eu_jev_route_configured_at(path: str) -> Iterator[None]: saved_registry: Final = dict(_registered_pass_through_routes) InitPassThroughEndpointHelpers.add_exact_path_route( app=FastAPI(), - path="/eu_jev/v1/systemone", + path=path, target="https://operator.example/decide", custom_headers=None, forward_headers=None, @@ -264,6 +265,18 @@ def configured_eu_jev_route() -> Iterator[None]: _registered_pass_through_routes.update(saved_registry) +@pytest.fixture +def configured_eu_jev_route() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone"): + yield + + +@pytest.fixture +def configured_eu_jev_route_with_trailing_slash() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone/"): + yield + + @pytest.fixture def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index af2910d54ff..f9d884da48d 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8543,7 +8543,7 @@ class TestEuJevPassthroughRoute: assert not route.called @pytest.fixture - def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str]], FastAPI]]: + def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str], str], FastAPI]]: from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, @@ -8557,7 +8557,7 @@ class TestEuJevPassthroughRoute: litellm.in_memory_llm_clients_cache.flush_cache() features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") - def build(methods: list[str]) -> FastAPI: + def build(methods: list[str], path: str) -> FastAPI: app: Final = FastAPI() reserve_lazy_slot(app, "llm_passthrough", features=features) attach_lazy_features(app, features) @@ -8566,7 +8566,7 @@ class TestEuJevPassthroughRoute: ) InitPassThroughEndpointHelpers.add_exact_path_route( app=app, - path="/eu_jev/v1/systemone", + path=path, target="https://operator.example/decide", custom_headers={"x-operator": "on"}, forward_headers=None, @@ -8587,11 +8587,11 @@ class TestEuJevPassthroughRoute: @pytest.mark.parametrize("root_path", ["", "/tenant-a"]) def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( - self, operator_app: Callable[[list[str]], FastAPI], root_path: str + self, operator_app: Callable[[list[str], str], FastAPI], root_path: str ) -> None: with ( respx.mock(assert_all_called=True) as upstream, - TestClient(operator_app(["POST"]), root_path=root_path) as client, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone"), root_path=root_path) as client, ): route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True}) @@ -8600,13 +8600,41 @@ class TestEuJevPassthroughRoute: assert route.calls.last.request.headers["x-operator"] == "on" def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused( - self, operator_app: Callable[[list[str]], FastAPI] + self, operator_app: Callable[[list[str], str], FastAPI] ) -> None: - with TestClient(operator_app(["GET"])) as client: + with TestClient(operator_app(["GET"], "/eu_jev/v1/systemone")) as client: response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) assert response.status_code == 405 + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_the_path_without_one( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("LITELLM_BUILTIN_PASS_THROUGH_ROUTES_FIRST", "true") + app: Final = operator_app(["GET", "PUT"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + responses: Final = (client.get("/eu_jev/v1/systemone"), client.put("/eu_jev/v1/systemone", json={})) + + assert [response.status_code for response in responses] == [404, 404] + assert not operator.called + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: From 7573ed653241db3d8648b93ad767a91b3c7afd9f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:23:03 -0700 Subject: [PATCH 15/15] test(eu_jev): annotate new test variables with Final --- .../test_typesafe_passthrough_logging_handler.py | 8 ++++---- .../test_llm_pass_through_endpoints.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index a20d6eb50e0..4011bc0294a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -370,7 +370,7 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou def test_eu_jev_decision_is_priced_from_its_registry_row(): model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] - response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( httpx_response=_response(), response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, logging_obj=_logging_obj(), @@ -399,7 +399,7 @@ def test_eu_jev_decision_is_priced_from_its_registry_row(): def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: unlisted_release: Final = "eu-jeff-99.0.0" assert f"eu_jev/{unlisted_release}" not in litellm.model_cost - response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( httpx_response=_response(), response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}}, logging_obj=_logging_obj(), @@ -424,7 +424,7 @@ def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: py "eu_jev/eu-jeff-latest", {**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0}, ) - response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( httpx_response=_response(), response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, logging_obj=_logging_obj(), @@ -442,7 +442,7 @@ def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: py def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): - normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( httpx_response=_response(), response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, request_body={"model": "eu-jeff-latest"}, diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index f9d884da48d..8f25e6e69df 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8355,10 +8355,10 @@ class TestEuJevPassthroughRoute: def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: body: Final = {"model": "eu-jeff-latest", "state": "x"} with respx.mock(assert_all_called=True) as upstream: - route = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").mock( return_value=httpx.Response(200, json={"id": "upstream_123"}) ) - response = client.post("/eu_jev/v1/systemone", json=body) + response: Final = client.post("/eu_jev/v1/systemone", json=body) assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) sent: Final = route.calls.last.request