From fe4ba0f5eeafe737a568ee81deddb3f2b7c8f371 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:48:05 +0200 Subject: [PATCH 1/6] feat(pass-through): add eu/jev decision API pass-through with spend tracking Adds a /eu_jev/{endpoint:path} pass-through route for eu/jev, an EU-hosted decision API that speaks the same POST /v1/systemone contract as TypeSafe. It reads EU_JEV_API_KEY and EU_JEV_API_BASE (defaults to https://jev.bevel.software), books spend through the shared decision logging handler, and prices eu_jev/eu-jeff-1.0.0 and eu_jev/eu-jeff-latest from the cost map --- gateway/routes/allowlist.py | 1 + ...odel_prices_and_context_window_backup.json | 22 ++ litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 222 ++++++++++++++++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 38 +++ .../pass_through_endpoints.py | 6 +- .../pass_through_endpoints/success_handler.py | 2 +- model_prices_and_context_window.json | 22 ++ ...st_typesafe_passthrough_logging_handler.py | 41 ++++ .../test_llm_pass_through_endpoints.py | 94 ++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 191 +++++++++++++++ 12 files changed, 637 insertions(+), 4 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cced49f788a..d365cbd1abf 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -107,6 +107,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/vllm/", "/mistral/", "/typesafe/", + "/eu_jev/", "/openrouter/", "/nvidia_nim/", "/groq/", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b744032fb6c..f71bb039417 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -70896,6 +70896,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index a909a7a993a..19c80691a87 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -230,6 +230,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/tinyfish/", "/transcribe", "/typesafe/", + "/eu_jev/", "/laya/", "/bespoke/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 20819a59a43..3cf5ca88f4c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -30372,6 +30372,228 @@ ] } }, + "/eu_jev/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/fal_ai/{endpoint}": { "delete": { "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d62c6b0d51f..36c0c86cf06 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -407,6 +407,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", + "/eu_jev", "/laya", "/bespoke", "/openrouter", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 464cfee270a..3d8b5f7f26e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -681,6 +681,44 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/eu_jev/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], + tags=["eu/jev Pass-through", "pass-through"], +) +async def eu_jev_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://jev.bevel.software" + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + eu_jev_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="eu_jev", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ + "Authorization": f"Bearer {eu_jev_api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( + await endpoint_func(request, fastapi_response, user_api_key_dict) + ) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b1c448b023b..5dcbece655d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -403,9 +403,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): @staticmethod def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) - if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( - "/v1/systemone" - ): + if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix( + "/" + ).endswith("/v1/systemone"): return EndpointType.DECISIONS if ( ("generateContent") in url diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 9c6c3111209..f0e0906d486 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -338,7 +338,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider in ("laya", "bespoke") + or custom_llm_provider in ("eu_jev", "laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b744032fb6c..f71bb039417 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -70896,6 +70896,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://jev.bevel.software/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 79209a32f28..9958805d6aa 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -366,3 +366,44 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou assert normalized["standard_logging_response_object"] is None assert "combined_usage_object" not in normalized["kwargs"] assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" + + +def test_eu_jev_decision_is_priced_from_its_registry_row(): + model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://jev.bevel.software/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-latest"}, + custom_llm_provider="eu_jev", + ) + + assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" + assert response["kwargs"]["custom_llm_provider"] == "eu_jev" + assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"]) + assert response["kwargs"]["response_cost"] > 0 + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 + + +def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + request_body={"model": "eu-jeff-latest"}, + logging_obj=_logging_obj(), + url_route="https://jev.bevel.software/v1/systemone", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="eu_jev", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev" + assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 21170abaeb1..9b7c9e38dd9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -49,6 +49,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + eu_jev_proxy_route, fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, @@ -8236,6 +8237,99 @@ class TestTypeSafePassthroughRoute: ) +class TestEuJevPassthroughRoute: + @staticmethod + def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = query_params or {} + request.json = AsyncMock(return_value=body) + return request + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: + body: Final = {"model": "eu-jeff-latest", "state": "x"} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response = client.post("/eu_jev/v1/systemone", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer eu-jev-test-key" + assert json.loads(sent.content) == body + + @pytest.mark.asyncio + async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + + async def fake_upstream(request, *_args): + target: Final = create_route.call_args.kwargs["target"] + upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params) + return Response(content=json.dumps({"upstream_query": parse_qs(upstream_url.query.decode())})) + + endpoint_func = AsyncMock(side_effect=fake_upstream) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + result = await eu_jev_proxy_route( + endpoint="v1/systemone", + request=self._request({"state": "x"}, {"trace": "yes"}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert json.loads(result.body) == {"upstream_query": {"trace": ["yes"]}} + endpoint_func.assert_awaited_once() + create_route.assert_called_once_with( + endpoint="v1/systemone", + target="https://eu-jev.example/base/v1/systemone", + custom_headers={ + "Authorization": "Bearer eu-jev-test-key", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + + @pytest.mark.asyncio + async def test_uses_the_hosted_api_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.delenv("EU_JEV_API_BASE", raising=False) + + endpoint_func = AsyncMock(return_value=Response()) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await eu_jev_proxy_route( + endpoint="v1/systemone", + request=self._request({"state": "x"}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == "https://jev.bevel.software/v1/systemone" + + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: @pytest.fixture diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f6eae1f98e..a2dc31c9140 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,6 +5333,42 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; + options?: never; + head?: never; + /** + * Eu Jev Proxy Route + * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; + trace?: never; + }; "/fal_ai/{endpoint}": { parameters: { query?: never; @@ -58817,6 +58853,161 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; fal_ai_proxy_route_fal_ai__endpoint__get: { parameters: { query?: never; From d58095896593c6d1ee09ac3360720d070d6197af Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:53:21 +0200 Subject: [PATCH 2/6] fix(pass-through): default eu_jev to its canonical eu-jev.com origin --- litellm/model_prices_and_context_window_backup.json | 4 ++-- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 2 +- model_prices_and_context_window.json | 4 ++-- .../test_typesafe_passthrough_logging_handler.py | 4 ++-- .../pass_through_endpoints/test_llm_pass_through_endpoints.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f71bb039417..ff293456e8d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -70902,7 +70902,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] @@ -70913,7 +70913,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 3d8b5f7f26e..0d7d3f78b53 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -693,7 +693,7 @@ async def eu_jev_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" - base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://jev.bevel.software" + base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" base_url: Final = httpx.URL(base_target_url) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f71bb039417..ff293456e8d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -70902,7 +70902,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] @@ -70913,7 +70913,7 @@ "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://jev.bevel.software/docs", + "source": "https://eu-jev.com/docs", "supported_endpoints": [ "/v1/systemone" ] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 9958805d6aa..dd7b82070f1 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -374,7 +374,7 @@ def test_eu_jev_decision_is_priced_from_its_registry_row(): httpx_response=_response(), response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, logging_obj=_logging_obj(), - url_route="https://jev.bevel.software/v1/systemone", + url_route="https://eu-jev.com/v1/systemone", result='{"answers": {}}', start_time=datetime.now(), end_time=datetime.now(), @@ -397,7 +397,7 @@ def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, request_body={"model": "eu-jeff-latest"}, logging_obj=_logging_obj(), - url_route="https://jev.bevel.software/v1/systemone", + url_route="https://eu-jev.com/v1/systemone", result="{}", start_time=datetime.now(), end_time=datetime.now(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9b7c9e38dd9..434e139b9c2 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8327,7 +8327,7 @@ class TestEuJevPassthroughRoute: user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), ) - assert create_route.call_args.kwargs["target"] == "https://jev.bevel.software/v1/systemone" + assert create_route.call_args.kwargs["target"] == "https://eu-jev.com/v1/systemone" @pytest.mark.parametrize("provider", ["laya", "bespoke"]) From 185378215cbe01237601a95cb485b93aa3d9e309 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:59:28 +0200 Subject: [PATCH 3/6] fix(pass-through): reject custom_body on the eu_jev route A key limited to other models could send an eu/jev request through custom_body, because auth checks the outer body while the shared pass-through forwards custom_body upstream. The route now answers 400 before anything is forwarded, the same way validate_oss_request guards the laya and bespoke routes --- .../llm_passthrough_endpoints.py | 2 ++ .../test_llm_pass_through_endpoints.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0d7d3f78b53..fa731c0e1fe 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -693,6 +693,8 @@ async def eu_jev_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + if "custom_body" in await read_request_body(request): + raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 434e139b9c2..c790f7f3b16 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8271,6 +8271,16 @@ class TestEuJevPassthroughRoute: assert sent.headers["authorization"] == "Bearer eu-jev-test-key" assert json.loads(sent.content) == body + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: + smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled}) + + assert response.status_code == 400 + assert "custom_body" in response.text + assert not route.called + @pytest.mark.asyncio async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") From 8c975c7d91b50f1085807508a289bb6c320d2ffb Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:59:44 +0200 Subject: [PATCH 4/6] fix(pass-through): give the eu_jev route request and 200 response schemas The OpenAPI docs gate flagged POST /eu_jev/{endpoint} for OA001 and OA003. The route now documents the eu/jev System One request body and its 200 answer, usage and meta shape, taken from the eu/jev OpenAPI document. The lazy OpenAPI snapshot and the dashboard API types are regenerated with python -m litellm.proxy._lazy_openapi_snapshot and npm run gen:api --- litellm/proxy/_lazy_openapi_snapshot.json | 1652 ++++++++++++++++- .../llm_passthrough_endpoints.py | 36 +- litellm/types/passthrough_endpoints/eu_jev.py | 100 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 694 ++++++- 4 files changed, 2456 insertions(+), 26 deletions(-) create mode 100644 litellm/types/passthrough_endpoints/eu_jev.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3cf5ca88f4c..2204448afd5 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -21808,6 +21808,39 @@ "title": "CachedTokensDetails", "type": "object" }, + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, "Click": { "additionalProperties": true, "description": "A click action.", @@ -22572,6 +22605,112 @@ "title": "ErrorResponse", "type": "object" }, + "EuJevDecisionResponse": { + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "description": "One answer per question, under the same names", + "title": "Answers", + "type": "object" + }, + "meta": { + "$ref": "#/components/schemas/EuJevMeta" + }, + "model": { + "description": "The release that answered, such as eu-jeff-1.0.0", + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/EuJevUsage" + } + }, + "required": [ + "model", + "answers", + "usage", + "meta" + ], + "title": "EuJevDecisionResponse", + "type": "object" + }, + "EuJevMeta": { + "properties": { + "cost_eur": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The exact charge in euros, on authenticated requests", + "title": "Cost Eur" + }, + "latency_ms": { + "description": "Server-side time for token quoting, credit reservation and inference", + "title": "Latency Ms", + "type": "integer" + }, + "mode": { + "const": "live", + "title": "Mode", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "mode", + "latency_ms" + ], + "title": "EuJevMeta", + "type": "object" + }, + "EuJevUsage": { + "properties": { + "input_tokens": { + "minimum": 0.0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "minimum": 0.0, + "title": "Output Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "EuJevUsage", + "type": "object" + }, "FileSearchTool": { "additionalProperties": true, "description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).", @@ -24211,6 +24350,26 @@ "title": "NamespaceTool", "type": "object" }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -27480,6 +27639,59 @@ "title": "Result", "type": "object" }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + }, "Screenshot": { "additionalProperties": true, "description": "A screenshot action.", @@ -30374,7 +30586,7 @@ }, "/eu_jev/{endpoint}": { "delete": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__delete", "parameters": [ { @@ -30387,11 +30599,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30418,7 +30914,7 @@ ] }, "get": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__get", "parameters": [ { @@ -30431,11 +30927,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30462,7 +31242,7 @@ ] }, "patch": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__patch", "parameters": [ { @@ -30475,11 +31255,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30506,7 +31570,7 @@ ] }, "post": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__post", "parameters": [ { @@ -30519,11 +31583,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" @@ -30550,7 +31898,7 @@ ] }, "put": { - "description": "[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is\nrejected with a 400.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", "operationId": "eu_jev_proxy_route_eu_jev__endpoint__put", "parameters": [ { @@ -30563,11 +31911,295 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index fa731c0e1fe..fb10ed664a6 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import ( model_deployments_support_stream_options, open_sse_before_first_byte, ) +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -122,6 +123,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, ) from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool +from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, @@ -681,10 +683,34 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( + EuJevDecisionRequest, + { + "model": "eu-jeff-latest", + "state": "I was charged twice. Can I get a refund?", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"}, + }, + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) + + @router.api_route( "/eu_jev/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], tags=["eu/jev Pass-through", "pass-through"], + responses={200: {"model": EuJevDecisionResponse}}, + openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, ) async def eu_jev_proxy_route( endpoint: str, @@ -692,7 +718,15 @@ async def eu_jev_proxy_route( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> Response: - """[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)""" + """ + Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + + `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + rejected with a 400. + + [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + """ if "custom_body" in await read_request_body(request): raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" diff --git a/litellm/types/passthrough_endpoints/eu_jev.py b/litellm/types/passthrough_endpoints/eu_jev.py new file mode 100644 index 00000000000..3b81d68deea --- /dev/null +++ b/litellm/types/passthrough_endpoints/eu_jev.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +EuJevEntry: TypeAlias = Annotated[ + DecisionsJSON | None, + Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"), +] + + +class EuJevChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: EuJevEntry + criteria: Annotated[ + Mapping[str, EuJevEntry], + Field( + min_length=2, + max_length=64, + description="Two to 64 options, option name to description. A null description uses the option name", + ), + ] + + model_config = ConfigDict(frozen=True) + + +class EuJevNoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: EuJevEntry + criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = ( + None + ) + + model_config = ConfigDict(frozen=True) + + +class EuJevScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: EuJevEntry + criteria: Annotated[ + Sequence[EuJevEntry], + Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevQuestion: TypeAlias = Annotated[ + EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion, + Field(discriminator="type"), +] + + +class EuJevDecisionRequest(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = ( + "eu-jeff-latest" + ) + state: Annotated[ + Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object], + Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"), + ] + questions: Annotated[ + Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion], + Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")] + + +class EuJevUsage(LiteLLMPydanticObjectBase): + input_tokens: Annotated[int, Field(ge=0)] + output_tokens: Annotated[int, Field(ge=0)] + + model_config = ConfigDict(frozen=True) + + +class EuJevMeta(LiteLLMPydanticObjectBase): + request_id: str + mode: Literal["live"] + latency_ms: Annotated[ + int, Field(description="Server-side time for token quoting, credit reservation and inference") + ] + cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None + + model_config = ConfigDict(frozen=True) + + +class EuJevDecisionResponse(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")] + answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")] + usage: EuJevUsage + meta: EuJevMeta + + model_config = ConfigDict(frozen=True) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a2dc31c9140..7649b2772cf 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5342,29 +5342,59 @@ export interface paths { }; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; options?: never; head?: never; /** * Eu Jev Proxy Route - * @description [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) */ patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; trace?: never; @@ -32115,6 +32145,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -58862,7 +58936,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58870,7 +59062,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58893,7 +59085,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58901,7 +59211,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58924,7 +59234,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58932,7 +59360,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58955,7 +59383,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58963,7 +59509,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ @@ -58986,7 +59532,125 @@ export interface operations { }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -58994,7 +59658,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["EuJevDecisionResponse"]; }; }; /** @description Validation Error */ From d3f8062e388b2a1dae214f5aba289036724db361 Mon Sep 17 00:00:00 2001 From: "Juan V." <69489757+edujuan@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:19:14 +0200 Subject: [PATCH 5/6] fix(pass-through): reject streaming on the eu_jev route A body with stream=true switched the route to streaming, which skips the decision spend tracking. Reject it with a 400, as validate_oss_request does for laya and bespoke. --- .../llm_passthrough_endpoints.py | 5 ++++- .../test_llm_pass_through_endpoints.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index fb10ed664a6..af4754c99e2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -727,8 +727,11 @@ async def eu_jev_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ - if "custom_body" in await read_request_body(request): + body: Final = await read_request_body(request) + if "custom_body" in body: raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") base_target_url: Final = get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com" encoded_endpoint: Final = httpx.URL(endpoint).path normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index c790f7f3b16..f3ef1dcd033 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -8281,6 +8281,17 @@ class TestEuJevPassthroughRoute: assert "custom_body" in response.text assert not route.called + def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post( + "/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True} + ) + + assert response.status_code == 400 + assert "Streaming" in response.text + assert not route.called + @pytest.mark.asyncio async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") From 27f804790494e3f25728dae84268a54aeaf3ab5f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:21:29 -0700 Subject: [PATCH 6/6] fix(pass-through): type the eu_jev request body like the other decision routes --- .../llm_passthrough_endpoints.py | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 855 ++++++++++++++++++ 2 files changed, 856 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 8dfffadc584..ecc9b46105c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -737,7 +737,7 @@ async def eu_jev_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) """ - body: Final = await read_request_body(request) + body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request)) if "custom_body" in body: raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") if body.get("stream"): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2b174a1141a..44f8932f212 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,6 +5333,72 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + get: operations["eu_jev_proxy_route_eu_jev__endpoint__get"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + put: operations["eu_jev_proxy_route_eu_jev__endpoint__put"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev__endpoint__post"]; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + delete: operations["eu_jev_proxy_route_eu_jev__endpoint__delete"]; + options?: never; + head?: never; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` is + * rejected with a 400. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + patch: operations["eu_jev_proxy_route_eu_jev__endpoint__patch"]; + trace?: never; + }; "/extraction": { parameters: { query?: never; @@ -32233,6 +32299,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -59146,6 +59256,751 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + eu_jev_proxy_route_eu_jev__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; extraction_extraction_post: { parameters: { query?: never;