This commit is contained in:
Juan V. 2026-10-29 01:41:27 +00:00 • committed by GitHub
commit 1ba18d6db5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 1830 additions and 27 deletions

View file

@ -354,6 +354,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
"/test",
"/debug/memory/summary",
"/api/event_logging/batch",
"/eu_jev/v1/systemone",
}
)

View file

@ -1752,6 +1752,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)

View file

@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
"AzureAIDecisionsConfig",
@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),

View file

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
_EU_JEV_PREFIX: Final = "eu_jev/"
def eu_jev_registry_model(requested_model: str | None) -> str:
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return release
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
requested_model: Final = decision_body.get("model")
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
hook_model: Final = hook_body.get("model")
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
if hook_model == eu_jev_scoped_model(client_body):
client_model: Final = client_body.get("model")
if isinstance(client_model, str):
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
return MappingProxyType(
{**without_model, **{key: value for key, value in client_body.items() if key == "model"}}
)
if isinstance(hook_model, str):
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
return hook_body

View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EuJevDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("EU_JEV_API_KEY",)
api_base_env = ("EU_JEV_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://eu-jev.com"

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
response_model: Final = (
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
)
logged_model: Final = response_model or request_model or "unknown"
registry_request_model: Final = (
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
)
logged_model: Final = response_model or registry_request_model or "unknown"
model_name: Final = f"{custom_llm_provider}/{logged_model}"
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_models: Final = (
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
if custom_llm_provider == "eu_jev"
else (response_model, registry_request_model)
)
candidate_model_keys: Final = tuple(
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (

View file

@ -71640,6 +71640,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

View file

@ -2241,6 +2241,32 @@
"generateContent": false
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": true,
"a2a": false,
"generateContent": false
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/tinyfish/",
"/transcribe",
"/typesafe/",
"/eu_jev/",
"/laya/",
"/bespoke/",
"/openrouter/",

View file

@ -23169,6 +23169,39 @@
"title": "CachedTokensDetails",
"type": "object"
},
"ChoiceAnswer": {
"additionalProperties": true,
"properties": {
"choice": {
"title": "Choice",
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
"type": "object"
},
"Click": {
"additionalProperties": true,
"description": "A click action.",
@ -23933,6 +23966,112 @@
"title": "ErrorResponse",
"type": "object"
},
"EuJevDecisionResponse": {
"properties": {
"answers": {
"additionalProperties": {
"discriminator": {
"mapping": {
"choice": "#/components/schemas/ChoiceAnswer",
"noul": "#/components/schemas/NoulAnswer",
"score": "#/components/schemas/ScoreAnswer"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/components/schemas/ChoiceAnswer"
},
{
"$ref": "#/components/schemas/NoulAnswer"
},
{
"$ref": "#/components/schemas/ScoreAnswer"
}
]
},
"description": "One answer per question, under the same names",
"title": "Answers",
"type": "object"
},
"meta": {
"$ref": "#/components/schemas/EuJevMeta"
},
"model": {
"description": "The release that answered, such as eu-jeff-1.0.0",
"title": "Model",
"type": "string"
},
"usage": {
"$ref": "#/components/schemas/EuJevUsage"
}
},
"required": [
"model",
"answers",
"usage",
"meta"
],
"title": "EuJevDecisionResponse",
"type": "object"
},
"EuJevMeta": {
"properties": {
"cost_eur": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "The exact charge in euros, on authenticated requests",
"title": "Cost Eur"
},
"latency_ms": {
"description": "Server-side time for token quoting, credit reservation and inference",
"title": "Latency Ms",
"type": "integer"
},
"mode": {
"const": "live",
"title": "Mode",
"type": "string"
},
"request_id": {
"title": "Request Id",
"type": "string"
}
},
"required": [
"request_id",
"mode",
"latency_ms"
],
"title": "EuJevMeta",
"type": "object"
},
"EuJevUsage": {
"properties": {
"input_tokens": {
"minimum": 0.0,
"title": "Input Tokens",
"type": "integer"
},
"output_tokens": {
"minimum": 0.0,
"title": "Output Tokens",
"type": "integer"
}
},
"required": [
"input_tokens",
"output_tokens"
],
"title": "EuJevUsage",
"type": "object"
},
"FileSearchTool": {
"additionalProperties": true,
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
@ -25593,6 +25732,26 @@
"title": "NamespaceTool",
"type": "object"
},
"NoulAnswer": {
"additionalProperties": true,
"properties": {
"noul": {
"title": "Noul",
"type": "number"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"noul"
],
"title": "NoulAnswer",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -28865,6 +29024,59 @@
"title": "Result",
"type": "object"
},
"ScoreAnswer": {
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
},
"legend": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
]
},
"title": "Legend",
"type": "object"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"score": {
"title": "Score",
"type": "number"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],
"title": "ScoreAnswer",
"type": "object"
},
"Screenshot": {
"additionalProperties": true,
"description": "A screenshot action.",
@ -31757,6 +31969,315 @@
]
}
},
"/eu_jev/v1/systemone": {
"post": {
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
"requestBody": {
"content": {
"application/json": {
"example": {
"model": "eu-jeff-latest",
"questions": {
"frustration": {
"criteria": [
"calm",
"mildly annoyed",
"angry"
],
"instructions": "How frustrated is the customer?",
"type": "score"
},
"is_refund_request": {
"instructions": "Is the customer asking for a refund?",
"type": "noul"
},
"team": {
"criteria": {
"billing": "Payments, invoices, and refunds",
"support": "Technical issues and bugs"
},
"instructions": "Which team should handle this?",
"type": "choice"
}
},
"state": "I was charged twice. Can I get a refund?"
},
"schema": {
"properties": {
"model": {
"default": "eu-jeff-latest",
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
"title": "Model",
"type": "string"
},
"questions": {
"additionalProperties": {
"discriminator": {
"propertyName": "type"
},
"oneOf": [
{
"properties": {
"criteria": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"description": "Two to 64 options, option name to description. A null description uses the option name",
"maxProperties": 64,
"minProperties": 2,
"title": "Criteria",
"type": "object"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevChoiceQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"anyOf": [
{
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
]
},
"propertyNames": {
"enum": [
"true",
"false"
]
},
"type": "object"
},
{
"type": "null"
}
],
"description": "Optional descriptions of what true and false mean",
"title": "Criteria"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions"
],
"title": "EuJevNoulQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"description": "Two to 10 level descriptions, ordered from low to high",
"items": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"maxItems": 10,
"minItems": 2,
"title": "Criteria",
"type": "array"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevScoreQuestion",
"type": "object"
}
]
},
"description": "One to eight questions, keyed by names of up to 80 characters",
"maxProperties": 8,
"minProperties": 1,
"propertyNames": {
"maxLength": 80,
"minLength": 1
},
"title": "Questions",
"type": "object"
},
"state": {
"anyOf": [
{
"maxLength": 100000,
"minLength": 1,
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
],
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
"title": "State"
}
},
"required": [
"state",
"questions"
],
"title": "EuJevDecisionRequest",
"type": "object"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/EuJevDecisionResponse"
}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Eu Jev Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/fal_ai/{endpoint}": {
"delete": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
@ -883,7 +884,6 @@ def route_in_additonal_public_routes(current_route: str):
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
```
"""
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.proxy_server import general_settings, premium_user
try:
@ -2173,6 +2173,13 @@ def _extract_model_candidates_from_request(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
if requested_model is not None and not isinstance(requested_model, str):
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
if route == "/cost/predict-cache":
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
return _dedupe_model_candidates(prediction_models)

View file

@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend")
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone"
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
# Writes that only touch the caller's own credential row (BYOK connections and
@ -503,7 +504,19 @@ class RouteChecks:
for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value:
if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"):
return True
return False
return RouteChecks.is_builtin_eu_jev_decision_route(route=route)
@staticmethod
def is_builtin_eu_jev_decision_route(route: str) -> bool:
if route.rstrip("/") != _EU_JEV_DECISION_ROUTE:
return False
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack
InitPassThroughEndpointHelpers,
)
return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
route=_EU_JEV_DECISION_ROUTE
)
@staticmethod
def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool:

View file

@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import LiteLLMRoutes
from litellm.proxy.auth.route_checks import RouteChecks
class BillableCategory(str, Enum):
@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None:
suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None)
if suffix_match is not None:
return suffix_match
if RouteChecks.is_builtin_eu_jev_decision_route(route=path):
return path
# Deep passthrough paths only: the bare prefix itself is not an inference call.
return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None)

View file

@ -89,6 +89,7 @@ from litellm.proxy.common_request_processing import (
open_sse_before_first_byte,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
_safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -117,6 +118,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
HttpPassThroughEndpointHelpers,
YieldsToConfiguredPassThroughRoute,
create_pass_through_route,
create_websocket_passthrough_route,
websocket_passthrough_request,
@ -130,6 +132,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
is_allowed_to_call_vector_store_endpoint,
)
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool
from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
@ -698,6 +701,85 @@ async def typesafe_proxy_route(
return await endpoint_func(request, fastapi_response, user_api_key_dict)
_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body(
EuJevDecisionRequest,
{
"model": "eu-jeff-latest",
"state": "I was charged twice. Can I get a refund?",
"questions": {
"team": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"},
},
"is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
"frustration": {
"type": "score",
"instructions": "How frustrated is the customer?",
"criteria": ["calm", "mildly annoyed", "angry"],
},
},
},
)
eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute)
@eu_jev_router.post(
"/eu_jev/v1/systemone",
tags=["eu/jev Pass-through", "pass-through"],
responses={200: {"model": EuJevDecisionResponse}},
openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY},
)
async def eu_jev_proxy_route(
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> Response:
"""
Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.
`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)
and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and
`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.
[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)
"""
body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request))
if "custom_body" in body:
raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests")
if body.get("stream"):
raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests")
eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev")
if eu_jev_credentials is None:
raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy")
base_url: Final = httpx.URL(
(eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com")
.rstrip("/")
.removesuffix("/v1")
)
updated_url: Final = base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"),
)
endpoint_func: Final = create_pass_through_route(
endpoint="v1/systemone",
target=str(updated_url),
custom_headers={
"Authorization": f"Bearer {eu_jev_credentials.api_key}",
"Content-Type": "application/json",
},
custom_llm_provider="eu_jev",
is_streaming_request=False,
)
return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python(
await endpoint_func(request, fastapi_response, user_api_key_dict)
)
router.include_router(eu_jev_router)
@router.post(
"/laya/v1/systemone",
tags=["Laya Pass-through", "pass-through"],

View file

@ -26,9 +26,11 @@ from fastapi import (
status,
)
from fastapi.responses import StreamingResponse
from fastapi.routing import APIRoute
from pydantic import TypeAdapter
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.routing import BaseRoute, Match, Route
from starlette.types import Scope
from starlette.websockets import WebSocketState
from typing_extensions import ReadOnly
from websockets.asyncio.client import connect
@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
from litellm.llms.oss_decision import validate_oss_request
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._lazy_features import lazy_owned_routes
@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import (
get_request_route,
request_dispatched_to_pass_through_endpoint,
)
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
@ -426,9 +430,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
parsed_url: Final = urlparse(url)
if custom_llm_provider == "chatgpt":
return EndpointType.OPENAI
if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith(
"/v1/systemone"
):
if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix(
"/"
).endswith("/v1/systemone"):
return EndpointType.DECISIONS
if (
("generateContent") in url
@ -1130,6 +1134,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]:
if eu_jev_client_body is None:
return MappingProxyType({})
return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)})
async def pass_through_request(
request: Request,
target: str,
@ -1248,6 +1262,8 @@ async def pass_through_request(
strip_client_guardrail_information(client_body)
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
_strip_client_pricing_overrides(client_body)
eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body)
client_body.update(_eu_jev_scoped_model_override(eu_jev_client_body))
_parsed_body = client_body
if custom_llm_provider in ("laya", "bespoke"):
decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body)
@ -1332,6 +1348,12 @@ async def pass_through_request(
)
_ = validate_oss_request(custom_llm_provider, decision_body)
_parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body)
if eu_jev_client_body is not None:
_parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage
eu_jev_upstream_body(
TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body
)
)
resolved_timeout: Final = resolve_pass_through_request_timeout(timeout)
async_client_obj: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.PassThroughEndpoint,
@ -2232,7 +2254,9 @@ def create_pass_through_route(
stream,
) = await _parse_request_data_by_content_type(request)
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path):
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or (
custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path)
):
raise HTTPException(
status_code=404,
detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.",
@ -3428,6 +3452,8 @@ class InitPassThroughEndpointHelpers:
return True
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route):
return True
# Fast path: check if any registered route key contains this path
# Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}"
@ -3504,6 +3530,25 @@ class InitPassThroughEndpointHelpers:
return None
@staticmethod
def has_configured_route_ignoring_trailing_slash(route: str) -> bool:
bare_route: Final = route.rstrip("/")
return any(
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None
for form in (bare_route, f"{bare_route}/")
)
class YieldsToConfiguredPassThroughRoute(APIRoute):
def matches(self, scope: Scope) -> tuple[Match, Scope]:
match, child_scope = super().matches(scope)
if match is not Match.FULL:
return match, child_scope
configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
route=get_request_route(Request(scope))
)
return (Match.NONE, {}) if configured else (match, child_scope)
def _get_combined_pass_through_endpoints(
pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint],

View file

@ -1,5 +1,6 @@
import json
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from fastapi import HTTPException
@ -10,6 +11,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
LiteLLM_ManagedVectorStore,
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.secret_managers.main import (
UnresolvableSecretReference,
get_secret_str,
@ -23,12 +25,23 @@ if TYPE_CHECKING:
from litellm.router import Router
@dataclass(frozen=True, slots=True)
class _PassThroughCredentials:
api_key: str
api_base: str | None
def _get_proxy_llm_router() -> "Router | None":
from litellm.proxy.proxy_server import llm_router
return llm_router
def _is_team_deployment(deployment: Mapping[str, object]) -> bool:
model_info: Final = deployment.get("model_info")
return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id"))
def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
value: Final = values.get(key) if values is not None else None
return value if isinstance(value, str) else None
@ -74,12 +87,13 @@ class PassthroughEndpointRouter:
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
deployment_api_key: Final = self._get_deployment_api_key(
deployment_credentials: Final = self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider,
region_name=region_name,
include_team_deployments=True,
)
if deployment_api_key is not None:
return _resolve_oidc_reference(deployment_api_key)
if deployment_credentials is not None:
return _resolve_oidc_reference(deployment_credentials.api_key)
verbose_router_logger.debug(
"No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider
)
@ -88,21 +102,43 @@ class PassthroughEndpointRouter:
)
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
def _get_deployment_api_key(
def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
deployment_credentials: Final = self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider,
region_name=None,
include_team_deployments=False,
)
api_key: Final = _resolve_oidc_reference(
deployment_credentials.api_key
if deployment_credentials is not None
else get_secret_str(
self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider)
)
)
if not api_key:
return None
return _PassThroughCredentials(
api_key=api_key,
api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
)
def _get_deployment_credentials(
self,
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
include_team_deployments: bool,
) -> _PassThroughCredentials | None:
llm_router: Final = self.llm_router_getter()
if llm_router is None:
return None
deployments: Final = llm_router.get_model_list() or ()
return next(
(
api_key
credentials
for deployment in deployments
if (
api_key := self._resolve_matching_deployment_api_key(
if (include_team_deployments or not _is_team_deployment(deployment))
and (
credentials := self._resolve_matching_deployment_credentials(
litellm_params=deployment["litellm_params"],
custom_llm_provider=custom_llm_provider,
region_name=region_name,
@ -113,12 +149,12 @@ class PassthroughEndpointRouter:
None,
)
def _resolve_matching_deployment_api_key(
def _resolve_matching_deployment_credentials(
self,
litellm_params: LiteLLMParamsTypedDict,
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
) -> _PassThroughCredentials | None:
if litellm_params.get("use_in_pass_through") is not True:
return None
if self._get_deployment_provider(litellm_params) != custom_llm_provider:
@ -134,7 +170,8 @@ class PassthroughEndpointRouter:
)
if deployment_region != region_name:
return None
return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None
def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None:
model: Final = litellm_params.get("model")

View file

@ -338,7 +338,7 @@ class PassThroughEndpointLogging:
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif (
self.is_typesafe_route(custom_llm_provider)
or custom_llm_provider in ("laya", "bespoke")
or custom_llm_provider in ("eu_jev", "laya", "bespoke")
or self.is_openrouter_decisions_route(url_route, custom_llm_provider)
):
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (

View file

@ -1553,6 +1553,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "EU_JEV",
"provider_display_name": "eu/jev",
"litellm_provider": "eu_jev",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://eu-jev.com",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "eu_jev/eu-jeff-latest"
},
{
"provider": "FalAI",
"provider_display_name": "Fal AI",

View file

@ -0,0 +1,100 @@
from collections.abc import Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from pydantic import ConfigDict, Field
from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer
from litellm.types.llms.base import LiteLLMPydanticObjectBase
EuJevEntry: TypeAlias = Annotated[
DecisionsJSON | None,
Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"),
]
class EuJevChoiceQuestion(LiteLLMPydanticObjectBase):
type: Literal["choice"]
instructions: EuJevEntry
criteria: Annotated[
Mapping[str, EuJevEntry],
Field(
min_length=2,
max_length=64,
description="Two to 64 options, option name to description. A null description uses the option name",
),
]
model_config = ConfigDict(frozen=True)
class EuJevNoulQuestion(LiteLLMPydanticObjectBase):
type: Literal["noul"]
instructions: EuJevEntry
criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = (
None
)
model_config = ConfigDict(frozen=True)
class EuJevScoreQuestion(LiteLLMPydanticObjectBase):
type: Literal["score"]
instructions: EuJevEntry
criteria: Annotated[
Sequence[EuJevEntry],
Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"),
]
model_config = ConfigDict(frozen=True)
EuJevQuestion: TypeAlias = Annotated[
EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion,
Field(discriminator="type"),
]
class EuJevDecisionRequest(LiteLLMPydanticObjectBase):
model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = (
"eu-jeff-latest"
)
state: Annotated[
Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object],
Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"),
]
questions: Annotated[
Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion],
Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"),
]
model_config = ConfigDict(frozen=True)
EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")]
class EuJevUsage(LiteLLMPydanticObjectBase):
input_tokens: Annotated[int, Field(ge=0)]
output_tokens: Annotated[int, Field(ge=0)]
model_config = ConfigDict(frozen=True)
class EuJevMeta(LiteLLMPydanticObjectBase):
request_id: str
mode: Literal["live"]
latency_ms: Annotated[
int, Field(description="Server-side time for token quoting, credit reservation and inference")
]
cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None
model_config = ConfigDict(frozen=True)
class EuJevDecisionResponse(LiteLLMPydanticObjectBase):
model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")]
answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")]
usage: EuJevUsage
meta: EuJevMeta
model_config = ConfigDict(frozen=True)

View file

@ -4469,6 +4469,7 @@ class LlmProviders(str, Enum):
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
BESPOKE = "bespoke"
EU_JEV = "eu_jev"
MISTRAL = "mistral"
MILVUS = "milvus"
GROQ = "groq"

View file

@ -9619,11 +9619,12 @@ class ProviderConfigManager:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI):
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV):
configs: Final = {
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,
LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig,
}
return configs[provider]()
return None

View file

@ -71640,6 +71640,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

View file

@ -2241,6 +2241,32 @@
"generateContent": false
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": true,
"a2a": false,
"generateContent": false
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -123,6 +123,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = (
),
("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"),
("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"),
("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"),
(
"openrouter",
"openrouter/typesafe/jev-1.13",

View file

@ -1028,6 +1028,40 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr
assert denied.value.status_code == 400
# eu-jev.com/docs (read 2026-10-21): an omitted model means eu-jeff-latest, and the legacy names
# jeff-latest and jeff-1.0.0 answer with the current release.
@pytest.mark.parametrize(
("request_data", "permission_model"),
[
({}, "eu_jev/eu-jeff-latest"),
({"model": None}, "eu_jev/eu-jeff-latest"),
({"model": "eu-jeff-latest"}, "eu_jev/eu-jeff-latest"),
({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"),
({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"),
({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"),
({"model": "eu_jev/eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"),
({"model": "eu_jev/jeff-latest"}, "eu_jev/eu-jeff-latest"),
],
)
@pytest.mark.parametrize("suffix", ["", "/"])
def test_eu_jev_decision_is_checked_against_the_model_eu_jev_runs(
request_data: dict[str, object], permission_model: str, suffix: str
) -> None:
assert get_model_from_request(request_data=request_data, route=f"/eu_jev/v1/systemone{suffix}") == permission_model
@pytest.mark.parametrize("model", [["eu-jeff-latest"], 7, {"name": "eu-jeff-latest"}])
def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> None:
with pytest.raises(HTTPException) as denied:
get_model_from_request(request_data={"model": model}, route="/eu_jev/v1/systemone")
assert denied.value.status_code == 400
@pytest.mark.usefixtures("configured_eu_jev_route")
def test_a_pass_through_configured_at_the_eu_jev_decision_path_keeps_the_requested_model() -> None:
assert get_model_from_request(request_data={"model": "jeff-latest"}, route="/eu_jev/v1/systemone") == "jeff-latest"
def test_laya_model_normalization_does_not_change_other_provider_routes() -> None:
assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest"
assert get_model_from_request(request_data={}, route="/laya/health") is None

View file

@ -662,6 +662,41 @@ def test_virtual_key_llm_api_routes_allows_model_info(route):
assert result is True
@pytest.mark.parametrize(
("route", "is_llm_route"),
[
("/eu_jev/v1/systemone", True),
("/eu_jev/v1/systemone/extra", False),
("/eu_jev/v1/systemone2", False),
("/eu_jev/custom", False),
],
)
def test_only_the_built_in_eu_jev_decision_path_is_an_llm_route(route: str, is_llm_route: bool) -> None:
assert RouteChecks.is_llm_api_route(route=route) is is_llm_route
@pytest.mark.usefixtures("configured_eu_jev_route")
def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_route() -> None:
assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False
@pytest.mark.usefixtures("configured_eu_jev_route_with_trailing_slash")
@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/"])
def test_a_pass_through_configured_at_the_eu_jev_decision_path_with_a_trailing_slash_is_not_an_llm_route(
route: str,
) -> None:
assert RouteChecks.is_llm_api_route(route=route) is False
@pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"])
def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None:
valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route])
assert RouteChecks.is_virtual_key_allowed_to_call_route(
route="/eu_jev/v1/systemone", valid_token=valid_token, request=_mock_request("POST")
)
@pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"])
def test_model_info_not_classified_as_llm_api(route):
"""Membership in `llm_api_routes` must not promote /model/info to an

View file

@ -8,17 +8,23 @@ import os
import tempfile
import warnings
from collections.abc import Awaitable, Callable, Iterator
from contextlib import contextmanager
from typing import Dict, Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import yaml
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prisma.errors import ClientNotConnectedError
import litellm
import litellm.proxy.proxy_server
from litellm._logging import verbose_proxy_logger
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
InitPassThroughEndpointHelpers,
_registered_pass_through_routes,
)
from litellm._service_logger import ServiceTypes
from litellm.integrations.otel.model.payloads import ServiceSpanData
from litellm.integrations.otel.model.spans import service_span_name
@ -239,6 +245,40 @@ def secret_vault_factory() -> type[FakeSecretVault]:
return FakeSecretVault
@contextmanager
def _eu_jev_route_configured_at(path: str) -> Iterator[None]:
saved_registry: Final = dict(_registered_pass_through_routes)
InitPassThroughEndpointHelpers.add_exact_path_route(
app=FastAPI(),
path=path,
target="https://operator.example/decide",
custom_headers=None,
forward_headers=None,
merge_query_params=None,
dependencies=None,
cost_per_request=None,
endpoint_id="operator-eu-jev",
methods=["POST"],
)
try:
yield
finally:
_registered_pass_through_routes.clear()
_registered_pass_through_routes.update(saved_registry)
@pytest.fixture
def configured_eu_jev_route() -> Iterator[None]:
with _eu_jev_route_configured_at("/eu_jev/v1/systemone"):
yield
@pytest.fixture
def configured_eu_jev_route_with_trailing_slash() -> Iterator[None]:
with _eu_jev_route_configured_at("/eu_jev/v1/systemone/"):
yield
@pytest.fixture
def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)

View file

@ -123,6 +123,7 @@ def test_is_pure_asgi_not_base_http_middleware():
("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")),
("/transcribe", (BillableCategory.LLM, "/transcribe")),
("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")),
("/eu_jev/v1/systemone", (BillableCategory.LLM, "/eu_jev/v1/systemone")),
("/mcp", (BillableCategory.MCP, "/mcp")),
("/mcp/", (BillableCategory.MCP, "/mcp")),
("/mcp/tools/list", (BillableCategory.MCP, "/mcp")),
@ -165,6 +166,8 @@ def test_classify_billable(path: str, expected: Tuple[BillableCategory, str]):
"/langfuse/api/public/ingestion",
# a bare provider prefix is not an inference call
"/bedrock",
"/eu_jev/v1/systemone/extra",
"/eu_jev/v1/systemone2",
"/v1/mcp",
"/v1/mcp/tools",
"/v1/mcp/server",
@ -184,6 +187,11 @@ def test_classify_non_billable_returns_none(path: str):
assert classify_billable_request(path) is None
@pytest.mark.usefixtures("configured_eu_jev_route")
def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_billed() -> None:
assert classify_billable_request("/eu_jev/v1/systemone") is None
@pytest.mark.parametrize(
"path",
[

View file

@ -366,3 +366,94 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou
assert normalized["standard_logging_response_object"] is None
assert "combined_usage_object" not in normalized["kwargs"]
assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917"
def test_eu_jev_decision_is_priced_from_its_registry_row():
model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]
response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}},
logging_obj=_logging_obj(),
url_route="https://eu-jev.com/v1/systemone",
result='{"answers": {}}',
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={"model": "eu-jeff-latest"},
custom_llm_provider="eu_jev",
)
assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0"
assert response["kwargs"]["custom_llm_provider"] == "eu_jev"
assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"])
assert response["kwargs"]["response_cost"] > 0
assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64
assert response["kwargs"]["combined_usage_object"].completion_tokens == 0
# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release,
# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it.
@pytest.mark.parametrize(
"request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}]
)
def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None:
unlisted_release: Final = "eu-jeff-99.0.0"
assert f"eu_jev/{unlisted_release}" not in litellm.model_cost
response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}},
logging_obj=_logging_obj(),
url_route="https://eu-jev.com/v1/systemone",
result='{"answers": {}}',
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body=request_body,
custom_llm_provider="eu_jev",
)
latest_cost: Final = litellm.model_cost["eu_jev/eu-jeff-latest"]["input_cost_per_token"]
assert response["kwargs"]["model"] == f"eu_jev/{unlisted_release}"
assert latest_cost > 0
assert response["kwargs"]["response_cost"] == pytest.approx(64 * latest_cost)
def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
litellm.model_cost,
"eu_jev/eu-jeff-latest",
{**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0},
)
response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
httpx_response=_response(),
response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}},
logging_obj=_logging_obj(),
url_route="https://eu-jev.com/v1/systemone",
result='{"answers": {}}',
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={"model": "eu-jeff-1.0.0"},
custom_llm_provider="eu_jev",
)
pinned_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]["input_cost_per_token"]
assert response["kwargs"]["response_cost"] == pytest.approx(64 * pinned_cost)
def test_success_handler_dispatches_eu_jev_to_the_shared_handler():
normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
httpx_response=_response(),
response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}},
request_body={"model": "eu-jeff-latest"},
logging_obj=_logging_obj(),
url_route="https://eu-jev.com/v1/systemone",
result="{}",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
custom_llm_provider="eu_jev",
)
assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev"
assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0"

View file

@ -15,7 +15,7 @@ from urllib.parse import parse_qs
import httpx
import pytest
import respx
from fastapi import HTTPException, Request, Response
from fastapi import FastAPI, HTTPException, Request, Response
from fastapi.routing import APIRoute
from fastapi.responses import StreamingResponse
from fastapi.testclient import TestClient
@ -51,6 +51,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
bedrock_proxy_route,
create_pass_through_route,
cursor_proxy_route,
eu_jev_proxy_route,
fal_ai_proxy_route,
get_azure_ai_search_index_from_endpoint,
get_vertex_base_url,
@ -8508,6 +8509,316 @@ class TestTypeSafePassthroughRoute:
)
class _RecordsPreCallModel(CustomLogger):
def __init__(self, switch_to: str | None = None) -> None:
super().__init__()
self.switch_to: Final = switch_to
self.models: Final[list[object]] = []
async def async_pre_call_hook(
self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str
) -> dict | None:
self.models.append(data.get("model"))
return None if self.switch_to is None else {**data, "model": self.switch_to}
class TestEuJevPassthroughRoute:
@pytest.fixture
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key")
monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base")
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
yield TestClient(app)
def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None:
body: Final = {"model": "eu-jeff-latest", "state": "x"}
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").mock(
return_value=httpx.Response(200, json={"id": "upstream_123"})
)
response: Final = client.post("/eu_jev/v1/systemone", json=body)
assert (response.status_code, response.json()) == (200, {"id": "upstream_123"})
sent: Final = route.calls.last.request
assert sent.headers["authorization"] == "Bearer eu-jev-test-key"
assert json.loads(sent.content) == body
@pytest.mark.parametrize(
("body", "hook_model"),
[
({"state": "x"}, "eu_jev/eu-jeff-latest"),
({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"),
({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"),
],
)
def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str
) -> None:
recorder: Final = _RecordsPreCallModel()
monkeypatch.setattr(litellm, "callbacks", [recorder])
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json=body)
assert response.status_code == 200
assert recorder.models == [hook_model]
assert json.loads(route.calls.last.request.content) == body
def test_forwards_the_release_a_pre_call_hook_switched_to(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")])
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"})
assert response.status_code == 200
assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"}
def test_a_provider_prefixed_model_is_limited_once_and_forwarded_without_the_prefix(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
recorder: Final = _RecordsPreCallModel()
monkeypatch.setattr(litellm, "callbacks", [recorder])
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu_jev/eu-jeff-1.0.0", "state": "x"})
assert response.status_code == 200
assert recorder.models == ["eu_jev/eu-jeff-1.0.0"]
assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"}
def test_a_pass_through_deployment_key_goes_to_that_deployment_base(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy import proxy_server
deployment_router: Final = litellm.Router(
model_list=[
{
"model_name": "eu-jev-friendly",
"litellm_params": {
"model": "eu_jev/eu-jeff-latest",
"api_key": "eu-jev-deployment-key",
"api_base": "https://eu-jev-deployment.example/v2",
"use_in_pass_through": True,
},
}
]
)
monkeypatch.setattr(proxy_server, "llm_router", deployment_router)
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev-deployment.example/v2/v1/systemone").respond(
200, json={"answers": {}}
)
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert response.status_code == 200
assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key"
def test_a_team_deployment_never_redirects_another_team_s_calls(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy import proxy_server
team_router: Final = litellm.Router(
model_list=[
{
"model_name": "team-a-eu-jev",
"litellm_params": {
"model": "eu_jev/eu-jeff-latest",
"api_key": "team-a-own-key",
"api_base": "https://team-a-server.example",
"use_in_pass_through": True,
},
"model_info": {"team_id": "team-a"},
}
]
)
monkeypatch.setattr(proxy_server, "llm_router", team_router)
with respx.mock(assert_all_called=False) as upstream:
team_server: Final = upstream.route(host="team-a-server.example").respond(200, json={"answers": {}})
eu_jev: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(
200, json={"answers": {}}
)
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "team b data"})
assert response.status_code == 200
assert not team_server.called
assert eu_jev.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key"
def test_a_base_ending_in_v1_is_not_doubled(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base/v1/")
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert response.status_code == 200
assert route.called
def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond(
200, json={"answers": {}}
)
response: Final = client.post("/eu_jev/v1/systemone?trace=yes", json={"state": "x"})
assert response.status_code == 200
assert route.called
def test_uses_the_hosted_api_when_base_is_unset(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("EU_JEV_API_BASE")
with respx.mock(assert_all_called=True) as upstream:
route: Final = upstream.post("https://eu-jev.com/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert response.status_code == 200
assert route.called
def test_answers_503_without_calling_upstream_when_the_key_is_unset(
self, client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("EU_JEV_API_KEY")
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.route(host="eu-jev.example").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert response.status_code == 503
assert "EU_JEV_API_KEY" in response.text
assert not route.called
@pytest.mark.parametrize(
("method", "path"),
[("GET", "/eu_jev/api/account/keys"), ("DELETE", "/eu_jev/v1/keys/abc"), ("GET", "/eu_jev/v1/systemone")],
)
def test_forwards_nothing_but_a_decision(self, client: TestClient, method: str, path: str) -> None:
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.route(host="eu-jev.example").respond(200, json={})
response: Final = client.request(method, path)
assert response.status_code in (404, 405)
assert not route.called
def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None:
smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}}
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled})
assert response.status_code == 400
assert "custom_body" in response.text
assert not route.called
def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None:
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}})
response: Final = client.post(
"/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True}
)
assert response.status_code == 400
assert "Streaming" in response.text
assert not route.called
@pytest.fixture
def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str], str], FastAPI]]:
from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
InitPassThroughEndpointHelpers,
_registered_pass_through_routes,
)
monkeypatch.delenv("EU_JEV_API_KEY", raising=False)
monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough")
def build(methods: list[str], path: str) -> FastAPI:
app: Final = FastAPI()
reserve_lazy_slot(app, "llm_passthrough", features=features)
attach_lazy_features(app, features)
monkeypatch.setitem(
app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")
)
InitPassThroughEndpointHelpers.add_exact_path_route(
app=app,
path=path,
target="https://operator.example/decide",
custom_headers={"x-operator": "on"},
forward_headers=None,
merge_query_params=None,
dependencies=None,
cost_per_request=None,
endpoint_id="operator-eu-jev",
methods=methods,
)
return app
saved_registry: Final = dict(_registered_pass_through_routes)
try:
yield build
finally:
_registered_pass_through_routes.clear()
_registered_pass_through_routes.update(saved_registry)
@pytest.mark.parametrize("root_path", ["", "/tenant-a"])
def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it(
self, operator_app: Callable[[list[str], str], FastAPI], root_path: str
) -> None:
with (
respx.mock(assert_all_called=True) as upstream,
TestClient(operator_app(["POST"], "/eu_jev/v1/systemone"), root_path=root_path) as client,
):
route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True})
response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True})
assert (response.status_code, response.json()) == (200, {"operator": True})
assert route.calls.last.request.headers["x-operator"] == "on"
def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused(
self, operator_app: Callable[[list[str], str], FastAPI]
) -> None:
with TestClient(operator_app(["GET"], "/eu_jev/v1/systemone")) as client:
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert response.status_code == 405
def test_a_pass_through_configured_with_a_trailing_slash_still_gets_the_path_without_one(
self, operator_app: Callable[[list[str], str], FastAPI]
) -> None:
with (
respx.mock(assert_all_called=True) as upstream,
TestClient(operator_app(["POST"], "/eu_jev/v1/systemone/")) as client,
):
route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True})
response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"})
assert (response.status_code, response.json()) == (200, {"operator": True})
assert route.calls.last.request.headers["x-operator"] == "on"
def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first(
self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
monkeypatch.setenv("LITELLM_BUILTIN_PASS_THROUGH_ROUTES_FIRST", "true")
app: Final = operator_app(["GET", "PUT"], "/eu_jev/v1/systemone")
InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev")
with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client:
operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True})
responses: Final = (client.get("/eu_jev/v1/systemone"), client.put("/eu_jev/v1/systemone", json={}))
assert [response.status_code for response in responses] == [404, 404]
assert not operator.called
@pytest.mark.parametrize("provider", ["laya", "bespoke"])
class TestOssDecisionPassthroughRoute:
@pytest.fixture

View file

@ -82,6 +82,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES: Final = MappingProxyType(
"/tinyfish/{endpoint:path}": frozenset({"GET", "POST"}),
"/laya/v1/systemone": frozenset({"POST"}),
"/bespoke/v1/systemone": frozenset({"POST"}),
"/eu_jev/v1/systemone": frozenset({"POST"}),
}
)

View file

@ -26,6 +26,7 @@ import pytest
from fastapi import FastAPI
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from starlette.requests import Request
from litellm.proxy._types import PassThroughGenericEndpoint, ProxyException
@ -137,3 +138,15 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation():
)
assert isinstance(result, UserAPIKeyAuth)
@pytest.mark.asyncio
@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/extra", "/eu_jev/v1/systemone2"])
async def test_no_eu_jev_route_reads_the_litellm_user_api_key_header(route: str) -> None:
request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]})
result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints(
request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer"
)
assert result == "sk-from-bearer"

View file

@ -1,6 +1,7 @@
import asyncio
import os
import unittest
from typing import Final
from unittest.mock import patch
import pytest
@ -60,6 +61,49 @@ def test_credential_loaded_after_deployment_registration_still_resolves():
)
def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base() -> None:
llm_router: Final = litellm.Router(
model_list=[
_flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"),
_flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"),
]
)
credentials: Final = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai")
assert credentials is not None
assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1")
def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env")
team_deployment: Final = {
**_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"),
"model_info": {"team_id": "team-a"},
}
passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment]))
credentials: Final = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai")
assert credentials is not None
assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None)
def test_credentials_for_a_provider_route_still_include_a_team_deployment_key() -> None:
team_deployment: Final = {
**_flagged_deployment("openai/gpt-4o", api_key="sk-team-a"),
"model_info": {"team_id": "team-a"},
}
passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment]))
assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-team-a"
def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key() -> None:
passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[]))
assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None
def test_credential_rotation_is_reflected_without_deployment_update():
CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")])
llm_router = litellm.Router(

View file

@ -458,9 +458,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo:
return entry
@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE)))
def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None:
entry: Final = _decisions_provider_entry(display_name)
@pytest.mark.parametrize(
("provider_key", "display_name", "provider"),
(
("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE),
("Nace", "Nace", LlmProviders.NACE),
("EU_JEV", "eu/jev", LlmProviders.EU_JEV),
),
)
def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None:
entry: Final = _decisions_provider_entry(provider_key)
assert entry.provider_display_name == display_name
assert entry.litellm_provider == provider.value

View file

@ -397,6 +397,22 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes():
assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway"
@pytest.mark.parametrize(
("path", "served"),
[
("/eu_jev/v1/systemone", True),
("/eu_jev/v1/systemone2", False),
("/eu_jev/v1/systemone/{subpath:path}", False),
("/eu_jev/custom", False),
],
)
def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None:
async def endpoint() -> None:
return None
assert _is_gateway_route(Route(path, endpoint, methods=["POST"])) is served
def test_every_app_mount_is_assigned_to_a_component():
"""Every Mount on the proxy app must be consciously assigned to a component.

View file

@ -5357,6 +5357,32 @@ export interface paths {
patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"];
trace?: never;
};
"/eu_jev/v1/systemone": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Eu Jev Proxy Route
* @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.
*
* `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)
* and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and
* `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)
*/
post: operations["eu_jev_proxy_route_eu_jev_v1_systemone_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/extraction": {
parameters: {
query?: never;
@ -32267,6 +32293,50 @@ export interface components {
[key: string]: unknown;
};
};
/** EuJevDecisionResponse */
EuJevDecisionResponse: {
/**
* Answers
* @description One answer per question, under the same names
*/
answers: {
[key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"];
};
meta: components["schemas"]["EuJevMeta"];
/**
* Model
* @description The release that answered, such as eu-jeff-1.0.0
*/
model: string;
usage: components["schemas"]["EuJevUsage"];
};
/** EuJevMeta */
EuJevMeta: {
/**
* Cost Eur
* @description The exact charge in euros, on authenticated requests
*/
cost_eur?: string | null;
/**
* Latency Ms
* @description Server-side time for token quoting, credit reservation and inference
*/
latency_ms: number;
/**
* Mode
* @constant
*/
mode: "live";
/** Request Id */
request_id: string;
};
/** EuJevUsage */
EuJevUsage: {
/** Input Tokens */
input_tokens: number;
/** Output Tokens */
output_tokens: number;
};
/**
* Eval
* @description Represents an evaluation from the OpenAI Evals API
@ -59445,6 +59515,144 @@ export interface operations {
};
};
};
eu_jev_proxy_route_eu_jev_v1_systemone_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
/**
* @example {
* "model": "eu-jeff-latest",
* "questions": {
* "frustration": {
* "criteria": [
* "calm",
* "mildly annoyed",
* "angry"
* ],
* "instructions": "How frustrated is the customer?",
* "type": "score"
* },
* "is_refund_request": {
* "instructions": "Is the customer asking for a refund?",
* "type": "noul"
* },
* "team": {
* "criteria": {
* "billing": "Payments, invoices, and refunds",
* "support": "Technical issues and bugs"
* },
* "instructions": "Which team should handle this?",
* "type": "choice"
* }
* },
* "state": "I was charged twice. Can I get a refund?"
* }
*/
"application/json": {
/**
* Model
* @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it
* @default eu-jeff-latest
*/
model?: string;
/**
* Questions
* @description One to eight questions, keyed by names of up to 80 characters
*/
questions: {
[key: string]: {
/**
* Criteria
* @description Two to 64 options, option name to description. A null description uses the option name
*/
criteria: {
[key: string]: string | {
[key: string]: unknown;
} | unknown[] | null;
};
/**
* Instructions
* @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference
*/
instructions: string | {
[key: string]: unknown;
} | unknown[] | null;
/**
* Type
* @constant
*/
type: "choice";
} | {
/**
* Criteria
* @description Optional descriptions of what true and false mean
*/
criteria?: {
[key: string]: string | {
[key: string]: unknown;
} | unknown[] | null;
} | null;
/**
* Instructions
* @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference
*/
instructions: string | {
[key: string]: unknown;
} | unknown[] | null;
/**
* Type
* @constant
*/
type: "noul";
} | {
/**
* Criteria
* @description Two to 10 level descriptions, ordered from low to high
*/
criteria: (string | {
[key: string]: unknown;
} | unknown[] | null)[];
/**
* Instructions
* @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference
*/
instructions: string | {
[key: string]: unknown;
} | unknown[] | null;
/**
* Type
* @constant
*/
type: "score";
};
};
/**
* State
* @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array
*/
state: string | {
[key: string]: unknown;
} | unknown[];
};
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["EuJevDecisionResponse"];
};
};
};
};
extraction_extraction_post: {
parameters: {
query?: never;