mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
feat(openrouter): price typesafe/jev-1.13 and add an openrouter decisions pass-through (#42301)
This commit is contained in:
parent
5d3d99eb9f
commit
1106b16745
13 changed files with 683 additions and 5 deletions
|
|
@ -100,6 +100,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
||||||
"/vllm/",
|
"/vllm/",
|
||||||
"/mistral/",
|
"/mistral/",
|
||||||
"/typesafe/",
|
"/typesafe/",
|
||||||
|
"/openrouter/",
|
||||||
"/nvidia_nim/",
|
"/nvidia_nim/",
|
||||||
"/groq/",
|
"/groq/",
|
||||||
"/voyage/",
|
"/voyage/",
|
||||||
|
|
|
||||||
|
|
@ -72883,6 +72883,16 @@
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
"openrouter/typesafe/jev-1.13": {
|
||||||
|
"input_cost_per_token": 4.2e-08,
|
||||||
|
"litellm_provider": "openrouter",
|
||||||
|
"max_input_tokens": 32000,
|
||||||
|
"max_output_tokens": 28800,
|
||||||
|
"max_tokens": 28800,
|
||||||
|
"mode": "evaluation",
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
"source": "https://openrouter.ai/typesafe/jev-1.13"
|
||||||
|
},
|
||||||
"typesafe/jev-1.13.0": {
|
"typesafe/jev-1.13.0": {
|
||||||
"input_cost_per_token": 4.2e-08,
|
"input_cost_per_token": 4.2e-08,
|
||||||
"litellm_provider": "typesafe",
|
"litellm_provider": "typesafe",
|
||||||
|
|
|
||||||
|
|
@ -212,6 +212,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
||||||
"/openai_passthrough/",
|
"/openai_passthrough/",
|
||||||
"/transcribe",
|
"/transcribe",
|
||||||
"/typesafe/",
|
"/typesafe/",
|
||||||
|
"/openrouter/",
|
||||||
"/vertex-ai/",
|
"/vertex-ai/",
|
||||||
"/vertex_ai/",
|
"/vertex_ai/",
|
||||||
"/vllm/",
|
"/vllm/",
|
||||||
|
|
|
||||||
|
|
@ -20729,6 +20729,223 @@
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/openrouter/{endpoint}": {
|
||||||
|
"delete": {
|
||||||
|
"operationId": "openrouter_proxy_route_openrouter__endpoint__delete",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"in": "path",
|
||||||
|
"name": "endpoint",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"title": "Endpoint",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
},
|
||||||
|
"422": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/HTTPValidationError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Validation Error"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Openrouter Proxy Route",
|
||||||
|
"tags": [
|
||||||
|
"llm_passthrough"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"get": {
|
||||||
|
"operationId": "openrouter_proxy_route_openrouter__endpoint__get",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"in": "path",
|
||||||
|
"name": "endpoint",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"title": "Endpoint",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
},
|
||||||
|
"422": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/HTTPValidationError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Validation Error"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Openrouter Proxy Route",
|
||||||
|
"tags": [
|
||||||
|
"llm_passthrough"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"patch": {
|
||||||
|
"operationId": "openrouter_proxy_route_openrouter__endpoint__patch",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"in": "path",
|
||||||
|
"name": "endpoint",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"title": "Endpoint",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
},
|
||||||
|
"422": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/HTTPValidationError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Validation Error"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Openrouter Proxy Route",
|
||||||
|
"tags": [
|
||||||
|
"llm_passthrough"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"post": {
|
||||||
|
"operationId": "openrouter_proxy_route_openrouter__endpoint__post",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"in": "path",
|
||||||
|
"name": "endpoint",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"title": "Endpoint",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
},
|
||||||
|
"422": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/HTTPValidationError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Validation Error"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Openrouter Proxy Route",
|
||||||
|
"tags": [
|
||||||
|
"llm_passthrough"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"put": {
|
||||||
|
"operationId": "openrouter_proxy_route_openrouter__endpoint__put",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"in": "path",
|
||||||
|
"name": "endpoint",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"title": "Endpoint",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful Response"
|
||||||
|
},
|
||||||
|
"422": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/HTTPValidationError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Validation Error"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"APIKeyHeader": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"summary": "Openrouter Proxy Route",
|
||||||
|
"tags": [
|
||||||
|
"llm_passthrough"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/transcribe": {
|
"/transcribe": {
|
||||||
"post": {
|
"post": {
|
||||||
"description": "AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`\nat `/transcribe` and the operation is read from the `X-Amz-Target` header, per the\nAWS JSON 1.1 protocol.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/transcribe)",
|
"description": "AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`\nat `/transcribe` and the operation is read from the `X-Amz-Target` header, per the\nAWS JSON 1.1 protocol.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/transcribe)",
|
||||||
|
|
|
||||||
|
|
@ -494,6 +494,7 @@ class LiteLLMRoutes(enum.Enum):
|
||||||
"/vllm",
|
"/vllm",
|
||||||
"/mistral",
|
"/mistral",
|
||||||
"/typesafe",
|
"/typesafe",
|
||||||
|
"/openrouter",
|
||||||
"/milvus",
|
"/milvus",
|
||||||
"/gigachat",
|
"/gigachat",
|
||||||
"/watsonx",
|
"/watsonx",
|
||||||
|
|
|
||||||
|
|
@ -579,6 +579,42 @@ async def typesafe_proxy_route(
|
||||||
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||||
|
|
||||||
|
|
||||||
|
@router.api_route(
|
||||||
|
"/openrouter/{endpoint:path}",
|
||||||
|
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
|
||||||
|
tags=["OpenRouter Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
|
||||||
|
)
|
||||||
|
async def openrouter_proxy_route(
|
||||||
|
endpoint: str,
|
||||||
|
request: Request,
|
||||||
|
fastapi_response: Response,
|
||||||
|
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||||
|
):
|
||||||
|
base_target_url: Final = get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1"
|
||||||
|
api_root: Final = base_target_url.removesuffix("/").removesuffix("/v1")
|
||||||
|
encoded_endpoint: Final = httpx.URL(endpoint).path
|
||||||
|
normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
|
||||||
|
base_url: Final = httpx.URL(api_root)
|
||||||
|
updated_url: Final = base_url.copy_with(
|
||||||
|
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
|
||||||
|
)
|
||||||
|
openrouter_api_key: Final = passthrough_endpoint_router.get_credentials(
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
region_name=None,
|
||||||
|
)
|
||||||
|
endpoint_func: Final = create_pass_through_route(
|
||||||
|
endpoint=endpoint,
|
||||||
|
target=str(updated_url),
|
||||||
|
custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
|
||||||
|
"Authorization": f"Bearer {openrouter_api_key}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
is_streaming_request=False,
|
||||||
|
)
|
||||||
|
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||||
|
|
||||||
|
|
||||||
@router.api_route(
|
@router.api_route(
|
||||||
"/milvus/{endpoint:path}",
|
"/milvus/{endpoint:path}",
|
||||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
|
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,7 @@ class TypeSafePassthroughLoggingHandler:
|
||||||
end_time: datetime,
|
end_time: datetime,
|
||||||
cache_hit: bool,
|
cache_hit: bool,
|
||||||
request_body: Mapping[str, object],
|
request_body: Mapping[str, object],
|
||||||
|
custom_llm_provider: str,
|
||||||
**kwargs: object,
|
**kwargs: object,
|
||||||
) -> PassThroughEndpointLoggingTypedDict:
|
) -> PassThroughEndpointLoggingTypedDict:
|
||||||
response: Final = _parse_typesafe_response(response_body)
|
response: Final = _parse_typesafe_response(response_body)
|
||||||
|
|
@ -72,12 +73,12 @@ class TypeSafePassthroughLoggingHandler:
|
||||||
request_model_value: Final = request_body.get("model")
|
request_model_value: Final = request_body.get("model")
|
||||||
request_model: Final = request_model_value if isinstance(request_model_value, str) else None
|
request_model: Final = request_model_value if isinstance(request_model_value, str) else None
|
||||||
logged_model: Final = response_model or request_model or "unknown"
|
logged_model: Final = response_model or request_model or "unknown"
|
||||||
model_name: Final = f"typesafe/{logged_model}"
|
model_name: Final = f"{custom_llm_provider}/{logged_model}"
|
||||||
usage: Final = response.usage or _TypeSafeUsage()
|
usage: Final = response.usage or _TypeSafeUsage()
|
||||||
input_tokens: Final = usage.input_tokens
|
input_tokens: Final = usage.input_tokens
|
||||||
output_tokens: Final = usage.output_tokens
|
output_tokens: Final = usage.output_tokens
|
||||||
candidate_model_keys: Final = tuple(
|
candidate_model_keys: Final = tuple(
|
||||||
f"typesafe/{model}" for model in (response_model, request_model) if model is not None
|
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
|
||||||
)
|
)
|
||||||
pricing: Final = _pricing_for(candidate_model_keys)
|
pricing: Final = _pricing_for(candidate_model_keys)
|
||||||
response_cost: Final = (
|
response_cost: Final = (
|
||||||
|
|
@ -91,13 +92,13 @@ class TypeSafePassthroughLoggingHandler:
|
||||||
updated_kwargs: Final = { # mutable-ok: pass-through logging contract requires mutable kwargs
|
updated_kwargs: Final = { # mutable-ok: pass-through logging contract requires mutable kwargs
|
||||||
**kwargs,
|
**kwargs,
|
||||||
"model": model_name,
|
"model": model_name,
|
||||||
"custom_llm_provider": "typesafe",
|
"custom_llm_provider": custom_llm_provider,
|
||||||
"response_cost": response_cost,
|
"response_cost": response_cost,
|
||||||
"combined_usage_object": usage_object,
|
"combined_usage_object": usage_object,
|
||||||
}
|
}
|
||||||
logging_obj.model_call_details.update(
|
logging_obj.model_call_details.update(
|
||||||
model=model_name,
|
model=model_name,
|
||||||
custom_llm_provider="typesafe",
|
custom_llm_provider=custom_llm_provider,
|
||||||
response_cost=response_cost,
|
response_cost=response_cost,
|
||||||
)
|
)
|
||||||
standard_logging_object: Final = get_standard_logging_object_payload(
|
standard_logging_object: Final = get_standard_logging_object_payload(
|
||||||
|
|
|
||||||
|
|
@ -311,7 +311,9 @@ class PassThroughEndpointLogging:
|
||||||
)
|
)
|
||||||
standard_logging_response_object = transcribe_handler_result["result"] # rebind-ok: elif-chain
|
standard_logging_response_object = transcribe_handler_result["result"] # rebind-ok: elif-chain
|
||||||
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
|
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
|
||||||
elif self.is_typesafe_route(custom_llm_provider):
|
elif self.is_typesafe_route(custom_llm_provider) or self.is_openrouter_decisions_route(
|
||||||
|
url_route, custom_llm_provider
|
||||||
|
):
|
||||||
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
|
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
|
||||||
TypeSafePassthroughLoggingHandler,
|
TypeSafePassthroughLoggingHandler,
|
||||||
)
|
)
|
||||||
|
|
@ -326,6 +328,7 @@ class PassThroughEndpointLogging:
|
||||||
end_time=end_time,
|
end_time=end_time,
|
||||||
cache_hit=cache_hit,
|
cache_hit=cache_hit,
|
||||||
request_body=request_body,
|
request_body=request_body,
|
||||||
|
custom_llm_provider=custom_llm_provider or "",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
standard_logging_response_object = typesafe_handler_result["result"]
|
standard_logging_response_object = typesafe_handler_result["result"]
|
||||||
|
|
@ -505,6 +508,9 @@ class PassThroughEndpointLogging:
|
||||||
def is_typesafe_route(self, custom_llm_provider: str | None) -> bool:
|
def is_typesafe_route(self, custom_llm_provider: str | None) -> bool:
|
||||||
return custom_llm_provider == "typesafe"
|
return custom_llm_provider == "typesafe"
|
||||||
|
|
||||||
|
def is_openrouter_decisions_route(self, url_route: str, custom_llm_provider: str | None) -> bool:
|
||||||
|
return custom_llm_provider == "openrouter" and urlparse(url_route).path.endswith("/alpha/decisions")
|
||||||
|
|
||||||
def is_langfuse_route(self, url_route: str):
|
def is_langfuse_route(self, url_route: str):
|
||||||
parsed_url: Final = urlparse(url_route)
|
parsed_url: Final = urlparse(url_route)
|
||||||
for route in self.TRACKED_LANGFUSE_ROUTES:
|
for route in self.TRACKED_LANGFUSE_ROUTES:
|
||||||
|
|
|
||||||
|
|
@ -165,6 +165,7 @@ class HttpJevClassifierClient:
|
||||||
end_time=end_time,
|
end_time=end_time,
|
||||||
cache_hit=False,
|
cache_hit=False,
|
||||||
request_body=MappingProxyType({"model": request.model}),
|
request_body=MappingProxyType({"model": request.model}),
|
||||||
|
custom_llm_provider="typesafe",
|
||||||
litellm_params=params,
|
litellm_params=params,
|
||||||
)
|
)
|
||||||
success_handlers: Final = logging_obj.dispatch_success_handlers(
|
success_handlers: Final = logging_obj.dispatch_success_handlers(
|
||||||
|
|
|
||||||
|
|
@ -72883,6 +72883,16 @@
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
"openrouter/typesafe/jev-1.13": {
|
||||||
|
"input_cost_per_token": 4.2e-08,
|
||||||
|
"litellm_provider": "openrouter",
|
||||||
|
"max_input_tokens": 32000,
|
||||||
|
"max_output_tokens": 28800,
|
||||||
|
"max_tokens": 28800,
|
||||||
|
"mode": "evaluation",
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
"source": "https://openrouter.ai/typesafe/jev-1.13"
|
||||||
|
},
|
||||||
"typesafe/jev-1.13.0": {
|
"typesafe/jev-1.13.0": {
|
||||||
"input_cost_per_token": 4.2e-08,
|
"input_cost_per_token": 4.2e-08,
|
||||||
"litellm_provider": "typesafe",
|
"litellm_provider": "typesafe",
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ def _handler_result(response_body: dict, request_body: dict) -> dict:
|
||||||
end_time=datetime.now(),
|
end_time=datetime.now(),
|
||||||
cache_hit=False,
|
cache_hit=False,
|
||||||
request_body=request_body,
|
request_body=request_body,
|
||||||
|
custom_llm_provider="typesafe",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -59,6 +60,7 @@ def test_uses_registry_pricing_and_standard_usage():
|
||||||
end_time=datetime.now(),
|
end_time=datetime.now(),
|
||||||
cache_hit=False,
|
cache_hit=False,
|
||||||
request_body={"model": "jev-latest"},
|
request_body={"model": "jev-latest"},
|
||||||
|
custom_llm_provider="typesafe",
|
||||||
)
|
)
|
||||||
|
|
||||||
expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"]
|
expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"]
|
||||||
|
|
@ -105,6 +107,7 @@ def test_records_model_provider_and_cost_on_logging_details():
|
||||||
end_time=datetime.now(),
|
end_time=datetime.now(),
|
||||||
cache_hit=False,
|
cache_hit=False,
|
||||||
request_body={"model": "jev-latest"},
|
request_body={"model": "jev-latest"},
|
||||||
|
custom_llm_provider="typesafe",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result["kwargs"]["model"] == "typesafe/jev-1.13.0"
|
assert result["kwargs"]["model"] == "typesafe/jev-1.13.0"
|
||||||
|
|
@ -132,3 +135,76 @@ def test_success_handler_dispatches_to_typesafe_handler():
|
||||||
|
|
||||||
assert normalized["kwargs"]["custom_llm_provider"] == "typesafe"
|
assert normalized["kwargs"]["custom_llm_provider"] == "typesafe"
|
||||||
assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0"
|
assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openrouter_decisions_response_is_priced_from_request_model_registry_row():
|
||||||
|
logging_obj = _logging_obj()
|
||||||
|
model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"]
|
||||||
|
response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
|
||||||
|
httpx_response=_response(),
|
||||||
|
response_body={
|
||||||
|
"model": "typesafe/jev-1.13-20260917",
|
||||||
|
"usage": {"input_tokens": 282, "output_tokens": 20},
|
||||||
|
},
|
||||||
|
logging_obj=logging_obj,
|
||||||
|
url_route="https://openrouter.ai/api/alpha/decisions",
|
||||||
|
result='{"answers": {}}',
|
||||||
|
start_time=datetime.now(),
|
||||||
|
end_time=datetime.now(),
|
||||||
|
cache_hit=False,
|
||||||
|
request_body={"model": "typesafe/jev-1.13"},
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"]
|
||||||
|
assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917"
|
||||||
|
assert response["kwargs"]["custom_llm_provider"] == "openrouter"
|
||||||
|
assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost)
|
||||||
|
assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282
|
||||||
|
assert response["kwargs"]["combined_usage_object"].completion_tokens == 20
|
||||||
|
assert response["kwargs"]["combined_usage_object"].total_tokens == 302
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_handler_dispatches_openrouter_to_the_shared_handler():
|
||||||
|
logging_obj = _logging_obj()
|
||||||
|
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
|
||||||
|
httpx_response=_response(),
|
||||||
|
response_body={
|
||||||
|
"model": "typesafe/jev-1.13-20260917",
|
||||||
|
"usage": {"input_tokens": 282, "output_tokens": 20},
|
||||||
|
},
|
||||||
|
request_body={"model": "typesafe/jev-1.13"},
|
||||||
|
logging_obj=logging_obj,
|
||||||
|
url_route="https://openrouter.ai/api/alpha/decisions",
|
||||||
|
result="{}",
|
||||||
|
start_time=datetime.now(),
|
||||||
|
end_time=datetime.now(),
|
||||||
|
cache_hit=False,
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert normalized["kwargs"]["custom_llm_provider"] == "openrouter"
|
||||||
|
assert normalized["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917"
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_routes():
|
||||||
|
logging_obj = _logging_obj()
|
||||||
|
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
|
||||||
|
httpx_response=_response(),
|
||||||
|
response_body={
|
||||||
|
"model": "typesafe/jev-1.13-20260917",
|
||||||
|
"usage": {"input_tokens": 282, "output_tokens": 20},
|
||||||
|
},
|
||||||
|
request_body={"model": "typesafe/jev-1.13"},
|
||||||
|
logging_obj=logging_obj,
|
||||||
|
url_route="https://openrouter.ai/api/v1/chat/completions",
|
||||||
|
result="{}",
|
||||||
|
start_time=datetime.now(),
|
||||||
|
end_time=datetime.now(),
|
||||||
|
cache_hit=False,
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert normalized["standard_logging_response_object"] is None
|
||||||
|
assert "combined_usage_object" not in normalized["kwargs"]
|
||||||
|
assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917"
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||||
mistral_proxy_route,
|
mistral_proxy_route,
|
||||||
relay_nvidia_nim_request,
|
relay_nvidia_nim_request,
|
||||||
openai_proxy_route,
|
openai_proxy_route,
|
||||||
|
openrouter_proxy_route,
|
||||||
typesafe_proxy_route,
|
typesafe_proxy_route,
|
||||||
vertex_discovery_proxy_route,
|
vertex_discovery_proxy_route,
|
||||||
vertex_proxy_route,
|
vertex_proxy_route,
|
||||||
|
|
@ -7114,3 +7115,144 @@ class TestTypeSafePassthroughRoute:
|
||||||
custom_llm_provider="typesafe",
|
custom_llm_provider="typesafe",
|
||||||
is_streaming_request=False,
|
is_streaming_request=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestOpenRouterPassthroughRoute:
|
||||||
|
@staticmethod
|
||||||
|
def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock:
|
||||||
|
request = MagicMock(spec=Request)
|
||||||
|
request.method = "POST"
|
||||||
|
request.query_params = query_params or {}
|
||||||
|
request.json = AsyncMock(return_value=body)
|
||||||
|
return request
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||||
|
from litellm.proxy.proxy_server import app
|
||||||
|
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key")
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base")
|
||||||
|
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
|
||||||
|
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||||
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||||
|
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
|
||||||
|
yield TestClient(app)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"method, body",
|
||||||
|
[
|
||||||
|
("GET", None),
|
||||||
|
("POST", {"state": "The sky is blue."}),
|
||||||
|
("PUT", {"state": "The sky is blue."}),
|
||||||
|
("DELETE", None),
|
||||||
|
("PATCH", {"state": "The sky is blue."}),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_forwards_every_method_and_body_upstream(
|
||||||
|
self, client: TestClient, method: str, body: dict[str, str] | None
|
||||||
|
) -> None:
|
||||||
|
with respx.mock(assert_all_called=True) as upstream:
|
||||||
|
route = upstream.request(method, "https://openrouter.example/base/alpha/decisions").mock(
|
||||||
|
return_value=httpx.Response(200, json={"id": "upstream_123"})
|
||||||
|
)
|
||||||
|
response = client.request(method, "/openrouter/alpha/decisions", json=body)
|
||||||
|
|
||||||
|
assert (response.status_code, response.json()) == (200, {"id": "upstream_123"})
|
||||||
|
sent: Final = route.calls.last.request
|
||||||
|
assert sent.headers["authorization"] == "Bearer openrouter-test-key"
|
||||||
|
assert json.loads(sent.content or b"{}") == (body or {})
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_forwards_target_auth_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key")
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base")
|
||||||
|
|
||||||
|
async def fake_upstream(request, *_args):
|
||||||
|
target: Final = create_route.call_args.kwargs["target"]
|
||||||
|
upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params)
|
||||||
|
return {"upstream_query": parse_qs(upstream_url.query.decode())}
|
||||||
|
|
||||||
|
endpoint_func = AsyncMock(side_effect=fake_upstream)
|
||||||
|
create_route = Mock(return_value=endpoint_func)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
|
||||||
|
create_route,
|
||||||
|
)
|
||||||
|
|
||||||
|
request = self._request({"state": "The sky is blue."}, {"trace": "yes"})
|
||||||
|
result = await openrouter_proxy_route(
|
||||||
|
endpoint="alpha/decisions",
|
||||||
|
request=request,
|
||||||
|
fastapi_response=MagicMock(spec=Response),
|
||||||
|
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {"upstream_query": {"trace": ["yes"]}}
|
||||||
|
endpoint_func.assert_awaited_once()
|
||||||
|
create_route.assert_called_once_with(
|
||||||
|
endpoint="alpha/decisions",
|
||||||
|
target="https://openrouter.example/base/alpha/decisions",
|
||||||
|
custom_headers={
|
||||||
|
"Authorization": "Bearer openrouter-test-key",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
custom_llm_provider="openrouter",
|
||||||
|
is_streaming_request=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_uses_default_target_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key")
|
||||||
|
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
|
||||||
|
|
||||||
|
endpoint_func = AsyncMock(return_value={"ok": True})
|
||||||
|
create_route = Mock(return_value=endpoint_func)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
|
||||||
|
create_route,
|
||||||
|
)
|
||||||
|
|
||||||
|
await openrouter_proxy_route(
|
||||||
|
endpoint="alpha/decisions",
|
||||||
|
request=self._request({"state": "The sky is blue."}),
|
||||||
|
fastapi_response=MagicMock(spec=Response),
|
||||||
|
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert create_route.call_args.kwargs["target"] == "https://openrouter.ai/api/alpha/decisions"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("endpoint", ["alpha/decisions", "v1/chat/completions"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"base_env, expected_root",
|
||||||
|
[
|
||||||
|
(None, "https://openrouter.ai/api"),
|
||||||
|
("https://openrouter.ai/api/v1", "https://openrouter.ai/api"),
|
||||||
|
("https://openrouter.example/base", "https://openrouter.example/base"),
|
||||||
|
("https://openrouter.example/base/v1/", "https://openrouter.example/base"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_derives_api_root_from_configured_base(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch, base_env: str | None, expected_root: str, endpoint: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key")
|
||||||
|
if base_env is None:
|
||||||
|
monkeypatch.delenv("OPENROUTER_API_BASE", raising=False)
|
||||||
|
else:
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_BASE", base_env)
|
||||||
|
|
||||||
|
endpoint_func = AsyncMock(return_value={"ok": True})
|
||||||
|
create_route = Mock(return_value=endpoint_func)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
|
||||||
|
create_route,
|
||||||
|
)
|
||||||
|
|
||||||
|
await openrouter_proxy_route(
|
||||||
|
endpoint=endpoint,
|
||||||
|
request=self._request({"state": "The sky is blue."}),
|
||||||
|
fastapi_response=MagicMock(spec=Response),
|
||||||
|
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert create_route.call_args.kwargs["target"] == f"{expected_root}/{endpoint}"
|
||||||
|
|
|
||||||
176
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
176
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -10646,6 +10646,27 @@ export interface paths {
|
||||||
patch: operations["openai_passthrough_route_openai_passthrough__endpoint__patch"];
|
patch: operations["openai_passthrough_route_openai_passthrough__endpoint__patch"];
|
||||||
trace?: never;
|
trace?: never;
|
||||||
};
|
};
|
||||||
|
"/openrouter/{endpoint}": {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path?: never;
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
/** Openrouter Proxy Route */
|
||||||
|
get: operations["openrouter_proxy_route_openrouter__endpoint__get"];
|
||||||
|
/** Openrouter Proxy Route */
|
||||||
|
put: operations["openrouter_proxy_route_openrouter__endpoint__put"];
|
||||||
|
/** Openrouter Proxy Route */
|
||||||
|
post: operations["openrouter_proxy_route_openrouter__endpoint__post"];
|
||||||
|
/** Openrouter Proxy Route */
|
||||||
|
delete: operations["openrouter_proxy_route_openrouter__endpoint__delete"];
|
||||||
|
options?: never;
|
||||||
|
head?: never;
|
||||||
|
/** Openrouter Proxy Route */
|
||||||
|
patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"];
|
||||||
|
trace?: never;
|
||||||
|
};
|
||||||
"/organization/daily/activity": {
|
"/organization/daily/activity": {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: never;
|
query?: never;
|
||||||
|
|
@ -56345,6 +56366,161 @@ export interface operations {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
openrouter_proxy_route_openrouter__endpoint__get: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
endpoint: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Validation Error */
|
||||||
|
422: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["HTTPValidationError"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
openrouter_proxy_route_openrouter__endpoint__put: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
endpoint: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Validation Error */
|
||||||
|
422: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["HTTPValidationError"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
openrouter_proxy_route_openrouter__endpoint__post: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
endpoint: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Validation Error */
|
||||||
|
422: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["HTTPValidationError"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
openrouter_proxy_route_openrouter__endpoint__delete: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
endpoint: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Validation Error */
|
||||||
|
422: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["HTTPValidationError"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
openrouter_proxy_route_openrouter__endpoint__patch: {
|
||||||
|
parameters: {
|
||||||
|
query?: never;
|
||||||
|
header?: never;
|
||||||
|
path: {
|
||||||
|
endpoint: string;
|
||||||
|
};
|
||||||
|
cookie?: never;
|
||||||
|
};
|
||||||
|
requestBody?: never;
|
||||||
|
responses: {
|
||||||
|
/** @description Successful Response */
|
||||||
|
200: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": unknown;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
/** @description Validation Error */
|
||||||
|
422: {
|
||||||
|
headers: {
|
||||||
|
[name: string]: unknown;
|
||||||
|
};
|
||||||
|
content: {
|
||||||
|
"application/json": components["schemas"]["HTTPValidationError"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
get_organization_daily_activity_organization_daily_activity_get: {
|
get_organization_daily_activity_organization_daily_activity_get: {
|
||||||
parameters: {
|
parameters: {
|
||||||
query?: {
|
query?: {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue