diff --git a/litellm/llms/databricks/embed/handler.py b/litellm/llms/databricks/embed/handler.py index 8b2fa768968..05807a118c3 100644 --- a/litellm/llms/databricks/embed/handler.py +++ b/litellm/llms/databricks/embed/handler.py @@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: # Check for custom user agent in optional_params or environment # This allows partners building on LiteLLM to set their own telemetry @@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase): aembedding=aembedding, custom_endpoint=True, headers=headers, + drop_params=drop_params, ) diff --git a/litellm/llms/openai_like/README.md b/litellm/llms/openai_like/README.md index c56f58b24c9..fb00b07c986 100644 --- a/litellm/llms/openai_like/README.md +++ b/litellm/llms/openai_like/README.md @@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers). +When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions: + +```json +{ + "your_provider": { + "supported_endpoints": ["/v1/chat/completions"], + "model_supported_endpoints": { + "model-a": ["/v1/messages"], + "model-b": ["/v1/responses"] + } + } +} +``` + ### How It Works 1. `json_loader.py` checks `supported_endpoints` for `/v1/responses` diff --git a/litellm/llms/openai_like/embedding/handler.py b/litellm/llms/openai_like/embedding/handler.py index 891f7876052..e6189d5e97e 100644 --- a/litellm/llms/openai_like/embedding/handler.py +++ b/litellm/llms/openai_like/embedding/handler.py @@ -8,7 +8,7 @@ from types import MappingProxyType from typing import Final import httpx -from pydantic import StrictFloat, StrictInt, StrictStr, TypeAdapter +from pydantic import TypeAdapter import litellm from litellm.llms.custom_httpx.http_handler import ( @@ -20,11 +20,26 @@ from litellm.types.utils import EmbeddingResponse from ..common_utils import OpenAILikeBase, OpenAILikeError -_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, StrictStr | StrictInt | StrictFloat]) +_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str]) +_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def _as_header_text(headers: Mapping[str, str | int | float]) -> Mapping[str, str]: - return MappingProxyType({name: str(value) for name, value in headers.items()}) +def _header_text(value: object) -> str | None: + if isinstance(value, bool) or not isinstance(value, str | int | float): + return None + return str(value) + + +def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]: + texts: Final = MappingProxyType({name: _header_text(value) for name, value in extra_headers.items()}) + malformed: Final = sorted(name for name, text in texts.items() if text is None) + if malformed and not drop_params: + raise litellm.UnsupportedParamsError( + message=f"extra_headers {', '.join(malformed)} must be text or a number. " + "To drop them from the call, set `litellm.drop_params = True`.", + model=model, + ) + return MappingProxyType({name: text for name, text in texts.items() if text is not None}) class OpenAILikeEmbeddingHandler(OpenAILikeBase): @@ -103,6 +118,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): aembedding=None, custom_endpoint: bool | None = None, headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: api_base, headers = self._validate_environment( api_base=api_base, @@ -117,10 +133,10 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): "input": input, **{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")}, } - extra_headers: Final = _as_header_text( - _HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}) + extra_headers: Final = _extra_headers_as_text( + _EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params ) - key_headers: Final = _as_header_text(_HEADERS_ADAPTER.validate_python(headers)) + key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers) extra_header_names: Final = frozenset(name.lower() for name in extra_headers) request_headers: Final = { **{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names}, diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 3076bc5bc76..658e4176f22 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -14,8 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) _SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool]) -_MODEL_ROW_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -_ENDPOINTS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]]) class SimpleProviderConfig: @@ -33,6 +32,9 @@ class SimpleProviderConfig: data.get("special_handling", {}) ) self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", []) + self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python( + data.get("model_supported_endpoints", {}) + ) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} ) @@ -104,7 +106,10 @@ class JSONProviderRegistry: provider: Final = cls._providers.get(slug) if provider is None: return False - return endpoint in provider.supported_endpoints or endpoint in _model_supported_endpoints(slug, model) + model_endpoints: Final = ( + provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else () + ) + return endpoint in provider.supported_endpoints or endpoint in model_endpoints @classmethod def list_providers(cls) -> list: @@ -112,16 +117,5 @@ class JSONProviderRegistry: return list(cls._providers.keys()) -def _model_supported_endpoints(slug: str, model: str | None) -> tuple[str, ...]: - import litellm - - if not model: - return () - row: Final = _MODEL_ROW_ADAPTER.validate_python( - litellm.model_cost.get(f"{slug}/{model.removeprefix(f'{slug}/')}", {}) - ) - return _ENDPOINTS_ADAPTER.validate_python(row.get("supported_endpoints") or ()) - - # Load on import JSONProviderRegistry.load() diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index c4f0456cebd..a71a9c3bb9b 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -235,7 +235,11 @@ "base_url": "https://router.hostinger.com/v1", "api_key_env": "HOSTINGER_ROUTER_API_KEY", "api_base_env": "HOSTINGER_ROUTER_API_BASE", - "supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"] + "supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"], + "model_supported_endpoints": { + "claude-sonnet-5": ["/v1/messages"], + "gpt-5.2": ["/v1/responses"] + } }, "sail": { "base_url": "https://api.sailresearch.com/v1", diff --git a/litellm/main.py b/litellm/main.py index f0ae4111580..fa84de8b7d1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7173,6 +7173,7 @@ def embedding( optional_params=optional_params, client=client, aembedding=aembedding, + drop_params=drop_unsupported_params, ) elif custom_llm_provider == "oci": if headers is None: diff --git a/tests/unit/llms/openai_like/test_hostinger_router_provider.py b/tests/unit/llms/openai_like/test_hostinger_router_provider.py index 47cf6d96ce0..8d9b2e05c51 100644 --- a/tests/unit/llms/openai_like/test_hostinger_router_provider.py +++ b/tests/unit/llms/openai_like/test_hostinger_router_provider.py @@ -1,3 +1,4 @@ +import copy import json from typing import Final @@ -265,6 +266,40 @@ def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configur assert route.calls.last.request.headers.get_list("x-org-id") == [sent] +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused: + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured}, + ) + + assert refused.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.parametrize("configured", [True, None, ["a", "b"]]) +def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object): + with respx.mock() as upstream: + route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE) + litellm.embedding( + model="hostinger_router/text-embedding-3-small", + input=["Say hello"], + api_key="hostinger-test-key", + extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"}, + drop_params=True, + ) + + request: Final = route.calls.last.request + assert "x-debug" not in request.headers + assert request.headers.get_list("x-trace-id") == ["trace-1"] + assert set(json.loads(request.content)) == {"model", "input"} + + def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False) monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key") @@ -393,3 +428,95 @@ def test_hostinger_router_responses_bridge_models_without_native_responses(): assert chat.call_count == 1 assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5" assert response.output[0].content[0].text == "Hello from Hostinger Router" + + +ANTHROPIC_MESSAGE_RESPONSE: Final = { + "id": "msg_hostinger", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "Hello from Hostinger Router"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sibling_model_info", + [ + {"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]}, + {"supported_endpoints": "/v1/messages"}, + ], +) +async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment( + sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"} + router: Final = litellm.Router( + model_list=[ + {"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info}, + {"model_name": "plain", "litellm_params": deployment_params}, + ] + ) + + with respx.mock(assert_all_called=False) as upstream: + native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond( + 403, json=UNSUPPORTED_ENDPOINT + ) + chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond( + 200, json=CHAT_COMPLETION_RESPONSE + ) + messages_response: Final = await router.aanthropic_messages( + model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32 + ) + responses_response: Final = await router.aresponses(model="plain", input="Say hello") + + assert native_messages.call_count == 0 + assert native_responses.call_count == 0 + assert chat.call_count == 2 + assert messages_response["content"][0]["text"] == "Hello from Hostinger Router" + assert responses_response.output[0].content[0].text == "Hello from Hostinger Router" + + +@pytest.mark.asyncio +async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": { + "model": "hostinger_router/claude-haiku-4-5", + "api_base": "https://router.hostinger.com/v1", + "api_key": "hostinger-test-key", + }, + "model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, + } + ] + ) + one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"} + + with respx.mock() as upstream: + native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond( + 200, json=ANTHROPIC_MESSAGE_RESPONSE + ) + await router.aanthropic_messages( + model="opted-in", + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]} + ], + max_tokens=32, + ) + + body: Final = json.loads(native.calls.last.request.content) + assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache