mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(openai_like): route Hostinger natives per model from providers.json, refuse non-text embeddings headers unless drop_params
This commit is contained in:
parent
c1d0b8f16c
commit
e8ab6a0d1f
7 changed files with 180 additions and 22 deletions
|
|
@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
# Check for custom user agent in optional_params or environment
|
||||
# This allows partners building on LiteLLM to set their own telemetry
|
||||
|
|
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
|
|||
aembedding=aembedding,
|
||||
custom_endpoint=True,
|
||||
headers=headers,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
|
|||
|
||||
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
|
||||
|
||||
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
|
||||
|
||||
```json
|
||||
{
|
||||
"your_provider": {
|
||||
"supported_endpoints": ["/v1/chat/completions"],
|
||||
"model_supported_endpoints": {
|
||||
"model-a": ["/v1/messages"],
|
||||
"model-b": ["/v1/responses"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import StrictFloat, StrictInt, StrictStr, TypeAdapter
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -20,11 +20,26 @@ from litellm.types.utils import EmbeddingResponse
|
|||
|
||||
from ..common_utils import OpenAILikeBase, OpenAILikeError
|
||||
|
||||
_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, StrictStr | StrictInt | StrictFloat])
|
||||
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
|
||||
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _as_header_text(headers: Mapping[str, str | int | float]) -> Mapping[str, str]:
|
||||
return MappingProxyType({name: str(value) for name, value in headers.items()})
|
||||
def _header_text(value: object) -> str | None:
|
||||
if isinstance(value, bool) or not isinstance(value, str | int | float):
|
||||
return None
|
||||
return str(value)
|
||||
|
||||
|
||||
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
|
||||
texts: Final = MappingProxyType({name: _header_text(value) for name, value in extra_headers.items()})
|
||||
malformed: Final = sorted(name for name, text in texts.items() if text is None)
|
||||
if malformed and not drop_params:
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"extra_headers {', '.join(malformed)} must be text or a number. "
|
||||
"To drop them from the call, set `litellm.drop_params = True`.",
|
||||
model=model,
|
||||
)
|
||||
return MappingProxyType({name: text for name, text in texts.items() if text is not None})
|
||||
|
||||
|
||||
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
||||
|
|
@ -103,6 +118,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
aembedding=None,
|
||||
custom_endpoint: bool | None = None,
|
||||
headers: dict | None = None,
|
||||
drop_params: bool = False,
|
||||
) -> EmbeddingResponse:
|
||||
api_base, headers = self._validate_environment(
|
||||
api_base=api_base,
|
||||
|
|
@ -117,10 +133,10 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
|
|||
"input": input,
|
||||
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
|
||||
}
|
||||
extra_headers: Final = _as_header_text(
|
||||
_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {})
|
||||
extra_headers: Final = _extra_headers_as_text(
|
||||
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
|
||||
)
|
||||
key_headers: Final = _as_header_text(_HEADERS_ADAPTER.validate_python(headers))
|
||||
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
|
||||
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
|
||||
request_headers: Final = {
|
||||
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
|
||||
|
|
|
|||
|
|
@ -14,8 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
|
|||
|
||||
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
|
||||
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
|
||||
_MODEL_ROW_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_ENDPOINTS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
|
||||
|
||||
|
||||
class SimpleProviderConfig:
|
||||
|
|
@ -33,6 +32,9 @@ class SimpleProviderConfig:
|
|||
data.get("special_handling", {})
|
||||
)
|
||||
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
|
||||
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
|
||||
data.get("model_supported_endpoints", {})
|
||||
)
|
||||
self.provider_info: ProviderSpecificModelInfo = (
|
||||
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
|
||||
)
|
||||
|
|
@ -104,7 +106,10 @@ class JSONProviderRegistry:
|
|||
provider: Final = cls._providers.get(slug)
|
||||
if provider is None:
|
||||
return False
|
||||
return endpoint in provider.supported_endpoints or endpoint in _model_supported_endpoints(slug, model)
|
||||
model_endpoints: Final = (
|
||||
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
|
||||
)
|
||||
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
|
||||
|
||||
@classmethod
|
||||
def list_providers(cls) -> list:
|
||||
|
|
@ -112,16 +117,5 @@ class JSONProviderRegistry:
|
|||
return list(cls._providers.keys())
|
||||
|
||||
|
||||
def _model_supported_endpoints(slug: str, model: str | None) -> tuple[str, ...]:
|
||||
import litellm
|
||||
|
||||
if not model:
|
||||
return ()
|
||||
row: Final = _MODEL_ROW_ADAPTER.validate_python(
|
||||
litellm.model_cost.get(f"{slug}/{model.removeprefix(f'{slug}/')}", {})
|
||||
)
|
||||
return _ENDPOINTS_ADAPTER.validate_python(row.get("supported_endpoints") or ())
|
||||
|
||||
|
||||
# Load on import
|
||||
JSONProviderRegistry.load()
|
||||
|
|
|
|||
|
|
@ -235,7 +235,11 @@
|
|||
"base_url": "https://router.hostinger.com/v1",
|
||||
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
|
||||
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"]
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
|
||||
"model_supported_endpoints": {
|
||||
"claude-sonnet-5": ["/v1/messages"],
|
||||
"gpt-5.2": ["/v1/responses"]
|
||||
}
|
||||
},
|
||||
"sail": {
|
||||
"base_url": "https://api.sailresearch.com/v1",
|
||||
|
|
|
|||
|
|
@ -7173,6 +7173,7 @@ def embedding(
|
|||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
drop_params=drop_unsupported_params,
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
if headers is None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -265,6 +266,40 @@ def test_hostinger_router_embeddings_send_numeric_extra_headers_as_text(configur
|
|||
assert route.calls.last.request.headers.get_list("x-org-id") == [sent]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_refuse_non_text_extra_headers(configured: object):
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="X-Debug") as refused:
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured},
|
||||
)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured", [True, None, ["a", "b"]])
|
||||
def test_hostinger_router_embeddings_drop_non_text_extra_headers_under_drop_params(configured: object):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://router.hostinger.com/v1/embeddings").respond(200, json=EMBEDDING_RESPONSE)
|
||||
litellm.embedding(
|
||||
model="hostinger_router/text-embedding-3-small",
|
||||
input=["Say hello"],
|
||||
api_key="hostinger-test-key",
|
||||
extra_headers={"X-Debug": configured, "X-Trace-Id": "trace-1"},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
assert "x-debug" not in request.headers
|
||||
assert request.headers.get_list("x-trace-id") == ["trace-1"]
|
||||
assert set(json.loads(request.content)) == {"model", "input"}
|
||||
|
||||
|
||||
def test_hostinger_router_embeddings_never_borrow_another_providers_key(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("HOSTINGER_ROUTER_API_KEY", raising=False)
|
||||
monkeypatch.setenv("OPENAI_LIKE_API_KEY", "other-provider-key")
|
||||
|
|
@ -393,3 +428,95 @@ def test_hostinger_router_responses_bridge_models_without_native_responses():
|
|||
assert chat.call_count == 1
|
||||
assert json.loads(chat.calls.last.request.content)["model"] == "claude-sonnet-5"
|
||||
assert response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
ANTHROPIC_MESSAGE_RESPONSE: Final = {
|
||||
"id": "msg_hostinger",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-4-5",
|
||||
"content": [{"type": "text", "text": "Hello from Hostinger Router"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"sibling_model_info",
|
||||
[
|
||||
{"supported_endpoints": ["/v1/chat/completions", "/v1/messages", "/v1/responses"]},
|
||||
{"supported_endpoints": "/v1/messages"},
|
||||
],
|
||||
)
|
||||
async def test_hostinger_router_endpoint_opt_in_stays_with_its_own_deployment(
|
||||
sibling_model_info: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
deployment_params: Final = {"model": "hostinger_router/gpt-5-nano", "api_key": "hostinger-test-key"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "opted-in", "litellm_params": deployment_params, "model_info": sibling_model_info},
|
||||
{"model_name": "plain", "litellm_params": deployment_params},
|
||||
]
|
||||
)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
native_messages: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
native_responses: Final = upstream.post("https://router.hostinger.com/v1/responses").respond(
|
||||
403, json=UNSUPPORTED_ENDPOINT
|
||||
)
|
||||
chat: Final = upstream.post("https://router.hostinger.com/v1/chat/completions").respond(
|
||||
200, json=CHAT_COMPLETION_RESPONSE
|
||||
)
|
||||
messages_response: Final = await router.aanthropic_messages(
|
||||
model="plain", messages=[{"role": "user", "content": "Say hello"}], max_tokens=32
|
||||
)
|
||||
responses_response: Final = await router.aresponses(model="plain", input="Say hello")
|
||||
|
||||
assert native_messages.call_count == 0
|
||||
assert native_responses.call_count == 0
|
||||
assert chat.call_count == 2
|
||||
assert messages_response["content"][0]["text"] == "Hello from Hostinger Router"
|
||||
assert responses_response.output[0].content[0].text == "Hello from Hostinger Router"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostinger_router_deployment_cache_ttl_opt_in_keeps_one_hour_ttl(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "opted-in",
|
||||
"litellm_params": {
|
||||
"model": "hostinger_router/claude-haiku-4-5",
|
||||
"api_base": "https://router.hostinger.com/v1",
|
||||
"api_key": "hostinger-test-key",
|
||||
},
|
||||
"model_info": {"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True},
|
||||
}
|
||||
]
|
||||
)
|
||||
one_hour_cache: Final = {"type": "ephemeral", "ttl": "1h"}
|
||||
|
||||
with respx.mock() as upstream:
|
||||
native: Final = upstream.post("https://router.hostinger.com/v1/messages").respond(
|
||||
200, json=ANTHROPIC_MESSAGE_RESPONSE
|
||||
)
|
||||
await router.aanthropic_messages(
|
||||
model="opted-in",
|
||||
messages=[
|
||||
{"role": "user", "content": [{"type": "text", "text": "Say hello", "cache_control": one_hour_cache}]}
|
||||
],
|
||||
max_tokens=32,
|
||||
)
|
||||
|
||||
body: Final = json.loads(native.calls.last.request.content)
|
||||
assert body["messages"][0]["content"][0]["cache_control"] == one_hour_cache
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue