mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
Merge pull request #38774 from BerriAI/litellm_fix_openai_embedding_encoding_format_omit
fix(embeddings): omit encoding_format when the client omits it on OpenAI-compatible calls
This commit is contained in:
Mateo Wang
GitHub
commit
6661e915a5
12 changed files with 286 additions and 255 deletions
|
|
@ -4,13 +4,12 @@ VLLM is a superset of OpenAI's `embedding` endpoint.
|
|||
|
||||
## `encoding_format`
|
||||
|
||||
For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request:
|
||||
For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request. `hosted_vllm/...` models use a separate handler that never adds the field on its own, so this resolution applies to the `openai/...`-style routes only:
|
||||
|
||||
1. Explicit value on the embedding call (`encoding_format=...`).
|
||||
2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry).
|
||||
3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env).
|
||||
4. Default **`float`**.
|
||||
|
||||
That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly.
|
||||
If none of those is set, or the winning value is the literal string `none`, the field is omitted from the upstream request entirely (LiteLLM also bypasses the OpenAI SDK's own base64 default), so OpenAI-compatible servers that reject `encoding_format` keep working.
|
||||
|
||||
To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params).
|
||||
To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params).
|
||||
|
|
|
|||
|
|
@ -12,9 +12,14 @@ if TYPE_CHECKING:
|
|||
|
||||
import openai
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from openai._base_client import make_request_options
|
||||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._types import RequestOptions
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from typing_extensions import overload
|
||||
|
||||
import litellm
|
||||
|
|
@ -329,6 +334,28 @@ class OpenAIChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
raise e
|
||||
|
||||
|
||||
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str] | None)
|
||||
_EXTRA_QUERY_ADAPTER: Final = TypeAdapter(dict[str, object] | None)
|
||||
_NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({})
|
||||
_SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body"))
|
||||
|
||||
|
||||
def _embedding_request_without_sdk_defaults(
|
||||
data: Mapping[str, object], timeout: float | httpx.Timeout
|
||||
) -> tuple[Mapping[str, object], RequestOptions]:
|
||||
body: Final = { # mutable-ok: the SDK json-encodes the body and needs a plain dict
|
||||
k: v for k, v in data.items() if k not in _SDK_OPTION_KEYS
|
||||
}
|
||||
extra_headers: Final = _EXTRA_HEADERS_ADAPTER.validate_python(data.get("extra_headers")) or _NO_EXTRA_HEADERS
|
||||
options: Final = make_request_options(
|
||||
extra_headers=types.MappingProxyType({**extra_headers, RAW_RESPONSE_HEADER: "true"}),
|
||||
extra_query=_EXTRA_QUERY_ADAPTER.validate_python(data.get("extra_query")),
|
||||
extra_body=data.get("extra_body"),
|
||||
timeout=timeout,
|
||||
)
|
||||
return body, options
|
||||
|
||||
|
||||
class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -1177,19 +1204,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
data: dict,
|
||||
timeout: float | httpx.Timeout,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
):
|
||||
"""
|
||||
Helper to:
|
||||
- call embeddings.create.with_raw_response when litellm.return_response_headers is True
|
||||
- call embeddings.create by default
|
||||
"""
|
||||
try:
|
||||
raw_response = await openai_aclient.embeddings.with_raw_response.create(**data, timeout=timeout)
|
||||
headers: Final = dict(raw_response.headers)
|
||||
response: Final = raw_response.parse()
|
||||
return headers, response
|
||||
except Exception as e:
|
||||
raise e
|
||||
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
|
||||
if "encoding_format" not in data:
|
||||
body, options = _embedding_request_without_sdk_defaults(data, timeout)
|
||||
bypass_response: Final = await openai_aclient.post(
|
||||
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
|
||||
)
|
||||
assert isinstance(bypass_response, LegacyAPIResponse)
|
||||
return bypass_response
|
||||
return await openai_aclient.embeddings.with_raw_response.create(**data, timeout=timeout)
|
||||
|
||||
@track_llm_api_timing()
|
||||
def make_sync_openai_embedding_request(
|
||||
|
|
@ -1198,20 +1221,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
data: dict,
|
||||
timeout: float | httpx.Timeout,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
):
|
||||
"""
|
||||
Helper to:
|
||||
- call embeddings.create.with_raw_response when litellm.return_response_headers is True
|
||||
- call embeddings.create by default
|
||||
"""
|
||||
try:
|
||||
raw_response = openai_client.embeddings.with_raw_response.create(**data, timeout=timeout)
|
||||
|
||||
headers: Final = dict(raw_response.headers)
|
||||
response: Final = raw_response.parse()
|
||||
return headers, response
|
||||
except Exception as e:
|
||||
raise e
|
||||
) -> LegacyAPIResponse[CreateEmbeddingResponse]:
|
||||
if "encoding_format" not in data:
|
||||
body, options = _embedding_request_without_sdk_defaults(data, timeout)
|
||||
bypass_response: Final = openai_client.post(
|
||||
"/embeddings", body=body, options=options, cast_to=CreateEmbeddingResponse
|
||||
)
|
||||
assert isinstance(bypass_response, LegacyAPIResponse)
|
||||
return bypass_response
|
||||
return openai_client.embeddings.with_raw_response.create(**data, timeout=timeout)
|
||||
|
||||
async def aembedding(
|
||||
self,
|
||||
|
|
@ -1236,14 +1254,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
client=client,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
headers, response = await self.make_openai_embedding_request(
|
||||
raw_response: Final = await self.make_openai_embedding_request(
|
||||
openai_aclient=openai_aclient,
|
||||
data=data,
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
headers: Final = dict(raw_response.headers)
|
||||
logging_obj.model_call_details["response_headers"] = headers
|
||||
stringified_response: Final = response.model_dump()
|
||||
stringified_response: Final = raw_response.parse().model_dump()
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=input,
|
||||
|
|
@ -1335,13 +1354,14 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
)
|
||||
|
||||
## embedding CALL
|
||||
headers: dict | None = None
|
||||
headers, sync_embedding_response = self.make_sync_openai_embedding_request(
|
||||
raw_response: Final = self.make_sync_openai_embedding_request(
|
||||
openai_client=openai_client,
|
||||
data=data,
|
||||
timeout=timeout,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
headers: Final = dict(raw_response.headers)
|
||||
sync_embedding_response: Final = raw_response.parse()
|
||||
|
||||
## LOGGING
|
||||
logging_obj.model_call_details["response_headers"] = headers
|
||||
|
|
|
|||
|
|
@ -6292,18 +6292,15 @@ def embedding(
|
|||
if headers is not None and headers != {}:
|
||||
optional_params["extra_headers"] = headers
|
||||
|
||||
if encoding_format is not None:
|
||||
optional_params["encoding_format"] = encoding_format
|
||||
requested_encoding_format: Final = (
|
||||
encoding_format
|
||||
or optional_params.get("encoding_format")
|
||||
or get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT")
|
||||
)
|
||||
if requested_encoding_format is None or requested_encoding_format.strip().lower() == "none":
|
||||
optional_params.pop("encoding_format", None)
|
||||
else:
|
||||
env_fmt: Final = get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT")
|
||||
if env_fmt is not None and env_fmt.strip().lower() == "none":
|
||||
optional_params.pop("encoding_format", None)
|
||||
else:
|
||||
_default_fmt: Final = optional_params.get("encoding_format") or env_fmt or "float"
|
||||
if _default_fmt.strip().lower() == "none":
|
||||
optional_params.pop("encoding_format", None)
|
||||
else:
|
||||
optional_params["encoding_format"] = _default_fmt
|
||||
optional_params["encoding_format"] = requested_encoding_format
|
||||
|
||||
api_version = None
|
||||
|
||||
|
|
|
|||
|
|
@ -3561,10 +3561,10 @@ def get_optional_params_embeddings(
|
|||
non_default_params=non_default_params, optional_params={}, kwargs=kwargs
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "gemini":
|
||||
# OpenAI SDKs (and litellm's own client) send encoding_format="float"
|
||||
# by default; float lists are exactly what the vertex API returns, so
|
||||
# the param is a no-op — don't reject the provider default. Other
|
||||
# values (e.g. "base64") stay on the unsupported-param path below.
|
||||
# OpenAI SDKs send encoding_format="float" by default; float lists are
|
||||
# exactly what the vertex API returns, so the param is a no-op and the
|
||||
# provider default is not rejected. Other values (e.g. "base64") stay
|
||||
# on the unsupported-param path below.
|
||||
if non_default_params.get("encoding_format") == "float":
|
||||
non_default_params.pop("encoding_format")
|
||||
supported_params = get_supported_openai_params(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
"limit": 809
|
||||
},
|
||||
"ANN201": {
|
||||
"limit": 2002
|
||||
"limit": 2001
|
||||
},
|
||||
"ANN202": {
|
||||
"limit": 841
|
||||
|
|
@ -240,10 +240,10 @@
|
|||
"limit": 96
|
||||
},
|
||||
"TRY201": {
|
||||
"limit": 405
|
||||
"limit": 403
|
||||
},
|
||||
"TRY203": {
|
||||
"limit": 113
|
||||
"limit": 111
|
||||
},
|
||||
"TRY300": {
|
||||
"limit": 855
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"limit": 733
|
||||
},
|
||||
"TQ002": {
|
||||
"limit": 742
|
||||
"limit": 741
|
||||
},
|
||||
"TQ003": {
|
||||
"limit": 62
|
||||
|
|
@ -21,6 +21,6 @@
|
|||
"limit": 117
|
||||
},
|
||||
"TQ008": {
|
||||
"limit": 11139
|
||||
"limit": 11135
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from io import BytesIO
|
|||
from unittest.mock import AsyncMock
|
||||
|
||||
|
||||
import httpx
|
||||
import litellm
|
||||
from litellm import completion, embedding
|
||||
import pytest
|
||||
|
|
@ -92,44 +93,54 @@ async def test_litellm_gateway_from_sdk_embedding(is_async):
|
|||
litellm.set_verbose = True
|
||||
litellm._turn_on_debug()
|
||||
|
||||
captured_bodies = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_bodies.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "my-vllm-model",
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
if is_async:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
openai_client = AsyncOpenAI(api_key="fake-key")
|
||||
mock_method = AsyncMock()
|
||||
patch_target = openai_client.embeddings.create
|
||||
openai_client = AsyncOpenAI(
|
||||
api_key="fake-key",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
response = await litellm.aembedding(
|
||||
model="litellm_proxy/my-vllm-model",
|
||||
input="Hello world",
|
||||
client=openai_client,
|
||||
api_base="my-custom-api-base",
|
||||
)
|
||||
else:
|
||||
from openai import OpenAI
|
||||
|
||||
openai_client = OpenAI(api_key="fake-key")
|
||||
mock_method = MagicMock()
|
||||
patch_target = openai_client.embeddings.create
|
||||
openai_client = OpenAI(
|
||||
api_key="fake-key",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
response = litellm.embedding(
|
||||
model="litellm_proxy/my-vllm-model",
|
||||
input="Hello world",
|
||||
client=openai_client,
|
||||
api_base="my-custom-api-base",
|
||||
)
|
||||
|
||||
with patch.object(patch_target.__self__, patch_target.__name__, new=mock_method):
|
||||
try:
|
||||
if is_async:
|
||||
await litellm.aembedding(
|
||||
model="litellm_proxy/my-vllm-model",
|
||||
input="Hello world",
|
||||
client=openai_client,
|
||||
api_base="my-custom-api-base",
|
||||
)
|
||||
else:
|
||||
litellm.embedding(
|
||||
model="litellm_proxy/my-vllm-model",
|
||||
input="Hello world",
|
||||
client=openai_client,
|
||||
api_base="my-custom-api-base",
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
request_body = captured_bodies[0]
|
||||
print("Request body - {}".format(request_body))
|
||||
|
||||
mock_method.assert_called_once()
|
||||
|
||||
print("Call KWARGS - {}".format(mock_method.call_args.kwargs))
|
||||
|
||||
assert "Hello world" == mock_method.call_args.kwargs["input"]
|
||||
assert "my-vllm-model" == mock_method.call_args.kwargs["model"]
|
||||
assert "Hello world" == request_body["input"]
|
||||
assert "my-vllm-model" == request_body["model"]
|
||||
assert "encoding_format" not in request_body
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
|
|
|
|||
|
|
@ -63,27 +63,39 @@ def test_embedding_nvidia_nim():
|
|||
litellm.set_verbose = True
|
||||
from openai import OpenAI
|
||||
|
||||
captured_bodies = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_bodies.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "nvidia/nv-embedqa-e5-v5",
|
||||
"usage": {"prompt_tokens": 6, "total_tokens": 6},
|
||||
},
|
||||
)
|
||||
|
||||
client = OpenAI(
|
||||
api_key="fake-api-key",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
with patch.object(client.embeddings.with_raw_response, "create") as mock_client:
|
||||
try:
|
||||
litellm.embedding(
|
||||
model="nvidia_nim/nvidia/nv-embedqa-e5-v5",
|
||||
input="What is the meaning of life?",
|
||||
input_type="passage",
|
||||
dimensions=1024,
|
||||
client=client,
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
mock_client.assert_called_once()
|
||||
request_body = mock_client.call_args.kwargs
|
||||
print("request_body: ", request_body)
|
||||
assert request_body["input"] == "What is the meaning of life?"
|
||||
assert request_body["model"] == "nvidia/nv-embedqa-e5-v5"
|
||||
assert request_body["extra_body"]["input_type"] == "passage"
|
||||
assert request_body["dimensions"] == 1024
|
||||
response = litellm.embedding(
|
||||
model="nvidia_nim/nvidia/nv-embedqa-e5-v5",
|
||||
input="What is the meaning of life?",
|
||||
input_type="passage",
|
||||
dimensions=1024,
|
||||
client=client,
|
||||
)
|
||||
request_body = captured_bodies[0]
|
||||
print("request_body: ", request_body)
|
||||
assert request_body["input"] == "What is the meaning of life?"
|
||||
assert request_body["model"] == "nvidia/nv-embedqa-e5-v5"
|
||||
assert request_body["input_type"] == "passage"
|
||||
assert request_body["dimensions"] == 1024
|
||||
assert "encoding_format" not in request_body
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_chat_completion_nvidia_nim_with_tools():
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import os
|
|||
import re
|
||||
import traceback
|
||||
|
||||
import httpx
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
|
|
@ -1255,56 +1257,42 @@ def test_jina_ai_img_embeddings(input_data, expected_payload_input):
|
|||
assert sent_data["input"] == expected_payload_input
|
||||
|
||||
|
||||
def test_encoding_format_defaults_to_float_for_openai_sdk(monkeypatch):
|
||||
def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch):
|
||||
"""
|
||||
When encoding_format is not provided, LiteLLM sends `float` for OpenAI-path embeddings.
|
||||
When encoding_format is not provided, LiteLLM leaves it out of the upstream request.
|
||||
|
||||
Optional global override: `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT`.
|
||||
"""
|
||||
monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False)
|
||||
with patch(
|
||||
"litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client"
|
||||
) as mock_get_client:
|
||||
# Create a mock client instance
|
||||
mock_client_instance = MagicMock()
|
||||
mock_get_client.return_value = mock_client_instance
|
||||
captured_bodies = []
|
||||
|
||||
# Mock the embeddings.with_raw_response.create method
|
||||
mock_response = MagicMock()
|
||||
mock_response.parse.return_value = MagicMock(
|
||||
model_dump=lambda: {
|
||||
"data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}],
|
||||
"model": "text-embedding-ada-002",
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_bodies.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "text-embedding-ada-002",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
}
|
||||
)
|
||||
mock_response.headers = {}
|
||||
|
||||
mock_client_instance.embeddings.with_raw_response.create.return_value = (
|
||||
mock_response
|
||||
},
|
||||
)
|
||||
|
||||
# Call the embedding function without encoding_format
|
||||
response = embedding(
|
||||
model="text-embedding-ada-002",
|
||||
input="Hello world",
|
||||
)
|
||||
client = openai.OpenAI(
|
||||
api_key="sk-test", http_client=httpx.Client(transport=httpx.MockTransport(handler))
|
||||
)
|
||||
|
||||
# Get the call arguments to verify what was sent to OpenAI SDK
|
||||
call_args = mock_client_instance.embeddings.with_raw_response.create.call_args
|
||||
assert (
|
||||
call_args is not None
|
||||
), "OpenAI SDK embeddings.create should have been called"
|
||||
response = embedding(
|
||||
model="text-embedding-ada-002",
|
||||
input="Hello world",
|
||||
api_key="sk-test",
|
||||
client=client,
|
||||
)
|
||||
|
||||
call_kwargs = call_args[1] # Get kwargs
|
||||
|
||||
assert "encoding_format" in call_kwargs
|
||||
assert (
|
||||
call_kwargs["encoding_format"] == "float"
|
||||
), "encoding_format should default to float when not provided by user"
|
||||
|
||||
print("✅ PASS: encoding_format='float' is correctly passed to OpenAI SDK")
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
assert "encoding_format" not in captured_bodies[0], (
|
||||
"encoding_format should be omitted from the upstream request when not provided by user"
|
||||
)
|
||||
|
||||
|
||||
def test_encoding_format_explicit_value_preserved():
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import traceback
|
|||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, AuthenticationError, BadRequestError, OpenAIError, RateLimitError
|
||||
from openai import AsyncAzureOpenAI, AsyncOpenAI, AuthenticationError, AzureOpenAI, BadRequestError, OpenAIError, RateLimitError
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
|
|
@ -895,7 +895,12 @@ def _pre_call_utils(
|
|||
):
|
||||
if call_type == "embedding":
|
||||
data["input"] = "Hello world!"
|
||||
mapped_target: Any = client.embeddings.with_raw_response
|
||||
if isinstance(client, (AzureOpenAI, AsyncAzureOpenAI)):
|
||||
mapped_target: Any = client.embeddings.with_raw_response
|
||||
patched_attr = "create"
|
||||
else:
|
||||
mapped_target = client
|
||||
patched_attr = "post"
|
||||
if sync_mode:
|
||||
original_function = litellm.embedding
|
||||
else:
|
||||
|
|
@ -905,6 +910,7 @@ def _pre_call_utils(
|
|||
if streaming is True:
|
||||
data["stream"] = True
|
||||
mapped_target = client.chat.completions.with_raw_response # type: ignore
|
||||
patched_attr = "create"
|
||||
if sync_mode:
|
||||
original_function = litellm.completion
|
||||
else:
|
||||
|
|
@ -914,12 +920,13 @@ def _pre_call_utils(
|
|||
if streaming is True:
|
||||
data["stream"] = True
|
||||
mapped_target = client.completions.with_raw_response # type: ignore
|
||||
patched_attr = "create"
|
||||
if sync_mode:
|
||||
original_function = litellm.text_completion
|
||||
else:
|
||||
original_function = litellm.atext_completion
|
||||
|
||||
return data, original_function, mapped_target
|
||||
return data, original_function, mapped_target, patched_attr
|
||||
|
||||
|
||||
def _pre_call_utils_httpx(
|
||||
|
|
@ -1003,7 +1010,7 @@ async def test_exception_with_headers(sync_mode, provider, model, call_type, str
|
|||
)
|
||||
|
||||
data = {"model": model}
|
||||
data, original_function, mapped_target = _pre_call_utils(
|
||||
data, original_function, mapped_target, patched_attr = _pre_call_utils(
|
||||
call_type=call_type,
|
||||
data=data,
|
||||
client=openai_client,
|
||||
|
|
@ -1049,7 +1056,7 @@ async def test_exception_with_headers(sync_mode, provider, model, call_type, str
|
|||
|
||||
with patch.object(
|
||||
mapped_target,
|
||||
"create",
|
||||
patched_attr,
|
||||
side_effect=_return_exception,
|
||||
):
|
||||
new_retry_after_mock_client = MagicMock(return_value=-1)
|
||||
|
|
|
|||
|
|
@ -2032,8 +2032,8 @@ def test_router_dynamic_cooldown_correct_retry_after_time():
|
|||
raise exception
|
||||
|
||||
with patch.object(
|
||||
openai_client.embeddings.with_raw_response,
|
||||
"create",
|
||||
openai_client,
|
||||
"post",
|
||||
side_effect=_return_exception,
|
||||
):
|
||||
new_retry_after_mock_client = MagicMock(return_value=-1)
|
||||
|
|
|
|||
|
|
@ -1,124 +1,121 @@
|
|||
from unittest.mock import MagicMock, patch
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from litellm import embedding
|
||||
import litellm
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"set_env, env_value, expected",
|
||||
[
|
||||
(False, None, "float"),
|
||||
(True, "base64", "base64"),
|
||||
],
|
||||
)
|
||||
def test_openai_embedding_encoding_format_default(
|
||||
monkeypatch, set_env, env_value, expected
|
||||
):
|
||||
monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False)
|
||||
if set_env:
|
||||
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_value)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.parse.return_value = MagicMock(
|
||||
model_dump=lambda: {
|
||||
"data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}],
|
||||
"model": "text-embedding-ada-002",
|
||||
"object": "list",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
}
|
||||
def _mock_openai_embedding_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post("https://api.openai.com/v1/embeddings").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
mock_response.headers = {}
|
||||
|
||||
with patch(
|
||||
"litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client"
|
||||
) as mock_get_client:
|
||||
mock_client_instance = MagicMock()
|
||||
mock_get_client.return_value = mock_client_instance
|
||||
mock_client_instance.embeddings.with_raw_response.create.return_value = (
|
||||
mock_response
|
||||
)
|
||||
|
||||
embedding(
|
||||
model="text-embedding-ada-002",
|
||||
input="Hello world",
|
||||
)
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_default_encoding_format_env(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False)
|
||||
|
||||
call_kwargs = (
|
||||
mock_client_instance.embeddings.with_raw_response.create.call_args[1]
|
||||
)
|
||||
assert call_kwargs["encoding_format"] == expected
|
||||
|
||||
def test_embedding_openai_omits_encoding_format_when_client_omits_it(respx_mock: respx.MockRouter) -> None:
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
response: Final = litellm.embedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert "encoding_format" not in request_body
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_embedding_openai_forwards_explicit_encoding_format(respx_mock: respx.MockRouter) -> None:
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
litellm.embedding(
|
||||
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", encoding_format="base64"
|
||||
)
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert request_body["encoding_format"] == "base64"
|
||||
|
||||
|
||||
def test_embedding_openai_explicit_encoding_format_wins_over_env_var(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", "float")
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
litellm.embedding(
|
||||
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", encoding_format="base64"
|
||||
)
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert request_body["encoding_format"] == "base64"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env_value", ["float", "base64"])
|
||||
def test_embedding_openai_env_var_sets_default_encoding_format(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, env_value: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_value)
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
litellm.embedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert request_body["encoding_format"] == env_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env_none", ["none", "NONE", " none "])
|
||||
def test_openai_embedding_encoding_format_env_none_omits_param(
|
||||
monkeypatch, env_none
|
||||
):
|
||||
"""LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT=none omits encoding_format (provider default)."""
|
||||
def test_embedding_openai_env_none_omits_encoding_format(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, env_none: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_none)
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.parse.return_value = MagicMock(
|
||||
model_dump=lambda: {
|
||||
"data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}],
|
||||
"model": "text-embedding-ada-002",
|
||||
"object": "list",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
}
|
||||
litellm.embedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert "encoding_format" not in request_body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aembedding_openai_omits_encoding_format_when_client_omits_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
mock_route: Final = _mock_openai_embedding_route(respx_mock)
|
||||
|
||||
response: Final = await litellm.aembedding(model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test")
|
||||
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.read())
|
||||
assert "encoding_format" not in request_body
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
|
||||
|
||||
|
||||
def test_embedding_openai_omitted_encoding_format_maps_provider_errors(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
respx_mock.post("https://api.openai.com/v1/embeddings").mock(
|
||||
return_value=httpx.Response(
|
||||
429,
|
||||
headers={"retry-after": "42", "x-should-retry": "false"},
|
||||
json={"error": {"message": "rate limited", "type": "rate_limit_error"}},
|
||||
)
|
||||
)
|
||||
mock_response.headers = {}
|
||||
|
||||
with patch(
|
||||
"litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client"
|
||||
) as mock_get_client:
|
||||
mock_client_instance = MagicMock()
|
||||
mock_get_client.return_value = mock_client_instance
|
||||
mock_client_instance.embeddings.with_raw_response.create.return_value = (
|
||||
mock_response
|
||||
with pytest.raises(litellm.RateLimitError) as exc_info:
|
||||
litellm.embedding(
|
||||
model="openai/text-embedding-3-small", input=["hello"], api_key="sk-test", max_retries=0
|
||||
)
|
||||
|
||||
embedding(
|
||||
model="text-embedding-ada-002",
|
||||
input="Hello world",
|
||||
)
|
||||
|
||||
call_kwargs = (
|
||||
mock_client_instance.embeddings.with_raw_response.create.call_args[1]
|
||||
)
|
||||
assert "encoding_format" not in call_kwargs
|
||||
|
||||
|
||||
def test_openai_embedding_encoding_format_explicit_overrides_env(monkeypatch):
|
||||
"""Request `encoding_format` wins over LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT."""
|
||||
monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", "float")
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.parse.return_value = MagicMock(
|
||||
model_dump=lambda: {
|
||||
"data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}],
|
||||
"model": "text-embedding-ada-002",
|
||||
"object": "list",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
}
|
||||
)
|
||||
mock_response.headers = {}
|
||||
|
||||
with patch(
|
||||
"litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client"
|
||||
) as mock_get_client:
|
||||
mock_client_instance = MagicMock()
|
||||
mock_get_client.return_value = mock_client_instance
|
||||
mock_client_instance.embeddings.with_raw_response.create.return_value = (
|
||||
mock_response
|
||||
)
|
||||
|
||||
embedding(
|
||||
model="text-embedding-ada-002",
|
||||
input="Hello world",
|
||||
encoding_format="base64",
|
||||
)
|
||||
|
||||
call_kwargs = (
|
||||
mock_client_instance.embeddings.with_raw_response.create.call_args[1]
|
||||
)
|
||||
assert call_kwargs["encoding_format"] == "base64"
|
||||
assert int(exc_info.value.litellm_response_headers["retry-after"]) == 42
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue