mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
test: migrate openai and small-provider translation tests
This commit is contained in:
yuneng
parent
ecc2f2c9ad
commit
7216244011
11 changed files with 861 additions and 1339 deletions
|
|
@ -0,0 +1,258 @@
|
|||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.openai_wire import responses_reply
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
TOKEN: Final = "scripted-openai-search-key"
|
||||
SEARCH_ANNOTATION: Final[dict[str, JsonValue]] = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
"start_index": 0,
|
||||
"end_index": 11,
|
||||
"url": "https://news.example.test/positive-story",
|
||||
"title": "Positive news",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _responses_peer(identity: str, model: str, text: str) -> Callable[[Request], Reply]:
|
||||
def respond(_: Request) -> Reply:
|
||||
return responses_reply(identity, model, text, stream=True)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _single_upstream_request(wire: Wire) -> Request:
|
||||
received: Final = wire.drain()
|
||||
assert len(received) == 1
|
||||
return received[0]
|
||||
|
||||
|
||||
def test_gpt_5_reasoning_streaming_sends_reasoning_effort(gateway: Gateway) -> None:
|
||||
provider_body: Final = {
|
||||
"model": "gpt-5-mini",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Think of a poem, and then write it."}],
|
||||
}
|
||||
],
|
||||
"reasoning": {"effort": "low"},
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
with wire_server(_responses_peer("reasoning-stream", "gpt-5-mini", "A small poem.")) as wire:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/responses/gpt-5-mini",
|
||||
api_key=TOKEN,
|
||||
api_base=wire.url,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Think of a poem, and then write it."}],
|
||||
"reasoning_effort": "low",
|
||||
"stream": True,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
request: Final = _single_upstream_request(wire)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (request.method, request.target) == ("POST", "/responses")
|
||||
assert json.loads(request.body) == provider_body
|
||||
assert "A small poem." in response.text
|
||||
|
||||
|
||||
def test_gpt_5_web_search_maps_builtin_tool(gateway: Gateway) -> None:
|
||||
provider_body: Final = {
|
||||
"model": "gpt-5",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "get price of nvda"}],
|
||||
}
|
||||
],
|
||||
"temperature": 1,
|
||||
"max_output_tokens": 8192,
|
||||
"tools": [{"type": "web_search"}],
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
with wire_server(_responses_peer("web-search", "gpt-5", "NVDA is listed at $123.")) as wire:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/responses/gpt-5", api_key=TOKEN, api_base=wire.url)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "get price of nvda"}],
|
||||
"stream": True,
|
||||
"temperature": 1,
|
||||
"max_tokens": 8192,
|
||||
"tools": [{"type": "web_search"}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
request: Final = _single_upstream_request(wire)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (request.method, request.target) == ("POST", "/responses")
|
||||
assert json.loads(request.body) == provider_body
|
||||
assert "NVDA is listed at $123." in response.text
|
||||
|
||||
|
||||
def test_openai_web_search_response_preserves_url_citation(gateway: Gateway) -> None:
|
||||
provider_body: Final = {
|
||||
"model": "gpt-5-search-api",
|
||||
"messages": [{"role": "user", "content": "What was a positive news story from today?"}],
|
||||
}
|
||||
provider_reply: Final = {
|
||||
"id": "chatcmpl-search",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5-search-api",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "A positive story was reported today.",
|
||||
"annotations": [SEARCH_ANNOTATION],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
|
||||
}
|
||||
|
||||
def search_peer(_: Request) -> Reply:
|
||||
return Reply(body=json.dumps(provider_reply).encode())
|
||||
|
||||
with wire_server(search_peer) as wire:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/gpt-5-search-api", api_key=TOKEN, api_base=wire.url)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "What was a positive news story from today?"}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
request: Final = _single_upstream_request(wire)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (request.method, request.target) == ("POST", "/chat/completions")
|
||||
assert json.loads(request.body) == provider_body
|
||||
annotations: Final = response.json()["choices"][0]["message"]["annotations"]
|
||||
assert annotations == [SEARCH_ANNOTATION]
|
||||
|
||||
|
||||
def test_openai_web_search_streaming_preserves_url_citation(gateway: Gateway) -> None:
|
||||
provider_body: Final = {
|
||||
"model": "gpt-5-search-api",
|
||||
"messages": [{"role": "user", "content": "What was a positive news story from today?"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
stream_body: Final = b"".join(
|
||||
(
|
||||
b"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-search-stream",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-5-search-api",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"content": "A positive story.",
|
||||
"annotations": [SEARCH_ANNOTATION],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode()
|
||||
+ b"\n\n",
|
||||
b'data: {"id":"chatcmpl-search-stream","object":"chat.completion.chunk","created":1,'
|
||||
b'"model":"gpt-5-search-api","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
)
|
||||
|
||||
def search_peer(_: Request) -> Reply:
|
||||
return Reply(content_type="text/event-stream", body=stream_body)
|
||||
|
||||
with wire_server(search_peer) as wire:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/gpt-5-search-api", api_key=TOKEN, api_base=wire.url)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "What was a positive news story from today?"}],
|
||||
"stream": True,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
request: Final = _single_upstream_request(wire)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (request.method, request.target) == ("POST", "/chat/completions")
|
||||
assert json.loads(request.body) == provider_body
|
||||
chunks: Final = tuple(
|
||||
json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: {")
|
||||
)
|
||||
annotations: Final = chunks[0]["choices"][0]["delta"]["annotations"]
|
||||
assert annotations == [SEARCH_ANNOTATION]
|
||||
|
||||
|
||||
def test_openai_codex_stream_uses_responses_endpoint(gateway: Gateway) -> None:
|
||||
provider_body: Final = {
|
||||
"model": "gpt-5.3-codex",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "Hey!"}],
|
||||
}
|
||||
],
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
with wire_server(_responses_peer("codex-stream", "gpt-5.3-codex", "Hello.")) as wire:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model="openai/gpt-5.3-codex", api_key=TOKEN, api_base=wire.url)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Hey!"}],
|
||||
"stream": True,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
request: Final = _single_upstream_request(wire)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (request.method, request.target) == ("POST", "/responses")
|
||||
assert json.loads(request.body) == provider_body
|
||||
assert "Hello." in response.text
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
"""
|
||||
Unit tests for litellm.compress().
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
CALL_TYPE = CallTypes.completion
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BM25 scorer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Message stubbing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Retrieval tool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# compress() — end-to-end
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding scorer — integration test (skipped without API key)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="Needs OPENAI_API_KEY")
|
||||
def test_embedding_scorer():
|
||||
result = litellm.compress(
|
||||
messages=[
|
||||
{"role": "user", "content": "Authentication code " * 2000},
|
||||
{"role": "user", "content": "Unrelated cooking recipes " * 2000},
|
||||
{"role": "user", "content": "Fix auth"},
|
||||
],
|
||||
model="gpt-4o",
|
||||
call_type=CALL_TYPE,
|
||||
compression_trigger=1000,
|
||||
embedding_model="text-embedding-3-small",
|
||||
)
|
||||
assert result["compression_ratio"] > 0
|
||||
assert len(result["cache"]) > 0
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
# Test implementations
|
||||
|
||||
|
||||
def test_completion_cost_deepseek():
|
||||
litellm.set_verbose = True
|
||||
model_name = "deepseek/deepseek-chat"
|
||||
messages_1 = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a history expert. The user will provide a series of questions, and your answers should be concise and start with `Answer:`",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "In what year did Qin Shi Huang unify the six states?",
|
||||
},
|
||||
{"role": "assistant", "content": "Answer: 221 BC"},
|
||||
{"role": "user", "content": "Who was the founder of the Han Dynasty?"},
|
||||
{"role": "assistant", "content": "Answer: Liu Bang"},
|
||||
{"role": "user", "content": "Who was the last emperor of the Tang Dynasty?"},
|
||||
{"role": "assistant", "content": "Answer: Li Zhu"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Who was the founding emperor of the Ming Dynasty?",
|
||||
},
|
||||
{"role": "assistant", "content": "Answer: Zhu Yuanzhang"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Who was the founding emperor of the Qing Dynasty?",
|
||||
},
|
||||
]
|
||||
|
||||
message_2 = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a history expert. The user will provide a series of questions, and your answers should be concise and start with `Answer:`",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "In what year did Qin Shi Huang unify the six states?",
|
||||
},
|
||||
{"role": "assistant", "content": "Answer: 221 BC"},
|
||||
{"role": "user", "content": "Who was the founder of the Han Dynasty?"},
|
||||
{"role": "assistant", "content": "Answer: Liu Bang"},
|
||||
{"role": "user", "content": "Who was the last emperor of the Tang Dynasty?"},
|
||||
{"role": "assistant", "content": "Answer: Li Zhu"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Who was the founding emperor of the Ming Dynasty?",
|
||||
},
|
||||
{"role": "assistant", "content": "Answer: Zhu Yuanzhang"},
|
||||
{"role": "user", "content": "When did the Shang Dynasty fall?"},
|
||||
]
|
||||
try:
|
||||
response_1 = litellm.completion(model=model_name, messages=messages_1)
|
||||
response_2 = litellm.completion(model=model_name, messages=message_2)
|
||||
# Add any assertions here to check the response
|
||||
print(response_2)
|
||||
assert response_2.usage.prompt_cache_hit_tokens is not None
|
||||
assert response_2.usage.prompt_cache_miss_tokens is not None
|
||||
assert (
|
||||
response_2.usage.prompt_tokens
|
||||
== response_2.usage.prompt_cache_miss_tokens
|
||||
+ response_2.usage.prompt_cache_hit_tokens
|
||||
)
|
||||
assert (
|
||||
response_2.usage._cache_read_input_tokens
|
||||
== response_2.usage.prompt_cache_hit_tokens
|
||||
)
|
||||
except litellm.APIError as e:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
|
||||
|
||||
from base_rerank_unit_tests import BaseLLMRerankTest
|
||||
import litellm
|
||||
|
||||
|
||||
class TestJinaAI(BaseLLMRerankTest):
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
return litellm.LlmProviders.JINA_AI
|
||||
|
||||
def get_base_rerank_call_args(self) -> dict:
|
||||
return {
|
||||
"model": "jina_ai/jina-reranker-v2-base-multilingual",
|
||||
}
|
||||
|
||||
|
||||
def test_jina_ai_embedding():
|
||||
litellm.embedding(
|
||||
model="jina_ai/jina-embeddings-v3",
|
||||
input=["a"],
|
||||
task="separation",
|
||||
dimensions=1024,
|
||||
)
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
"""
|
||||
Tests for Lambda AI provider integration
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.lambda_ai.chat.transformation import LambdaAIChatConfig
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lambda_ai_completion_call():
|
||||
"""Test completion call with Lambda AI provider (requires LAMBDA_API_KEY)"""
|
||||
# Skip if no API key is available
|
||||
if not os.getenv("LAMBDA_API_KEY"):
|
||||
pytest.skip("LAMBDA_API_KEY not set")
|
||||
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model="lambda_ai/llama3.1-8b-instruct",
|
||||
messages=[{"role": "user", "content": "Hello, this is a test"}],
|
||||
max_tokens=10,
|
||||
)
|
||||
assert response.choices[0].message.content
|
||||
assert response.model
|
||||
assert response.usage
|
||||
except Exception as e:
|
||||
# If the API key is invalid or there's a network issue, that's okay
|
||||
# The important thing is that the provider was recognized
|
||||
if "lambda_ai" not in str(e) and "provider" not in str(e).lower():
|
||||
# Re-raise if it's not a provider-related error
|
||||
raise
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -79,6 +79,52 @@ def _assert_deepseek_request(request: httpx.Request, messages: list[dict[str, st
|
|||
assert actual_data["stream"] is stream
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_deepseek_httpx_transport")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_completion_cost_deepseek_reports_prompt_cache_usage(respx_mock: respx.MockRouter) -> None:
|
||||
messages: Final = [{"role": "user", "content": "A cached prompt"}]
|
||||
route: Final = respx_mock.post(DEEPSEEK_CHAT_COMPLETIONS_URL).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-deepseek-cache",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "deepseek-chat",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Answer"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 12,
|
||||
"prompt_cache_hit_tokens": 4,
|
||||
"prompt_cache_miss_tokens": 6,
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="deepseek/deepseek-chat",
|
||||
messages=messages,
|
||||
api_key=DEEPSEEK_API_KEY,
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert request_body == {"model": "deepseek-chat", "messages": messages}
|
||||
assert response.usage.prompt_cache_hit_tokens == 4
|
||||
assert response.usage.prompt_cache_miss_tokens == 6
|
||||
assert response.usage.prompt_tokens == 10
|
||||
assert response.usage._cache_read_input_tokens == response.usage.prompt_cache_hit_tokens
|
||||
|
||||
|
||||
def _function_tool(name: str) -> dict:
|
||||
return {
|
||||
"type": "function",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
|
|
@ -136,6 +140,44 @@ class TestJinaAIEmbeddingTransform:
|
|||
assert result == expected_result
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_jina_ai_embedding_forwards_task_and_dimensions_and_parses_values(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://api.jina.ai/v1/embeddings").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "jina-embeddings-v3",
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
||||
"usage": {"prompt_tokens": 3, "total_tokens": 3},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.embedding(
|
||||
model="jina_ai/jina-embeddings-v3",
|
||||
input=["a"],
|
||||
task="separation",
|
||||
dimensions=1024,
|
||||
api_key="jina-test-key",
|
||||
api_base="https://api.jina.ai/v1",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert request_body == {
|
||||
"model": "jina-embeddings-v3",
|
||||
"input": ["a"],
|
||||
"task": "separation",
|
||||
"dimensions": 1024,
|
||||
}
|
||||
assert response.data[0]["embedding"] == [0.1, 0.2]
|
||||
assert response.usage.prompt_tokens == 3
|
||||
|
||||
|
||||
def _transform(raw_response: httpx.Response) -> EmbeddingResponse:
|
||||
return JinaAIEmbeddingConfig().transform_embedding_response(
|
||||
model="jina-embeddings-v3",
|
||||
|
|
|
|||
|
|
@ -1096,6 +1096,30 @@ def test_openai_pdf_url_is_downloaded_into_file_data() -> None:
|
|||
assert base64.b64decode(file_content["file_data"].split(",", maxsplit=1)[1]) == pdf_content
|
||||
|
||||
|
||||
def test_vision_with_custom_model_preserves_base64_image_in_request() -> None:
|
||||
image_data: Final = b"test-image"
|
||||
image_uri: Final = f"data:image/png;base64,{base64.b64encode(image_data).decode('utf-8')}"
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": image_uri}},
|
||||
],
|
||||
}
|
||||
]
|
||||
request: Final = OpenAIGPTConfig().transform_request(
|
||||
model="my-custom-model",
|
||||
messages=messages,
|
||||
optional_params={"max_tokens": 10},
|
||||
litellm_params={"api_base": "https://my-custom.api.openai.com"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request == {"model": "my-custom-model", "messages": messages, "max_tokens": 10}
|
||||
assert request["messages"][0]["content"][1]["image_url"]["url"] == image_uri
|
||||
|
||||
|
||||
class TestToolMessageImageHoisting:
|
||||
"""transform_request moves tool-message images into a following user message
|
||||
(OpenAI-compatible APIs only accept text in role:"tool" messages)."""
|
||||
|
|
|
|||
|
|
@ -1,13 +1,21 @@
|
|||
import asyncio, importlib, os
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from pydantic import JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm import create_thread, get_thread
|
||||
from litellm.caching.caching import Cache, LiteLLMCacheType
|
||||
from litellm.google_genai.adapters.transformation import GoogleGenAIStreamWrapper
|
||||
from litellm.llms.openai.openai import (
|
||||
AssistantEventHandler,
|
||||
AsyncAssistantEventHandler,
|
||||
|
|
@ -19,8 +27,7 @@ from litellm.llms.openai.openai import (
|
|||
SyncCursorPage,
|
||||
Thread,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse, ModelResponse
|
||||
from litellm import create_thread, get_thread
|
||||
from litellm.types.utils import Delta, ImageResponse, ModelResponse, ModelResponseStream, StreamingChoices
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
from openai.types.beta.assistant import Assistant
|
||||
|
|
@ -273,6 +280,432 @@ async def test_acompletion_streams_tool_call_arguments_over_injected_transport()
|
|||
assert rebuilt.choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def _openai_sse_chunk(chunk_id: str, model: str, choices: list[JsonValue]) -> bytes:
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": model,
|
||||
"choices": choices,
|
||||
}
|
||||
return f"data: {json.dumps(body)}\n\n".encode()
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_o1_parallel_tool_calls_are_dropped_for_unsupported_models(respx_mock: respx.MockRouter) -> None:
|
||||
model: Final = "o1"
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-o-series",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "done"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "foo"}],
|
||||
parallel_tool_calls=True,
|
||||
drop_params=True,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert request_body == {"model": model, "messages": [{"role": "user", "content": "foo"}]}
|
||||
assert response.choices[0].message.content == "done"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_prediction_parameter_participates_in_completion_cache_key(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
request_body: Final = json.loads(request.content)
|
||||
prediction_content: Final = request_body["prediction"]["content"]
|
||||
response_id: Final = (
|
||||
"chatcmpl-prediction-one" if prediction_content == "prediction one" else "chatcmpl-prediction-two"
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": response_id,
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4o",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": prediction_content},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond)
|
||||
messages: Final = [{"role": "user", "content": "replace the username"}]
|
||||
first_prediction: Final = {"type": "content", "content": "prediction one"}
|
||||
second_prediction: Final = {"type": "content", "content": "prediction two"}
|
||||
|
||||
first: Final = litellm.completion(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
prediction=first_prediction,
|
||||
caching=True,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
cached: Final = litellm.completion(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
prediction=first_prediction,
|
||||
caching=True,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
distinct: Final = litellm.completion(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
prediction=second_prediction,
|
||||
caching=True,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
request_bodies: Final = tuple(json.loads(call.request.content) for call in route.calls)
|
||||
|
||||
assert first.id == cached.id == "chatcmpl-prediction-one"
|
||||
assert distinct.id == "chatcmpl-prediction-two"
|
||||
assert route.call_count == 2
|
||||
assert request_bodies == (
|
||||
{"model": "gpt-4o", "messages": messages, "prediction": first_prediction},
|
||||
{"model": "gpt-4o", "messages": messages, "prediction": second_prediction},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_openai_tool_calling_sends_schema_and_parses_function_call(respx_mock: respx.MockRouter) -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_stock_price",
|
||||
"description": "Get the current stock price for a given ticker symbol.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"ticker": {"type": "string", "description": "The stock ticker symbol."}},
|
||||
"required": ["ticker"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-tool-call",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4.1",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-tsla",
|
||||
"type": "function",
|
||||
"function": {"name": "get_stock_price", "arguments": '{"ticker":"TSLA"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 8, "completion_tokens": 3, "total_tokens": 11},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="openai/gpt-4.1",
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "What is TSLA stock price at today?"}]}],
|
||||
temperature=0.5,
|
||||
max_tokens=1600,
|
||||
tools=tools,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
tool_call: Final = response.choices[0].message.tool_calls[0]
|
||||
|
||||
assert request_body == {
|
||||
"model": "gpt-4.1",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "What is TSLA stock price at today?"}]}],
|
||||
"temperature": 0.5,
|
||||
"max_tokens": 1600,
|
||||
"tools": tools,
|
||||
}
|
||||
assert (tool_call.id, tool_call.function.name, tool_call.function.arguments) == (
|
||||
"call-tsla",
|
||||
"get_stock_price",
|
||||
'{"ticker":"TSLA"}',
|
||||
)
|
||||
assert response.choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_via_gemini_streaming_bridge_translates_openai_chunks() -> None:
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-bridge",
|
||||
object="chat.completion.chunk",
|
||||
created=0,
|
||||
model="gpt-3.5-turbo",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(role="assistant", content="A starship crossed the sky."),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
async def stream() -> AsyncIterator[ModelResponseStream]:
|
||||
yield chunk
|
||||
|
||||
wrapper: Final = GoogleGenAIStreamWrapper(stream())
|
||||
chunks: Final = tuple([item async for item in wrapper])
|
||||
|
||||
assert chunks == (
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "A starship crossed the sky."}], "role": "model"},
|
||||
"finishReason": None,
|
||||
"index": 0,
|
||||
"safetyRatings": [],
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_streaming_content_with_n_greater_than_one_preserves_choice_indices(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
stream_body: Final = b"".join(
|
||||
(
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-content",
|
||||
"gpt-4o",
|
||||
[{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-content",
|
||||
"gpt-4o",
|
||||
[{"index": 1, "delta": {"role": "assistant", "content": "Hi"}, "finish_reason": None}],
|
||||
),
|
||||
_openai_sse_chunk("chatcmpl-n-content", "gpt-4o", [{"index": 0, "delta": {}, "finish_reason": "stop"}]),
|
||||
_openai_sse_chunk("chatcmpl-n-content", "gpt-4o", [{"index": 1, "delta": {}, "finish_reason": "stop"}]),
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=stream_body)
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="openai/gpt-4o",
|
||||
messages=[{"role": "user", "content": "Say hello in one word"}],
|
||||
stream=True,
|
||||
n=2,
|
||||
max_tokens=10,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
chunks: Final = tuple(response)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
indices: Final = tuple(chunk.choices[0].index for chunk in chunks if chunk.choices)
|
||||
content_by_index: Final = {
|
||||
index: "".join(
|
||||
chunk.choices[0].delta.content or ""
|
||||
for chunk in chunks
|
||||
if chunk.choices and chunk.choices[0].index == index
|
||||
)
|
||||
for index in (0, 1)
|
||||
}
|
||||
|
||||
assert route.call_count == 1
|
||||
assert request_body == {
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "Say hello in one word"}],
|
||||
"stream": True,
|
||||
"n": 2,
|
||||
"max_tokens": 10,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
assert frozenset(indices) == frozenset({0, 1})
|
||||
assert content_by_index == {0: "Hello", 1: "Hi"}
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_streaming_tool_calls_with_n_greater_than_one_preserves_choice_indices(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location", "unit"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
stream_body: Final = b"".join(
|
||||
(
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call-0",
|
||||
"type": "function",
|
||||
"function": {"name": "get_current_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[{"index": 0, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[
|
||||
{
|
||||
"index": 1,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_current_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[{"index": 1, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[
|
||||
{
|
||||
"index": 2,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call-2",
|
||||
"type": "function",
|
||||
"function": {"name": "get_current_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
),
|
||||
_openai_sse_chunk(
|
||||
"chatcmpl-n-tools",
|
||||
"gpt-4o",
|
||||
[{"index": 2, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=stream_body)
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="openai/gpt-4o",
|
||||
messages=[{"role": "user", "content": "What is the weather in San Francisco?"}],
|
||||
tools=tools,
|
||||
stream=True,
|
||||
n=3,
|
||||
api_key="sk-openai-test",
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
)
|
||||
chunks: Final = tuple(response)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
indices: Final = tuple(chunk.choices[0].index for chunk in chunks if chunk.choices)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert request_body == {
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "What is the weather in San Francisco?"}],
|
||||
"tools": tools,
|
||||
"stream": True,
|
||||
"n": 3,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
assert frozenset(indices) == frozenset({0, 1, 2})
|
||||
|
||||
|
||||
_PROVIDER_HEADERS: Final = {"x-request-id": "req_openai", "x-ratelimit-remaining-requests": "41"}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@ Unit tests for litellm.compress().
|
|||
"""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.compression.scoring.bm25 import bm25_score_messages
|
||||
|
|
@ -437,6 +441,56 @@ def test_embedding_scorer_forwards_embedding_model_params(monkeypatch):
|
|||
assert captured["api_base"] == "https://example-embeddings.test"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_embedding_scorer_compresses_context_using_requested_embedding_model(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
auth_context: Final = "Authentication code " * 2000
|
||||
recipe_context: Final = "Unrelated cooking recipes " * 2000
|
||||
messages: Final = [
|
||||
{"role": "user", "content": auth_context},
|
||||
{"role": "user", "content": recipe_context},
|
||||
{"role": "user", "content": "Fix auth"},
|
||||
]
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/embeddings").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"model": "text-embedding-3-small",
|
||||
"data": [
|
||||
{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]},
|
||||
{"object": "embedding", "index": 1, "embedding": [0.9, 0.1]},
|
||||
{"object": "embedding", "index": 2, "embedding": [0.0, 1.0]},
|
||||
{"object": "embedding", "index": 3, "embedding": [1.0, 0.0]},
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "total_tokens": 4},
|
||||
},
|
||||
)
|
||||
)
|
||||
result: Final = litellm.compress(
|
||||
messages=messages,
|
||||
model="gpt-4o",
|
||||
call_type=CALL_TYPE,
|
||||
compression_trigger=1000,
|
||||
embedding_model="text-embedding-3-small",
|
||||
embedding_model_params={"api_key": "sk-test"},
|
||||
)
|
||||
request_body: Final = json.loads(route.calls[0].request.content)
|
||||
|
||||
assert request_body == {
|
||||
"model": "text-embedding-3-small",
|
||||
"input": [
|
||||
"Fix auth",
|
||||
auth_context[:15000] + "\n...\n" + auth_context[-15000:],
|
||||
recipe_context[:15000] + "\n...\n" + recipe_context[-15000:],
|
||||
"Fix auth",
|
||||
],
|
||||
}
|
||||
assert result["compression_ratio"] > 0
|
||||
assert len(result["cache"]) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Embedding scorer — integration test (skipped without API key)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue