mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
test: remove remaining llm translation suite
This commit is contained in:
yuneng
parent
81275ff472
commit
0658f7633c
49 changed files with 18 additions and 1778 deletions
|
|
@ -1,4 +1,4 @@
|
|||
Unit tests for individual LLM providers.
|
||||
Shared VCR documentation for provider tests.
|
||||
|
||||
Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI.
|
||||
|
||||
|
|
@ -25,7 +25,6 @@ The same VCR cache is used by other test directories that exercise live
|
|||
provider APIs. The reusable conftest plumbing lives in
|
||||
`tests/_vcr_conftest_common.py` and is wired into:
|
||||
|
||||
- `tests/llm_translation/`
|
||||
- `tests/audio_tests/`
|
||||
- `tests/batches_tests/`
|
||||
- `tests/guardrails_tests/`
|
||||
|
|
@ -67,5 +66,5 @@ make test-llm-translation-flush-vcr-cache
|
|||
Skip the cache entirely (every call goes live, no recording):
|
||||
|
||||
```bash
|
||||
LITELLM_VCR_DISABLE=1 uv run pytest tests/llm_translation/test_<file>.py
|
||||
LITELLM_VCR_DISABLE=1 uv run pytest tests/audio_tests/test_<file>.py
|
||||
```
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
"""Shared VCR (Redis-backed) plumbing imported by per-directory conftests.
|
||||
|
||||
See ``tests/llm_translation/Readme.md`` for the full design and
|
||||
``tests/llm_translation/conftest.py`` for the reference wiring."""
|
||||
See ``tests/VCR_TESTING.md`` for the full design and
|
||||
``tests/_vcr_conftest_common.py`` for the shared wiring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2]
|
|||
WEATHER_WAV: Final = (
|
||||
Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav"
|
||||
)
|
||||
DUMMY_PDF: Final = Path(__file__).resolve().parent.parent.parent / "llm_translation" / "fixtures" / "dummy.pdf"
|
||||
DUMMY_PDF: Final = Path(__file__).resolve().parents[2] / "fixtures" / "dummy.pdf"
|
||||
DUMMY_PDF_URL: Final = (
|
||||
"https://cdn.jsdelivr.net/gh/BerriAI/litellm"
|
||||
"@d769e81c90d453240c61fc572cdb27fae06a89d0"
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
import httpx
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
import os
|
||||
from litellm._uuid import uuid
|
||||
|
||||
import litellm
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
pwd = os.path.dirname(os.path.realpath(__file__))
|
||||
print(pwd)
|
||||
|
||||
file_path = os.path.join(pwd, "gettysburg.wav")
|
||||
|
||||
audio_file = open(file_path, "rb")
|
||||
|
||||
class BaseLLMAudioTranscriptionTest(ABC):
|
||||
@abstractmethod
|
||||
def get_base_audio_transcription_call_args(self) -> dict:
|
||||
"""Must return the base audio transcription call args"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
"""Must return the custom llm provider"""
|
||||
pass
|
||||
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import asyncio
|
||||
import httpx
|
||||
import json
|
||||
import pytest
|
||||
from typing import Any, Dict, List
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
get_supported_openai_params,
|
||||
get_optional_params,
|
||||
)
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
file_data = (Path(__file__).parent.parent / "white_100x100.png").read_bytes()
|
||||
|
||||
encoded_file = base64.b64encode(file_data).decode("utf-8")
|
||||
base64_image = f"data:image/png;base64,{encoded_file}"
|
||||
|
||||
class BaseLLMEmbeddingTest(ABC):
|
||||
"""
|
||||
Abstract base test class that enforces a common test across all test classes.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_base_embedding_call_args(self) -> dict:
|
||||
"""Must return the base embedding call args"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
"""Must return the custom llm provider"""
|
||||
pass
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
async def test_basic_embedding(self, sync_mode):
|
||||
litellm.set_verbose = True
|
||||
embedding_call_args = self.get_base_embedding_call_args()
|
||||
if sync_mode is True:
|
||||
response = litellm.embedding(
|
||||
**embedding_call_args,
|
||||
input=["hello", "world"],
|
||||
)
|
||||
|
||||
print("embedding response: ", response)
|
||||
else:
|
||||
response = await litellm.aembedding(
|
||||
**embedding_call_args,
|
||||
input=["hello", "world"],
|
||||
)
|
||||
|
||||
print("async embedding response: ", response)
|
||||
|
||||
from openai.types.create_embedding_response import CreateEmbeddingResponse
|
||||
|
||||
CreateEmbeddingResponse.model_validate(response.model_dump())
|
||||
|
||||
|
|
@ -1,571 +0,0 @@
|
|||
import json
|
||||
import pytest
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
from unittest.mock import MagicMock, Mock
|
||||
import os
|
||||
import inspect
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
get_supported_openai_params,
|
||||
)
|
||||
from typing import Union
|
||||
from litellm.types.utils import Usage, ModelResponse
|
||||
|
||||
# test_example.py
|
||||
from abc import ABC, abstractmethod
|
||||
from openai import OpenAI
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
|
||||
|
||||
class BaseLLMChatTest(ABC):
|
||||
"""
|
||||
Abstract base test class that enforces a common test across all test classes.
|
||||
"""
|
||||
|
||||
@property
|
||||
def completion_function(self):
|
||||
return litellm.completion
|
||||
|
||||
@property
|
||||
def async_completion_function(self):
|
||||
return litellm.acompletion
|
||||
|
||||
@abstractmethod
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
"""Must return the base completion call args"""
|
||||
pass
|
||||
|
||||
def get_base_completion_call_args_with_reasoning_model(self) -> dict:
|
||||
"""Must return the base completion call args with reasoning_effort"""
|
||||
return {}
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _handle_rate_limits(self):
|
||||
"""Fixture to handle rate limit errors for all test methods"""
|
||||
try:
|
||||
yield
|
||||
except litellm.RateLimitError:
|
||||
pytest.skip("Rate limit exceeded")
|
||||
except litellm.InternalServerError:
|
||||
pytest.skip("Model is overloaded")
|
||||
|
||||
def test_streaming(self):
|
||||
"""Check if litellm handles streaming correctly"""
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
from typing import Optional
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
# litellm.set_verbose = True
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Hello, how are you?"}],
|
||||
}
|
||||
]
|
||||
try:
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=messages,
|
||||
stream=True,
|
||||
)
|
||||
assert response is not None
|
||||
assert isinstance(response, CustomStreamWrapper)
|
||||
except litellm.InternalServerError:
|
||||
pytest.skip("Model is overloaded")
|
||||
|
||||
# for OpenAI the content contains the JSON schema, so we need to assert that the content is not None
|
||||
chunks = []
|
||||
created_at: Optional[int] = None
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
chunks.append(chunk)
|
||||
if isinstance(chunk, ModelResponseStream):
|
||||
if created_at is None:
|
||||
created_at = chunk.created
|
||||
assert chunk.created == created_at
|
||||
|
||||
resp = litellm.stream_chunk_builder(chunks=chunks)
|
||||
print(resp)
|
||||
|
||||
# assert resp.usage.prompt_tokens > 0
|
||||
# assert resp.usage.completion_tokens > 0
|
||||
# assert resp.usage.total_tokens > 0
|
||||
|
||||
def test_web_search(self):
|
||||
from litellm.utils import supports_web_search
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
|
||||
if not supports_web_search(base_completion_call_args["model"], None):
|
||||
pytest.skip("Model does not support web search")
|
||||
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather like in Boston today?"}
|
||||
],
|
||||
web_search_options={},
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
|
||||
print(f"response={response}")
|
||||
|
||||
def test_url_context(self):
|
||||
from litellm.utils import supports_url_context
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
|
||||
if not supports_url_context(base_completion_call_args["model"], None):
|
||||
pytest.skip("Model does not support url context")
|
||||
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Summarize the content of this URL: https://en.wikipedia.org/wiki/Artificial_intelligence",
|
||||
}
|
||||
],
|
||||
tools=[{"urlContext": {}}],
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
print(f"response={response}")
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_handling(self, pdf_messages, sync_mode):
|
||||
from litellm.utils import supports_pdf_input
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
image_content = [
|
||||
{"type": "text", "text": "What's this file about?"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_data": pdf_messages,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
image_messages = [{"role": "user", "content": image_content}]
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
|
||||
if not supports_pdf_input(base_completion_call_args["model"], None):
|
||||
pytest.skip("Model does not support image input")
|
||||
|
||||
if sync_mode:
|
||||
response = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=image_messages,
|
||||
)
|
||||
else:
|
||||
response = await self.async_completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=image_messages,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_pdf_handling_with_file_id(self):
|
||||
from litellm.utils import supports_pdf_input
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
image_content = [
|
||||
{"type": "text", "text": "What's this file about?"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {
|
||||
"file_id": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/tests/llm_translation/fixtures/dummy.pdf"
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
image_messages = [{"role": "user", "content": image_content}]
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
|
||||
if not supports_pdf_input(base_completion_call_args["model"], None):
|
||||
pytest.skip("Model does not support image input")
|
||||
|
||||
response = await self.async_completion_function(
|
||||
**base_completion_call_args,
|
||||
messages=image_messages,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
|
||||
@pytest.fixture
|
||||
def tool_call_no_arguments(self):
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_2c384bc6-de46-4f29-8adc-60dd5805d305",
|
||||
"function": {"name": "Get-FAQ", "arguments": "{}"},
|
||||
"type": "function",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
def pdf_messages(self):
|
||||
import base64
|
||||
import os
|
||||
|
||||
# Use local PDF file instead of external URL to avoid flaky tests
|
||||
test_dir = os.path.dirname(__file__)
|
||||
pdf_path = os.path.join(test_dir, "fixtures", "dummy.pdf")
|
||||
|
||||
with open(pdf_path, "rb") as f:
|
||||
file_data = f.read()
|
||||
|
||||
encoded_file = base64.b64encode(file_data).decode("utf-8")
|
||||
url = f"data:application/pdf;base64,{encoded_file}"
|
||||
|
||||
return url
|
||||
|
||||
@pytest.mark.flaky(retries=3, delay=1)
|
||||
def test_basic_tool_calling(self):
|
||||
try:
|
||||
from litellm import completion, ModelResponse
|
||||
|
||||
litellm.set_verbose = True
|
||||
litellm.turn_on_debug()
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
if not supports_function_calling(base_completion_call_args["model"], None):
|
||||
print("Model does not support function calling")
|
||||
pytest.skip("Model does not support function calling")
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in Boston today in fahrenheit?",
|
||||
}
|
||||
]
|
||||
request_args = {
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
}
|
||||
request_args.update(self.get_base_completion_call_args())
|
||||
response: ModelResponse = completion(**request_args) # type: ignore
|
||||
print(f"response: {response}")
|
||||
|
||||
assert response is not None
|
||||
|
||||
# if the provider did not return any tool calls do not make a subsequent llm api call
|
||||
if response.choices[0].message.content is not None:
|
||||
try:
|
||||
json.loads(response.choices[0].message.content)
|
||||
pytest.fail(f"Tool call returned in content instead of tool_calls")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
pass
|
||||
if (
|
||||
"<thinking>" in response.choices[0].message.content
|
||||
and "</thinking>" in response.choices[0].message.content
|
||||
):
|
||||
pytest.fail(
|
||||
"Thinking block returned in content instead of separate reasoning_content"
|
||||
)
|
||||
if response.choices[0].message.tool_calls is None:
|
||||
return
|
||||
# Add any assertions here to check the response
|
||||
|
||||
assert isinstance(
|
||||
response.choices[0].message.tool_calls[0].function.name, str
|
||||
)
|
||||
assert isinstance(
|
||||
response.choices[0].message.tool_calls[0].function.arguments, str
|
||||
)
|
||||
assert (
|
||||
response.choices[0].finish_reason == "tool_calls"
|
||||
), f"finish_reason: {response.choices[0].finish_reason}, expected: tool_calls"
|
||||
messages.append(
|
||||
response.choices[0].message.model_dump()
|
||||
) # Add assistant tool invokes
|
||||
tool_result = (
|
||||
'{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}'
|
||||
)
|
||||
# Add user submitted tool results in the OpenAI format
|
||||
messages.append(
|
||||
{
|
||||
"tool_call_id": response.choices[0].message.tool_calls[0].id,
|
||||
"role": "tool",
|
||||
"name": response.choices[0].message.tool_calls[0].function.name,
|
||||
"content": tool_result,
|
||||
}
|
||||
)
|
||||
# In the second response, Claude should deduce answer from tool results
|
||||
request_2_args = {
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
}
|
||||
request_2_args.update(self.get_base_completion_call_args())
|
||||
second_response: ModelResponse = completion(**request_2_args) # type: ignore
|
||||
print(f"second response: {second_response}")
|
||||
assert second_response is not None
|
||||
|
||||
# either content or tool calls should be present
|
||||
assert (
|
||||
second_response.choices[0].message.content is not None
|
||||
or second_response.choices[0].message.tool_calls is not None
|
||||
)
|
||||
except litellm.ServiceUnavailableError:
|
||||
pytest.skip("Model is overloaded")
|
||||
except litellm.InternalServerError:
|
||||
pytest.skip("Model is overloaded")
|
||||
except litellm.RateLimitError:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
def test_function_calling_with_tool_response(self):
|
||||
from litellm.utils import supports_function_calling
|
||||
from litellm import completion
|
||||
|
||||
litellm.turn_on_debug()
|
||||
try:
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
if not supports_function_calling(base_completion_call_args["model"], None):
|
||||
print("Model does not support function calling")
|
||||
pytest.skip("Model does not support function calling")
|
||||
|
||||
def get_weather(city: str):
|
||||
return f"City: {city}, Weather: Sunny with 34 degree Celcius"
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather in a city",
|
||||
"parameters": {
|
||||
"$id": "https://some/internal/name",
|
||||
"$schema": "https://json-schema.org/draft-07/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city to get the weather for",
|
||||
}
|
||||
},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
messages = [{"content": "How is the weather in Mumbai?", "role": "user"}]
|
||||
response, iteration = "", 0
|
||||
while True:
|
||||
if response:
|
||||
break
|
||||
# Create a streaming response with tool calling enabled
|
||||
stream = completion(
|
||||
**base_completion_call_args,
|
||||
messages=messages,
|
||||
tools=TOOLS,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
final_tool_calls = {}
|
||||
for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
print(delta)
|
||||
if delta.content:
|
||||
response += delta.content
|
||||
elif delta.tool_calls:
|
||||
for tool_call in chunk.choices[0].delta.tool_calls or []:
|
||||
index = tool_call.index
|
||||
if index not in final_tool_calls:
|
||||
final_tool_calls[index] = tool_call
|
||||
else:
|
||||
final_tool_calls[
|
||||
index
|
||||
].function.arguments += tool_call.function.arguments
|
||||
if final_tool_calls:
|
||||
for tool_call in final_tool_calls.values():
|
||||
if tool_call.function.name == "get_weather":
|
||||
city = json.loads(tool_call.function.arguments)["city"]
|
||||
tool_response = get_weather(city)
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [tool_call],
|
||||
"content": None,
|
||||
}
|
||||
)
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": tool_response,
|
||||
}
|
||||
)
|
||||
iteration += 1
|
||||
if iteration > 2:
|
||||
print("Something went wrong!")
|
||||
break
|
||||
|
||||
print(response)
|
||||
except litellm.ServiceUnavailableError:
|
||||
pass
|
||||
|
||||
class BaseOSeriesModelsTest(ABC): # test across azure/openai
|
||||
@abstractmethod
|
||||
def get_base_completion_call_args(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_client(self) -> OpenAI:
|
||||
pass
|
||||
|
||||
class BaseAnthropicChatTest(ABC):
|
||||
"""
|
||||
Ensures consistent result across anthropic model usage
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
"""Must return the base completion call args"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_base_completion_call_args_with_thinking(self) -> dict:
|
||||
"""Must return the base completion call args"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def completion_function(self):
|
||||
return litellm.completion
|
||||
|
||||
class BaseReasoningLLMTests(ABC):
|
||||
"""
|
||||
Base class for testing reasoning llms
|
||||
|
||||
- test that the responses contain reasoning_content
|
||||
- test that the usage contains reasoning_tokens
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
"""Must return the base completion call args"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def completion_function(self):
|
||||
return litellm.completion
|
||||
|
||||
def test_non_streaming_reasoning_effort(self):
|
||||
"""
|
||||
Base test for non-streaming reasoning effort
|
||||
|
||||
- Assert that `reasoning_content` is not None from response message
|
||||
- Assert that `reasoning_tokens` is greater than 0 from usage
|
||||
"""
|
||||
litellm.turn_on_debug()
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
response: ModelResponse = self.completion_function(
|
||||
**base_completion_call_args, reasoning_effort="low"
|
||||
)
|
||||
|
||||
# user gets `reasoning_content` in the response message
|
||||
assert response.choices[0].message.reasoning_content is not None
|
||||
assert isinstance(response.choices[0].message.reasoning_content, str)
|
||||
|
||||
# user get `reasoning_tokens`
|
||||
assert response.usage.completion_tokens_details.reasoning_tokens > 0
|
||||
|
||||
def test_streaming_reasoning_effort(self):
|
||||
"""
|
||||
Base test for streaming reasoning effort
|
||||
|
||||
- Assert that `reasoning_content` is not None from streaming response
|
||||
- Assert that `reasoning_tokens` is greater than 0 from usage
|
||||
"""
|
||||
# litellm.turn_on_debug()
|
||||
base_completion_call_args = self.get_base_completion_call_args()
|
||||
response: CustomStreamWrapper = self.completion_function(
|
||||
**base_completion_call_args,
|
||||
reasoning_effort="low",
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
resoning_content: str = ""
|
||||
usage: Usage = None
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
if hasattr(chunk.choices[0].delta, "reasoning_content"):
|
||||
resoning_content += chunk.choices[0].delta.reasoning_content
|
||||
if hasattr(chunk, "usage"):
|
||||
usage = chunk.usage
|
||||
|
||||
assert resoning_content is not None
|
||||
assert len(resoning_content) > 0
|
||||
|
||||
print(f"usage: {usage}")
|
||||
assert usage.completion_tokens_details.reasoning_tokens > 0
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
import asyncio
|
||||
import httpx
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
get_supported_openai_params,
|
||||
get_optional_params,
|
||||
)
|
||||
|
||||
# test_example.py
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
def assert_response_shape(response, custom_llm_provider):
|
||||
expected_response_shape = {"id": str, "results": list, "meta": dict}
|
||||
|
||||
expected_results_shape = {"index": int, "relevance_score": float}
|
||||
|
||||
expected_meta_shape = {"api_version": dict, "billed_units": dict}
|
||||
|
||||
expected_api_version_shape = {"version": str}
|
||||
|
||||
expected_billed_units_shape = {"search_units": int}
|
||||
expected_billed_units_total_tokens_shape = {"total_tokens": int}
|
||||
|
||||
assert isinstance(response.id, expected_response_shape["id"])
|
||||
assert isinstance(response.results, expected_response_shape["results"])
|
||||
for result in response.results:
|
||||
assert isinstance(result["index"], expected_results_shape["index"])
|
||||
assert isinstance(
|
||||
result["relevance_score"], expected_results_shape["relevance_score"]
|
||||
)
|
||||
assert isinstance(response.meta, expected_response_shape["meta"])
|
||||
|
||||
if custom_llm_provider == "cohere":
|
||||
|
||||
assert isinstance(
|
||||
response.meta["api_version"], expected_meta_shape["api_version"]
|
||||
)
|
||||
assert isinstance(
|
||||
response.meta["api_version"]["version"],
|
||||
expected_api_version_shape["version"],
|
||||
)
|
||||
assert isinstance(
|
||||
response.meta["billed_units"], expected_meta_shape["billed_units"]
|
||||
)
|
||||
if "total_tokens" in response.meta["billed_units"]:
|
||||
assert isinstance(
|
||||
response.meta["billed_units"]["total_tokens"],
|
||||
expected_billed_units_total_tokens_shape["total_tokens"],
|
||||
)
|
||||
else:
|
||||
assert isinstance(
|
||||
response.meta["billed_units"]["search_units"],
|
||||
expected_billed_units_shape["search_units"],
|
||||
)
|
||||
|
||||
class BaseLLMRerankTest(ABC):
|
||||
"""
|
||||
Abstract base test class that enforces a common test across all test classes.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_base_rerank_call_args(self) -> dict:
|
||||
"""Must return the base rerank call args"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
"""Must return the custom llm provider"""
|
||||
pass
|
||||
|
||||
def get_expected_cost(self) -> float:
|
||||
"""
|
||||
Override this method to set the expected cost for the rerank call.
|
||||
Default is None, which means the test will check cost > 0.
|
||||
Return 0.0 for free models.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
# conftest.py
|
||||
#
|
||||
# xdist-compatible test isolation for llm_translation tests.
|
||||
# Mirrors the pattern in tests/local_testing/conftest.py:
|
||||
# - Function-scoped fixture resets litellm globals to true defaults
|
||||
# - Module-scoped reload only in single-process mode
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
from collections.abc import Generator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm # noqa: E402
|
||||
|
||||
from tests._vcr_conftest_common import ( # noqa: E402,F401
|
||||
VerboseReporterState,
|
||||
_pin_multipart_boundary,
|
||||
apply_vcr_auto_marker_to_items,
|
||||
emit_cassette_cache_session_banner,
|
||||
emit_vcr_classification_summary,
|
||||
emit_vcr_diagnostic_log,
|
||||
guard_vcr_patch_points,
|
||||
install_live_call_probe,
|
||||
record_vcr_outcome,
|
||||
register_persister_if_enabled,
|
||||
reset_vcr_diag_dir,
|
||||
vcr_config_dict,
|
||||
)
|
||||
from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def fake_openai_endpoint():
|
||||
ensure_fake_openai_endpoint()
|
||||
yield
|
||||
|
||||
|
||||
# Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles
|
||||
# the vast majority of respx-vs-vcrpy conflicts automatically. The entries
|
||||
# below are the persister's, the WebSocket VCR's, and the cassette patch-leak
|
||||
# guard's own unit-test files, which exercise ``save_cassette`` /
|
||||
# ``load_cassette`` against fakeredis or enter cassettes themselves and must
|
||||
# not run under a live cassette context.
|
||||
_VCR_AUTO_MARKER_SKIP_FILES = frozenset(
|
||||
{"test_vcr_redis_persister.py", "test_ws_vcr.py", "test_vcr_leak_guard.py"}
|
||||
)
|
||||
|
||||
|
||||
_verbose_state = VerboseReporterState()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vcr_config():
|
||||
return vcr_config_dict()
|
||||
|
||||
|
||||
def pytest_recording_configure(config, vcr):
|
||||
register_persister_if_enabled(vcr)
|
||||
|
||||
|
||||
@pytest.hookimpl(hookwrapper=True)
|
||||
def pytest_runtest_makereport(item, call):
|
||||
outcome = yield
|
||||
rep = outcome.get_result()
|
||||
setattr(item, f"rep_{rep.when}", rep)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _vcr_outcome_gate(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.hookimpl(wrapper=True, trylast=True)
|
||||
def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, object, object]:
|
||||
try:
|
||||
result: Final = yield
|
||||
except BaseException:
|
||||
guard_vcr_patch_points(item, teardown_failed=True)
|
||||
raise
|
||||
guard_vcr_patch_points(item, teardown_failed=False)
|
||||
return result
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
_verbose_state.remember_pluginmanager(config)
|
||||
reset_vcr_diag_dir()
|
||||
|
||||
|
||||
def pytest_runtest_logreport(report):
|
||||
_verbose_state.maybe_emit_verdict(report)
|
||||
|
||||
|
||||
def pytest_terminal_summary(terminalreporter, exitstatus, config):
|
||||
emit_cassette_cache_session_banner(terminalreporter)
|
||||
emit_vcr_classification_summary(terminalreporter)
|
||||
emit_vcr_diagnostic_log(terminalreporter)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Capture TRUE defaults at conftest import time (before test modules pollute).
|
||||
# ---------------------------------------------------------------------------
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"set_verbose": getattr(litellm, "set_verbose", False),
|
||||
"cache": getattr(litellm, "cache", None),
|
||||
"allowed_fails": getattr(litellm, "allowed_fails", 3),
|
||||
"disable_aiohttp_transport": getattr(litellm, "disable_aiohttp_transport", False),
|
||||
"force_ipv4": getattr(litellm, "force_ipv4", False),
|
||||
"drop_params": getattr(litellm, "drop_params", None),
|
||||
"modify_params": getattr(litellm, "modify_params", False),
|
||||
"api_base": getattr(litellm, "api_base", None),
|
||||
"api_key": getattr(litellm, "api_key", None),
|
||||
"cohere_key": getattr(litellm, "cohere_key", None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_and_teardown(event_loop): # Add event_loop as a dependency
|
||||
|
||||
import litellm
|
||||
|
||||
# ---- Save current state (for teardown restore) ----
|
||||
original_state = {}
|
||||
for attr in (
|
||||
"callbacks",
|
||||
"success_callback",
|
||||
"failure_callback",
|
||||
"_async_success_callback",
|
||||
"_async_failure_callback",
|
||||
):
|
||||
if hasattr(litellm, attr):
|
||||
val = getattr(litellm, attr)
|
||||
original_state[attr] = val.copy() if val else []
|
||||
|
||||
for attr in _SCALAR_DEFAULTS:
|
||||
if hasattr(litellm, attr):
|
||||
original_state[attr] = getattr(litellm, attr)
|
||||
|
||||
# ---- Reset to true defaults before the test ----
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue())
|
||||
importlib.reload(litellm)
|
||||
|
||||
# Set the event loop from the fixture
|
||||
asyncio.set_event_loop(event_loop)
|
||||
|
||||
yield
|
||||
|
||||
# ---- Teardown ----
|
||||
for attr, original_value in original_state.items():
|
||||
if hasattr(litellm, attr):
|
||||
setattr(litellm, attr, original_value)
|
||||
|
||||
# Clean up any pending tasks
|
||||
pending = asyncio.all_tasks(event_loop)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
|
||||
# Run the event loop until all tasks are cancelled
|
||||
if pending:
|
||||
event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
apply_vcr_auto_marker_to_items(
|
||||
items,
|
||||
skip_files=_VCR_AUTO_MARKER_SKIP_FILES,
|
||||
)
|
||||
|
||||
custom_logger_tests = [
|
||||
item for item in items if "custom_logger" in item.parent.name
|
||||
]
|
||||
other_tests = [item for item in items if "custom_logger" not in item.parent.name]
|
||||
|
||||
custom_logger_tests.sort(key=lambda x: x.name)
|
||||
other_tests.sort(key=lambda x: x.name)
|
||||
|
||||
items[:] = custom_logger_tests + other_tests
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
# Databricks Configuration Template for LiteLLM Testing
|
||||
# =====================================================
|
||||
#
|
||||
# Copy this file to your preferred location and fill in your credentials:
|
||||
# cp databricks_config.template.txt /path/to/databricks_config.txt
|
||||
#
|
||||
# Then update the CONFIG_FILE path in test_databricks_integration.py
|
||||
#
|
||||
# Lines starting with # are comments and will be ignored
|
||||
# Only lines with KEY=VALUE format (where VALUE is not empty) will be read
|
||||
|
||||
# ==============================================================================
|
||||
# DATABRICKS WORKSPACE CONFIGURATION (Required)
|
||||
# ==============================================================================
|
||||
|
||||
# Your Databricks workspace URL (without /serving-endpoints suffix)
|
||||
# Example: https://adb-1234567890123456.7.azuredatabricks.net
|
||||
DATABRICKS_HOST=
|
||||
|
||||
# API Base URL for serving endpoints (usually {host}/serving-endpoints)
|
||||
# Example: https://adb-1234567890123456.7.azuredatabricks.net/serving-endpoints
|
||||
DATABRICKS_API_BASE=
|
||||
|
||||
# ==============================================================================
|
||||
# AUTHENTICATION METHOD 1: OAuth M2M (Recommended for Production)
|
||||
# Use Service Principal credentials
|
||||
# ==============================================================================
|
||||
|
||||
# Service Principal Application/Client ID
|
||||
# Example: 12345678-1234-1234-1234-123456789012
|
||||
DATABRICKS_CLIENT_ID=
|
||||
|
||||
# Service Principal Secret
|
||||
# Example: your-client-secret-value
|
||||
DATABRICKS_CLIENT_SECRET=
|
||||
|
||||
# ==============================================================================
|
||||
# AUTHENTICATION METHOD 2: Personal Access Token (PAT)
|
||||
# For development and testing
|
||||
# ==============================================================================
|
||||
|
||||
# Personal Access Token (starts with 'dapi')
|
||||
# Example: dapi_your_token_here
|
||||
DATABRICKS_API_KEY=
|
||||
|
||||
# ==============================================================================
|
||||
# MODEL CONFIGURATION
|
||||
# ==============================================================================
|
||||
|
||||
# Model to use for testing chat completions
|
||||
# Example: databricks-gpt-oss-120b, databricks-meta-llama-3-1-70b-instruct
|
||||
TEST_CHAT_MODEL=databricks-gpt-oss-120b
|
||||
|
||||
# Model to use for testing embeddings (optional)
|
||||
# Example: databricks-bge-large-en
|
||||
TEST_EMBEDDING_MODEL=databricks-bge-large-en
|
||||
|
||||
# ==============================================================================
|
||||
# OPTIONAL: Custom User-Agent for Partner Attribution Testing
|
||||
# ==============================================================================
|
||||
|
||||
# Custom user agent string to test partner attribution
|
||||
# Example: mycompany/1.0.0
|
||||
# This will result in User-Agent: mycompany_litellm/{version}
|
||||
# Leave empty to use default: litellm/{version}
|
||||
CUSTOM_USER_AGENT=
|
||||
|
||||
# ==============================================================================
|
||||
# TEST SETTINGS
|
||||
# ==============================================================================
|
||||
|
||||
# Which authentication method to test: oauth, pat, sdk, or all
|
||||
# oauth = Use DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET
|
||||
# pat = Use DATABRICKS_API_KEY
|
||||
# sdk = Use Databricks SDK automatic authentication (~/.databrickscfg)
|
||||
# all = Test all three methods (oauth, pat, sdk) in sequence
|
||||
TEST_AUTH_METHOD=pat
|
||||
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,441 +0,0 @@
|
|||
"""
|
||||
Base test class for LiteLLM Realtime API E2E tests.
|
||||
|
||||
Provides common test infrastructure for testing realtime WebSocket connections
|
||||
across different providers (OpenAI, xAI, etc.)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
class RealTimeWebSocketClient:
|
||||
"""
|
||||
Mock WebSocket client for testing realtime connections.
|
||||
Captures messages sent from the backend and provides a simple interface
|
||||
for testing connection success.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.messages_sent = []
|
||||
self.messages_received = []
|
||||
self.received_initial_event = False
|
||||
self.connection_successful = False
|
||||
self.close_code = None
|
||||
self.close_reason = None
|
||||
# Required by realtime_streaming.py - import exceptions module
|
||||
from websockets import exceptions as websockets_exceptions
|
||||
|
||||
self.exceptions = websockets_exceptions
|
||||
|
||||
async def accept(self):
|
||||
"""Accept the WebSocket connection"""
|
||||
pass
|
||||
|
||||
async def send_text(self, message):
|
||||
"""Receive message from backend and store it"""
|
||||
self.messages_sent.append(message)
|
||||
try:
|
||||
if isinstance(message, bytes):
|
||||
message_str = message.decode("utf-8")
|
||||
else:
|
||||
message_str = message
|
||||
|
||||
msg_data = json.loads(message_str)
|
||||
msg_type = msg_data.get("type", "unknown")
|
||||
|
||||
# Pretty print API response
|
||||
print(f"\n{'='*80}")
|
||||
print(
|
||||
f"API RESPONSE #{len(self.messages_received) + 1} - Event: {msg_type}"
|
||||
)
|
||||
print(f"{'='*80}")
|
||||
print(json.dumps(msg_data, indent=2, sort_keys=False))
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
self.messages_received.append(msg_data)
|
||||
|
||||
# Check for initial connection event
|
||||
if not self.received_initial_event and self._is_initial_event(msg_type):
|
||||
self.received_initial_event = True
|
||||
self.connection_successful = True
|
||||
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
||||
# Non-JSON messages are acceptable
|
||||
print(f"\n[Non-JSON message: {e}]")
|
||||
print(f"Raw content: {str(message)[:200]}\n")
|
||||
pass
|
||||
|
||||
def _is_initial_event(self, msg_type: str) -> bool:
|
||||
"""Check if message type is an initial connection event"""
|
||||
# OpenAI and xAI send "session.created"; some providers send "conversation.created"
|
||||
return msg_type in ["session.created", "conversation.created"]
|
||||
|
||||
async def receive_text(self):
|
||||
"""
|
||||
Wait briefly for messages, then close connection.
|
||||
This allows the backend forwarding task to send messages.
|
||||
"""
|
||||
print(f"\nWaiting for connection to establish...")
|
||||
max_wait = 5.0
|
||||
check_interval = 0.1
|
||||
waited = 0.0
|
||||
|
||||
while waited < max_wait:
|
||||
if self.connection_successful:
|
||||
print(f"Connection successful after {waited:.1f}s\n")
|
||||
break
|
||||
await asyncio.sleep(check_interval)
|
||||
waited += check_interval
|
||||
|
||||
if not self.connection_successful:
|
||||
print(f"Warning: No initial event received after {max_wait}s\n")
|
||||
|
||||
# If we have a pending message to send, send it now
|
||||
if hasattr(self, "_pending_client_message") and self._pending_client_message:
|
||||
print(f"Sending client message to backend...\n")
|
||||
# This simulates receiving a message from the client that needs to be forwarded to backend
|
||||
# We return it as if it came from the client
|
||||
msg = self._pending_client_message
|
||||
self._pending_client_message = None
|
||||
return msg
|
||||
|
||||
# Close connection to end the test
|
||||
print(f"\n{'='*80}")
|
||||
print(f"TEST COMPLETE - Closing connection")
|
||||
print(f"Total messages received from API: {len(self.messages_received)}")
|
||||
print(f"{'='*80}\n")
|
||||
raise websockets.exceptions.ConnectionClosed(None, None)
|
||||
|
||||
def queue_client_message(self, message: str):
|
||||
"""Queue a message to be sent from 'client' to backend"""
|
||||
self._pending_client_message = message
|
||||
|
||||
async def close(self, code=1000, reason=""):
|
||||
"""Close the WebSocket"""
|
||||
self.close_code = code
|
||||
self.close_reason = reason
|
||||
|
||||
@property
|
||||
def headers(self):
|
||||
return {}
|
||||
|
||||
|
||||
class BaseRealtimeTest(ABC):
|
||||
"""
|
||||
Abstract base test class for realtime API tests.
|
||||
|
||||
Child classes must implement:
|
||||
- get_model(): Return the model name to test
|
||||
- get_api_key_env_var(): Return the environment variable name for the API key
|
||||
- get_initial_event_type(): Return the expected initial event type (e.g., "session.created")
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_model(self) -> str:
|
||||
"""Return the model name to test (e.g., 'gpt-4o-realtime-preview-2024-10-01')"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_api_key_env_var(self) -> str:
|
||||
"""Return the environment variable name for the API key (e.g., 'OPENAI_API_KEY')"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_initial_event_type(self) -> Union[str, Tuple[str, ...]]:
|
||||
"""Return the expected initial event type(s).
|
||||
|
||||
May return a single event type (e.g. ``'session.created'``) or a tuple
|
||||
of acceptable types when the upstream provider can legitimately emit
|
||||
more than one initial event (e.g. xAI's Grok Voice Agent has shipped
|
||||
both ``conversation.created`` and ``session.created``).
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_skip_reason(self) -> str:
|
||||
"""Return the skip reason when API key is missing"""
|
||||
return f"No {self.get_api_key_env_var()} provided"
|
||||
|
||||
def should_skip(self) -> bool:
|
||||
"""Check if tests should be skipped due to missing API key"""
|
||||
return os.environ.get(self.get_api_key_env_var()) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_connection(self):
|
||||
"""
|
||||
Test basic realtime WebSocket connection.
|
||||
Verifies that:
|
||||
1. Connection is established successfully
|
||||
2. Initial event is received
|
||||
3. Messages are properly forwarded
|
||||
"""
|
||||
litellm.turn_on_debug()
|
||||
if self.should_skip():
|
||||
pytest.skip(self.get_skip_reason())
|
||||
|
||||
websocket_client = RealTimeWebSocketClient()
|
||||
caught_exception = None
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"STARTING REALTIME CONNECTION TEST")
|
||||
print(f"Model: {self.get_model()}")
|
||||
print(f"API Key Env Var: {self.get_api_key_env_var()}")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
try:
|
||||
await litellm._arealtime(
|
||||
model=self.get_model(),
|
||||
websocket=websocket_client,
|
||||
api_key=os.environ.get(self.get_api_key_env_var()),
|
||||
timeout=60,
|
||||
)
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"\nException: {type(e).__name__}: {e}\n")
|
||||
caught_exception = e
|
||||
|
||||
# Build debug info
|
||||
error_details = []
|
||||
error_details.append(f"messages_sent: {len(websocket_client.messages_sent)}")
|
||||
error_details.append(
|
||||
f"messages_received: {len(websocket_client.messages_received)}"
|
||||
)
|
||||
error_details.append(f"close_code: {websocket_client.close_code}")
|
||||
error_details.append(f"close_reason: {websocket_client.close_reason}")
|
||||
if caught_exception:
|
||||
error_details.append(
|
||||
f"exception: {type(caught_exception).__name__}: {caught_exception}"
|
||||
)
|
||||
|
||||
# Assertions
|
||||
assert (
|
||||
websocket_client.connection_successful
|
||||
), f"Failed to connect. Debug: {'; '.join(error_details)}"
|
||||
assert websocket_client.received_initial_event, f"Did not receive initial event"
|
||||
assert len(websocket_client.messages_received) > 0, "No messages received"
|
||||
|
||||
# Verify initial event
|
||||
initial_event = websocket_client.messages_received[0]
|
||||
expected_event_type = self.get_initial_event_type()
|
||||
if isinstance(expected_event_type, str):
|
||||
allowed_event_types: Tuple[str, ...] = (expected_event_type,)
|
||||
else:
|
||||
allowed_event_types = tuple(expected_event_type)
|
||||
assert (
|
||||
initial_event["type"] in allowed_event_types
|
||||
), f"Expected one of {allowed_event_types}, got {initial_event.get('type')}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_with_query_params(self):
|
||||
"""
|
||||
Test realtime connection with explicit query parameters.
|
||||
Verifies that query params are properly passed to the backend.
|
||||
"""
|
||||
litellm.turn_on_debug()
|
||||
if self.should_skip():
|
||||
pytest.skip(self.get_skip_reason())
|
||||
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
websocket_client = RealTimeWebSocketClient()
|
||||
caught_exception = None
|
||||
|
||||
# Strip provider prefix from model name for query params
|
||||
model_name = self.get_model()
|
||||
if "/" in model_name:
|
||||
model_name = model_name.split("/", 1)[1]
|
||||
|
||||
query_params: RealtimeQueryParams = {"model": model_name}
|
||||
|
||||
try:
|
||||
await litellm._arealtime(
|
||||
model=self.get_model(),
|
||||
websocket=websocket_client,
|
||||
api_key=os.environ.get(self.get_api_key_env_var()),
|
||||
query_params=query_params,
|
||||
timeout=60,
|
||||
)
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
pass
|
||||
except Exception as e:
|
||||
caught_exception = e
|
||||
|
||||
# Build debug info
|
||||
error_details = []
|
||||
error_details.append(f"messages_sent: {len(websocket_client.messages_sent)}")
|
||||
error_details.append(
|
||||
f"messages_received: {len(websocket_client.messages_received)}"
|
||||
)
|
||||
if caught_exception:
|
||||
error_details.append(
|
||||
f"exception: {type(caught_exception).__name__}: {caught_exception}"
|
||||
)
|
||||
|
||||
# Skip on transient failures
|
||||
if (
|
||||
not websocket_client.connection_successful
|
||||
and websocket_client.close_code is not None
|
||||
):
|
||||
pytest.skip(f"Transient connection failure: {'; '.join(error_details)}")
|
||||
|
||||
# Assertions
|
||||
assert (
|
||||
websocket_client.connection_successful
|
||||
), f"Failed to connect. Debug: {'; '.join(error_details)}"
|
||||
assert len(websocket_client.messages_received) > 0, "No messages received"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_user_message(self):
|
||||
"""
|
||||
Test sending an actual user message and receiving responses.
|
||||
This creates a more realistic conversation flow.
|
||||
"""
|
||||
if self.should_skip():
|
||||
pytest.skip(self.get_skip_reason())
|
||||
|
||||
litellm.turn_on_debug()
|
||||
|
||||
# Create a custom websocket client that sends a message
|
||||
class InteractiveWebSocketClient(RealTimeWebSocketClient):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.sent_user_message = False
|
||||
self.response_messages = []
|
||||
self.wait_for_responses = True
|
||||
|
||||
async def receive_text(self):
|
||||
"""Enhanced receive that sends a user message after connection"""
|
||||
print(f"\n{'='*80}")
|
||||
print(f"CLIENT-SIDE RECEIVE HANDLER")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Wait for initial connection
|
||||
max_wait = 5.0
|
||||
check_interval = 0.1
|
||||
waited = 0.0
|
||||
|
||||
while waited < max_wait:
|
||||
if self.connection_successful:
|
||||
print(f"Connection established after {waited:.1f}s\n")
|
||||
break
|
||||
await asyncio.sleep(check_interval)
|
||||
waited += check_interval
|
||||
|
||||
# Step 1: Send a user message after connection is established
|
||||
if self.connection_successful and not self.sent_user_message:
|
||||
self.sent_user_message = True
|
||||
user_msg_data = {
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Say hi back to me!"}
|
||||
],
|
||||
},
|
||||
}
|
||||
user_msg = json.dumps(user_msg_data)
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"STEP 1: SENDING USER MESSAGE TO BACKEND")
|
||||
print(f"{'='*80}")
|
||||
print(json.dumps(user_msg_data, indent=2))
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
return user_msg
|
||||
|
||||
# Step 2: Trigger the response after user message is acknowledged
|
||||
if not hasattr(self, "triggered_response"):
|
||||
self.triggered_response = True
|
||||
# Wait a bit for the user message to be processed
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
response_create_data = {"type": "response.create"}
|
||||
response_create = json.dumps(response_create_data)
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"STEP 2: TRIGGERING LLM RESPONSE")
|
||||
print(f"{'='*80}")
|
||||
print(json.dumps(response_create_data, indent=2))
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
return response_create
|
||||
|
||||
# Step 3: Wait for LLM responses
|
||||
if self.wait_for_responses:
|
||||
print(f"\nSTEP 3: Waiting 5 seconds for LLM to respond...\n")
|
||||
await asyncio.sleep(5.0)
|
||||
self.wait_for_responses = False
|
||||
|
||||
# Collect response info
|
||||
for msg in self.messages_received:
|
||||
msg_type = msg.get("type", "unknown")
|
||||
if msg_type not in ["conversation.created", "ping"]:
|
||||
self.response_messages.append(msg)
|
||||
|
||||
print(
|
||||
f"\nReceived {len(self.response_messages)} response messages (excluding init/ping)\n"
|
||||
)
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"CLOSING CONNECTION")
|
||||
print(f"Total messages received: {len(self.messages_received)}")
|
||||
print(f"{'='*80}\n")
|
||||
raise websockets.exceptions.ConnectionClosed(None, None)
|
||||
|
||||
websocket_client = InteractiveWebSocketClient()
|
||||
caught_exception = None
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"STARTING INTERACTIVE MESSAGE TEST")
|
||||
print(f"Model: {self.get_model()}")
|
||||
print(f"Message: 'Say hi back to me!'")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
try:
|
||||
await litellm._arealtime(
|
||||
model=self.get_model(),
|
||||
websocket=websocket_client,
|
||||
api_key=os.environ.get(self.get_api_key_env_var()),
|
||||
timeout=60,
|
||||
)
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"\nException: {type(e).__name__}: {e}\n")
|
||||
caught_exception = e
|
||||
|
||||
# Print results
|
||||
print(f"\n{'='*80}")
|
||||
print(f"TEST RESULTS SUMMARY")
|
||||
print(f"{'='*80}")
|
||||
print(f"Connection successful: {websocket_client.connection_successful}")
|
||||
print(f"User message sent: {websocket_client.sent_user_message}")
|
||||
print(f"Total messages received: {len(websocket_client.messages_received)}")
|
||||
print(
|
||||
f"Response messages (excluding init/ping): {len(websocket_client.response_messages)}"
|
||||
)
|
||||
|
||||
if websocket_client.response_messages:
|
||||
print(f"\nResponse Event Types:")
|
||||
for i, msg in enumerate(websocket_client.response_messages, 1):
|
||||
print(f" {i}. {msg.get('type', 'unknown')}")
|
||||
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Skip if no responses (might be timing issue)
|
||||
if not websocket_client.response_messages:
|
||||
pytest.skip("No response messages received (might be timing/network issue)")
|
||||
|
||||
assert websocket_client.connection_successful, "Failed to establish connection"
|
||||
assert websocket_client.sent_user_message, "Failed to send user message"
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
"""WebSocket VCR wiring for the realtime suite.
|
||||
|
||||
This directory inherits the HTTP VCR machinery from
|
||||
``tests/llm_translation/conftest.py`` (which only intercepts httpx/aiohttp and
|
||||
is therefore a no-op for realtime WebSocket traffic). The autouse fixture below
|
||||
adds the WebSocket layer: it patches ``websockets.connect`` for the duration of
|
||||
each test so realtime frames are recorded to, or replayed from, the same
|
||||
cassette Redis under a ``litellm:vcr:wscassette:`` prefix.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
|
||||
|
||||
from tests._vcr_conftest_common import ( # noqa: E402
|
||||
vcr_disabled,
|
||||
vcr_outcome_logging_enabled,
|
||||
)
|
||||
from tests._ws_vcr import ( # noqa: E402
|
||||
WsVcrController,
|
||||
build_ws_cassette_client,
|
||||
load_ws_cassette,
|
||||
replay_timeout_seconds,
|
||||
save_ws_cassette,
|
||||
ws_redis_key_for,
|
||||
)
|
||||
|
||||
_ws_cassette_client: Optional[object] = None
|
||||
|
||||
|
||||
def _get_ws_cassette_client() -> Optional[object]:
|
||||
global _ws_cassette_client
|
||||
if _ws_cassette_client is None:
|
||||
_ws_cassette_client = build_ws_cassette_client()
|
||||
return _ws_cassette_client
|
||||
|
||||
|
||||
def _emit_verdict(request: pytest.FixtureRequest, verdict: str) -> None:
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
return
|
||||
reporter = request.config.pluginmanager.getplugin("terminalreporter")
|
||||
if reporter is None:
|
||||
return
|
||||
reporter.write_line(f"{verdict} :: {request.node.nodeid}")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _ws_vcr(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch):
|
||||
if vcr_disabled():
|
||||
yield
|
||||
return
|
||||
|
||||
import websockets
|
||||
|
||||
client = _get_ws_cassette_client()
|
||||
if client is None:
|
||||
yield
|
||||
return
|
||||
|
||||
key = ws_redis_key_for(request.node.nodeid)
|
||||
cassette = load_ws_cassette(client, key)
|
||||
controller = WsVcrController(
|
||||
original_connect=websockets.connect,
|
||||
cassette=cassette,
|
||||
timeout=replay_timeout_seconds(),
|
||||
)
|
||||
monkeypatch.setattr(websockets, "connect", controller.connect)
|
||||
|
||||
yield
|
||||
|
||||
rep_call = getattr(request.node, "rep_call", None)
|
||||
passed = bool(rep_call and rep_call.passed)
|
||||
|
||||
if controller.recorded:
|
||||
built = controller.built_cassette()
|
||||
if built is not None:
|
||||
save_ws_cassette(client, key, built, passed=passed)
|
||||
|
||||
if vcr_outcome_logging_enabled():
|
||||
_emit_verdict(request, controller.verdict())
|
||||
|
||||
if controller.errors and passed:
|
||||
raise controller.errors[0]
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
"""
|
||||
OpenAI Realtime API E2E Tests (using base class)
|
||||
|
||||
Tests OpenAI's Realtime API through LiteLLM's realtime interface.
|
||||
Uses the base test class to ensure consistent behavior across providers.
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest
|
||||
|
||||
|
||||
class TestOpenAIRealtime(BaseRealtimeTest):
|
||||
"""
|
||||
E2E tests for OpenAI Realtime API using base test class.
|
||||
"""
|
||||
|
||||
def get_model(self) -> str:
|
||||
# OpenAI shut down the entire gpt-4o-realtime-preview family
|
||||
# (including the undated alias) on 2026-05-07. gpt-realtime is the
|
||||
# current GA realtime model.
|
||||
return "gpt-realtime"
|
||||
|
||||
def get_api_key_env_var(self) -> str:
|
||||
return "OPENAI_API_KEY"
|
||||
|
||||
def get_initial_event_type(self) -> str:
|
||||
return "session.created"
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
"""
|
||||
xAI Realtime API E2E Tests
|
||||
|
||||
Tests xAI's Grok Voice Agent API through LiteLLM's realtime interface.
|
||||
Uses the base test class to ensure consistent behavior across providers.
|
||||
"""
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest
|
||||
|
||||
|
||||
class TestXAIRealtime(BaseRealtimeTest):
|
||||
"""
|
||||
E2E tests for xAI Realtime API.
|
||||
|
||||
xAI's Grok Voice Agent API is OpenAI-compatible:
|
||||
- Endpoint: wss://api.x.ai/v1/realtime
|
||||
- Model: grok-4-1-fast-non-reasoning
|
||||
- Initial event: historically "conversation.created"; xAI has since shipped
|
||||
"session.created" (matching OpenAI). Accept either to avoid spurious
|
||||
failures whenever xAI flips the wire format.
|
||||
"""
|
||||
|
||||
def get_model(self) -> str:
|
||||
return "xai/grok-4-1-fast-non-reasoning"
|
||||
|
||||
def get_api_key_env_var(self) -> str:
|
||||
return "XAI_API_KEY"
|
||||
|
||||
def get_initial_event_type(self) -> Tuple[str, ...]:
|
||||
return ("conversation.created", "session.created")
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest
|
||||
|
||||
|
||||
class TestDeepgramAudioTranscription(BaseLLMAudioTranscriptionTest):
|
||||
def get_base_audio_transcription_call_args(self) -> dict:
|
||||
return {
|
||||
"model": "deepgram/nova-2",
|
||||
}
|
||||
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
return litellm.LlmProviders.DEEPGRAM
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest
|
||||
|
||||
os.environ.setdefault("ELEVENLABS_API_KEY", "test-elevenlabs-key")
|
||||
|
||||
|
||||
class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest):
|
||||
def get_base_audio_transcription_call_args(self) -> dict:
|
||||
return {
|
||||
"model": "elevenlabs/scribe_v1",
|
||||
}
|
||||
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
return litellm.LlmProviders.ELEVENLABS
|
||||
|
||||
class TestElevenLabsTextToSpeechTransformation:
|
||||
@pytest.fixture(scope="class")
|
||||
def config(self):
|
||||
from litellm.llms.elevenlabs.text_to_speech.transformation import (
|
||||
ElevenLabsTextToSpeechConfig,
|
||||
)
|
||||
|
||||
return ElevenLabsTextToSpeechConfig()
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
|
||||
|
||||
# sys.path.insert(
|
||||
# 0, os.path.abspath("../..")
|
||||
# ) # noqa
|
||||
# ) # Adds the parent directory to the system path
|
||||
|
||||
from base_llm_unit_tests import BaseLLMChatTest
|
||||
|
||||
class TestGroq(BaseLLMChatTest):
|
||||
test_web_search = None
|
||||
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
return {
|
||||
"model": "groq/openai/gpt-oss-120b",
|
||||
}
|
||||
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
import asyncio
|
||||
import traceback
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import litellm.types
|
||||
import litellm.types.utils
|
||||
from litellm.llms.anthropic.chat import ModelResponseIterator
|
||||
|
||||
load_dotenv()
|
||||
import io
|
||||
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.llms.anthropic.common_utils import process_anthropic_headers
|
||||
from httpx import Headers
|
||||
from base_llm_unit_tests import BaseLLMChatTest
|
||||
|
||||
|
||||
@pytest.mark.flaky(retries=3, delay=2)
|
||||
class TestMistralCompletion(BaseLLMChatTest):
|
||||
test_basic_tool_calling = None
|
||||
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.set_verbose = True
|
||||
return {"model": "mistral/mistral-medium-latest"}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
import os
|
||||
from typing import Dict
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
from tests.llm_translation.base_audio_transcription_unit_tests import (
|
||||
BaseLLMAudioTranscriptionTest,
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("MISTRAL_API_KEY"),
|
||||
reason="MISTRAL_API_KEY not set, skipping Mistral audio transcription tests",
|
||||
)
|
||||
class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest):
|
||||
def get_base_audio_transcription_call_args(self) -> Dict:
|
||||
return {
|
||||
"model": "mistral/voxtral-mini-latest",
|
||||
}
|
||||
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
return litellm.LlmProviders.MISTRAL
|
||||
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
"""
|
||||
Test TogetherAI LLM
|
||||
"""
|
||||
|
||||
from base_llm_unit_tests import BaseLLMChatTest
|
||||
from tests._live_test_helpers import cheapest_together_chat_model
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
class TestTogetherAI(BaseLLMChatTest):
|
||||
test_basic_tool_calling = None
|
||||
test_function_calling_with_tool_response = None
|
||||
|
||||
def get_base_completion_call_args(self) -> dict:
|
||||
litellm.set_verbose = True
|
||||
return {"model": cheapest_together_chat_model(function_calling=True, response_schema=True)}
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
#
|
||||
# Wires OCR tests into the Redis-backed VCR cache so live provider
|
||||
# calls (Mistral OCR, Azure AI OCR, Azure Document Intelligence,
|
||||
# Vertex AI OCR) are replayed for 24h. See tests/llm_translation/Readme.md
|
||||
# Vertex AI OCR) are replayed for 24h. See tests/VCR_TESTING.md
|
||||
# for the design overview.
|
||||
|
||||
import pytest
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ def logger(monkeypatch: pytest.MonkeyPatch) -> RecordingLogger:
|
|||
|
||||
|
||||
TESTS_DIR: Final = Path(__file__).resolve().parents[1]
|
||||
PDF_PATH: Final = TESTS_DIR / "llm_translation" / "fixtures" / "dummy.pdf"
|
||||
PDF_PATH: Final = TESTS_DIR / "fixtures" / "dummy.pdf"
|
||||
PNG_PATH: Final = TESTS_DIR / "image_gen_tests" / "test_image.png"
|
||||
PINNED_CDN: Final = "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0"
|
||||
PDF_URL: Final = f"{PINNED_CDN}/tests/llm_translation/fixtures/dummy.pdf"
|
||||
|
|
@ -299,4 +299,3 @@ async def test_ocr(case: Case, monkeypatch: pytest.MonkeyPatch, logger: Recordin
|
|||
_assert_ocr_response(response, case.provider.model, case.document.expected_text)
|
||||
_assert_logged(await logger.wait_for_call(), response, case.provider.model, response.model, case.call)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Wires search tests into the Redis-backed VCR cache so live provider
|
||||
# calls (Brave, DataForSEO, DuckDuckGo, Exa, Firecrawl, Google PSE,
|
||||
# Linkup, Parallel.ai, Perplexity, SearchAPI, Searxng, Serper, Tavily)
|
||||
# are replayed for 24h. See tests/llm_translation/Readme.md for the
|
||||
# are replayed for 24h. See tests/VCR_TESTING.md for the
|
||||
# design overview.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ e.g. every `transform_retrieve_batch_response` returns a real `LiteLLMBatch`
|
|||
with `object == "batch"`, a valid status, and an int `created_at`.
|
||||
|
||||
This module is intentionally NOT named `test_*`: it holds no standalone tests
|
||||
and must not be collected on its own. It mirrors the established repo pattern in
|
||||
`tests/llm_translation/base_*_unit_tests.py`.
|
||||
and must not be collected on its own. Provider tests import and subclass this
|
||||
contract suite from their own unit-test files.
|
||||
|
||||
Providers that do NOT implement BaseBatchesConfig (e.g. vertex_ai, whose
|
||||
transformation is a standalone class with a different shape) cannot use this and
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Tests for GigaChat LiteLLM Provider
|
||||
|
||||
Tests message transformation, parameter handling, and response transformation.
|
||||
Run with: pytest tests/llm_translation/test_gigachat.py -v
|
||||
Run with: pytest tests/unit/llms/gigachat/chat/test_transformation.py -v
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
|
|||
|
|
@ -79,7 +79,6 @@ def test_mistral_audio_transcription_request_transform():
|
|||
os.path.dirname(__file__),
|
||||
"../../../../..",
|
||||
"tests",
|
||||
"llm_translation",
|
||||
"gettysburg.wav",
|
||||
)
|
||||
audio_file = open(wav_path, "rb")
|
||||
|
|
@ -109,7 +108,6 @@ def test_mistral_audio_transcription_request_with_diarize():
|
|||
os.path.dirname(__file__),
|
||||
"../../../../..",
|
||||
"tests",
|
||||
"llm_translation",
|
||||
"gettysburg.wav",
|
||||
)
|
||||
audio_file = open(wav_path, "rb")
|
||||
|
|
|
|||
|
|
@ -95,7 +95,6 @@ def _open_test_audio():
|
|||
os.path.dirname(__file__),
|
||||
"../../../..",
|
||||
"tests",
|
||||
"llm_translation",
|
||||
"gettysburg.wav",
|
||||
)
|
||||
return open(wav_path, "rb")
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 535 KiB After Width: | Height: | Size: 535 KiB |
|
Before Width: | Height: | Size: 1.5 MiB After Width: | Height: | Size: 1.5 MiB |
|
|
@ -1358,10 +1358,8 @@ def vertex_client():
|
|||
@pytest.fixture
|
||||
def encoded_images():
|
||||
image_paths = [
|
||||
"./tests/llm_translation/duck.png",
|
||||
# "./duck.png",
|
||||
"./tests/llm_translation/guinea.png",
|
||||
# "./guinea.png",
|
||||
"./tests/unit/llms/vertex_ai/fixtures/duck.png",
|
||||
"./tests/unit/llms/vertex_ai/fixtures/guinea.png",
|
||||
]
|
||||
return [encode_image_to_base64(path) for path in image_paths]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,15 @@
|
|||
"""
|
||||
Integration tests for Volcengine embedding following LiteLLM testing patterns
|
||||
Based on the BaseLLMEmbeddingTest framework
|
||||
"""
|
||||
"""Tests for Volcengine embedding."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
|
||||
# Add parent directory to path for imports
|
||||
|
||||
from tests.llm_translation.base_embedding_unit_tests import BaseLLMEmbeddingTest
|
||||
import litellm
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
|
||||
class TestVolcEngineEmbedding(BaseLLMEmbeddingTest):
|
||||
class TestVolcEngineEmbedding:
|
||||
"""Test Volcengine embedding integration following LiteLLM patterns"""
|
||||
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
|
|
|
|||
|
|
@ -8,10 +8,9 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from tests.llm_translation.base_embedding_unit_tests import BaseLLMEmbeddingTest
|
||||
|
||||
|
||||
class TestVoyageAI(BaseLLMEmbeddingTest):
|
||||
class TestVoyageAI:
|
||||
def get_custom_llm_provider(self) -> litellm.LlmProviders:
|
||||
return litellm.LlmProviders.VOYAGE
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ def create_skill_zip(skill_name: str):
|
|||
|
||||
The zip file is automatically cleaned up after use.
|
||||
"""
|
||||
test_dir = Path(__file__).parents[2] / "llm_translation" / "test_skills_data"
|
||||
test_dir = Path(__file__).parent / "test_skills_data"
|
||||
skill_dir = test_dir / skill_name
|
||||
|
||||
# Create a zip file containing the skill directory
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ def test_redis_key_is_stable_across_working_directories(tmp_path, monkeypatch):
|
|||
monkeypatch.chdir(repo_root)
|
||||
key_from_root = redis_key_for(abs_cassette)
|
||||
|
||||
monkeypatch.chdir(os.path.join(repo_root, "tests", "llm_translation"))
|
||||
monkeypatch.chdir(os.path.join(repo_root, "tests", "audio_tests"))
|
||||
key_from_subdir = redis_key_for(abs_cassette)
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue