diff --git a/tests/llm_translation/Readme.md b/tests/VCR_TESTING.md similarity index 94% rename from tests/llm_translation/Readme.md rename to tests/VCR_TESTING.md index aef28d38782..7db8da2c34e 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/VCR_TESTING.md @@ -1,4 +1,4 @@ -Unit tests for individual LLM providers. +Shared VCR documentation for provider tests. Name of the test file is the name of the LLM provider - e.g. `test_openai.py` is for OpenAI. @@ -25,7 +25,6 @@ The same VCR cache is used by other test directories that exercise live provider APIs. The reusable conftest plumbing lives in `tests/_vcr_conftest_common.py` and is wired into: -- `tests/llm_translation/` - `tests/audio_tests/` - `tests/batches_tests/` - `tests/guardrails_tests/` @@ -67,5 +66,5 @@ make test-llm-translation-flush-vcr-cache Skip the cache entirely (every call goes live, no recording): ```bash -LITELLM_VCR_DISABLE=1 uv run pytest tests/llm_translation/test_.py +LITELLM_VCR_DISABLE=1 uv run pytest tests/audio_tests/test_.py ``` diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 36ae70497e7..1774442cddc 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -1,7 +1,7 @@ """Shared VCR (Redis-backed) plumbing imported by per-directory conftests. -See ``tests/llm_translation/Readme.md`` for the full design and -``tests/llm_translation/conftest.py`` for the reference wiring.""" +See ``tests/VCR_TESTING.md`` for the full design and +``tests/_vcr_conftest_common.py`` for the shared wiring.""" from __future__ import annotations diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py index 852e10636d0..8aa5f680972 100644 --- a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -57,7 +57,7 @@ pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] WEATHER_WAV: Final = ( Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" ) -DUMMY_PDF: Final = Path(__file__).resolve().parent.parent.parent / "llm_translation" / "fixtures" / "dummy.pdf" +DUMMY_PDF: Final = Path(__file__).resolve().parents[2] / "fixtures" / "dummy.pdf" DUMMY_PDF_URL: Final = ( "https://cdn.jsdelivr.net/gh/BerriAI/litellm" "@d769e81c90d453240c61fc572cdb27fae06a89d0" diff --git a/tests/llm_translation/fixtures/dummy.pdf b/tests/fixtures/dummy.pdf similarity index 100% rename from tests/llm_translation/fixtures/dummy.pdf rename to tests/fixtures/dummy.pdf diff --git a/tests/llm_translation/base_audio_transcription_unit_tests.py b/tests/llm_translation/base_audio_transcription_unit_tests.py deleted file mode 100644 index bdf9f998d57..00000000000 --- a/tests/llm_translation/base_audio_transcription_unit_tests.py +++ /dev/null @@ -1,28 +0,0 @@ -import httpx -import json -from typing import Any, Dict, List -from unittest.mock import MagicMock, Mock, patch -import os -from litellm._uuid import uuid - -import litellm -from abc import ABC, abstractmethod - -pwd = os.path.dirname(os.path.realpath(__file__)) -print(pwd) - -file_path = os.path.join(pwd, "gettysburg.wav") - -audio_file = open(file_path, "rb") - -class BaseLLMAudioTranscriptionTest(ABC): - @abstractmethod - def get_base_audio_transcription_call_args(self) -> dict: - """Must return the base audio transcription call args""" - pass - - @abstractmethod - def get_custom_llm_provider(self) -> litellm.LlmProviders: - """Must return the custom llm provider""" - pass - diff --git a/tests/llm_translation/base_embedding_unit_tests.py b/tests/llm_translation/base_embedding_unit_tests.py deleted file mode 100644 index 7efd08896c8..00000000000 --- a/tests/llm_translation/base_embedding_unit_tests.py +++ /dev/null @@ -1,64 +0,0 @@ -import asyncio -import httpx -import json -import pytest -from typing import Any, Dict, List -from unittest.mock import MagicMock, Mock, patch - -import litellm -from litellm.exceptions import BadRequestError -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.utils import ( - CustomStreamWrapper, - get_supported_openai_params, - get_optional_params, -) -import base64 -from pathlib import Path - -from abc import ABC, abstractmethod - -file_data = (Path(__file__).parent.parent / "white_100x100.png").read_bytes() - -encoded_file = base64.b64encode(file_data).decode("utf-8") -base64_image = f"data:image/png;base64,{encoded_file}" - -class BaseLLMEmbeddingTest(ABC): - """ - Abstract base test class that enforces a common test across all test classes. - """ - - @abstractmethod - def get_base_embedding_call_args(self) -> dict: - """Must return the base embedding call args""" - pass - - @abstractmethod - def get_custom_llm_provider(self) -> litellm.LlmProviders: - """Must return the custom llm provider""" - pass - - @pytest.mark.asyncio() - @pytest.mark.parametrize("sync_mode", [True, False]) - async def test_basic_embedding(self, sync_mode): - litellm.set_verbose = True - embedding_call_args = self.get_base_embedding_call_args() - if sync_mode is True: - response = litellm.embedding( - **embedding_call_args, - input=["hello", "world"], - ) - - print("embedding response: ", response) - else: - response = await litellm.aembedding( - **embedding_call_args, - input=["hello", "world"], - ) - - print("async embedding response: ", response) - - from openai.types.create_embedding_response import CreateEmbeddingResponse - - CreateEmbeddingResponse.model_validate(response.model_dump()) - diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py deleted file mode 100644 index d8d53dcc45c..00000000000 --- a/tests/llm_translation/base_llm_unit_tests.py +++ /dev/null @@ -1,571 +0,0 @@ -import json -import pytest -import sys -from typing import Any, Dict, List -from unittest.mock import MagicMock, Mock -import os -import inspect - -import litellm -from litellm.exceptions import BadRequestError -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.utils import ( - CustomStreamWrapper, - get_supported_openai_params, -) -from typing import Union -from litellm.types.utils import Usage, ModelResponse - -# test_example.py -from abc import ABC, abstractmethod -from openai import OpenAI - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) - -class BaseLLMChatTest(ABC): - """ - Abstract base test class that enforces a common test across all test classes. - """ - - @property - def completion_function(self): - return litellm.completion - - @property - def async_completion_function(self): - return litellm.acompletion - - @abstractmethod - def get_base_completion_call_args(self) -> dict: - """Must return the base completion call args""" - pass - - def get_base_completion_call_args_with_reasoning_model(self) -> dict: - """Must return the base completion call args with reasoning_effort""" - return {} - - @pytest.fixture(autouse=True) - def _handle_rate_limits(self): - """Fixture to handle rate limit errors for all test methods""" - try: - yield - except litellm.RateLimitError: - pytest.skip("Rate limit exceeded") - except litellm.InternalServerError: - pytest.skip("Model is overloaded") - - def test_streaming(self): - """Check if litellm handles streaming correctly""" - from litellm.types.utils import ModelResponseStream - from typing import Optional - - base_completion_call_args = self.get_base_completion_call_args() - # litellm.set_verbose = True - messages = [ - { - "role": "user", - "content": [{"type": "text", "text": "Hello, how are you?"}], - } - ] - try: - response = self.completion_function( - **base_completion_call_args, - messages=messages, - stream=True, - ) - assert response is not None - assert isinstance(response, CustomStreamWrapper) - except litellm.InternalServerError: - pytest.skip("Model is overloaded") - - # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None - chunks = [] - created_at: Optional[int] = None - for chunk in response: - print(chunk) - chunks.append(chunk) - if isinstance(chunk, ModelResponseStream): - if created_at is None: - created_at = chunk.created - assert chunk.created == created_at - - resp = litellm.stream_chunk_builder(chunks=chunks) - print(resp) - - # assert resp.usage.prompt_tokens > 0 - # assert resp.usage.completion_tokens > 0 - # assert resp.usage.total_tokens > 0 - - def test_web_search(self): - from litellm.utils import supports_web_search - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - litellm.turn_on_debug() - - base_completion_call_args = self.get_base_completion_call_args() - - if not supports_web_search(base_completion_call_args["model"], None): - pytest.skip("Model does not support web search") - - response = self.completion_function( - **base_completion_call_args, - messages=[ - {"role": "user", "content": "What's the weather like in Boston today?"} - ], - web_search_options={}, - max_tokens=100, - ) - - assert response is not None - - print(f"response={response}") - - def test_url_context(self): - from litellm.utils import supports_url_context - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - litellm.turn_on_debug() - - base_completion_call_args = self.get_base_completion_call_args() - - if not supports_url_context(base_completion_call_args["model"], None): - pytest.skip("Model does not support url context") - - response = self.completion_function( - **base_completion_call_args, - messages=[ - { - "role": "user", - "content": "Summarize the content of this URL: https://en.wikipedia.org/wiki/Artificial_intelligence", - } - ], - tools=[{"urlContext": {}}], - ) - - assert response is not None - print(f"response={response}") - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_pdf_handling(self, pdf_messages, sync_mode): - from litellm.utils import supports_pdf_input - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - litellm.turn_on_debug() - - image_content = [ - {"type": "text", "text": "What's this file about?"}, - { - "type": "file", - "file": { - "file_data": pdf_messages, - }, - }, - ] - - image_messages = [{"role": "user", "content": image_content}] - - base_completion_call_args = self.get_base_completion_call_args() - - if not supports_pdf_input(base_completion_call_args["model"], None): - pytest.skip("Model does not support image input") - - if sync_mode: - response = self.completion_function( - **base_completion_call_args, - messages=image_messages, - ) - else: - response = await self.async_completion_function( - **base_completion_call_args, - messages=image_messages, - ) - - assert response is not None - - @pytest.mark.asyncio - async def test_async_pdf_handling_with_file_id(self): - from litellm.utils import supports_pdf_input - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - litellm.turn_on_debug() - - image_content = [ - {"type": "text", "text": "What's this file about?"}, - { - "type": "file", - "file": { - "file_id": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/tests/llm_translation/fixtures/dummy.pdf" - }, - }, - ] - - image_messages = [{"role": "user", "content": image_content}] - - base_completion_call_args = self.get_base_completion_call_args() - - if not supports_pdf_input(base_completion_call_args["model"], None): - pytest.skip("Model does not support image input") - - response = await self.async_completion_function( - **base_completion_call_args, - messages=image_messages, - ) - - assert response is not None - - @pytest.fixture - def tool_call_no_arguments(self): - return { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_2c384bc6-de46-4f29-8adc-60dd5805d305", - "function": {"name": "Get-FAQ", "arguments": "{}"}, - "type": "function", - } - ], - } - - @pytest.fixture - def pdf_messages(self): - import base64 - import os - - # Use local PDF file instead of external URL to avoid flaky tests - test_dir = os.path.dirname(__file__) - pdf_path = os.path.join(test_dir, "fixtures", "dummy.pdf") - - with open(pdf_path, "rb") as f: - file_data = f.read() - - encoded_file = base64.b64encode(file_data).decode("utf-8") - url = f"data:application/pdf;base64,{encoded_file}" - - return url - - @pytest.mark.flaky(retries=3, delay=1) - def test_basic_tool_calling(self): - try: - from litellm import completion, ModelResponse - - litellm.set_verbose = True - litellm.turn_on_debug() - from litellm.utils import supports_function_calling - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - base_completion_call_args = self.get_base_completion_call_args() - if not supports_function_calling(base_completion_call_args["model"], None): - print("Model does not support function calling") - pytest.skip("Model does not support function calling") - - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in fahrenheit?", - } - ] - request_args = { - "messages": messages, - "tools": tools, - } - request_args.update(self.get_base_completion_call_args()) - response: ModelResponse = completion(**request_args) # type: ignore - print(f"response: {response}") - - assert response is not None - - # if the provider did not return any tool calls do not make a subsequent llm api call - if response.choices[0].message.content is not None: - try: - json.loads(response.choices[0].message.content) - pytest.fail(f"Tool call returned in content instead of tool_calls") - except Exception as e: - print(f"Error: {e}") - pass - if ( - "" in response.choices[0].message.content - and "" in response.choices[0].message.content - ): - pytest.fail( - "Thinking block returned in content instead of separate reasoning_content" - ) - if response.choices[0].message.tool_calls is None: - return - # Add any assertions here to check the response - - assert isinstance( - response.choices[0].message.tool_calls[0].function.name, str - ) - assert isinstance( - response.choices[0].message.tool_calls[0].function.arguments, str - ) - assert ( - response.choices[0].finish_reason == "tool_calls" - ), f"finish_reason: {response.choices[0].finish_reason}, expected: tool_calls" - messages.append( - response.choices[0].message.model_dump() - ) # Add assistant tool invokes - tool_result = ( - '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' - ) - # Add user submitted tool results in the OpenAI format - messages.append( - { - "tool_call_id": response.choices[0].message.tool_calls[0].id, - "role": "tool", - "name": response.choices[0].message.tool_calls[0].function.name, - "content": tool_result, - } - ) - # In the second response, Claude should deduce answer from tool results - request_2_args = { - "messages": messages, - "tools": tools, - } - request_2_args.update(self.get_base_completion_call_args()) - second_response: ModelResponse = completion(**request_2_args) # type: ignore - print(f"second response: {second_response}") - assert second_response is not None - - # either content or tool calls should be present - assert ( - second_response.choices[0].message.content is not None - or second_response.choices[0].message.tool_calls is not None - ) - except litellm.ServiceUnavailableError: - pytest.skip("Model is overloaded") - except litellm.InternalServerError: - pytest.skip("Model is overloaded") - except litellm.RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - def test_function_calling_with_tool_response(self): - from litellm.utils import supports_function_calling - from litellm import completion - - litellm.turn_on_debug() - try: - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - base_completion_call_args = self.get_base_completion_call_args() - if not supports_function_calling(base_completion_call_args["model"], None): - print("Model does not support function calling") - pytest.skip("Model does not support function calling") - - def get_weather(city: str): - return f"City: {city}, Weather: Sunny with 34 degree Celcius" - - TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather in a city", - "parameters": { - "$id": "https://some/internal/name", - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to get the weather for", - } - }, - "required": ["city"], - "additionalProperties": False, - }, - "strict": True, - }, - } - ] - - messages = [{"content": "How is the weather in Mumbai?", "role": "user"}] - response, iteration = "", 0 - while True: - if response: - break - # Create a streaming response with tool calling enabled - stream = completion( - **base_completion_call_args, - messages=messages, - tools=TOOLS, - stream=True, - ) - - final_tool_calls = {} - for chunk in stream: - delta = chunk.choices[0].delta - print(delta) - if delta.content: - response += delta.content - elif delta.tool_calls: - for tool_call in chunk.choices[0].delta.tool_calls or []: - index = tool_call.index - if index not in final_tool_calls: - final_tool_calls[index] = tool_call - else: - final_tool_calls[ - index - ].function.arguments += tool_call.function.arguments - if final_tool_calls: - for tool_call in final_tool_calls.values(): - if tool_call.function.name == "get_weather": - city = json.loads(tool_call.function.arguments)["city"] - tool_response = get_weather(city) - messages.append( - { - "role": "assistant", - "tool_calls": [tool_call], - "content": None, - } - ) - messages.append( - { - "role": "tool", - "tool_call_id": tool_call.id, - "content": tool_response, - } - ) - iteration += 1 - if iteration > 2: - print("Something went wrong!") - break - - print(response) - except litellm.ServiceUnavailableError: - pass - -class BaseOSeriesModelsTest(ABC): # test across azure/openai - @abstractmethod - def get_base_completion_call_args(self): - pass - - @abstractmethod - def get_client(self) -> OpenAI: - pass - -class BaseAnthropicChatTest(ABC): - """ - Ensures consistent result across anthropic model usage - """ - - @abstractmethod - def get_base_completion_call_args(self) -> dict: - """Must return the base completion call args""" - pass - - @abstractmethod - def get_base_completion_call_args_with_thinking(self) -> dict: - """Must return the base completion call args""" - pass - - @property - def completion_function(self): - return litellm.completion - -class BaseReasoningLLMTests(ABC): - """ - Base class for testing reasoning llms - - - test that the responses contain reasoning_content - - test that the usage contains reasoning_tokens - """ - - @abstractmethod - def get_base_completion_call_args(self) -> dict: - """Must return the base completion call args""" - pass - - @property - def completion_function(self): - return litellm.completion - - def test_non_streaming_reasoning_effort(self): - """ - Base test for non-streaming reasoning effort - - - Assert that `reasoning_content` is not None from response message - - Assert that `reasoning_tokens` is greater than 0 from usage - """ - litellm.turn_on_debug() - base_completion_call_args = self.get_base_completion_call_args() - response: ModelResponse = self.completion_function( - **base_completion_call_args, reasoning_effort="low" - ) - - # user gets `reasoning_content` in the response message - assert response.choices[0].message.reasoning_content is not None - assert isinstance(response.choices[0].message.reasoning_content, str) - - # user get `reasoning_tokens` - assert response.usage.completion_tokens_details.reasoning_tokens > 0 - - def test_streaming_reasoning_effort(self): - """ - Base test for streaming reasoning effort - - - Assert that `reasoning_content` is not None from streaming response - - Assert that `reasoning_tokens` is greater than 0 from usage - """ - # litellm.turn_on_debug() - base_completion_call_args = self.get_base_completion_call_args() - response: CustomStreamWrapper = self.completion_function( - **base_completion_call_args, - reasoning_effort="low", - stream=True, - stream_options={"include_usage": True}, - ) - - resoning_content: str = "" - usage: Usage = None - for chunk in response: - print(chunk) - if hasattr(chunk.choices[0].delta, "reasoning_content"): - resoning_content += chunk.choices[0].delta.reasoning_content - if hasattr(chunk, "usage"): - usage = chunk.usage - - assert resoning_content is not None - assert len(resoning_content) > 0 - - print(f"usage: {usage}") - assert usage.completion_tokens_details.reasoning_tokens > 0 diff --git a/tests/llm_translation/base_rerank_unit_tests.py b/tests/llm_translation/base_rerank_unit_tests.py deleted file mode 100644 index dbf7d0fd1ff..00000000000 --- a/tests/llm_translation/base_rerank_unit_tests.py +++ /dev/null @@ -1,85 +0,0 @@ -import asyncio -import httpx -import json -from typing import Any, Dict, List -from unittest.mock import MagicMock, Mock, patch - -import litellm -from litellm.exceptions import BadRequestError -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.utils import ( - CustomStreamWrapper, - get_supported_openai_params, - get_optional_params, -) - -# test_example.py -from abc import ABC, abstractmethod - -def assert_response_shape(response, custom_llm_provider): - expected_response_shape = {"id": str, "results": list, "meta": dict} - - expected_results_shape = {"index": int, "relevance_score": float} - - expected_meta_shape = {"api_version": dict, "billed_units": dict} - - expected_api_version_shape = {"version": str} - - expected_billed_units_shape = {"search_units": int} - expected_billed_units_total_tokens_shape = {"total_tokens": int} - - assert isinstance(response.id, expected_response_shape["id"]) - assert isinstance(response.results, expected_response_shape["results"]) - for result in response.results: - assert isinstance(result["index"], expected_results_shape["index"]) - assert isinstance( - result["relevance_score"], expected_results_shape["relevance_score"] - ) - assert isinstance(response.meta, expected_response_shape["meta"]) - - if custom_llm_provider == "cohere": - - assert isinstance( - response.meta["api_version"], expected_meta_shape["api_version"] - ) - assert isinstance( - response.meta["api_version"]["version"], - expected_api_version_shape["version"], - ) - assert isinstance( - response.meta["billed_units"], expected_meta_shape["billed_units"] - ) - if "total_tokens" in response.meta["billed_units"]: - assert isinstance( - response.meta["billed_units"]["total_tokens"], - expected_billed_units_total_tokens_shape["total_tokens"], - ) - else: - assert isinstance( - response.meta["billed_units"]["search_units"], - expected_billed_units_shape["search_units"], - ) - -class BaseLLMRerankTest(ABC): - """ - Abstract base test class that enforces a common test across all test classes. - """ - - @abstractmethod - def get_base_rerank_call_args(self) -> dict: - """Must return the base rerank call args""" - pass - - @abstractmethod - def get_custom_llm_provider(self) -> litellm.LlmProviders: - """Must return the custom llm provider""" - pass - - def get_expected_cost(self) -> float: - """ - Override this method to set the expected cost for the rerank call. - Default is None, which means the test will check cost > 0. - Return 0.0 for free models. - """ - return None - diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py deleted file mode 100644 index a88dcf4ae3e..00000000000 --- a/tests/llm_translation/conftest.py +++ /dev/null @@ -1,194 +0,0 @@ -# conftest.py -# -# xdist-compatible test isolation for llm_translation tests. -# Mirrors the pattern in tests/local_testing/conftest.py: -# - Function-scoped fixture resets litellm globals to true defaults -# - Module-scoped reload only in single-process mode - -import asyncio -import importlib -from collections.abc import Generator -from typing import Final - -import pytest - - -import litellm # noqa: E402 - -from tests._vcr_conftest_common import ( # noqa: E402,F401 - VerboseReporterState, - _pin_multipart_boundary, - apply_vcr_auto_marker_to_items, - emit_cassette_cache_session_banner, - emit_vcr_classification_summary, - emit_vcr_diagnostic_log, - guard_vcr_patch_points, - install_live_call_probe, - record_vcr_outcome, - register_persister_if_enabled, - reset_vcr_diag_dir, - vcr_config_dict, -) -from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 - - -@pytest.fixture(scope="session", autouse=True) -def fake_openai_endpoint(): - ensure_fake_openai_endpoint() - yield - - -# Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles -# the vast majority of respx-vs-vcrpy conflicts automatically. The entries -# below are the persister's, the WebSocket VCR's, and the cassette patch-leak -# guard's own unit-test files, which exercise ``save_cassette`` / -# ``load_cassette`` against fakeredis or enter cassettes themselves and must -# not run under a live cassette context. -_VCR_AUTO_MARKER_SKIP_FILES = frozenset( - {"test_vcr_redis_persister.py", "test_ws_vcr.py", "test_vcr_leak_guard.py"} -) - - -_verbose_state = VerboseReporterState() - - -@pytest.fixture(scope="module") -def vcr_config(): - return vcr_config_dict() - - -def pytest_recording_configure(config, vcr): - register_persister_if_enabled(vcr) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - rep = outcome.get_result() - setattr(item, f"rep_{rep.when}", rep) - - -@pytest.fixture(autouse=True) -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - - -@pytest.hookimpl(wrapper=True, trylast=True) -def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, object, object]: - try: - result: Final = yield - except BaseException: - guard_vcr_patch_points(item, teardown_failed=True) - raise - guard_vcr_patch_points(item, teardown_failed=False) - return result - - -def pytest_configure(config): - _verbose_state.remember_pluginmanager(config) - reset_vcr_diag_dir() - - -def pytest_runtest_logreport(report): - _verbose_state.maybe_emit_verdict(report) - - -def pytest_terminal_summary(terminalreporter, exitstatus, config): - emit_cassette_cache_session_banner(terminalreporter) - emit_vcr_classification_summary(terminalreporter) - emit_vcr_diagnostic_log(terminalreporter) - - -# --------------------------------------------------------------------------- -# Capture TRUE defaults at conftest import time (before test modules pollute). -# --------------------------------------------------------------------------- -_SCALAR_DEFAULTS = { - "num_retries": getattr(litellm, "num_retries", None), - "set_verbose": getattr(litellm, "set_verbose", False), - "cache": getattr(litellm, "cache", None), - "allowed_fails": getattr(litellm, "allowed_fails", 3), - "disable_aiohttp_transport": getattr(litellm, "disable_aiohttp_transport", False), - "force_ipv4": getattr(litellm, "force_ipv4", False), - "drop_params": getattr(litellm, "drop_params", None), - "modify_params": getattr(litellm, "modify_params", False), - "api_base": getattr(litellm, "api_base", None), - "api_key": getattr(litellm, "api_key", None), - "cohere_key": getattr(litellm, "cohere_key", None), -} - - -@pytest.fixture(scope="session") -def event_loop(): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(event_loop): # Add event_loop as a dependency - - import litellm - - # ---- Save current state (for teardown restore) ---- - original_state = {} - for attr in ( - "callbacks", - "success_callback", - "failure_callback", - "_async_success_callback", - "_async_failure_callback", - ): - if hasattr(litellm, attr): - val = getattr(litellm, attr) - original_state[attr] = val.copy() if val else [] - - for attr in _SCALAR_DEFAULTS: - if hasattr(litellm, attr): - original_state[attr] = getattr(litellm, attr) - - # ---- Reset to true defaults before the test ---- - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - importlib.reload(litellm) - - # Set the event loop from the fixture - asyncio.set_event_loop(event_loop) - - yield - - # ---- Teardown ---- - for attr, original_value in original_state.items(): - if hasattr(litellm, attr): - setattr(litellm, attr, original_value) - - # Clean up any pending tasks - pending = asyncio.all_tasks(event_loop) - for task in pending: - task.cancel() - - # Run the event loop until all tasks are cancelled - if pending: - event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - - -def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items( - items, - skip_files=_VCR_AUTO_MARKER_SKIP_FILES, - ) - - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - items[:] = custom_logger_tests + other_tests diff --git a/tests/llm_translation/databricks_config.template.txt b/tests/llm_translation/databricks_config.template.txt deleted file mode 100644 index 7352fdbc773..00000000000 --- a/tests/llm_translation/databricks_config.template.txt +++ /dev/null @@ -1,78 +0,0 @@ -# Databricks Configuration Template for LiteLLM Testing -# ===================================================== -# -# Copy this file to your preferred location and fill in your credentials: -# cp databricks_config.template.txt /path/to/databricks_config.txt -# -# Then update the CONFIG_FILE path in test_databricks_integration.py -# -# Lines starting with # are comments and will be ignored -# Only lines with KEY=VALUE format (where VALUE is not empty) will be read - -# ============================================================================== -# DATABRICKS WORKSPACE CONFIGURATION (Required) -# ============================================================================== - -# Your Databricks workspace URL (without /serving-endpoints suffix) -# Example: https://adb-1234567890123456.7.azuredatabricks.net -DATABRICKS_HOST= - -# API Base URL for serving endpoints (usually {host}/serving-endpoints) -# Example: https://adb-1234567890123456.7.azuredatabricks.net/serving-endpoints -DATABRICKS_API_BASE= - -# ============================================================================== -# AUTHENTICATION METHOD 1: OAuth M2M (Recommended for Production) -# Use Service Principal credentials -# ============================================================================== - -# Service Principal Application/Client ID -# Example: 12345678-1234-1234-1234-123456789012 -DATABRICKS_CLIENT_ID= - -# Service Principal Secret -# Example: your-client-secret-value -DATABRICKS_CLIENT_SECRET= - -# ============================================================================== -# AUTHENTICATION METHOD 2: Personal Access Token (PAT) -# For development and testing -# ============================================================================== - -# Personal Access Token (starts with 'dapi') -# Example: dapi_your_token_here -DATABRICKS_API_KEY= - -# ============================================================================== -# MODEL CONFIGURATION -# ============================================================================== - -# Model to use for testing chat completions -# Example: databricks-gpt-oss-120b, databricks-meta-llama-3-1-70b-instruct -TEST_CHAT_MODEL=databricks-gpt-oss-120b - -# Model to use for testing embeddings (optional) -# Example: databricks-bge-large-en -TEST_EMBEDDING_MODEL=databricks-bge-large-en - -# ============================================================================== -# OPTIONAL: Custom User-Agent for Partner Attribution Testing -# ============================================================================== - -# Custom user agent string to test partner attribution -# Example: mycompany/1.0.0 -# This will result in User-Agent: mycompany_litellm/{version} -# Leave empty to use default: litellm/{version} -CUSTOM_USER_AGENT= - -# ============================================================================== -# TEST SETTINGS -# ============================================================================== - -# Which authentication method to test: oauth, pat, sdk, or all -# oauth = Use DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET -# pat = Use DATABRICKS_API_KEY -# sdk = Use Databricks SDK automatic authentication (~/.databrickscfg) -# all = Test all three methods (oauth, pat, sdk) in sequence -TEST_AUTH_METHOD=pat - diff --git a/tests/llm_translation/dog.wav b/tests/llm_translation/dog.wav deleted file mode 100644 index 3ca0b533b51..00000000000 Binary files a/tests/llm_translation/dog.wav and /dev/null differ diff --git a/tests/llm_translation/gettysburg.wav b/tests/llm_translation/gettysburg.wav deleted file mode 100644 index 9690f521e84..00000000000 Binary files a/tests/llm_translation/gettysburg.wav and /dev/null differ diff --git a/tests/llm_translation/log.xt b/tests/llm_translation/log.xt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/llm_translation/realtime/__init__.py b/tests/llm_translation/realtime/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/llm_translation/realtime/base_realtime_tests.py b/tests/llm_translation/realtime/base_realtime_tests.py deleted file mode 100644 index 82665235f67..00000000000 --- a/tests/llm_translation/realtime/base_realtime_tests.py +++ /dev/null @@ -1,441 +0,0 @@ -""" -Base test class for LiteLLM Realtime API E2E tests. - -Provides common test infrastructure for testing realtime WebSocket connections -across different providers (OpenAI, xAI, etc.) -""" - -import asyncio -import json -import os -from abc import ABC, abstractmethod -from typing import Optional, Tuple, Union - -import pytest -import websockets - - -import litellm - - -class RealTimeWebSocketClient: - """ - Mock WebSocket client for testing realtime connections. - Captures messages sent from the backend and provides a simple interface - for testing connection success. - """ - - def __init__(self): - self.messages_sent = [] - self.messages_received = [] - self.received_initial_event = False - self.connection_successful = False - self.close_code = None - self.close_reason = None - # Required by realtime_streaming.py - import exceptions module - from websockets import exceptions as websockets_exceptions - - self.exceptions = websockets_exceptions - - async def accept(self): - """Accept the WebSocket connection""" - pass - - async def send_text(self, message): - """Receive message from backend and store it""" - self.messages_sent.append(message) - try: - if isinstance(message, bytes): - message_str = message.decode("utf-8") - else: - message_str = message - - msg_data = json.loads(message_str) - msg_type = msg_data.get("type", "unknown") - - # Pretty print API response - print(f"\n{'='*80}") - print( - f"API RESPONSE #{len(self.messages_received) + 1} - Event: {msg_type}" - ) - print(f"{'='*80}") - print(json.dumps(msg_data, indent=2, sort_keys=False)) - print(f"{'='*80}\n") - - self.messages_received.append(msg_data) - - # Check for initial connection event - if not self.received_initial_event and self._is_initial_event(msg_type): - self.received_initial_event = True - self.connection_successful = True - - except (json.JSONDecodeError, UnicodeDecodeError) as e: - # Non-JSON messages are acceptable - print(f"\n[Non-JSON message: {e}]") - print(f"Raw content: {str(message)[:200]}\n") - pass - - def _is_initial_event(self, msg_type: str) -> bool: - """Check if message type is an initial connection event""" - # OpenAI and xAI send "session.created"; some providers send "conversation.created" - return msg_type in ["session.created", "conversation.created"] - - async def receive_text(self): - """ - Wait briefly for messages, then close connection. - This allows the backend forwarding task to send messages. - """ - print(f"\nWaiting for connection to establish...") - max_wait = 5.0 - check_interval = 0.1 - waited = 0.0 - - while waited < max_wait: - if self.connection_successful: - print(f"Connection successful after {waited:.1f}s\n") - break - await asyncio.sleep(check_interval) - waited += check_interval - - if not self.connection_successful: - print(f"Warning: No initial event received after {max_wait}s\n") - - # If we have a pending message to send, send it now - if hasattr(self, "_pending_client_message") and self._pending_client_message: - print(f"Sending client message to backend...\n") - # This simulates receiving a message from the client that needs to be forwarded to backend - # We return it as if it came from the client - msg = self._pending_client_message - self._pending_client_message = None - return msg - - # Close connection to end the test - print(f"\n{'='*80}") - print(f"TEST COMPLETE - Closing connection") - print(f"Total messages received from API: {len(self.messages_received)}") - print(f"{'='*80}\n") - raise websockets.exceptions.ConnectionClosed(None, None) - - def queue_client_message(self, message: str): - """Queue a message to be sent from 'client' to backend""" - self._pending_client_message = message - - async def close(self, code=1000, reason=""): - """Close the WebSocket""" - self.close_code = code - self.close_reason = reason - - @property - def headers(self): - return {} - - -class BaseRealtimeTest(ABC): - """ - Abstract base test class for realtime API tests. - - Child classes must implement: - - get_model(): Return the model name to test - - get_api_key_env_var(): Return the environment variable name for the API key - - get_initial_event_type(): Return the expected initial event type (e.g., "session.created") - """ - - @abstractmethod - def get_model(self) -> str: - """Return the model name to test (e.g., 'gpt-4o-realtime-preview-2024-10-01')""" - pass - - @abstractmethod - def get_api_key_env_var(self) -> str: - """Return the environment variable name for the API key (e.g., 'OPENAI_API_KEY')""" - pass - - @abstractmethod - def get_initial_event_type(self) -> Union[str, Tuple[str, ...]]: - """Return the expected initial event type(s). - - May return a single event type (e.g. ``'session.created'``) or a tuple - of acceptable types when the upstream provider can legitimately emit - more than one initial event (e.g. xAI's Grok Voice Agent has shipped - both ``conversation.created`` and ``session.created``). - """ - pass - - def get_skip_reason(self) -> str: - """Return the skip reason when API key is missing""" - return f"No {self.get_api_key_env_var()} provided" - - def should_skip(self) -> bool: - """Check if tests should be skipped due to missing API key""" - return os.environ.get(self.get_api_key_env_var()) is None - - @pytest.mark.asyncio - async def test_realtime_connection(self): - """ - Test basic realtime WebSocket connection. - Verifies that: - 1. Connection is established successfully - 2. Initial event is received - 3. Messages are properly forwarded - """ - litellm.turn_on_debug() - if self.should_skip(): - pytest.skip(self.get_skip_reason()) - - websocket_client = RealTimeWebSocketClient() - caught_exception = None - - print(f"\n{'='*80}") - print(f"STARTING REALTIME CONNECTION TEST") - print(f"Model: {self.get_model()}") - print(f"API Key Env Var: {self.get_api_key_env_var()}") - print(f"{'='*80}\n") - - try: - await litellm._arealtime( - model=self.get_model(), - websocket=websocket_client, - api_key=os.environ.get(self.get_api_key_env_var()), - timeout=60, - ) - except websockets.exceptions.ConnectionClosed: - pass - except Exception as e: - print(f"\nException: {type(e).__name__}: {e}\n") - caught_exception = e - - # Build debug info - error_details = [] - error_details.append(f"messages_sent: {len(websocket_client.messages_sent)}") - error_details.append( - f"messages_received: {len(websocket_client.messages_received)}" - ) - error_details.append(f"close_code: {websocket_client.close_code}") - error_details.append(f"close_reason: {websocket_client.close_reason}") - if caught_exception: - error_details.append( - f"exception: {type(caught_exception).__name__}: {caught_exception}" - ) - - # Assertions - assert ( - websocket_client.connection_successful - ), f"Failed to connect. Debug: {'; '.join(error_details)}" - assert websocket_client.received_initial_event, f"Did not receive initial event" - assert len(websocket_client.messages_received) > 0, "No messages received" - - # Verify initial event - initial_event = websocket_client.messages_received[0] - expected_event_type = self.get_initial_event_type() - if isinstance(expected_event_type, str): - allowed_event_types: Tuple[str, ...] = (expected_event_type,) - else: - allowed_event_types = tuple(expected_event_type) - assert ( - initial_event["type"] in allowed_event_types - ), f"Expected one of {allowed_event_types}, got {initial_event.get('type')}" - - @pytest.mark.asyncio - async def test_realtime_with_query_params(self): - """ - Test realtime connection with explicit query parameters. - Verifies that query params are properly passed to the backend. - """ - litellm.turn_on_debug() - if self.should_skip(): - pytest.skip(self.get_skip_reason()) - - from litellm.types.realtime import RealtimeQueryParams - - websocket_client = RealTimeWebSocketClient() - caught_exception = None - - # Strip provider prefix from model name for query params - model_name = self.get_model() - if "/" in model_name: - model_name = model_name.split("/", 1)[1] - - query_params: RealtimeQueryParams = {"model": model_name} - - try: - await litellm._arealtime( - model=self.get_model(), - websocket=websocket_client, - api_key=os.environ.get(self.get_api_key_env_var()), - query_params=query_params, - timeout=60, - ) - except websockets.exceptions.ConnectionClosed: - pass - except Exception as e: - caught_exception = e - - # Build debug info - error_details = [] - error_details.append(f"messages_sent: {len(websocket_client.messages_sent)}") - error_details.append( - f"messages_received: {len(websocket_client.messages_received)}" - ) - if caught_exception: - error_details.append( - f"exception: {type(caught_exception).__name__}: {caught_exception}" - ) - - # Skip on transient failures - if ( - not websocket_client.connection_successful - and websocket_client.close_code is not None - ): - pytest.skip(f"Transient connection failure: {'; '.join(error_details)}") - - # Assertions - assert ( - websocket_client.connection_successful - ), f"Failed to connect. Debug: {'; '.join(error_details)}" - assert len(websocket_client.messages_received) > 0, "No messages received" - - @pytest.mark.asyncio - async def test_send_user_message(self): - """ - Test sending an actual user message and receiving responses. - This creates a more realistic conversation flow. - """ - if self.should_skip(): - pytest.skip(self.get_skip_reason()) - - litellm.turn_on_debug() - - # Create a custom websocket client that sends a message - class InteractiveWebSocketClient(RealTimeWebSocketClient): - def __init__(self): - super().__init__() - self.sent_user_message = False - self.response_messages = [] - self.wait_for_responses = True - - async def receive_text(self): - """Enhanced receive that sends a user message after connection""" - print(f"\n{'='*80}") - print(f"CLIENT-SIDE RECEIVE HANDLER") - print(f"{'='*80}\n") - - # Wait for initial connection - max_wait = 5.0 - check_interval = 0.1 - waited = 0.0 - - while waited < max_wait: - if self.connection_successful: - print(f"Connection established after {waited:.1f}s\n") - break - await asyncio.sleep(check_interval) - waited += check_interval - - # Step 1: Send a user message after connection is established - if self.connection_successful and not self.sent_user_message: - self.sent_user_message = True - user_msg_data = { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "Say hi back to me!"} - ], - }, - } - user_msg = json.dumps(user_msg_data) - - print(f"\n{'='*80}") - print(f"STEP 1: SENDING USER MESSAGE TO BACKEND") - print(f"{'='*80}") - print(json.dumps(user_msg_data, indent=2)) - print(f"{'='*80}\n") - - return user_msg - - # Step 2: Trigger the response after user message is acknowledged - if not hasattr(self, "triggered_response"): - self.triggered_response = True - # Wait a bit for the user message to be processed - await asyncio.sleep(0.5) - - response_create_data = {"type": "response.create"} - response_create = json.dumps(response_create_data) - - print(f"\n{'='*80}") - print(f"STEP 2: TRIGGERING LLM RESPONSE") - print(f"{'='*80}") - print(json.dumps(response_create_data, indent=2)) - print(f"{'='*80}\n") - - return response_create - - # Step 3: Wait for LLM responses - if self.wait_for_responses: - print(f"\nSTEP 3: Waiting 5 seconds for LLM to respond...\n") - await asyncio.sleep(5.0) - self.wait_for_responses = False - - # Collect response info - for msg in self.messages_received: - msg_type = msg.get("type", "unknown") - if msg_type not in ["conversation.created", "ping"]: - self.response_messages.append(msg) - - print( - f"\nReceived {len(self.response_messages)} response messages (excluding init/ping)\n" - ) - - print(f"\n{'='*80}") - print(f"CLOSING CONNECTION") - print(f"Total messages received: {len(self.messages_received)}") - print(f"{'='*80}\n") - raise websockets.exceptions.ConnectionClosed(None, None) - - websocket_client = InteractiveWebSocketClient() - caught_exception = None - - print(f"\n{'='*80}") - print(f"STARTING INTERACTIVE MESSAGE TEST") - print(f"Model: {self.get_model()}") - print(f"Message: 'Say hi back to me!'") - print(f"{'='*80}\n") - - try: - await litellm._arealtime( - model=self.get_model(), - websocket=websocket_client, - api_key=os.environ.get(self.get_api_key_env_var()), - timeout=60, - ) - except websockets.exceptions.ConnectionClosed: - pass - except Exception as e: - print(f"\nException: {type(e).__name__}: {e}\n") - caught_exception = e - - # Print results - print(f"\n{'='*80}") - print(f"TEST RESULTS SUMMARY") - print(f"{'='*80}") - print(f"Connection successful: {websocket_client.connection_successful}") - print(f"User message sent: {websocket_client.sent_user_message}") - print(f"Total messages received: {len(websocket_client.messages_received)}") - print( - f"Response messages (excluding init/ping): {len(websocket_client.response_messages)}" - ) - - if websocket_client.response_messages: - print(f"\nResponse Event Types:") - for i, msg in enumerate(websocket_client.response_messages, 1): - print(f" {i}. {msg.get('type', 'unknown')}") - - print(f"{'='*80}\n") - - # Skip if no responses (might be timing issue) - if not websocket_client.response_messages: - pytest.skip("No response messages received (might be timing/network issue)") - - assert websocket_client.connection_successful, "Failed to establish connection" - assert websocket_client.sent_user_message, "Failed to send user message" diff --git a/tests/llm_translation/realtime/conftest.py b/tests/llm_translation/realtime/conftest.py deleted file mode 100644 index e305131593e..00000000000 --- a/tests/llm_translation/realtime/conftest.py +++ /dev/null @@ -1,89 +0,0 @@ -"""WebSocket VCR wiring for the realtime suite. - -This directory inherits the HTTP VCR machinery from -``tests/llm_translation/conftest.py`` (which only intercepts httpx/aiohttp and -is therefore a no-op for realtime WebSocket traffic). The autouse fixture below -adds the WebSocket layer: it patches ``websockets.connect`` for the duration of -each test so realtime frames are recorded to, or replayed from, the same -cassette Redis under a ``litellm:vcr:wscassette:`` prefix. -""" - -from __future__ import annotations - -import os -import sys -from typing import Optional - -import pytest - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))) - -from tests._vcr_conftest_common import ( # noqa: E402 - vcr_disabled, - vcr_outcome_logging_enabled, -) -from tests._ws_vcr import ( # noqa: E402 - WsVcrController, - build_ws_cassette_client, - load_ws_cassette, - replay_timeout_seconds, - save_ws_cassette, - ws_redis_key_for, -) - -_ws_cassette_client: Optional[object] = None - - -def _get_ws_cassette_client() -> Optional[object]: - global _ws_cassette_client - if _ws_cassette_client is None: - _ws_cassette_client = build_ws_cassette_client() - return _ws_cassette_client - - -def _emit_verdict(request: pytest.FixtureRequest, verdict: str) -> None: - if os.environ.get("PYTEST_XDIST_WORKER"): - return - reporter = request.config.pluginmanager.getplugin("terminalreporter") - if reporter is None: - return - reporter.write_line(f"{verdict} :: {request.node.nodeid}") - - -@pytest.fixture(autouse=True) -def _ws_vcr(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch): - if vcr_disabled(): - yield - return - - import websockets - - client = _get_ws_cassette_client() - if client is None: - yield - return - - key = ws_redis_key_for(request.node.nodeid) - cassette = load_ws_cassette(client, key) - controller = WsVcrController( - original_connect=websockets.connect, - cassette=cassette, - timeout=replay_timeout_seconds(), - ) - monkeypatch.setattr(websockets, "connect", controller.connect) - - yield - - rep_call = getattr(request.node, "rep_call", None) - passed = bool(rep_call and rep_call.passed) - - if controller.recorded: - built = controller.built_cassette() - if built is not None: - save_ws_cassette(client, key, built, passed=passed) - - if vcr_outcome_logging_enabled(): - _emit_verdict(request, controller.verdict()) - - if controller.errors and passed: - raise controller.errors[0] diff --git a/tests/llm_translation/realtime/test_openai_realtime_simple.py b/tests/llm_translation/realtime/test_openai_realtime_simple.py deleted file mode 100644 index 93451a6617e..00000000000 --- a/tests/llm_translation/realtime/test_openai_realtime_simple.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -OpenAI Realtime API E2E Tests (using base class) - -Tests OpenAI's Realtime API through LiteLLM's realtime interface. -Uses the base test class to ensure consistent behavior across providers. -""" - - -import pytest - - -from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest - - -class TestOpenAIRealtime(BaseRealtimeTest): - """ - E2E tests for OpenAI Realtime API using base test class. - """ - - def get_model(self) -> str: - # OpenAI shut down the entire gpt-4o-realtime-preview family - # (including the undated alias) on 2026-05-07. gpt-realtime is the - # current GA realtime model. - return "gpt-realtime" - - def get_api_key_env_var(self) -> str: - return "OPENAI_API_KEY" - - def get_initial_event_type(self) -> str: - return "session.created" diff --git a/tests/llm_translation/realtime/test_xai_realtime.py b/tests/llm_translation/realtime/test_xai_realtime.py deleted file mode 100644 index 19cf8624c48..00000000000 --- a/tests/llm_translation/realtime/test_xai_realtime.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -xAI Realtime API E2E Tests - -Tests xAI's Grok Voice Agent API through LiteLLM's realtime interface. -Uses the base test class to ensure consistent behavior across providers. -""" - -from typing import Tuple - -import pytest - - -from tests.llm_translation.realtime.base_realtime_tests import BaseRealtimeTest - - -class TestXAIRealtime(BaseRealtimeTest): - """ - E2E tests for xAI Realtime API. - - xAI's Grok Voice Agent API is OpenAI-compatible: - - Endpoint: wss://api.x.ai/v1/realtime - - Model: grok-4-1-fast-non-reasoning - - Initial event: historically "conversation.created"; xAI has since shipped - "session.created" (matching OpenAI). Accept either to avoid spurious - failures whenever xAI flips the wire format. - """ - - def get_model(self) -> str: - return "xai/grok-4-1-fast-non-reasoning" - - def get_api_key_env_var(self) -> str: - return "XAI_API_KEY" - - def get_initial_event_type(self) -> Tuple[str, ...]: - return ("conversation.created", "session.created") diff --git a/tests/llm_translation/test_deepgram.py b/tests/llm_translation/test_deepgram.py deleted file mode 100644 index 855d570488b..00000000000 --- a/tests/llm_translation/test_deepgram.py +++ /dev/null @@ -1,15 +0,0 @@ - -import pytest - -import litellm -from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest - - -class TestDeepgramAudioTranscription(BaseLLMAudioTranscriptionTest): - def get_base_audio_transcription_call_args(self) -> dict: - return { - "model": "deepgram/nova-2", - } - - def get_custom_llm_provider(self) -> litellm.LlmProviders: - return litellm.LlmProviders.DEEPGRAM diff --git a/tests/llm_translation/test_elevenlabs.py b/tests/llm_translation/test_elevenlabs.py deleted file mode 100644 index fb73843a1fc..00000000000 --- a/tests/llm_translation/test_elevenlabs.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import pytest - -import litellm -from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest - -os.environ.setdefault("ELEVENLABS_API_KEY", "test-elevenlabs-key") - - -class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest): - def get_base_audio_transcription_call_args(self) -> dict: - return { - "model": "elevenlabs/scribe_v1", - } - - def get_custom_llm_provider(self) -> litellm.LlmProviders: - return litellm.LlmProviders.ELEVENLABS - -class TestElevenLabsTextToSpeechTransformation: - @pytest.fixture(scope="class") - def config(self): - from litellm.llms.elevenlabs.text_to_speech.transformation import ( - ElevenLabsTextToSpeechConfig, - ) - - return ElevenLabsTextToSpeechConfig() diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py deleted file mode 100644 index ea633b5cc84..00000000000 --- a/tests/llm_translation/test_groq.py +++ /dev/null @@ -1,17 +0,0 @@ - - -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # noqa -# ) # Adds the parent directory to the system path - -from base_llm_unit_tests import BaseLLMChatTest - -class TestGroq(BaseLLMChatTest): - test_web_search = None - - def get_base_completion_call_args(self) -> dict: - return { - "model": "groq/openai/gpt-oss-120b", - } - diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py deleted file mode 100644 index 3e709840daa..00000000000 --- a/tests/llm_translation/test_mistral_api.py +++ /dev/null @@ -1,31 +0,0 @@ -import asyncio -import traceback - -from dotenv import load_dotenv - -import litellm.types -import litellm.types.utils -from litellm.llms.anthropic.chat import ModelResponseIterator - -load_dotenv() -import io - -from typing import Optional -from unittest.mock import MagicMock, patch - -import pytest - -import litellm - -from litellm.llms.anthropic.common_utils import process_anthropic_headers -from httpx import Headers -from base_llm_unit_tests import BaseLLMChatTest - - -@pytest.mark.flaky(retries=3, delay=2) -class TestMistralCompletion(BaseLLMChatTest): - test_basic_tool_calling = None - - def get_base_completion_call_args(self) -> dict: - litellm.set_verbose = True - return {"model": "mistral/mistral-medium-latest"} diff --git a/tests/llm_translation/test_mistral_audio_transcription_transformation.py b/tests/llm_translation/test_mistral_audio_transcription_transformation.py deleted file mode 100644 index 6328e52cbed..00000000000 --- a/tests/llm_translation/test_mistral_audio_transcription_transformation.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -from typing import Dict - -import litellm -import pytest - -from tests.llm_translation.base_audio_transcription_unit_tests import ( - BaseLLMAudioTranscriptionTest, -) - -@pytest.mark.skipif( - not os.getenv("MISTRAL_API_KEY"), - reason="MISTRAL_API_KEY not set, skipping Mistral audio transcription tests", -) -class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest): - def get_base_audio_transcription_call_args(self) -> Dict: - return { - "model": "mistral/voxtral-mini-latest", - } - - def get_custom_llm_provider(self) -> litellm.LlmProviders: - return litellm.LlmProviders.MISTRAL - diff --git a/tests/llm_translation/test_together_ai.py b/tests/llm_translation/test_together_ai.py deleted file mode 100644 index b78fbeca9e8..00000000000 --- a/tests/llm_translation/test_together_ai.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Test TogetherAI LLM -""" - -from base_llm_unit_tests import BaseLLMChatTest -from tests._live_test_helpers import cheapest_together_chat_model -import json -from datetime import datetime -from unittest.mock import AsyncMock - -import litellm -import pytest - -class TestTogetherAI(BaseLLMChatTest): - test_basic_tool_calling = None - test_function_calling_with_tool_response = None - - def get_base_completion_call_args(self) -> dict: - litellm.set_verbose = True - return {"model": cheapest_together_chat_model(function_calling=True, response_schema=True)} diff --git a/tests/ocr_tests/conftest.py b/tests/ocr_tests/conftest.py index 58edc3e43da..3fcff304823 100644 --- a/tests/ocr_tests/conftest.py +++ b/tests/ocr_tests/conftest.py @@ -2,7 +2,7 @@ # # Wires OCR tests into the Redis-backed VCR cache so live provider # calls (Mistral OCR, Azure AI OCR, Azure Document Intelligence, -# Vertex AI OCR) are replayed for 24h. See tests/llm_translation/Readme.md +# Vertex AI OCR) are replayed for 24h. See tests/VCR_TESTING.md # for the design overview. import pytest diff --git a/tests/ocr_tests/test_ocr_matrix.py b/tests/ocr_tests/test_ocr_matrix.py index 3f8c9b402fc..17118852a45 100644 --- a/tests/ocr_tests/test_ocr_matrix.py +++ b/tests/ocr_tests/test_ocr_matrix.py @@ -76,7 +76,7 @@ def logger(monkeypatch: pytest.MonkeyPatch) -> RecordingLogger: TESTS_DIR: Final = Path(__file__).resolve().parents[1] -PDF_PATH: Final = TESTS_DIR / "llm_translation" / "fixtures" / "dummy.pdf" +PDF_PATH: Final = TESTS_DIR / "fixtures" / "dummy.pdf" PNG_PATH: Final = TESTS_DIR / "image_gen_tests" / "test_image.png" PINNED_CDN: Final = "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0" PDF_URL: Final = f"{PINNED_CDN}/tests/llm_translation/fixtures/dummy.pdf" @@ -299,4 +299,3 @@ async def test_ocr(case: Case, monkeypatch: pytest.MonkeyPatch, logger: Recordin _assert_ocr_response(response, case.provider.model, case.document.expected_text) _assert_logged(await logger.wait_for_call(), response, case.provider.model, response.model, case.call) - diff --git a/tests/search_tests/conftest.py b/tests/search_tests/conftest.py index deef6527a8f..9e3aa6bed1c 100644 --- a/tests/search_tests/conftest.py +++ b/tests/search_tests/conftest.py @@ -3,7 +3,7 @@ # Wires search tests into the Redis-backed VCR cache so live provider # calls (Brave, DataForSEO, DuckDuckGo, Exa, Firecrawl, Google PSE, # Linkup, Parallel.ai, Perplexity, SearchAPI, Searxng, Serper, Tavily) -# are replayed for 24h. See tests/llm_translation/Readme.md for the +# are replayed for 24h. See tests/VCR_TESTING.md for the # design overview. diff --git a/tests/unit/llms/base_llm/batches/base_batches_config_test.py b/tests/unit/llms/base_llm/batches/base_batches_config_test.py index 5195dd8ba44..121ec5285a6 100644 --- a/tests/unit/llms/base_llm/batches/base_batches_config_test.py +++ b/tests/unit/llms/base_llm/batches/base_batches_config_test.py @@ -11,8 +11,8 @@ e.g. every `transform_retrieve_batch_response` returns a real `LiteLLMBatch` with `object == "batch"`, a valid status, and an int `created_at`. This module is intentionally NOT named `test_*`: it holds no standalone tests -and must not be collected on its own. It mirrors the established repo pattern in -`tests/llm_translation/base_*_unit_tests.py`. +and must not be collected on its own. Provider tests import and subclass this +contract suite from their own unit-test files. Providers that do NOT implement BaseBatchesConfig (e.g. vertex_ai, whose transformation is a standalone class with a different shape) cannot use this and diff --git a/tests/unit/llms/gigachat/chat/test_transformation.py b/tests/unit/llms/gigachat/chat/test_transformation.py index d784a4979e1..9116a4d1979 100644 --- a/tests/unit/llms/gigachat/chat/test_transformation.py +++ b/tests/unit/llms/gigachat/chat/test_transformation.py @@ -2,7 +2,7 @@ Tests for GigaChat LiteLLM Provider Tests message transformation, parameter handling, and response transformation. -Run with: pytest tests/llm_translation/test_gigachat.py -v +Run with: pytest tests/unit/llms/gigachat/chat/test_transformation.py -v """ import asyncio diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index 68875ff6d32..b475cf7e3ff 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -79,7 +79,6 @@ def test_mistral_audio_transcription_request_transform(): os.path.dirname(__file__), "../../../../..", "tests", - "llm_translation", "gettysburg.wav", ) audio_file = open(wav_path, "rb") @@ -109,7 +108,6 @@ def test_mistral_audio_transcription_request_with_diarize(): os.path.dirname(__file__), "../../../../..", "tests", - "llm_translation", "gettysburg.wav", ) audio_file = open(wav_path, "rb") diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index 407e1d19fb3..f313f8bd873 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -95,7 +95,6 @@ def _open_test_audio(): os.path.dirname(__file__), "../../../..", "tests", - "llm_translation", "gettysburg.wav", ) return open(wav_path, "rb") diff --git a/tests/llm_translation/duck.png b/tests/unit/llms/vertex_ai/fixtures/duck.png similarity index 100% rename from tests/llm_translation/duck.png rename to tests/unit/llms/vertex_ai/fixtures/duck.png diff --git a/tests/llm_translation/guinea.png b/tests/unit/llms/vertex_ai/fixtures/guinea.png similarity index 100% rename from tests/llm_translation/guinea.png rename to tests/unit/llms/vertex_ai/fixtures/guinea.png diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 94dc7dc861d..5a1c2529f77 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -1358,10 +1358,8 @@ def vertex_client(): @pytest.fixture def encoded_images(): image_paths = [ - "./tests/llm_translation/duck.png", - # "./duck.png", - "./tests/llm_translation/guinea.png", - # "./guinea.png", + "./tests/unit/llms/vertex_ai/fixtures/duck.png", + "./tests/unit/llms/vertex_ai/fixtures/guinea.png", ] return [encode_image_to_base64(path) for path in image_paths] diff --git a/tests/unit/llms/volcengine/test_volcengine_embedding.py b/tests/unit/llms/volcengine/test_volcengine_embedding.py index 0122bc50695..795c77022b9 100644 --- a/tests/unit/llms/volcengine/test_volcengine_embedding.py +++ b/tests/unit/llms/volcengine/test_volcengine_embedding.py @@ -1,19 +1,15 @@ -""" -Integration tests for Volcengine embedding following LiteLLM testing patterns -Based on the BaseLLMEmbeddingTest framework -""" +"""Tests for Volcengine embedding.""" from unittest.mock import MagicMock, patch import pytest # Add parent directory to path for imports -from tests.llm_translation.base_embedding_unit_tests import BaseLLMEmbeddingTest import litellm from litellm.types.utils import EmbeddingResponse -class TestVolcEngineEmbedding(BaseLLMEmbeddingTest): +class TestVolcEngineEmbedding: """Test Volcengine embedding integration following LiteLLM patterns""" def get_custom_llm_provider(self) -> litellm.LlmProviders: diff --git a/tests/unit/llms/voyage/embedding/test_transformation_contextual.py b/tests/unit/llms/voyage/embedding/test_transformation_contextual.py index d6c37957e04..9fd630283bd 100644 --- a/tests/unit/llms/voyage/embedding/test_transformation_contextual.py +++ b/tests/unit/llms/voyage/embedding/test_transformation_contextual.py @@ -8,10 +8,9 @@ import pytest import litellm from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from tests.llm_translation.base_embedding_unit_tests import BaseLLMEmbeddingTest -class TestVoyageAI(BaseLLMEmbeddingTest): +class TestVoyageAI: def get_custom_llm_provider(self) -> litellm.LlmProviders: return litellm.LlmProviders.VOYAGE diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator.zip b/tests/unit/skills/test_skills_data/slack-gif-creator.zip similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator.zip rename to tests/unit/skills/test_skills_data/slack-gif-creator.zip diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md b/tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/SKILL.md rename to tests/unit/skills/test_skills_data/slack-gif-creator/SKILL.md diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/core/__init__.py b/tests/unit/skills/test_skills_data/slack-gif-creator/core/__init__.py similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/core/__init__.py rename to tests/unit/skills/test_skills_data/slack-gif-creator/core/__init__.py diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/core/easing.py b/tests/unit/skills/test_skills_data/slack-gif-creator/core/easing.py similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/core/easing.py rename to tests/unit/skills/test_skills_data/slack-gif-creator/core/easing.py diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/core/frame_composer.py b/tests/unit/skills/test_skills_data/slack-gif-creator/core/frame_composer.py similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/core/frame_composer.py rename to tests/unit/skills/test_skills_data/slack-gif-creator/core/frame_composer.py diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/core/gif_builder.py b/tests/unit/skills/test_skills_data/slack-gif-creator/core/gif_builder.py similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/core/gif_builder.py rename to tests/unit/skills/test_skills_data/slack-gif-creator/core/gif_builder.py diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/core/validators.py b/tests/unit/skills/test_skills_data/slack-gif-creator/core/validators.py similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/core/validators.py rename to tests/unit/skills/test_skills_data/slack-gif-creator/core/validators.py diff --git a/tests/llm_translation/test_skills_data/slack-gif-creator/requirements.txt b/tests/unit/skills/test_skills_data/slack-gif-creator/requirements.txt similarity index 100% rename from tests/llm_translation/test_skills_data/slack-gif-creator/requirements.txt rename to tests/unit/skills/test_skills_data/slack-gif-creator/requirements.txt diff --git a/tests/llm_translation/test_skills_data/test-delete-skill/SKILL.md b/tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md similarity index 100% rename from tests/llm_translation/test_skills_data/test-delete-skill/SKILL.md rename to tests/unit/skills/test_skills_data/test-delete-skill/SKILL.md diff --git a/tests/llm_translation/test_skills_data/test-skill-litellm/SKILL.md b/tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md similarity index 100% rename from tests/llm_translation/test_skills_data/test-skill-litellm/SKILL.md rename to tests/unit/skills/test_skills_data/test-skill-litellm/SKILL.md diff --git a/tests/llm_translation/test_skills_data/test-skill.zip b/tests/unit/skills/test_skills_data/test-skill.zip similarity index 100% rename from tests/llm_translation/test_skills_data/test-skill.zip rename to tests/unit/skills/test_skills_data/test-skill.zip diff --git a/tests/unit/skills/test_skills_db.py b/tests/unit/skills/test_skills_db.py index 20ffed6fec1..3f8bc61f306 100644 --- a/tests/unit/skills/test_skills_db.py +++ b/tests/unit/skills/test_skills_db.py @@ -42,7 +42,7 @@ def create_skill_zip(skill_name: str): The zip file is automatically cleaned up after use. """ - test_dir = Path(__file__).parents[2] / "llm_translation" / "test_skills_data" + test_dir = Path(__file__).parent / "test_skills_data" skill_dir = test_dir / skill_name # Create a zip file containing the skill directory diff --git a/tests/unit/test_vcr_redis_persister.py b/tests/unit/test_vcr_redis_persister.py index 5d48a69ea43..e589a00271d 100644 --- a/tests/unit/test_vcr_redis_persister.py +++ b/tests/unit/test_vcr_redis_persister.py @@ -116,7 +116,7 @@ def test_redis_key_is_stable_across_working_directories(tmp_path, monkeypatch): monkeypatch.chdir(repo_root) key_from_root = redis_key_for(abs_cassette) - monkeypatch.chdir(os.path.join(repo_root, "tests", "llm_translation")) + monkeypatch.chdir(os.path.join(repo_root, "tests", "audio_tests")) key_from_subdir = redis_key_for(abs_cassette) monkeypatch.chdir(tmp_path)