diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh index 5ce8b6c84ba..e9e5dd3d66b 100755 --- a/.circleci/scripts/unit_selection.sh +++ b/.circleci/scripts/unit_selection.sh @@ -7,6 +7,8 @@ legacy_flags=( caching-local enterprise-package enterprise-routing + llm-other-providers + llm-vertex-ai mcp-integration misc proxy-db-auth-checks @@ -50,6 +52,8 @@ legacy_paths() { echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py echo tests/unit/enterprise/proxy/test_managed_files_access_check.py echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;; + llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;; + llm-vertex-ai) echo tests/unit/llms/vertex_ai ;; mcp-integration) echo tests/unit/experimental_mcp_client echo tests/unit/proxy/_experimental/mcp_server diff --git a/.circleci/tests.yml b/.circleci/tests.yml index 10ee19f146a..994d67da64d 100644 --- a/.circleci/tests.yml +++ b/.circleci/tests.yml @@ -354,6 +354,21 @@ workflows: - proxy-db-endpoints-and-responses base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-llm-vertex-ai + flag: llm-vertex-ai + shards: 2 + workers: 1 + reruns: 2 + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> + - unit: + name: unit-llm-other-providers + flag: llm-other-providers + shards: 3 + reruns: 2 + base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> + pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - unit: name: unit-misc flag: misc diff --git a/.github/merge-smoke-tests.json b/.github/merge-smoke-tests.json index 8ed7b917460..a563424c230 100644 --- a/.github/merge-smoke-tests.json +++ b/.github/merge-smoke-tests.json @@ -1,8 +1,8 @@ { "cases": { - "CHAT-JSON": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport", - "CHAT-TEXT-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport", - "CHAT-TOOL-STREAM": "tests/test_litellm/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport", + "CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport", + "CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport", + "CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport", "MODEL-ALLOW": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_allows_listed_model_for_key", "MODEL-DENY": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]", "COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones", diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 91b54f4ee70..2fa05879350 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -89,6 +89,7 @@ jobs: - shard: Vertex AI artifact-name: llm-vertex-ai test-path: "tests/test_litellm/llms/vertex_ai" + unit-flag: llm-vertex-ai workers: 1 reruns: 2 timeout-minutes: 20 @@ -97,6 +98,7 @@ jobs: - shard: All Other Providers artifact-name: llm-other-providers test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + unit-flag: llm-other-providers workers: 2 reruns: 2 timeout-minutes: 20 diff --git a/Makefile b/Makefile index 62e6ae53275..e86047b1987 100644 --- a/Makefile +++ b/Makefile @@ -314,7 +314,7 @@ test-unit: install-test-deps # Matrix test targets (matching CI workflow groups) test-unit-llms: install-test-deps - $(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/unit/llms --tb=short -vv -n 4 --durations=20 test-unit-proxy-guardrails: install-test-deps $(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20 diff --git a/tests/llm_translation/test_bedrock_gpt_oss.py b/tests/llm_translation/test_bedrock_gpt_oss.py index 4af81ee81f7..b264c16601f 100644 --- a/tests/llm_translation/test_bedrock_gpt_oss.py +++ b/tests/llm_translation/test_bedrock_gpt_oss.py @@ -22,7 +22,7 @@ class TestBedrockGPTOSS(BaseLLMChatTest): """Bedrock GPT-OSS intermittently emits truncated toolUse.input deltas on the live endpoint, which makes the inherited live integration test flaky. The accumulation side is covered deterministically by - tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py::test_transform_tool_calls_index; + tests/unit/llms/bedrock/chat/test_invoke_handler.py::test_transform_tool_calls_index; the GPT-OSS-specific request-body transformation is covered by test_function_calling_request_body_gpt_oss below. """ diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 3a5e2209f1e..2d79f8a6af6 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -324,7 +324,7 @@ def test_parallel_function_call_anthropic_error_msg(model, messages): Anthropic (and Bedrock Invoke via ``AnthropicConfig.transform_request``) inject a dummy tool so CLIs work with ``modify_params`` left off. Bedrock Converse's no-raise behavior is covered offline in - ``tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py`` + ``tests/unit/llms/bedrock/chat/test_converse_transformation.py`` (see #24158, #27138), which needs no live credentials. """ # Force modify_params off as a clean baseline: it exercises the Anthropic diff --git a/tests/local_testing/test_handler_gc_does_not_close_client.py b/tests/local_testing/test_handler_gc_does_not_close_client.py index 1a6ab1b1827..63c5694dd89 100644 --- a/tests/local_testing/test_handler_gc_does_not_close_client.py +++ b/tests/local_testing/test_handler_gc_does_not_close_client.py @@ -17,7 +17,7 @@ body can still arrive, released once the caller is done with the response. Nothing here re-tests the shapes ``_handler_may_close_client`` covers -- a borrowed ``handler.client``, a caller-supplied client, an evicted-but-held -client. Those are pinned in ``tests/test_litellm/llms/custom_httpx/ +client. Those are pinned in ``tests/unit/llms/custom_httpx/ test_http_handler.py``. What is uncovered there is the in-flight response, so no test here may keep the client in a local: that inflates the very refcount under test, and the test then passes on a broken handler. They hold weak references diff --git a/tests/local_testing/test_sagemaker_nova_integration.py b/tests/local_testing/test_sagemaker_nova_integration.py index beeb1fa2db3..95f28fe9892 100644 --- a/tests/local_testing/test_sagemaker_nova_integration.py +++ b/tests/local_testing/test_sagemaker_nova_integration.py @@ -4,7 +4,7 @@ Integration tests for SageMaker Nova provider. These tests require a live SageMaker Nova endpoint and AWS credentials. They are skipped by default — run manually with: - pytest tests/test_litellm/llms/sagemaker/test_sagemaker_nova_integration.py -v --no-header -rN + pytest tests/local_testing/test_sagemaker_nova_integration.py -v --no-header -rN Prerequisites: export AWS_PROFILE= # or set AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY @@ -251,7 +251,7 @@ class TestSagemakerNova2LiteIntegration: Run with: export SAGEMAKER_NOVA2_LITE_ENDPOINT= - pytest tests/test_litellm/llms/sagemaker/test_sagemaker_nova_integration.py::TestSagemakerNova2LiteIntegration -v + pytest tests/local_testing/test_sagemaker_nova_integration.py::TestSagemakerNova2LiteIntegration -v """ def test_should_accept_reasoning_effort_low(self): diff --git a/tests/search_tests/test_bing_grounding_search.py b/tests/search_tests/test_bing_grounding_search.py index 3d1737477a1..f532158e462 100644 --- a/tests/search_tests/test_bing_grounding_search.py +++ b/tests/search_tests/test_bing_grounding_search.py @@ -85,7 +85,7 @@ class TestBingGroundingSearch(BaseSearchTest): class TestBingGroundingSearchTransformation: """ Full-stack tests through `litellm.search` / `litellm.asearch` with the HTTP layer mocked. - Transformation details are unit-tested in tests/test_litellm/llms/azure/search/. + Transformation details are unit-tested in tests/unit/llms/azure/search/. """ @pytest.fixture(autouse=True) diff --git a/tests/search_tests/test_nimble_search.py b/tests/search_tests/test_nimble_search.py index df432f8ae84..3426fc712f4 100644 --- a/tests/search_tests/test_nimble_search.py +++ b/tests/search_tests/test_nimble_search.py @@ -58,7 +58,7 @@ class TestNimbleSearch(BaseSearchTest): class TestNimbleSearchTransformation: """ Full-stack tests through `litellm.search` / `litellm.asearch` with the HTTP layer mocked. - Transformation details are unit-tested in tests/test_litellm/llms/nimble/search/. + Transformation details are unit-tested in tests/unit/llms/nimble/search/. """ @pytest.fixture(autouse=True) diff --git a/tests/test_litellm/integrations/test_helicone.py b/tests/test_litellm/integrations/test_helicone.py index 64960de050a..99cb1380dd7 100644 --- a/tests/test_litellm/integrations/test_helicone.py +++ b/tests/test_litellm/integrations/test_helicone.py @@ -13,7 +13,7 @@ def _claude_mapping(messages, response_obj): def test_claude_mapping_serializes_custom_tool_calls(monkeypatch): """ Stub the anthropic module unconditionally: the SDK may be absent (it lives in the - proxy-runtime extra), and the tests/test_litellm/llms/anthropic test package can + proxy-runtime extra), and the tests/unit/llms/anthropic test package can shadow it on sys.path, so an import probe proves nothing about the real SDK. """ stub = types.ModuleType("anthropic") diff --git a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py index 7a69b676667..f692259db2e 100644 --- a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py +++ b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py @@ -9,171 +9,6 @@ import os import pytest -from litellm.llms.cometapi.chat.transformation import ( - CometAPIChatCompletionStreamingHandler, - CometAPIConfig, -) -from litellm.llms.cometapi.common_utils import CometAPIException - - -class TestCometAPIChatCompletionStreamingHandler: - def test_chunk_parser_successful(self): - handler = CometAPIChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - # Test input chunk - chunk = { - "id": "test_id", - "created": 1234567890, - "model": "gpt-3.5-turbo", - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, - "choices": [ - {"delta": {"content": "test content", "reasoning": "test reasoning"}} - ], - } - - # Parse chunk - result = handler.chunk_parser(chunk) - - # Verify response - assert result.id == "test_id" - assert result.object == "chat.completion.chunk" - assert result.created == 1234567890 - assert result.model == "gpt-3.5-turbo" - assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] - assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] - assert result.usage.total_tokens == chunk["usage"]["total_tokens"] - assert len(result.choices) == 1 - assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" - - def test_chunk_parser_error_response(self): - handler = CometAPIChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - # Test error chunk - error_chunk = { - "error": { - "message": "test error", - "code": 400, - } - } - - # Verify error handling - with pytest.raises(CometAPIException) as exc_info: - handler.chunk_parser(error_chunk) - - assert "CometAPI Error: test error" in str(exc_info.value) - assert exc_info.value.status_code == 400 - - def test_chunk_parser_key_error(self): - handler = CometAPIChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - # Test invalid chunk missing required fields - invalid_chunk = {"incomplete": "data"} - - # Verify KeyError handling - with pytest.raises(CometAPIException) as exc_info: - handler.chunk_parser(invalid_chunk) - - assert "KeyError" in str(exc_info.value) - assert exc_info.value.status_code == 400 - - -class TestCometAPIConfig: - def test_transform_request_basic(self): - """Test basic request transformation""" - config = CometAPIConfig() - - transformed_request = config.transform_request( - model="cometapi/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello, world!"}], - optional_params={}, - litellm_params={}, - headers={}, - ) - - assert transformed_request["model"] == "cometapi/gpt-3.5-turbo" - assert transformed_request["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - - def test_transform_request_with_extra_body(self): - """Test request transformation with extra_body parameters""" - config = CometAPIConfig() - - transformed_request = config.transform_request( - model="cometapi/gpt-4", - messages=[{"role": "user", "content": "Hello, world!"}], - optional_params={"extra_body": {"custom_param": "custom_value"}}, - litellm_params={}, - headers={}, - ) - - # Validate that extra_body parameters are merged into the request - assert transformed_request["custom_param"] == "custom_value" - assert transformed_request["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - - def test_cache_control_flag_removal(self): - """Test cache control flag removal from messages""" - config = CometAPIConfig() - - transformed_request = config.transform_request( - model="cometapi/gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Hello, world!", - "cache_control": {"type": "ephemeral"}, - } - ], - optional_params={}, - litellm_params={}, - headers={}, - ) - - # CometAPI should remove cache_control flags by default - assert transformed_request["messages"][0].get("cache_control") is None - - def test_map_openai_params(self): - """Test OpenAI parameter mapping""" - config = CometAPIConfig() - - non_default_params = { - "temperature": 0.7, - "max_tokens": 100, - "top_p": 0.9, - } - - mapped_params = config.map_openai_params( - non_default_params=non_default_params, - optional_params={}, - model="cometapi/gpt-3.5-turbo", - drop_params=False, - ) - - assert mapped_params["temperature"] == 0.7 - assert mapped_params["max_tokens"] == 100 - assert mapped_params["top_p"] == 0.9 - - def test_get_error_class(self): - """Test error class creation""" - config = CometAPIConfig() - - error = config.get_error_class( - error_message="Test error", - status_code=400, - headers={"Content-Type": "application/json"}, - ) - - assert isinstance(error, CometAPIException) - assert error.message == "Test error" - assert error.status_code == 400 # Integration test example (requires real API key) diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py deleted file mode 100644 index a3391a2c585..00000000000 --- a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -from typing import Final - -import httpx -import respx - -import litellm - - -def test_completion_merges_leading_system_and_developer_messages_for_chat_template_models( - respx_mock: respx.MockRouter, -): - upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( - return_value=httpx.Response( - status_code=200, - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": "my-custom-model", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, - }, - ) - ) - - response: Final = litellm.completion( - model="databricks/my-custom-model", - messages=[ - {"role": "system", "content": "You are terse."}, - {"role": "developer", "content": "Skills: none."}, - {"role": "user", "content": "Hello"}, - ], - api_base="https://example.databricks.test/serving-endpoints", - api_key="fake-databricks-api-key", - num_retries=0, - ) - - assert upstream.call_count == 1 - request_body: Final = json.loads(upstream.calls[0].request.read()) - assert request_body["messages"] == [ - {"role": "system", "content": "You are terse.\n\nSkills: none."}, - {"role": "user", "content": "Hello"}, - ] - assert response.choices[0].message.content == "Answer" - - -def test_completion_merges_system_messages_when_one_has_empty_content(respx_mock: respx.MockRouter): - upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( - return_value=httpx.Response( - status_code=200, - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": "my-custom-model", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, - }, - ) - ) - - litellm.completion( - model="databricks/my-custom-model", - messages=[ - {"role": "system", "content": "You are terse."}, - {"role": "system", "content": ""}, - {"role": "user", "content": "Hello"}, - ], - api_base="https://example.databricks.test/serving-endpoints", - api_key="fake-databricks-api-key", - num_retries=0, - ) - - request_body: Final = json.loads(upstream.calls[0].request.read()) - assert request_body["messages"] == [ - {"role": "system", "content": "You are terse."}, - {"role": "user", "content": "Hello"}, - ] diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_integration.py b/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_integration.py deleted file mode 100644 index 5b013681864..00000000000 --- a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_integration.py +++ /dev/null @@ -1,433 +0,0 @@ -""" -Integration tests for DeepInfra rerank functionality. -Tests the full rerank flow following the repository patterns. -""" - -import asyncio -import json -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -import litellm - - -def assert_response_shape(response, custom_llm_provider): - """Helper function to validate response structure specific to DeepInfra.""" - assert hasattr(response, "id") - assert hasattr(response, "results") - assert hasattr(response, "meta") - assert isinstance(response.results, list) - - for result in response.results: - assert "index" in result - assert "relevance_score" in result - assert isinstance(result["index"], int) - assert isinstance(result["relevance_score"], (int, float)) - - # Check meta structure - assert "tokens" in response.meta - assert "billed_units" in response.meta - assert "input_tokens" in response.meta["tokens"] - assert "total_tokens" in response.meta["billed_units"] - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_basic_rerank_deepinfra(mock_sync_post, mock_async_post, sync_mode): - """Test basic DeepInfra rerank functionality.""" - # Mock response data that matches DeepInfra API format - mock_response_data = { - "scores": [0.9, 0.1], - "input_tokens": 25, - "request_id": "deepinfra-request-123", - "inference_status": { - "status": "success", - "runtime_ms": 150, - "cost": 0.0001, - "tokens_generated": 0, - "tokens_input": 25, - }, - } - - def return_val(): - return mock_response_data - - api_key = "test_deepinfra_api_key" - api_base = "https://api.deepinfra.com" - - if sync_mode: - # Create mock response object for sync - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_sync_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - top_n=2, - custom_llm_provider="deepinfra", - api_key=api_key, - api_base=api_base, - ) - mock_sync_post.assert_called_once() - else: - # Create mock response object for async - mock_response = AsyncMock() - - def return_val(): - return mock_response_data - - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_async_post.return_value = mock_response - - response = asyncio.run( - litellm.arerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - top_n=2, - custom_llm_provider="deepinfra", - api_key=api_key, - api_base=api_base, - ) - ) - mock_async_post.assert_called_once() - - # Verify response structure - assert response.id == "deepinfra-request-123" - assert response.results is not None - assert len(response.results) == 2 - assert response.results[0]["index"] == 0 - assert response.results[0]["relevance_score"] == 0.9 - assert response.results[1]["index"] == 1 - assert response.results[1]["relevance_score"] == 0.1 - - # Verify metadata - assert response.meta["tokens"]["input_tokens"] == 25 - assert response.meta["billed_units"]["total_tokens"] == 25 - - # Verify hidden params specific to DeepInfra - assert response._hidden_params["status"] == "success" - assert response._hidden_params["runtime_ms"] == 150 - assert response._hidden_params["cost"] == 0.0001 - # Note: The model name is processed and the 'deepinfra/' prefix is removed - assert response._hidden_params["model"] == "Qwen/Qwen3-Reranker-0.6B" - - assert_response_shape(response, custom_llm_provider="deepinfra") - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_with_queries_param( - mock_sync_post, mock_async_post, sync_mode -): - """Test DeepInfra rerank with multiple queries parameter.""" - mock_response_data = { - "scores": [0.8, 0.6, 0.2], - "input_tokens": 35, - "request_id": "deepinfra-multi-query-123", - "inference_status": {"status": "success", "runtime_ms": 200}, - } - - def return_val(): - return mock_response_data - - if sync_mode: - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_sync_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-4B", - query="hello", - documents=["hello", "world", "test"], - queries=["hello", "hi there"], # DeepInfra specific param - custom_llm_provider="deepinfra", - api_key="test_key", - api_base="https://api.deepinfra.com", - ) - - mock_sync_post.assert_called_once() - # Verify that queries parameter was passed in request - call_data = json.loads(mock_sync_post.call_args.kwargs["data"]) - assert "queries" in call_data - assert call_data["queries"] == ["hello", "hi there"] - else: - mock_response = AsyncMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_async_post.return_value = mock_response - - response = asyncio.run( - litellm.arerank( - model="deepinfra/Qwen/Qwen3-Reranker-4B", - query="hello", - documents=["hello", "world", "test"], - queries=["hello", "hi there"], - custom_llm_provider="deepinfra", - api_key="test_key", - api_base="https://api.deepinfra.com", - ) - ) - - mock_async_post.assert_called_once() - call_data = json.loads(mock_async_post.call_args.kwargs["data"]) - assert "queries" in call_data - assert call_data["queries"] == ["hello", "hi there"] - - assert response.results is not None - assert len(response.results) == 3 - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_with_service_tier(mock_post): - """Test DeepInfra rerank with service_tier parameter.""" - mock_response_data = { - "scores": [0.95, 0.75], - "input_tokens": 30, - "request_id": "deepinfra-premium-123", - } - - def return_val(): - return mock_response_data - - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-8B", - query="premium search", - documents=["doc1", "doc2"], - service_tier="premium", # DeepInfra specific param - custom_llm_provider="deepinfra", - api_key="test_key", - api_base="https://api.deepinfra.com", - ) - - mock_post.assert_called_once() - - # Verify URL - call_url = mock_post.call_args.kwargs["url"] - assert "api.deepinfra.com/inference/Qwen/Qwen3-Reranker-8B" in call_url - - # Verify request contains service_tier - call_data = json.loads(mock_post.call_args.kwargs["data"]) - assert call_data["service_tier"] == "premium" - - assert response.results is not None - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_with_env_vars(mock_post, monkeypatch): - """Test DeepInfra rerank with environment variable configuration.""" - monkeypatch.setenv("DEEPINFRA_API_KEY", "env_test_key") - monkeypatch.setenv("DEEPINFRA_API_BASE", "https://custom-deepinfra.com") - - mock_response_data = { - "scores": [0.88, 0.22], - "input_tokens": 28, - "request_id": "env-test-123", - } - - def return_val(): - return mock_response_data - - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - custom_llm_provider="deepinfra", - ) - - mock_post.assert_called_once() - - # Verify headers contain env API key - headers = mock_post.call_args.kwargs.get("headers", {}) - assert "Bearer env_test_key" in headers.get("Authorization", "") - - assert response.results is not None - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_error_handling(mock_post): - """Test DeepInfra rerank error handling.""" - error_response = {"detail": {"error": "Invalid API key"}} - - def return_val(): - return error_response - - mock_response = MagicMock() - mock_response.status_code = 401 - mock_response.json = return_val - mock_response.text = json.dumps(error_response) - mock_response.headers = {"content-type": "application/json"} - mock_post.return_value = mock_response - - # The current implementation handles errors gracefully, so we expect a successful response - # with the error information in the hidden params - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - custom_llm_provider="deepinfra", - api_key="invalid_key", - api_base="https://api.deepinfra.com", - ) - - # Verify that the response contains error information - assert ( - response._hidden_params["status"] == "unknown" - ) # Default status when error occurs - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_defaults_api_base_when_missing(mock_post, monkeypatch): - """With no api_base anywhere, the call still goes out against DeepInfra's own base.""" - monkeypatch.delenv("DEEPINFRA_API_BASE", raising=False) - - mock_response = MagicMock() - mock_response.json = lambda: {"scores": [0.9, 0.1], "input_tokens": 20} - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - custom_llm_provider="deepinfra", - api_key="test_key", - # api_base is intentionally missing - ) - - assert "api.deepinfra.com" in mock_post.call_args.kwargs["url"] - assert [result["relevance_score"] for result in response.results] == [0.9, 0.1] - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_request_format(mock_post): - """Test that the request is properly formatted for DeepInfra API.""" - mock_response_data = {"scores": [0.9, 0.1], "input_tokens": 20} - - def return_val(): - return mock_response_data - - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="test query", - documents=["doc1", "doc2"], - custom_llm_provider="deepinfra", - api_key="test_key", - api_base="https://api.deepinfra.com", - instruction="custom instruction", - webhook="https://webhook.example.com", - ) - - mock_post.assert_called_once() - - # Verify URL format - call_url = mock_post.call_args.kwargs["url"] - assert call_url == "https://api.deepinfra.com/inference/Qwen/Qwen3-Reranker-0.6B" - - # Verify headers - headers = mock_post.call_args.kwargs["headers"] - assert headers["Authorization"] == "Bearer test_key" - assert headers["accept"] == "application/json" - assert headers["content-type"] == "application/json" - - # Verify request body format - request_data = json.loads(mock_post.call_args.kwargs["data"]) - assert request_data["queries"] == [ - "test query", - "test query", - ] # DeepInfra requires queries to match documents length - assert request_data["documents"] == ["doc1", "doc2"] - assert request_data["instruction"] == "custom instruction" - assert request_data["webhook"] == "https://webhook.example.com" - - assert response.results is not None - - -def test_deepinfra_rerank_models(): - """Test that DeepInfra Qwen rerank models are recognized.""" - # These should not raise errors during model validation - models = [ - "deepinfra/Qwen/Qwen3-Reranker-0.6B", - "deepinfra/Qwen/Qwen3-Reranker-4B", - "deepinfra/Qwen/Qwen3-Reranker-8B", - ] - - for model in models: - resolved_model, provider, _, api_base = litellm.get_llm_provider(model=model) - assert provider == "deepinfra" - assert resolved_model == model.removeprefix("deepinfra/") - assert api_base == "https://api.deepinfra.com/v1/openai" - - -@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") -def test_deepinfra_rerank_minimal_response(mock_post): - """Test handling of minimal DeepInfra response.""" - # Minimal response with just scores - mock_response_data = {"scores": [0.7, 0.3]} - - def return_val(): - return mock_response_data - - mock_response = MagicMock() - mock_response.json = return_val - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.text = json.dumps(mock_response_data) - mock_post.return_value = mock_response - - response = litellm.rerank( - model="deepinfra/Qwen/Qwen3-Reranker-0.6B", - query="hello", - documents=["hello", "world"], - custom_llm_provider="deepinfra", - api_key="test_key", - api_base="https://api.deepinfra.com", - ) - - # Should handle minimal response gracefully - assert response.results is not None - assert len(response.results) == 2 - assert response.results[0]["relevance_score"] == 0.7 - assert response.results[1]["relevance_score"] == 0.3 - - # Should have default values for missing fields - assert response.meta["tokens"]["input_tokens"] == 0 # Default when missing - assert response._hidden_params["status"] == "unknown" # Default when missing diff --git a/tests/test_litellm/llms/gemini/files/__init__.py b/tests/test_litellm/llms/gemini/files/__init__.py deleted file mode 100644 index f48fe7dbe2b..00000000000 --- a/tests/test_litellm/llms/gemini/files/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for Gemini files functionality""" diff --git a/tests/test_litellm/llms/gemini/videos/__init__.py b/tests/test_litellm/llms/gemini/videos/__init__.py deleted file mode 100644 index e0780c08321..00000000000 --- a/tests/test_litellm/llms/gemini/videos/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Gemini Video Generation Tests diff --git a/tests/test_litellm/llms/manus/__init__.py b/tests/test_litellm/llms/manus/__init__.py deleted file mode 100644 index c9121a7b2a4..00000000000 --- a/tests/test_litellm/llms/manus/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Manus provider tests diff --git a/tests/test_litellm/llms/manus/responses/__init__.py b/tests/test_litellm/llms/manus/responses/__init__.py deleted file mode 100644 index ea7ebb64d55..00000000000 --- a/tests/test_litellm/llms/manus/responses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Manus Responses API tests diff --git a/tests/test_litellm/llms/minimax/__init__.py b/tests/test_litellm/llms/minimax/__init__.py deleted file mode 100644 index 451f542f4ad..00000000000 --- a/tests/test_litellm/llms/minimax/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# MiniMax tests diff --git a/tests/test_litellm/llms/minimax/chat/__init__.py b/tests/test_litellm/llms/minimax/chat/__init__.py deleted file mode 100644 index 4a7916ae6cf..00000000000 --- a/tests/test_litellm/llms/minimax/chat/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# MiniMax chat tests diff --git a/tests/test_litellm/llms/minimax/messages/__init__.py b/tests/test_litellm/llms/minimax/messages/__init__.py deleted file mode 100644 index de5a80602ea..00000000000 --- a/tests/test_litellm/llms/minimax/messages/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# MiniMax messages tests diff --git a/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index d1eb6241ceb..db77eabba23 100644 --- a/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -1,19 +1,9 @@ import os from typing import Dict -from unittest.mock import MagicMock -import httpx import litellm import pytest -from litellm.llms.base_llm.audio_transcription.transformation import ( - BaseAudioTranscriptionConfig, -) -from litellm.llms.mistral.audio_transcription.transformation import ( - MistralAudioTranscriptionConfig, -) -from litellm.types.utils import TranscriptionResponse -from litellm.utils import ProviderConfigManager from tests.llm_translation.base_audio_transcription_unit_tests import ( BaseLLMAudioTranscriptionTest, ) @@ -37,184 +27,3 @@ class TestMistralAudioTranscription(BaseLLMAudioTranscriptionTest): "Async audio transcription test for Mistral is skipped in this suite; " "async test plugins (e.g. pytest-asyncio/anyio) are not configured here." ) - - -def test_mistral_audio_transcription_config_installed(): - """Ensure Mistral audio transcription config is registered with ProviderConfigManager.""" - config = ProviderConfigManager.get_provider_audio_transcription_config( - model="mistral/voxtral-mini-latest", - provider=litellm.LlmProviders.MISTRAL, - ) - assert config is not None - assert isinstance(config, BaseAudioTranscriptionConfig) - assert isinstance(config, MistralAudioTranscriptionConfig) - - -def test_mistral_audio_transcription_get_complete_url(): - config = MistralAudioTranscriptionConfig() - url = config.get_complete_url( - api_base=None, - api_key="fake-key", - model="voxtral-mini-latest", - optional_params={}, - litellm_params={}, - ) - assert url == "https://api.mistral.ai/v1/audio/transcriptions" - - -def test_mistral_audio_transcription_get_complete_url_custom_base(): - config = MistralAudioTranscriptionConfig() - url = config.get_complete_url( - api_base="https://custom.api.example.com/v1/", - api_key="fake-key", - model="voxtral-mini-latest", - optional_params={}, - litellm_params={}, - ) - assert url == "https://custom.api.example.com/v1/audio/transcriptions" - - -def test_mistral_audio_transcription_validate_environment(): - config = MistralAudioTranscriptionConfig() - headers = config.validate_environment( - headers={}, - model="voxtral-mini-latest", - messages=[], - optional_params={}, - litellm_params={}, - api_key="test-key-123", - ) - assert headers["Authorization"] == "Bearer test-key-123" - assert headers["accept"] == "application/json" - - -def test_mistral_audio_transcription_supported_params(): - config = MistralAudioTranscriptionConfig() - params = config.get_supported_openai_params("voxtral-mini-latest") - assert "language" in params - assert "temperature" in params - assert "response_format" in params - assert "timestamp_granularities" in params - - -def test_mistral_audio_transcription_request_transform(): - config = MistralAudioTranscriptionConfig() - - wav_path = os.path.join( - os.path.dirname(__file__), - "../../../../..", - "tests", - "llm_translation", - "gettysburg.wav", - ) - audio_file = open(wav_path, "rb") - - result = config.transform_audio_transcription_request( - model="voxtral-mini-latest", - audio_file=audio_file, - optional_params={"language": "en", "temperature": 0.0}, - litellm_params={}, - ) - - audio_file.close() - - assert isinstance(result.data, dict) - assert result.data["model"] == "voxtral-mini-latest" - assert result.data["language"] == "en" - assert result.data["temperature"] == 0.0 - assert result.files is not None - assert "file" in result.files - - -def test_mistral_audio_transcription_request_with_diarize(): - """Test that Mistral-specific params like diarize are passed through.""" - config = MistralAudioTranscriptionConfig() - - wav_path = os.path.join( - os.path.dirname(__file__), - "../../../../..", - "tests", - "llm_translation", - "gettysburg.wav", - ) - audio_file = open(wav_path, "rb") - - result = config.transform_audio_transcription_request( - model="voxtral-mini-latest", - audio_file=audio_file, - optional_params={"diarize": True}, - litellm_params={}, - ) - - audio_file.close() - - assert isinstance(result.data, dict) - assert result.data["diarize"] == "true" - - -def test_mistral_audio_transcription_response_transform(): - config = MistralAudioTranscriptionConfig() - - mock_response = MagicMock(spec=httpx.Response) - mock_response.json.return_value = {"text": "Four score and seven years ago..."} - - response = config.transform_audio_transcription_response(mock_response) - - assert isinstance(response, TranscriptionResponse) - assert response.text == "Four score and seven years ago..." - - -def test_mistral_audio_transcription_response_transform_diarized(): - """Test that diarized responses preserve segments and language.""" - config = MistralAudioTranscriptionConfig() - - mock_response = MagicMock(spec=httpx.Response) - mock_response.json.return_value = { - "model": "voxtral-mini-latest", - "text": "Hello, how are you? I am fine.", - "language": None, - "segments": [ - { - "text": "Hello, how are you?", - "start": 0.3, - "end": 2.1, - "speaker_id": "speaker_1", - "type": "transcription_segment", - }, - { - "text": "I am fine.", - "start": 2.5, - "end": 3.8, - "speaker_id": "speaker_2", - "type": "transcription_segment", - }, - ], - "usage": { - "prompt_audio_seconds": 4, - "prompt_tokens": 5, - "total_tokens": 50, - "completion_tokens": 20, - }, - } - - response = config.transform_audio_transcription_response(mock_response) - - assert isinstance(response, TranscriptionResponse) - assert response.text == "Hello, how are you? I am fine." - assert response["segments"] is not None - assert len(response["segments"]) == 2 - assert response["segments"][0]["speaker_id"] == "speaker_1" - assert response["segments"][1]["speaker_id"] == "speaker_2" - assert response["language"] is None - - -def test_mistral_audio_transcription_response_transform_empty(): - config = MistralAudioTranscriptionConfig() - - mock_response = MagicMock(spec=httpx.Response) - mock_response.json.return_value = {} - - response = config.transform_audio_transcription_response(mock_response) - - assert isinstance(response, TranscriptionResponse) - assert response.text == "" diff --git a/tests/test_litellm/llms/openai_like/test_json_providers.py b/tests/test_litellm/llms/openai_like/test_json_providers.py index d84cc8d3237..55703063fae 100644 --- a/tests/test_litellm/llms/openai_like/test_json_providers.py +++ b/tests/test_litellm/llms/openai_like/test_json_providers.py @@ -3,321 +3,12 @@ Tests for JSON-based provider configuration system. """ import os -import sys -from unittest.mock import patch -try: - import pytest -except ImportError: - # pytest not available, will run as standalone script - pytest = None - -# Add workspace to path -workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) -sys.path.insert(0, workspace_path) +import pytest import litellm -class TestJSONProviderLoader: - """Test JSON provider loading and configuration""" - - def test_load_json_providers(self): - """Test that JSON providers load correctly""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - # Verify publicai is loaded - assert JSONProviderRegistry.exists("publicai") - - # Get publicai config - publicai = JSONProviderRegistry.get("publicai") - assert publicai is not None - assert publicai.base_url == "https://api.publicai.co/v1" - assert publicai.api_key_env == "PUBLICAI_API_KEY" - assert publicai.api_base_env == "PUBLICAI_API_BASE" - assert publicai.param_mappings.get("max_completion_tokens") == "max_tokens" - - def test_dynamic_config_generation(self): - """Test dynamic config class creation""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("publicai") - config_class = create_config_class(provider) - config = config_class() - - # Test API info resolution - api_base, api_key = config._get_openai_compatible_provider_info(None, None) - assert api_base == "https://api.publicai.co/v1" - - # Test with custom base - api_base, api_key = config._get_openai_compatible_provider_info( - "https://custom.api.com", "test-key" - ) - assert api_base == "https://custom.api.com" - assert api_key == "test-key" - - def test_parameter_mapping(self): - """Test parameter mapping works""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("publicai") - config_class = create_config_class(provider) - config = config_class() - - # Test parameter mapping - optional_params = {} - non_default_params = {"max_completion_tokens": 100, "temperature": 0.7} - result = config.map_openai_params( - non_default_params, optional_params, "gpt-4", False - ) - - # max_completion_tokens should be mapped to max_tokens - assert "max_tokens" in result - assert result["max_tokens"] == 100 - assert "max_completion_tokens" not in result - - # temperature should be passed through - assert result["temperature"] == 0.7 - - def test_supported_params(self): - """Test that config returns supported params""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("publicai") - config_class = create_config_class(provider) - config = config_class() - - # Get supported params - supported = config.get_supported_openai_params("gpt-4") - - # Should have standard OpenAI params - assert isinstance(supported, list) - assert len(supported) > 0 - - def test_tool_params_excluded_when_function_calling_not_supported(self): - """Test that tool-related params are excluded for models that don't support - function calling. Regression test for https://github.com/BerriAI/litellm/issues/21125 - """ - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("publicai") - config_class = create_config_class(provider) - config = config_class() - - # Mock supports_function_calling to return False - with patch("litellm.utils.supports_function_calling", return_value=False): - supported = config.get_supported_openai_params("some-model-without-fc") - - tool_params = [ - "tools", - "tool_choice", - "function_call", - "functions", - "parallel_tool_calls", - ] - for param in tool_params: - assert ( - param not in supported - ), f"'{param}' should not be in supported params when function calling is not supported" - - # Non-tool params should still be present - assert "temperature" in supported - assert "max_tokens" in supported - assert "stop" in supported - - def test_tool_params_included_when_function_calling_supported(self): - """Test that tool-related params are included for models that support function calling.""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("publicai") - config_class = create_config_class(provider) - config = config_class() - - # Mock supports_function_calling to return True - with patch("litellm.utils.supports_function_calling", return_value=True): - supported = config.get_supported_openai_params("some-model-with-fc") - - assert "tools" in supported - assert "tool_choice" in supported - - def test_provider_resolution(self): - """Test that provider resolution finds JSON providers""" - from litellm.litellm_core_utils.get_llm_provider_logic import ( - get_llm_provider, - ) - - model, provider, api_key, api_base = get_llm_provider( - model="publicai/gpt-4", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "gpt-4" - assert provider == "publicai" - assert api_base == "https://api.publicai.co/v1" - - def test_provider_config_manager(self): - """Test that ProviderConfigManager returns JSON-based configs""" - from litellm import LlmProviders - from litellm.utils import ProviderConfigManager - - config = ProviderConfigManager.get_provider_chat_config( - model="gpt-4", provider=LlmProviders.PUBLICAI - ) - - assert config is not None - assert config.custom_llm_provider == "publicai" - - -class TestPinstripes: - """Tests for Pinstripes JSON-configured provider""" - - def test_pinstripes_json_config_exists(self): - """Test that pinstripes is configured in providers.json""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - assert JSONProviderRegistry.exists("pinstripes") - - pinstripes = JSONProviderRegistry.get("pinstripes") - assert pinstripes is not None - assert pinstripes.base_url == "https://pinstripes.io/v1" - assert pinstripes.api_key_env == "PINSTRIPES_API_KEY" - assert pinstripes.param_mappings.get("max_completion_tokens") == "max_tokens" - - def test_pinstripes_provider_resolution(self): - """Test that provider resolution finds pinstripes and returns the default base URL""" - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="pinstripes/ps/glm-4.5-air", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "ps/glm-4.5-air" - assert provider == "pinstripes" - assert api_base == "https://pinstripes.io/v1" - - def test_pinstripes_dynamic_config(self): - """Test dynamic config class creation for pinstripes""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("pinstripes") - config_class = create_config_class(provider) - config = config_class() - - api_base, api_key = config._get_openai_compatible_provider_info(None, None) - assert api_base == "https://pinstripes.io/v1" - - api_base, api_key = config._get_openai_compatible_provider_info( - "https://custom.pinstripes.io/v1", "test-key" - ) - assert api_base == "https://custom.pinstripes.io/v1" - assert api_key == "test-key" - - def test_pinstripes_parameter_mapping(self): - """Test that max_completion_tokens is mapped to max_tokens for pinstripes""" - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("pinstripes") - config_class = create_config_class(provider) - config = config_class() - - optional_params = {} - non_default_params = {"max_completion_tokens": 100, "temperature": 0.7} - result = config.map_openai_params( - non_default_params, optional_params, "ps/glm-4.5-air", False - ) - - assert "max_tokens" in result - assert result["max_tokens"] == 100 - assert "max_completion_tokens" not in result - assert result["temperature"] == 0.7 - - -class TestDarkbloom: - def test_darkbloom_json_config_exists(self): - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - darkbloom = JSONProviderRegistry.get("darkbloom") - assert darkbloom is not None - assert darkbloom.base_url == "https://api.darkbloom.dev/v1" - assert darkbloom.api_key_env == "DARKBLOOM_API_KEY" - assert darkbloom.api_base_env == "DARKBLOOM_API_BASE" - assert darkbloom.param_mappings.get("max_completion_tokens") == "max_tokens" - - def test_darkbloom_provider_resolution(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="darkbloom/gemma-4-26b", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "gemma-4-26b" - assert provider == "darkbloom" - assert api_key is None - assert api_base == "https://api.darkbloom.dev/v1" - - def test_darkbloom_dynamic_config(self): - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("darkbloom") - config_class = create_config_class(provider) - config = config_class() - - api_base, api_key = config._get_openai_compatible_provider_info(None, None) - assert api_base == "https://api.darkbloom.dev/v1" - - api_base, api_key = config._get_openai_compatible_provider_info( - "https://custom.darkbloom.dev/v1", "test-key" - ) - assert api_base == "https://custom.darkbloom.dev/v1" - assert api_key == "test-key" - - def test_darkbloom_complete_url_appends_endpoint(self): - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("darkbloom") - config_class = create_config_class(provider) - config = config_class() - - url = config.get_complete_url( - api_base="https://api.darkbloom.dev/v1", - api_key="test-key", - model="darkbloom/gemma-4-26b", - optional_params={}, - litellm_params={}, - stream=True, - ) - - assert url == "https://api.darkbloom.dev/v1/chat/completions" - - def test_darkbloom_provider_config_manager(self): - from litellm import LlmProviders - from litellm.utils import ProviderConfigManager - - config = ProviderConfigManager.get_provider_chat_config( - model="gemma-4-26b", provider=LlmProviders.DARKBLOOM - ) - - assert config is not None - assert config.custom_llm_provider == "darkbloom" - - class TestPublicAIIntegration: """Integration tests for PublicAI provider""" @@ -457,55 +148,3 @@ class TestPublicAIIntegration: pytest.fail(f"Content list conversion test failed: {str(e)}") else: raise - - -if __name__ == "__main__": - # Run basic tests - print("Testing JSON Provider System...") - - test_loader = TestJSONProviderLoader() - print("\n1. Testing JSON provider loading...") - test_loader.test_load_json_providers() - print(" ✓ JSON providers loaded") - - print("\n2. Testing dynamic config generation...") - test_loader.test_dynamic_config_generation() - print(" ✓ Dynamic config works") - - print("\n3. Testing parameter mapping...") - test_loader.test_parameter_mapping() - print(" ✓ Parameter mapping works") - - print("\n4. Testing excluded params...") - test_loader.test_excluded_params() - print(" ✓ Excluded params work") - - print("\n5. Testing provider resolution...") - test_loader.test_provider_resolution() - print(" ✓ Provider resolution works") - - print("\n6. Testing provider config manager...") - test_loader.test_provider_config_manager() - print(" ✓ Config manager works") - - print("\n" + "=" * 50) - print("PublicAI Integration Tests...") - print("=" * 50) - - test_integration = TestPublicAIIntegration() - - print("\n7. Testing basic completion...") - test_integration.test_publicai_completion_basic() - - print("\n8. Testing streaming...") - test_integration.test_publicai_completion_with_streaming() - - print("\n9. Testing parameter mapping...") - test_integration.test_publicai_parameter_mapping() - - print("\n10. Testing content list conversion...") - test_integration.test_publicai_content_list_conversion() - - print("\n" + "=" * 50) - print("✓ All tests passed!") - print("=" * 50) diff --git a/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py b/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py index 8104fb12943..580994f60b8 100644 --- a/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py +++ b/tests/test_litellm/llms/openai_like/test_xiaomi_mimo.py @@ -4,86 +4,12 @@ Related to issue #18794 """ import os -import sys -from unittest.mock import MagicMock, patch -try: - import pytest -except ImportError: - pytest = None - -# Add workspace to path -workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) -sys.path.insert(0, workspace_path) +import pytest import litellm -class TestXiaomiMiMoProviderConfig: - """Test Xiaomi MiMo provider configuration""" - - def test_xiaomi_mimo_in_provider_list(self): - """Test that xiaomi_mimo is in the provider list (fixes #18794)""" - from litellm import LlmProviders - - # Verify xiaomi_mimo is in the enum - assert hasattr(LlmProviders, "XIAOMI_MIMO") - assert LlmProviders.XIAOMI_MIMO.value == "xiaomi_mimo" - - # Verify it's in the provider list - assert "xiaomi_mimo" in litellm.provider_list - - def test_xiaomi_mimo_json_config_exists(self): - """Test that xiaomi_mimo is configured in providers.json""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - # Verify xiaomi_mimo is loaded - assert JSONProviderRegistry.exists("xiaomi_mimo") - - # Get xiaomi_mimo config - xiaomi_mimo = JSONProviderRegistry.get("xiaomi_mimo") - assert xiaomi_mimo is not None - assert xiaomi_mimo.base_url == "https://api.xiaomimimo.com/v1" - assert xiaomi_mimo.api_key_env == "XIAOMI_MIMO_API_KEY" - assert xiaomi_mimo.param_mappings.get("max_completion_tokens") == "max_tokens" - - def test_xiaomi_mimo_provider_resolution(self): - """Test that provider resolution finds xiaomi_mimo""" - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="xiaomi_mimo/mimo-v2-flash", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "mimo-v2-flash" - assert provider == "xiaomi_mimo" - assert api_base == "https://api.xiaomimimo.com/v1" - - def test_xiaomi_mimo_router_config(self): - """Test that xiaomi_mimo can be used in Router configuration (fixes #18794)""" - from litellm import Router - - # This should not raise "Unsupported provider - xiaomi_mimo" - router = Router( - model_list=[ - { - "model_name": "mimo-v2-flash", - "litellm_params": { - "model": "xiaomi_mimo/mimo-v2-flash", - "api_key": "test-key", - }, - } - ] - ) - - # Verify the deployment was created successfully - assert len(router.model_list) == 1 - assert router.model_list[0]["model_name"] == "mimo-v2-flash" - - class TestXiaomiMiMoIntegration: """Integration tests for Xiaomi MiMo provider""" @@ -128,30 +54,3 @@ class TestXiaomiMiMoIntegration: pytest.fail(f"Xiaomi MiMo completion failed: {str(e)}") else: raise - - -if __name__ == "__main__": - # Run basic tests - print("Testing Xiaomi MiMo Provider...") - - test_config = TestXiaomiMiMoProviderConfig() - - print("\n1. Testing provider in list...") - test_config.test_xiaomi_mimo_in_provider_list() - print(" ✓ xiaomi_mimo in provider list") - - print("\n2. Testing JSON config...") - test_config.test_xiaomi_mimo_json_config_exists() - print(" ✓ xiaomi_mimo JSON config loaded") - - print("\n3. Testing provider resolution...") - test_config.test_xiaomi_mimo_provider_resolution() - print(" ✓ Provider resolution works") - - print("\n4. Testing router configuration...") - test_config.test_xiaomi_mimo_router_config() - print(" ✓ Router configuration works (issue #18794 fixed)") - - print("\n" + "=" * 50) - print("✓ All configuration tests passed!") - print("=" * 50) diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index c8751fb2d95..8cc46dc98d0 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -54,61 +54,3 @@ def test_ovhcloud_audio_transcription_config_installed(): assert config is not None assert isinstance(config, BaseAudioTranscriptionConfig) - - - -class TestOVHCloudDurationFieldMigration: - """Tests for OVHCloud duration -> seconds field migration.""" - - def test_seconds_field_mapped_to_duration(self): - """New `seconds` field should be normalized to `duration`.""" - from litellm.llms.ovhcloud.audio_transcription.transformation import ( - OVHCloudAudioTranscriptionConfig, - ) - from unittest.mock import MagicMock - - config = OVHCloudAudioTranscriptionConfig() - mock_response = MagicMock() - mock_response.json.return_value = { - "text": "Hello world", - "seconds": 3.14, - } - - result = config.transform_audio_transcription_response(mock_response) - - assert result.text == "Hello world" - assert result._hidden_params["duration"] == 3.14 - - def test_legacy_duration_field_still_works(self): - """Legacy `duration` field should still be accepted.""" - from litellm.llms.ovhcloud.audio_transcription.transformation import ( - OVHCloudAudioTranscriptionConfig, - ) - from unittest.mock import MagicMock - - config = OVHCloudAudioTranscriptionConfig() - mock_response = MagicMock() - mock_response.json.return_value = { - "text": "Hello world", - "duration": 2.71, - } - - result = config.transform_audio_transcription_response(mock_response) - - assert result.text == "Hello world" - assert result._hidden_params["duration"] == 2.71 - - - - def test_seconds_zero_mapped_to_duration(self): - """seconds=0.0 must not be treated as falsy and lost.""" - from litellm.llms.ovhcloud.audio_transcription.transformation import ( - OVHCloudAudioTranscriptionConfig, - ) - from unittest.mock import MagicMock - - config = OVHCloudAudioTranscriptionConfig() - mock_response = MagicMock() - mock_response.json.return_value = {"text": "silence", "seconds": 0.0} - result = config.transform_audio_transcription_response(mock_response) - assert result._hidden_params["duration"] == 0.0 \ No newline at end of file diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py index 057ab9ede9a..34954587ed0 100644 --- a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py +++ b/tests/test_litellm/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -6,174 +6,12 @@ import os import pytest -from litellm.llms.ovhcloud.utils import OVHCloudException -from litellm.utils import get_optional_params -from litellm.llms.ovhcloud.chat.transformation import ( - OVHCloudChatCompletionStreamingHandler, - OVHCloudChatConfig, -) -config = OVHCloudChatConfig() model = "ovhcloud/Mistral-7B-Instruct-v0.3" -class TestOvhCloudChatCompletionStreamingHandler: - def test_chunk_parser_successful(self): - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - chunk = { - "id": "test_id", - "created": 1234567890, - "model": "gpt-oss-20b", - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, - "choices": [ - {"delta": {"content": "test content", "reasoning": "test reasoning"}} - ], - } - - result = handler.chunk_parser(chunk) - - assert result.id == "test_id" - assert result.object == "chat.completion.chunk" - assert result.created == 1234567890 - assert result.model == "gpt-oss-20b" - assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] - assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] - assert result.usage.total_tokens == chunk["usage"]["total_tokens"] - assert len(result.choices) == 1 - assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" - - def test_chunk_parser_error_response(self): - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - error_chunk = { - "error": { - "message": "test error", - "code": 400, - } - } - - with pytest.raises(OVHCloudException) as exc_info: - handler.chunk_parser(error_chunk) - - assert "OVHCloud Error: test error" in str(exc_info.value) - assert exc_info.value.status_code == 400 - - def test_chunk_parser_key_error(self): - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=None, sync_stream=True - ) - - invalid_chunk = {"incomplete": "data"} - - with pytest.raises(OVHCloudException) as exc_info: - handler.chunk_parser(invalid_chunk) - - assert "KeyError" in str(exc_info.value) - assert exc_info.value.status_code == 400 - - -class TestOVHCloudConfig: - def test_transform_request_basic(self): - """Test basic request transformation""" - transformed_request = config.transform_request( - model, - messages=[{"role": "user", "content": "Hello, world!"}], - optional_params={}, - litellm_params={}, - headers={}, - ) - - assert transformed_request["model"] == model - assert transformed_request["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - - def test_transform_request_with_extra_body(self): - """Test request transformation with extra_body parameters""" - transformed_request = config.transform_request( - model, - messages=[{"role": "user", "content": "Hello, world!"}], - optional_params={"extra_body": {"custom_param": "custom_value"}}, - litellm_params={}, - headers={}, - ) - - assert transformed_request["custom_param"] == "custom_value" - assert transformed_request["messages"] == [ - {"role": "user", "content": "Hello, world!"} - ] - - def test_map_openai_params(self): - """Test OpenAI parameter mapping""" - non_default_params = { - "temperature": 0.7, - "max_tokens": 100, - "top_p": 0.9, - } - - mapped_params = config.map_openai_params( - non_default_params=non_default_params, - optional_params={}, - model=model, - drop_params=False, - ) - - assert mapped_params["temperature"] == 0.7 - assert mapped_params["max_tokens"] == 100 - assert mapped_params["top_p"] == 0.9 - - def test_get_error_class(self): - """Test error class creation""" - error = config.get_error_class( - error_message="Test error", - status_code=400, - headers={"Content-Type": "application/json"}, - ) - - assert isinstance(error, OVHCloudException) - assert error.message == "Test error" - assert error.status_code == 400 - - @pytest.mark.parametrize( - "model", - [ - "Meta-Llama-3_3-70B-Instruct", - "Meta-Llama-3_1-70B-Instruct", - "Mixtral-8x7B-Instruct-v0.1", - "gpt-oss-120b", - "some-model-not-in-the-cost-map", - ], - ) - def test_tools_not_filtered_by_static_model_map(self, model): - """ - OVHCloud AI Endpoints are OpenAI-compatible; tools/tool_choice must pass - through for any model. The server is responsible for rejecting unsupported - tool calls — LiteLLM must not strip them based on a stale static catalog. - """ - - params = get_optional_params( - model=model, - custom_llm_provider="ovhcloud", - tools=[ - { - "type": "function", - "function": {"name": "x", "parameters": {}}, - } - ], - tool_choice="auto", - ) - - assert "tools" in params - assert "tool_choice" in params - - def test_ovhcloud_integration(): from litellm import completion @@ -285,78 +123,3 @@ def test_ovhcloud_with_custom_base_url(): if __name__ == "__main__": pytest.main([__file__, "-v"]) - - -class TestOVHCloudReasoningFieldMigration: - """Tests for OVHCloud reasoning_content -> reasoning field migration.""" - - def test_streaming_new_reasoning_field(self): - """New `reasoning` field should be mapped to `reasoning_content`.""" - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=iter([]), - sync_stream=True, - ) - chunk = { - "id": "test-id", - "created": 1234567890, - "model": "test-model", - "choices": [ - { - "delta": { - "role": "assistant", - "reasoning": "Let me think...", - }, - "index": 0, - } - ], - } - result = handler.chunk_parser(chunk) - assert result.choices[0]["delta"]["reasoning_content"] == "Let me think..." - - def test_streaming_legacy_reasoning_content_unchanged(self): - """Legacy `reasoning_content` field should pass through untouched.""" - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=iter([]), - sync_stream=True, - ) - chunk = { - "id": "test-id", - "created": 1234567890, - "model": "test-model", - "choices": [ - { - "delta": { - "role": "assistant", - "reasoning_content": "Already correct field.", - }, - "index": 0, - } - ], - } - result = handler.chunk_parser(chunk) - assert result.choices[0]["delta"]["reasoning_content"] == "Already correct field." - - def test_streaming_both_fields_legacy_wins(self): - """When both fields present, existing `reasoning_content` is not overwritten.""" - handler = OVHCloudChatCompletionStreamingHandler( - streaming_response=iter([]), - sync_stream=True, - ) - chunk = { - "id": "test-id", - "created": 1234567890, - "model": "test-model", - "choices": [ - { - "delta": { - "reasoning": "new field", - "reasoning_content": "legacy field", - }, - "index": 0, - } - ], - } - result = handler.chunk_parser(chunk) - assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" - - diff --git a/tests/test_litellm/llms/reducto/__init__.py b/tests/test_litellm/llms/reducto/__init__.py deleted file mode 100644 index 8b137891791..00000000000 --- a/tests/test_litellm/llms/reducto/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/test_litellm/llms/s3_vectors/__init__.py b/tests/test_litellm/llms/s3_vectors/__init__.py deleted file mode 100644 index d4b0c4d8550..00000000000 --- a/tests/test_litellm/llms/s3_vectors/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# S3 Vectors tests diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/__init__.py b/tests/test_litellm/llms/s3_vectors/vector_stores/__init__.py deleted file mode 100644 index 231735c1de7..00000000000 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# S3 Vectors vector store tests diff --git a/tests/test_litellm/llms/soniox/__init__.py b/tests/test_litellm/llms/soniox/__init__.py deleted file mode 100644 index b2cd496d66a..00000000000 --- a/tests/test_litellm/llms/soniox/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Soniox provider tests.""" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 4679b978f78..d3a7ba7a1bd 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1,1681 +1,13 @@ -import base64 - import pytest from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_gemini_tool_call_result, ) -from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - _transform_request_body, - check_if_part_exists_in_parts, - _get_highest_media_resolution, - _extract_max_media_resolution_from_messages, -) from litellm.types.llms.vertex_ai import BlobType -from litellm.types.utils import Message - - -def test_check_if_part_exists_in_parts(): - parts = [ - {"text": "Hello", "thought": True}, - {"text": "World", "thought": False}, - ] - part = {"text": "Hello", "thought": True} - new_part = {"text": "Hello World", "thought": True} - assert check_if_part_exists_in_parts(parts, part) - assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) - assert check_if_part_exists_in_parts(parts, new_part, ["text"]) - - -def test_check_if_part_exists_in_parts_camel_case_snake_case(): - """Test that function handles both camelCase and snake_case key variations""" - # Test snake_case to camelCase matching - parts_with_snake_case = [ - { - "function_call": { - "name": "get_current_weather", - "args": {"location": "San Francisco, CA"}, - } - }, - {"text": "Some other content"}, - ] - - part_with_camel_case = { - "functionCall": { - "name": "get_current_weather", - "args": {"location": "San Francisco, CA"}, - } - } - - # Should find match between function_call and functionCall - assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) - - # Test camelCase to snake_case matching - parts_with_camel_case = [ - {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} - ] - - part_with_snake_case = { - "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} - } - - # Should find match between functionCall and function_call - assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) - - # Test no match when values differ - part_with_different_values = { - "function_call": {"name": "different_function", "args": {"x": 5}} - } - - assert not check_if_part_exists_in_parts( - parts_with_snake_case, part_with_different_values - ) - - # Test multiple keys with mixed casing - parts_mixed = [ - { - "function_call": {"name": "test"}, - "thoughtSignature": "reasoning", - "text": "content", - } - ] - - part_mixed_casing = { - "functionCall": {"name": "test"}, - "thought_signature": "reasoning", - "text": "content", - } - - assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) - - -def test_cached_content_respects_modify_params_for_cache_incompatible_fields(): - """Regression: cachedContent drops system/tools/toolConfig only when modify_params=True.""" - import litellm - - cache_name = "projects/p/locations/us-central1/cachedContents/abc123" - messages = [ - {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "hi"}, - ] - optional_params = { - "tools": [ - { - "functionDeclarations": [ - {"name": "get_weather", "description": "Get weather"}, - ] - } - ], - "tool_choice": {"functionCallingConfig": {"mode": "AUTO"}}, - } - - original_modify_params = litellm.modify_params - try: - # With modify_params=False (default), keep fields even with cachedContent. - litellm.modify_params = False - result = _transform_request_body( - messages=list(messages), - model="gemini-2.5-pro", - optional_params=dict(optional_params), - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=cache_name, - ) - assert result.get("cachedContent") == cache_name - assert "system_instruction" in result - assert "tools" in result - assert "toolConfig" in result - assert "contents" in result - - # With modify_params=True, drop cache-incompatible fields. - litellm.modify_params = True - result_modify_true = _transform_request_body( - messages=list(messages), - model="gemini-2.5-pro", - optional_params=dict(optional_params), - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=cache_name, - ) - assert result_modify_true.get("cachedContent") == cache_name - assert "system_instruction" not in result_modify_true - assert "tools" not in result_modify_true - assert "toolConfig" not in result_modify_true - assert "contents" in result_modify_true - - # Without cache, fields are always included. - result_no_cache = _transform_request_body( - messages=list(messages), - model="gemini-2.5-pro", - optional_params=dict(optional_params), - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=None, - ) - assert "system_instruction" in result_no_cache - assert "tools" in result_no_cache - assert "toolConfig" in result_no_cache - finally: - litellm.modify_params = original_modify_params - - -# Tests for issue #14556: Labels field provider-aware filtering -def test_google_genai_excludes_labels(): - """Test that Google GenAI/AI Studio endpoints exclude labels when custom_llm_provider='gemini'""" - messages = [{"role": "user", "content": "test"}] - optional_params = {"labels": {"project": "test", "team": "ai"}} - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="gemini", - litellm_params=litellm_params, - cached_content=None, - ) - - # Google GenAI/AI Studio should NOT include labels - assert "labels" not in result - assert "contents" in result - - -def test_vertex_ai_includes_labels(): - """Test that Vertex AI endpoints include labels when custom_llm_provider='vertex_ai'""" - messages = [{"role": "user", "content": "test"}] - optional_params = {"labels": {"project": "test", "team": "ai"}} - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - # Vertex AI SHOULD include labels - assert "labels" in result - assert result["labels"] == {"project": "test", "team": "ai"} - - -def test_service_tier_forwarded_to_vertex_ai(): - """Test that service_tier in optional_params is mapped to serviceTier in request body.""" - messages = [{"role": "user", "content": "test"}] - optional_params = {"service_tier": "flex"} - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - assert "serviceTier" in result - assert result["serviceTier"] == "flex" - - -def test_extra_body_cache_not_forwarded_to_vertex_ai(): - """ - 'cache' inside extra_body is a LiteLLM-internal proxy caching control. - It must NOT be forwarded to the Vertex AI request body. - - Regression test for: "Invalid JSON payload received. Unknown name \"cache\": Cannot find field." - Vertex AI enforces a strict JSON schema and rejects any unknown field. - """ - messages = [{"role": "user", "content": "test"}] - optional_params = { - "extra_body": { - "cache": {"use-cache": True, "ttl": 86400}, # LiteLLM-internal - "some_vertex_param": "value", # legitimate provider extra - }, - } - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - # 'cache' must be stripped — Vertex AI has no such field - assert "cache" not in result, ( - "extra_body.cache must not be forwarded to Vertex AI. " - 'Vertex AI rejects it with 400: Unknown name "cache": Cannot find field.' - ) - - # Other legitimate extra_body keys should still pass through - assert "some_vertex_param" in result - assert result["some_vertex_param"] == "value" - - # Core request fields must be present - assert "contents" in result - - -def test_extra_body_tags_not_forwarded_to_vertex_ai(): - """ - 'tags' inside extra_body is a LiteLLM-internal param for logging/tracking. - It must NOT be forwarded to the Vertex AI request body. - Documented in litellm_proxy.md: "Send tags by including them in the extra_body parameter" - """ - messages = [{"role": "user", "content": "test"}] - optional_params = { - "extra_body": { - "tags": ["user:alice", "env:prod"], - "custom_param": "allowed", - }, - } - litellm_params = {} - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params=litellm_params, - cached_content=None, - ) - - assert "tags" not in result - assert "custom_param" in result - assert result["custom_param"] == "allowed" - - -def test_extra_body_google_maps_rewrites_json_response_format(): - messages = [{"role": "user", "content": "test"}] - optional_params = { - "response_mime_type": "application/json", - "response_schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - "extra_body": { - "tools": [{"googleMaps": {}}], - }, - } - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=None, - ) - - generation_config = result["generationConfig"] - assert "response_mime_type" not in generation_config - assert generation_config["responseFormat"] == { - "text": { - "mimeType": "APPLICATION_JSON", - "schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - } - } - - -def test_extra_body_generation_config_cannot_restore_google_maps_json_mime_type(): - messages = [{"role": "user", "content": "test"}] - optional_params = { - "tools": [{"googleMaps": {}}], - "response_mime_type": "application/json", - "extra_body": { - "generationConfig": { - "response_mime_type": "application/json", - "response_json_schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - }, - }, - } - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params, - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=None, - ) - - generation_config = result["generationConfig"] - assert "response_mime_type" not in generation_config - assert "response_json_schema" not in generation_config - assert generation_config["responseFormat"] == { - "text": { - "mimeType": "APPLICATION_JSON", - "schema": { - "type": "object", - "properties": {"answer": {"type": "string"}}, - }, - } - } - - -def test_metadata_to_labels_vertex_only(): - """Test that metadata->labels conversion only happens for Vertex AI""" - messages = [{"role": "user", "content": "test"}] - optional_params = {} - litellm_params = { - "metadata": { - "requester_metadata": {"user": "john_doe", "project": "test-project"} - } - } - - # Google GenAI/AI Studio should not include labels from metadata - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params.copy(), - custom_llm_provider="gemini", - litellm_params=litellm_params.copy(), - cached_content=None, - ) - assert "labels" not in result - - # Vertex AI should include labels from metadata - result = _transform_request_body( - messages=messages, - model="gemini-2.5-pro", - optional_params=optional_params.copy(), - custom_llm_provider="vertex_ai", - litellm_params=litellm_params.copy(), - cached_content=None, - ) - assert "labels" in result - assert result["labels"] == {"user": "john_doe", "project": "test-project"} - - -def test_empty_content_handling(): - """Test that empty content strings are properly handled in Gemini message transformation""" - # Test with empty content in user message - messages = [{"content": "", "role": "user"}] - - contents = _gemini_convert_messages_with_history(messages=messages) - - # Verify that the content was properly transformed - assert len(contents) == 1 - assert contents[0]["role"] == "user" - assert len(contents[0]["parts"]) == 1 - assert "text" in contents[0]["parts"][0] - assert contents[0]["parts"][0]["text"] == "" - - -def test_thought_signature_extraction_from_response(): - """Test that thought signatures are extracted from Gemini response parts and stored in provider_specific_fields""" - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.types.llms.vertex_ai import HttpxPartType - - # Test case: Single function call with thought signature - test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - - parts_with_signature = [ - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "Paris"}, - }, - thoughtSignature=test_signature, - ) - ] - - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts_with_signature, - cumulative_tool_call_idx=0, - is_function_call=False, - ) - - # Verify thought signature is stored in provider_specific_fields - assert tools is not None - assert len(tools) == 1 - assert "provider_specific_fields" in tools[0] - assert tools[0]["provider_specific_fields"]["thought_signature"] == test_signature - - -def test_thought_signature_parallel_function_calls(): - """Test that only the first function call in parallel calls has thought signature""" - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.types.llms.vertex_ai import HttpxPartType - - test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - - # Parallel function calls - only first has signature - parts_parallel = [ - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "Paris"}, - }, - thoughtSignature=test_signature, # First FC has signature - ), - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "London"}, - }, - # Second FC has no signature (parallel call) - ), - ] - - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts_parallel, - cumulative_tool_call_idx=0, - is_function_call=False, - ) - - # Verify only first tool call has thought signature - assert tools is not None - assert len(tools) == 2 - assert "provider_specific_fields" in tools[0] - assert tools[0]["provider_specific_fields"]["thought_signature"] == test_signature - # Second tool call should not have thought signature - assert "provider_specific_fields" not in tools[ - 1 - ] or "thought_signature" not in tools[1].get("provider_specific_fields", {}) - - -def test_thought_signature_preservation_in_conversion(): - """Test that thought signatures are preserved when converting assistant messages back to Gemini format""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - - # Assistant message with tool calls containing thought signatures - assistant_message = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - }, - "index": 0, - "provider_specific_fields": { - "thought_signature": test_signature, - }, - }, - { - "id": "call_def456", - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "London"}', - }, - "index": 1, - # No thought signature for parallel call - }, - ], - } - - gemini_parts = convert_to_gemini_tool_call_invoke(assistant_message) - - # Verify thought signature is preserved in first function call part - assert len(gemini_parts) == 2 - assert "function_call" in gemini_parts[0] - assert "thoughtSignature" in gemini_parts[0] - assert gemini_parts[0]["thoughtSignature"] == test_signature - - # Verify second function call part does not have thought signature - assert "function_call" in gemini_parts[1] - assert "thoughtSignature" not in gemini_parts[1] - - -def test_thought_signature_sequential_function_calls(): - """Test that each sequential function call preserves its own thought signature""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - signature_1 = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - signature_2 = "DifferentSignatureForSecondCall1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ" - - # Sequential function calls - each has its own signature - # This simulates a multi-step conversation where each step has a signature - assistant_message_step1 = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_step1", - "type": "function", - "function": { - "name": "check_flight", - "arguments": '{"flight": "AA100"}', - }, - "index": 0, - "provider_specific_fields": { - "thought_signature": signature_1, - }, - }, - ], - } - - assistant_message_step2 = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_step2", - "type": "function", - "function": { - "name": "book_taxi", - "arguments": '{"destination": "airport"}', - }, - "index": 0, - "provider_specific_fields": { - "thought_signature": signature_2, - }, - }, - ], - } - - gemini_parts_step1 = convert_to_gemini_tool_call_invoke(assistant_message_step1) - gemini_parts_step2 = convert_to_gemini_tool_call_invoke(assistant_message_step2) - - # Verify each step preserves its own signature - assert len(gemini_parts_step1) == 1 - assert gemini_parts_step1[0]["thoughtSignature"] == signature_1 - - assert len(gemini_parts_step2) == 1 - assert gemini_parts_step2[0]["thoughtSignature"] == signature_2 - - -def test_thought_signature_with_function_call_mode(): - """Test thought signature extraction in function_call mode (is_function_call=True)""" - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.types.llms.vertex_ai import HttpxPartType - - test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - - parts_with_signature = [ - HttpxPartType( - functionCall={ - "name": "get_current_weather", - "args": {"location": "Tokyo"}, - }, - thoughtSignature=test_signature, - ) - ] - - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts_with_signature, - cumulative_tool_call_idx=0, - is_function_call=True, - ) - - # Verify thought signature is stored in function's provider_specific_fields - assert function is not None - # Function should be dict-like (TypedDict or dict) - assert hasattr(function, "__getitem__") or isinstance(function, dict) - assert "provider_specific_fields" in function - assert function["provider_specific_fields"]["thought_signature"] == test_signature - assert tools is None - - -def test_dummy_signature_added_for_gemini_3_conversation_history(): - """Test that dummy signatures are added when transferring conversation history from older models (like gemini-2.5-flash) to gemini-3.""" - import base64 - - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - # Simulate conversation history from gemini-2.5-flash (no thought signature) - assistant_message_from_older_model = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - }, - "index": 0, - # No provider_specific_fields - older model doesn't provide signatures - }, - ], - } - - # Convert to Gemini format for gemini-3-pro-preview (should add dummy signature) - gemini_parts = convert_to_gemini_tool_call_invoke( - assistant_message_from_older_model, model="gemini-3-pro-preview" - ) - - # Verify dummy signature is added - assert len(gemini_parts) == 1 - assert "function_call" in gemini_parts[0] - assert "thoughtSignature" in gemini_parts[0] - - # Verify it's the expected dummy signature (base64 encoded "skip_thought_signature_validator") - expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( - "utf-8" - ) - assert gemini_parts[0]["thoughtSignature"] == expected_dummy - - -def test_dummy_signature_not_added_for_gemini_2_5(): - """Test that dummy signatures are NOT added when target model is not gemini-3.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - # Simulate conversation history from gemini-2.5-flash (no thought signature) - assistant_message = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - }, - "index": 0, - # No provider_specific_fields - }, - ], - } - - # Convert to Gemini format for gemini-2.5-flash (should NOT add dummy signature) - gemini_parts = convert_to_gemini_tool_call_invoke( - assistant_message, model="gemini-2.5-flash" - ) - - # Verify no dummy signature is added for non-gemini-3 models - assert len(gemini_parts) == 1 - assert "function_call" in gemini_parts[0] - assert "thoughtSignature" not in gemini_parts[0] - - -def test_dummy_signature_not_added_when_signature_exists(): - """Test that dummy signatures are NOT added when a real signature already exists.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - real_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - - # Assistant message with existing thought signature - assistant_message_with_signature = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - "provider_specific_fields": { - "thought_signature": real_signature, - }, - }, - "index": 0, - }, - ], - } - - # Convert to Gemini format for gemini-3-pro-preview - gemini_parts = convert_to_gemini_tool_call_invoke( - assistant_message_with_signature, model="gemini-3-pro-preview" - ) - - # Verify real signature is preserved, not replaced with dummy - assert len(gemini_parts) == 1 - assert "function_call" in gemini_parts[0] - assert "thoughtSignature" in gemini_parts[0] - assert gemini_parts[0]["thoughtSignature"] == real_signature - - -def test_dummy_signature_with_function_call_mode(): - """Test that dummy signatures are added for function_call mode when converting to gemini-3.""" - import base64 - - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - # Assistant message with function_call (not tool_calls) and no signature - assistant_message_function_call = { - "role": "assistant", - "content": None, - "function_call": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - # No provider_specific_fields - }, - } - - # Convert to Gemini format for gemini-3-pro-preview - gemini_parts = convert_to_gemini_tool_call_invoke( - assistant_message_function_call, model="gemini-3-pro-preview" - ) - - # Verify dummy signature is added - assert len(gemini_parts) == 1 - assert "function_call" in gemini_parts[0] - assert "thoughtSignature" in gemini_parts[0] - - # Verify it's the expected dummy signature - expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( - "utf-8" - ) - assert gemini_parts[0]["thoughtSignature"] == expected_dummy - - -def _parallel_tool_calls(*signatures): - return [ - { - "id": f"call_{idx}", - "type": "function", - "function": { - "name": f"tool_{idx}", - "arguments": '{"location": "Paris"}', - **( - {"provider_specific_fields": {"thought_signature": signature}} - if signature is not None - else {} - ), - }, - "index": idx, - } - for idx, signature in enumerate(signatures) - ] - - -def _parallel_tool_calls_signed_via_id(*signatures): - """Parallel tool calls in the shape LiteLLM actually hands back to clients. - - The signature rides in the tool call id behind __thought__, which is what an - OpenAI-format client echoes back on the next turn. - """ - from litellm.litellm_core_utils.prompt_templates.factory import ( - _encode_tool_call_id_with_signature, - ) - - return [ - { - "id": _encode_tool_call_id_with_signature(f"call_{idx}", signature), - "type": "function", - "function": {"name": f"tool_{idx}", "arguments": '{"location": "Paris"}'}, - "index": idx, - } - for idx, signature in enumerate(signatures) - ] - - -REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n" -PLACEHOLDER_SIGNATURE = base64.b64encode(b"skip_thought_signature_validator").decode( - "utf-8" -) - - -def test_dummy_signature_only_on_first_parallel_tool_call(): - """Google documents the placeholder as a last resort that degrades quality, so an unsigned - parallel turn replayed to gemini-3 gets a budget of exactly one.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(None, None, None), - }, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - assert "thoughtSignature" not in gemini_parts[2] - - -def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty(): - """Gemini signs only the first of N parallel function calls, so a faithful replay has - nothing to attach to the siblings.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None, None), - }, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - assert "thoughtSignature" not in gemini_parts[2] - - -def test_real_signature_on_later_parallel_tool_call_is_preserved(): - """Clients may reorder or drop calls, so a signature that lands on a non-first call is - still the model's own and must survive the round trip.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(None, REAL_THOUGHT_SIGNATURE), - }, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE - assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - - -def test_no_signatures_on_parallel_tool_calls_for_gemini_2_5(): - """Non-gemini-3 models never get a placeholder signature, on any call.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(None, None), - }, - model="gemini-2.5-flash", - ) - - assert len(gemini_parts) == 2 - assert all("thoughtSignature" not in part for part in gemini_parts) - - -def test_signature_embedded_in_tool_call_id_only_on_first_parallel_call(): - """The production shape: the signature arrives inside the first call's id, siblings have bare ids.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls_signed_via_id( - REAL_THOUGHT_SIGNATURE, None, None - ), - }, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - assert "thoughtSignature" not in gemini_parts[2] - - -def test_tool_level_provider_specific_fields_signature_leaves_siblings_empty(): - """A signature on the tool call itself, rather than on its function, behaves the same way.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - tool_calls = _parallel_tool_calls(None, None) - tool_calls[0]["provider_specific_fields"] = { - "thought_signature": REAL_THOUGHT_SIGNATURE - } - - gemini_parts = convert_to_gemini_tool_call_invoke( - {"role": "assistant", "content": None, "tool_calls": tool_calls}, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - - -def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry(): - """A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not - consume the one placeholder slot and leave the real first function call bare.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - tool_calls = [ - {"id": "call_custom", "type": "custom", "custom": {"name": "noop", "input": ""}} - ] + _parallel_tool_calls(None, None) - - gemini_parts = convert_to_gemini_tool_call_invoke( - {"role": "assistant", "content": None, "tool_calls": tool_calls}, - model="gemini-3-pro-preview", - ) - - assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - - -def test_no_placeholder_when_model_is_unknown(): - """Without a model there is nothing to prove the target needs a placeholder, so none is added.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(None, None), - }, - ) - - assert len(gemini_parts) == 2 - assert all("thoughtSignature" not in part for part in gemini_parts) - - -def test_real_signature_forwarded_to_gemini_2_5_without_placeholder_siblings(): - """Older models still receive a real signature that a client replays, and still get no placeholder.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None), - }, - model="gemini-2.5-flash", - ) - - assert len(gemini_parts) == 2 - assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - - -def test_parallel_tool_call_history_replayed_through_full_message_conversion(): - """End to end through the message-history converter, the path a real /chat/completions replay takes.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - messages = [ - {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls_signed_via_id( - REAL_THOUGHT_SIGNATURE, None, None - ), - }, - ] - - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-3-pro-preview" - ) - - model_parts = contents[1]["parts"] - assert len(model_parts) == 3 - assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in model_parts[1] - assert "thoughtSignature" not in model_parts[2] - - -@pytest.mark.parametrize( - "model", - ["gemini-3.5-flash", "vertex_ai/gemini-3.5-flash", "gemini/gemini-3.5-flash"], -) -def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): - """A native gemini-3.5 parallel turn replays with zero skip_thought_signature_validator parts. - - Fabricating the placeholder alongside a real signature is what produced empty text responses - on gemini-3.5 parallel function calling, so the whole payload has to stay placeholder-free. - """ - import json - - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - messages = [ - {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls_signed_via_id( - REAL_THOUGHT_SIGNATURE, None, None - ), - }, - ] - - contents = _gemini_convert_messages_with_history(messages=messages, model=model) - - model_parts = contents[1]["parts"] - assert len(model_parts) == 3 - assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE - assert "thoughtSignature" not in model_parts[1] - assert "thoughtSignature" not in model_parts[2] - assert PLACEHOLDER_SIGNATURE not in json.dumps(contents) - - -@pytest.mark.parametrize( - "model", - [ - "gemini-3-pro-preview", - "gemini-3-flash-preview", - "gemini-3.1-pro-preview", - "gemini-3.5-flash", - "gemini-3.6-flash", - "gemini-3.7-flash", - "gemini-3.8-flash", - "vertex_ai/gemini-3.5-flash", - "vertex_ai/gemini-3.7-flash", - "vertex_ai/gemini-3.8-flash", - "gemini/gemini-3.5-flash", - "gemini/gemini-3.7-flash", - "gemini/gemini-3.8-flash", - ], -) -def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): - """The gemini-3 gate is a substring match, so every family member and prefix form has to - land on the same one-placeholder budget rather than only the versions we happened to try.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - convert_to_gemini_tool_call_invoke, - ) - - gemini_parts = convert_to_gemini_tool_call_invoke( - { - "role": "assistant", - "content": None, - "tool_calls": _parallel_tool_calls(None, None, None), - }, - model=model, - ) - - assert len(gemini_parts) == 3 - assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE - assert "thoughtSignature" not in gemini_parts[1] - assert "thoughtSignature" not in gemini_parts[2] - - -def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): - """Text-part and function-call signatures are collected by separate code paths, so scoping the - placeholder must not disturb a real signature that arrived on the text part.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": "Checking all three cities.", - "provider_specific_fields": {"thought_signatures": ["real_25_signature"]}, - "tool_calls": _parallel_tool_calls(None, None, None), - } - - parts = _gemini_convert_messages_with_history( - messages=[msg], model="gemini-3-pro-preview" - )[0]["parts"] - - assert parts[0]["text"] == "Checking all three cities." - assert parts[0]["thoughtSignature"] == "real_25_signature" - assert parts[1]["thoughtSignature"] == PLACEHOLDER_SIGNATURE - assert "thoughtSignature" not in parts[2] - assert "thoughtSignature" not in parts[3] - - -# Tests for media_resolution (detail parameter) handling - Issue #17084 -class TestMediaResolution: - """Tests for media_resolution handling in Gemini 2.x models""" - - def test_get_highest_media_resolution_high_wins(self): - """Test that 'high' resolution takes precedence over 'low'""" - assert _get_highest_media_resolution("low", "high") == "high" - assert _get_highest_media_resolution("high", "low") == "high" - assert _get_highest_media_resolution(None, "high") == "high" - assert _get_highest_media_resolution("high", None) == "high" - - def test_get_highest_media_resolution_low_over_none(self): - """Test that 'low' resolution takes precedence over None""" - assert _get_highest_media_resolution(None, "low") == "low" - assert _get_highest_media_resolution("low", None) == "low" - - def test_get_highest_media_resolution_same_values(self): - """Test handling of same resolution values""" - assert _get_highest_media_resolution("high", "high") == "high" - assert _get_highest_media_resolution("low", "low") == "low" - assert _get_highest_media_resolution(None, None) is None - - def test_get_highest_media_resolution_medium(self): - """Test that 'medium' resolution is correctly ranked between 'low' and 'high'""" - assert _get_highest_media_resolution("low", "medium") == "medium" - assert _get_highest_media_resolution("medium", "low") == "medium" - assert _get_highest_media_resolution("medium", "high") == "high" - assert _get_highest_media_resolution("high", "medium") == "high" - assert _get_highest_media_resolution(None, "medium") == "medium" - assert _get_highest_media_resolution("medium", None) == "medium" - - def test_get_highest_media_resolution_ultra_high(self): - """Test that 'ultra_high' resolution takes precedence over all others""" - assert _get_highest_media_resolution("high", "ultra_high") == "ultra_high" - assert _get_highest_media_resolution("ultra_high", "high") == "ultra_high" - assert _get_highest_media_resolution("medium", "ultra_high") == "ultra_high" - assert _get_highest_media_resolution("low", "ultra_high") == "ultra_high" - assert _get_highest_media_resolution(None, "ultra_high") == "ultra_high" - assert _get_highest_media_resolution("ultra_high", None) == "ultra_high" - - def test_extract_max_media_resolution_single_image_high(self): - """Test extraction of media resolution from single image with detail=high""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abc123", - "detail": "high", - }, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) == "high" - - def test_extract_max_media_resolution_single_image_low(self): - """Test extraction of media resolution from single image with detail=low""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abc123", - "detail": "low", - }, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) == "low" - - def test_extract_max_media_resolution_no_detail(self): - """Test extraction when no detail parameter is provided""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": {"url": "data:image/png;base64,abc123"}, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) is None - - def test_extract_max_media_resolution_multiple_images_mixed(self): - """Test that highest resolution is returned when multiple images have different details""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Compare these images"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abc123", - "detail": "low", - }, - }, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,def456", - "detail": "high", - }, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) == "high" - - def test_extract_max_media_resolution_text_only(self): - """Test extraction from messages with no images""" - messages = [ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": "I'm doing well!"}, - ] - assert _extract_max_media_resolution_from_messages(messages) is None - - def test_transform_request_body_gemini_2x_adds_media_resolution(self): - """Test that media_resolution is added to generationConfig for Gemini 2.x models""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgo=", - "detail": "high", - }, - }, - ], - } - ] - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-flash", - optional_params={}, - custom_llm_provider="gemini", - litellm_params={}, - cached_content=None, - ) - - assert "generationConfig" in result - assert "mediaResolution" in result["generationConfig"] - assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_HIGH" - - def test_transform_request_body_gemini_2x_low_resolution(self): - """Test that low media_resolution is correctly added for Gemini 2.x""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgo=", - "detail": "low", - }, - }, - ], - } - ] - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-flash", - optional_params={}, - custom_llm_provider="gemini", - litellm_params={}, - cached_content=None, - ) - - assert "generationConfig" in result - assert "mediaResolution" in result["generationConfig"] - assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_LOW" - - def test_transform_request_body_gemini_3_no_global_media_resolution(self): - """Test that Gemini 3 models don't add media_resolution to generationConfig (they use per-part)""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgo=", - "detail": "high", - }, - }, - ], - } - ] - - result = _transform_request_body( - messages=messages, - model="gemini-3-pro-preview", - optional_params={}, - custom_llm_provider="gemini", - litellm_params={}, - cached_content=None, - ) - - # Gemini 3 should NOT have mediaResolution in generationConfig - # (it's handled per-part in the content transformation) - if "generationConfig" in result: - assert "mediaResolution" not in result["generationConfig"] - - def test_transform_request_body_no_detail_no_media_resolution(self): - """Test that no mediaResolution is added when detail is not specified""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, - }, - ], - } - ] - - result = _transform_request_body( - messages=messages, - model="gemini-2.5-flash", - optional_params={}, - custom_llm_provider="gemini", - litellm_params={}, - cached_content=None, - ) - - # When no detail is specified, mediaResolution should not be in generationConfig - if "generationConfig" in result: - assert "mediaResolution" not in result["generationConfig"] - - def test_extract_max_media_resolution_file_type_with_detail(self): - """Test that detail is extracted from file content type, not just image_url""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this file?"}, - { - "type": "file", - "file": { - "url": "data:image/png;base64,abc123", - "detail": "high", - }, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) == "high" - - def test_extract_max_media_resolution_mixed_image_and_file(self): - """Test that highest detail is returned across both image_url and file types""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Compare these"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,abc123", - "detail": "low", - }, - }, - { - "type": "file", - "file": { - "url": "data:image/png;base64,def456", - "detail": "high", - }, - }, - ], - } - ] - assert _extract_max_media_resolution_from_messages(messages) == "high" - - def test_transform_request_body_gemini_1x_no_media_resolution(self): - """Test that Gemini 1.x models don't get mediaResolution in generationConfig""" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this?"}, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgo=", - "detail": "high", - }, - }, - ], - } - ] - - result = _transform_request_body( - messages=messages, - model="gemini-1.5-pro", - optional_params={}, - custom_llm_provider="gemini", - litellm_params={}, - cached_content=None, - ) - - # Gemini 1.x should NOT have mediaResolution (not supported) - if "generationConfig" in result: - assert "mediaResolution" not in result["generationConfig"] - - -# Tests for VideoMetadata support across all Gemini models (Issue #25474) -class TestVideoMetadataAllGeminiModels: - """Tests that video_metadata (fps, start_offset, end_offset) works for all Gemini models""" - - def _make_video_messages(self, video_metadata: dict) -> list: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Analyze this video"}, - { - "type": "file", - "file": { - "file_id": "gs://bucket/video.mp4", - "format": "video/mp4", - "video_metadata": video_metadata, - }, - }, - ], - } - ] - - def _get_file_part(self, contents: list) -> dict: - for part in contents[0]["parts"]: - if "file_data" in part: - return part - raise AssertionError("No file part found in contents") - - def test_video_metadata_fps_gemini_2_5_flash(self): - """Gemini 2.5 Flash: fps in video_metadata should be forwarded (Issue #25474)""" - messages = self._make_video_messages({"fps": 5}) - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-2.5-flash" - ) - file_part = self._get_file_part(contents) - assert "video_metadata" in file_part - assert file_part["video_metadata"]["fps"] == 5 - - def test_video_metadata_fps_gemini_2_5_pro(self): - """Gemini 2.5 Pro: fps in video_metadata should be forwarded (Issue #25474)""" - messages = self._make_video_messages({"fps": 10}) - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-2.5-pro" - ) - file_part = self._get_file_part(contents) - assert "video_metadata" in file_part - assert file_part["video_metadata"]["fps"] == 10 - - def test_video_metadata_offsets_gemini_2_5_flash(self): - """Gemini 2.5 Flash: start_offset/end_offset converted to camelCase (Issue #25474)""" - messages = self._make_video_messages( - {"start_offset": "5s", "end_offset": "30s"} - ) - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-2.5-flash" - ) - file_part = self._get_file_part(contents) - assert "video_metadata" in file_part - vm = file_part["video_metadata"] - assert vm["startOffset"] == "5s" - assert vm["endOffset"] == "30s" - - def test_video_metadata_all_fields_gemini_2_5_flash(self): - """Gemini 2.5 Flash: all video_metadata fields forwarded correctly (Issue #25474)""" - messages = self._make_video_messages( - {"fps": 5, "start_offset": "10s", "end_offset": "60s"} - ) - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-2.5-flash" - ) - file_part = self._get_file_part(contents) - assert "video_metadata" in file_part - vm = file_part["video_metadata"] - assert vm["fps"] == 5 - assert vm["startOffset"] == "10s" - assert vm["endOffset"] == "60s" - - def test_video_metadata_gemini_1_5_pro(self): - """Gemini 1.5 Pro: video_metadata should also be forwarded (Issue #25474)""" - messages = self._make_video_messages({"fps": 2}) - contents = _gemini_convert_messages_with_history( - messages=messages, model="gemini-1.5-pro" - ) - file_part = self._get_file_part(contents) - assert "video_metadata" in file_part - assert file_part["video_metadata"]["fps"] == 2 - - -def test_convert_tool_response_with_base64_image(): - """Test tool response with base64 data URI image.""" - # Create a small test image (1x1 red pixel PNG) - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - image_data_uri = f"data:image/png;base64,{test_image_base64}" - - # Create tool message with image - tool_message = { - "role": "tool", - "tool_call_id": "call_test123", - "content": [ - { - "type": "text", - "text": '{"url": "https://example.com", "status": "success"}', - }, - {"type": "input_image", "image_url": image_data_uri}, - ], - } - - # Mock last message with tool calls - last_message_with_tool_calls = { - "tool_calls": [ - { - "id": "call_test123", - "function": {"name": "click_at", "arguments": '{"x": 100, "y": 200}'}, - } - ] - } - - # Convert tool response with nested multimodal functionResponse.parts. - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) - - assert isinstance(result, list), "Should return a parts list when media is present" - assert len(result) == 1, "Should return one function_response part" - result_part = result[0] - assert "function_response" in result_part - assert "inline_data" not in result_part - function_response = result_part["function_response"] - assert function_response["name"] == "click_at" - assert "response" in function_response - # Verify JSON response is parsed correctly - assert "url" in function_response["response"] - assert function_response["response"]["url"] == "https://example.com" - - # Check inline_data is nested under functionResponse.parts. - assert "parts" in function_response - assert len(function_response["parts"]) == 1 - inline_data: BlobType = function_response["parts"][0]["inline_data"] - assert "data" in inline_data - assert "mime_type" in inline_data - assert inline_data["mime_type"] == "image/png" - assert inline_data["data"] == test_image_base64 - - -def test_gemini_history_nests_multimodal_tool_response_parts(): - """Full history conversion should not emit sibling inline_data tool result parts.""" - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - messages = [ - {"role": "user", "content": "Get me an image"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_get_image", - "type": "function", - "function": {"name": "get_image", "arguments": "{}"}, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_get_image", - "content": [ - {"type": "text", "text": '{"image_ref": "inline"}'}, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": test_image_base64, - }, - }, - ], - }, - ] - - contents = _gemini_convert_messages_with_history(messages=messages) - - tool_response_parts = contents[-1]["parts"] - assert len(tool_response_parts) == 1 - assert "inline_data" not in tool_response_parts[0] - function_response = tool_response_parts[0]["function_response"] - assert function_response["parts"] == [ - { - "inline_data": { - "data": test_image_base64, - "mime_type": "image/png", - } - } - ] def test_convert_tool_response_with_url_image(): """Test tool response with HTTP URL image (will download and convert).""" - import pytest - # Use a publicly accessible test image URL test_image_url = "https://via.placeholder.com/1x1.png" @@ -1701,13 +33,9 @@ def test_convert_tool_response_with_url_image(): } try: - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) + result = convert_to_gemini_tool_call_result(tool_message, last_message_with_tool_calls) - assert isinstance( - result, list - ), "Should return a parts list when media is present" + assert isinstance(result, list), "Should return a parts list when media is present" assert len(result) == 1, "Should return one function_response part" result_part = result[0] assert "function_response" in result_part @@ -1724,1060 +52,3 @@ def test_convert_tool_response_with_url_image(): except Exception as e: # Skip test if URL download fails (no internet connection, etc.) pytest.skip(f"Failed to download image from URL: {e}") - - -def test_convert_tool_response_text_only(): - """Test tool response with only text (no image).""" - tool_message = { - "role": "tool", - "tool_call_id": "call_test789", - "content": [ - {"type": "text", "text": '{"status": "completed", "result": "success"}'} - ], - } - - last_message_with_tool_calls = { - "tool_calls": [ - { - "id": "call_test789", - "function": {"name": "wait_5_seconds", "arguments": "{}"}, - } - ] - } - - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) - - # Should be a single part (no list) when no image - assert not isinstance(result, list), "Should return single part when no image" - - # Check function_response exists - assert "function_response" in result - function_response = result["function_response"] - assert function_response["name"] == "wait_5_seconds" - # Verify JSON response is parsed correctly - assert "status" in function_response["response"] - assert function_response["response"]["status"] == "completed" - - # Check inline_data does NOT exist (no image provided) - assert "inline_data" not in result - - -def test_file_data_field_order(): - """ - Test that file_data fields are in the correct order (mime_type before file_uri). - - The Gemini API is sensitive to field order in the file_data object. - This test verifies that mime_type comes before file_uri in both: - 1. Dictionary key order - 2. JSON serialization - - Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order. - """ - import json - - from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - - # Test with HTTPS URL and explicit format (audio file) - file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123" - format = "audio/mpeg" - - result = _process_gemini_media(image_url=file_url, format=format) - - # Verify the result has file_data - assert "file_data" in result - file_data = result["file_data"] - - # Verify both fields are present - assert "mime_type" in file_data - assert "file_uri" in file_data - assert file_data["mime_type"] == "audio/mpeg" - assert file_data["file_uri"] == file_url - - # Verify field order by checking dictionary keys - # In Python 3.7+, dict maintains insertion order - file_data_keys = list(file_data.keys()) - assert file_data_keys.index("mime_type") < file_data_keys.index( - "file_uri" - ), "mime_type must come before file_uri in the file_data dict" - - # Also verify by serializing to JSON string - json_str = json.dumps(file_data) - mime_type_pos = json_str.find('"mime_type"') - file_uri_pos = json_str.find('"file_uri"') - assert ( - mime_type_pos < file_uri_pos - ), "mime_type must appear before file_uri in JSON serialization" - - -def test_file_data_field_order_gcs_urls(): - """Test that GCS URLs also maintain correct field order.""" - import json - - from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - - # Test with GCS URL - gcs_url = "gs://bucket/audio.mp3" - - result = _process_gemini_media(image_url=gcs_url) - - # Verify the result has file_data - assert "file_data" in result - file_data = result["file_data"] - - # Verify both fields are present - assert "mime_type" in file_data - assert "file_uri" in file_data - - # Verify field order - file_data_keys = list(file_data.keys()) - assert file_data_keys.index("mime_type") < file_data_keys.index( - "file_uri" - ), "mime_type must come before file_uri in the file_data dict" - - -def test_gemini_files_api_uri_without_format(): - """ - Test that Gemini Files API URIs work WITHOUT an explicit format/mime_type. - - When a user uploads a file via the Gemini Files API and then references it - by URI (https://generativelanguage.googleapis.com/v1beta/files/...), - the file is already on Google's servers. These URLs return 403 when - fetched directly, so _process_gemini_media must NOT try to resolve the - MIME type via HTTP. Instead it should pass the URI through as file_data - and let the Gemini API resolve the type from its stored metadata. - - Related issue: https://github.com/BerriAI/litellm/issues/24907 - """ - from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - - file_url = "https://generativelanguage.googleapis.com/v1beta/files/37eh7rsw1vfe" - - # Should NOT raise — previously this hit the generic https:// handler - # which called _get_image_mime_type_from_url() and got a 403. - result = _process_gemini_media(image_url=file_url) - - assert "file_data" in result - file_data = result["file_data"] - assert file_data["file_uri"] == file_url - # When no format is provided, mime_type should be absent so the - # Gemini API infers it from the stored file metadata. - assert "mime_type" not in file_data - - -def test_gemini_files_api_uri_with_format(): - """ - Test that Gemini Files API URIs correctly forward an explicit format. - - Related issue: https://github.com/BerriAI/litellm/issues/24907 - """ - from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media - - file_url = "https://generativelanguage.googleapis.com/v1beta/files/n1vhxa28lyaw" - - result = _process_gemini_media(image_url=file_url, format="text/plain") - - assert "file_data" in result - file_data = result["file_data"] - assert file_data["file_uri"] == file_url - assert file_data["mime_type"] == "text/plain" - - -def test_extract_file_data_with_path_object(): - """ - Test that filename is correctly extracted from Path objects for MIME type detection. - - When uploading files using Path objects (e.g., Path("speech.mp3")), the filename - must be extracted to enable proper MIME type detection. Without this, files get - uploaded with 'application/octet-stream' instead of the correct MIME type. - - Related issue: Files uploaded with wrong MIME type cause Gemini API to reject - requests where the specified format doesn't match the uploaded file's MIME type. - """ - import os - import tempfile - from pathlib import Path - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - # Create a temporary MP3 file - with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp: - tmp.write(b"fake mp3 content") - tmp_path = tmp.name - - try: - # Test with Path object - path_obj = Path(tmp_path) - extracted = extract_file_data(path_obj) - - # Verify filename was extracted - assert extracted["filename"] is not None - assert extracted["filename"].endswith(".mp3") - - # Verify MIME type was correctly detected - assert ( - extracted["content_type"] == "audio/mpeg" - ), f"Expected 'audio/mpeg' but got '{extracted['content_type']}'" - - # Verify content was read - assert extracted["content"] == b"fake mp3 content" - - finally: - # Clean up temporary file - os.unlink(tmp_path) - - -def test_extract_file_data_with_pathlib_path(): - """Test that filename is correctly extracted from pathlib.Path inputs. - Bare str paths are rejected — when this runs in a proxy request handler - the value is attacker-controlled and opening it as a path is an LFI.""" - import os - import tempfile - from pathlib import Path - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: - tmp.write(b"fake wav content") - tmp_path = Path(tmp.name) - - try: - extracted = extract_file_data(tmp_path) - - assert extracted["filename"] is not None - assert extracted["filename"].endswith(".wav") - assert extracted["content_type"] in [ - "audio/wav", - "audio/x-wav", - ], f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'" - assert extracted["content"] == b"fake wav content" - finally: - os.unlink(str(tmp_path)) - - -def test_extract_file_data_with_tuple_format(): - """Test that tuple format (with explicit content_type) still works correctly.""" - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - # Test with tuple format: (filename, content, content_type) - filename = "test_audio.mp3" - content = b"test audio content" - content_type = "audio/mpeg" - - extracted = extract_file_data((filename, content, content_type)) - - # Verify all fields are correct - assert extracted["filename"] == filename - assert extracted["content"] == content - assert extracted["content_type"] == content_type - - -def test_extract_file_data_fallback_to_octet_stream(): - """Unknown file types fall back to application/octet-stream.""" - import os - import tempfile - from pathlib import Path - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - extract_file_data, - ) - - with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp: - tmp.write(b"unknown content") - tmp_path = Path(tmp.name) - - try: - extracted = extract_file_data(tmp_path) - - assert extracted["filename"] is not None - assert extracted["filename"].endswith(".xyz123") - assert ( - extracted["content_type"] == "application/octet-stream" - ), f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" - finally: - os.unlink(str(tmp_path)) - - -def test_convert_tool_response_with_pdf_file(): - """Test tool response with PDF file content using file_data field.""" - # Create a minimal test PDF (base64 encoded) - test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" - file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" - - # Create tool message with file - tool_message = { - "role": "tool", - "tool_call_id": "call_pdf_test", - "content": [ - {"type": "text", "text": '{"status": "success", "pages": 1}'}, - {"type": "file", "file_data": file_data_uri}, - ], - } - - # Mock last message with tool calls - last_message_with_tool_calls = { - "tool_calls": [ - { - "id": "call_pdf_test", - "function": { - "name": "analyze_document", - "arguments": '{"path": "/tmp/doc.pdf"}', - }, - } - ] - } - - # Convert tool response with nested multimodal functionResponse.parts. - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) - - assert isinstance(result, list), "Should return a parts list when media is present" - assert len(result) == 1, "Should return one function_response part" - result_part = result[0] - assert "function_response" in result_part - assert "inline_data" not in result_part - function_response = result_part["function_response"] - assert function_response["name"] == "analyze_document" - assert "response" in function_response - # Verify JSON response is parsed correctly - assert "status" in function_response["response"] - assert function_response["response"]["status"] == "success" - - # Check inline_data is nested under functionResponse.parts. - assert "parts" in function_response - assert len(function_response["parts"]) == 1 - inline_data: BlobType = function_response["parts"][0]["inline_data"] - assert "data" in inline_data - assert "mime_type" in inline_data - assert inline_data["mime_type"] == "application/pdf" - assert inline_data["data"] == test_pdf_base64 - - -def test_convert_tool_response_with_input_file_type(): - """Test tool response with input_file content type (Responses API format).""" - # Create a minimal test PDF (base64 encoded) - test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" - file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" - - # Create tool message with input_file type - tool_message = { - "role": "tool", - "tool_call_id": "call_input_file_test", - "content": [{"type": "input_file", "file_data": file_data_uri}], - } - - # Mock last message with tool calls - last_message_with_tool_calls = { - "tool_calls": [ - { - "id": "call_input_file_test", - "function": {"name": "read_file", "arguments": "{}"}, - } - ] - } - - # Convert tool response - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) - - # Check inline_data is nested under functionResponse.parts. - assert isinstance(result, list), "Should return a parts list when media is present" - assert len(result) == 1, "Should return one function_response part" - function_response = result[0]["function_response"] - assert ( - function_response["parts"][0]["inline_data"]["mime_type"] == "application/pdf" - ) - - -def test_convert_tool_response_with_nested_file_object(): - """Test tool response with file content using nested file object format.""" - # Create a minimal test PDF (base64 encoded) - test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" - file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" - - # Create tool message with nested file object (OpenAI Agents SDK format) - tool_message = { - "role": "tool", - "tool_call_id": "call_nested_test", - "content": [{"type": "file", "file": {"file_data": file_data_uri}}], - } - - # Mock last message with tool calls - last_message_with_tool_calls = { - "tool_calls": [ - { - "id": "call_nested_test", - "function": {"name": "process_document", "arguments": "{}"}, - } - ] - } - - # Convert tool response - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) - - # Check inline_data is nested under functionResponse.parts. - assert isinstance(result, list), "Should return a parts list when media is present" - assert len(result) == 1, "Should return one function_response part" - function_response = result[0]["function_response"] - inline_data: BlobType = function_response["parts"][0]["inline_data"] - assert "data" in inline_data - assert "mime_type" in inline_data - assert inline_data["mime_type"] == "application/pdf" - assert inline_data["data"] == test_pdf_base64 - - -def test_assistant_message_with_images_field(): - """ - Test that assistant messages with images field are properly converted to Gemini format. - - This handles the case where an assistant message contains generated images in the - `images` field (e.g., from image generation models like gemini-2.5-flash-image). - The images should be converted to inline_data parts in the Gemini format. - """ - # Create a small test image (1x1 red pixel PNG) - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - image_data_uri = f"data:image/png;base64,{test_image_base64}" - - # Create messages with assistant message containing images field - messages = [ - { - "role": "user", - "content": "Generate an image of a banana wearing a costume that says LiteLLM", - }, - { - "role": "assistant", - "content": "Here's your banana in a LiteLLM costume!", - "images": [ - { - "image_url": {"url": image_data_uri, "detail": "auto"}, - "index": 0, - "type": "image_url", - } - ], - }, - ] - - # Convert messages to Gemini format - contents = _gemini_convert_messages_with_history(messages=messages) - - # Verify structure - assert len(contents) == 2, f"Expected 2 content blocks, got {len(contents)}" - - # Verify user message - assert contents[0]["role"] == "user" - assert len(contents[0]["parts"]) == 1 - assert ( - contents[0]["parts"][0]["text"] - == "Generate an image of a banana wearing a costume that says LiteLLM" - ) - - # Verify assistant message - assert contents[1]["role"] == "model" - assert ( - len(contents[1]["parts"]) == 2 - ), f"Expected 2 parts (text + image), got {len(contents[1]['parts'])}" - - # Find text part and inline_data part - text_part = None - inline_data_part = None - for part in contents[1]["parts"]: - if "text" in part: - text_part = part - elif "inline_data" in part: - inline_data_part = part - - # Verify text part - assert text_part is not None, "Missing text part in assistant message" - assert text_part["text"] == "Here's your banana in a LiteLLM costume!" - - # Verify inline_data part (image) - assert inline_data_part is not None, "Missing inline_data part in assistant message" - inline_data: BlobType = inline_data_part["inline_data"] - assert "data" in inline_data - assert "mime_type" in inline_data - assert inline_data["mime_type"] == "image/png" - assert inline_data["data"] == test_image_base64 - - -def test_assistant_message_with_multiple_images(): - """Test that assistant messages with multiple images are properly converted.""" - # Create two test images - test_image1_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - test_image2_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" - image1_data_uri = f"data:image/png;base64,{test_image1_base64}" - image2_data_uri = f"data:image/jpeg;base64,{test_image2_base64}" - - messages = [ - {"role": "user", "content": "Generate two images"}, - { - "role": "assistant", - "content": "Here are your images:", - "images": [ - { - "image_url": {"url": image1_data_uri, "detail": "auto"}, - "index": 0, - "type": "image_url", - }, - { - "image_url": {"url": image2_data_uri, "detail": "high"}, - "index": 1, - "type": "image_url", - }, - ], - }, - ] - - # Convert messages to Gemini format - contents = _gemini_convert_messages_with_history(messages=messages) - - # Verify assistant message has 3 parts (1 text + 2 images) - assert contents[1]["role"] == "model" - assert ( - len(contents[1]["parts"]) == 3 - ), f"Expected 3 parts (text + 2 images), got {len(contents[1]['parts'])}" - - # Count inline_data parts - inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] - assert ( - len(inline_data_parts) == 2 - ), f"Expected 2 inline_data parts, got {len(inline_data_parts)}" - - # Verify first image - assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" - assert inline_data_parts[0]["inline_data"]["data"] == test_image1_base64 - - # Verify second image - assert inline_data_parts[1]["inline_data"]["mime_type"] == "image/jpeg" - assert inline_data_parts[1]["inline_data"]["data"] == test_image2_base64 - - -def test_assistant_message_with_images_using_message_object(): - """Test that Message objects with images field are properly converted.""" - # Create a small test image - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - image_data_uri = f"data:image/png;base64,{test_image_base64}" - - # Create messages using Message object (as returned by LiteLLM) - user_message = {"role": "user", "content": "Generate an image"} - - assistant_message = Message( - content="Here's your image!", - role="assistant", - tool_calls=None, - function_call=None, - images=[ - { - "image_url": {"url": image_data_uri, "detail": "auto"}, - "index": 0, - "type": "image_url", - } - ], - ) - - messages = [user_message, assistant_message] - - # Convert messages to Gemini format - contents = _gemini_convert_messages_with_history(messages=messages) - - # Verify assistant message has both text and image - assert contents[1]["role"] == "model" - assert len(contents[1]["parts"]) == 2 - - # Verify image was converted - inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] - assert len(inline_data_parts) == 1 - assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" - assert inline_data_parts[0]["inline_data"]["data"] == test_image_base64 - - -def test_assistant_message_with_images_in_conversation_history(): - """ - Test multi-turn conversation where assistant message with images is in history. - - This simulates the real use case where: - 1. User asks for image generation - 2. Assistant generates image (with images field) - 3. User asks follow-up question about the image - """ - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - image_data_uri = f"data:image/png;base64,{test_image_base64}" - - messages = [ - {"role": "user", "content": "Generate an image of a cat"}, - { - "role": "assistant", - "content": "Here's a cat image:", - "images": [ - { - "image_url": {"url": image_data_uri, "detail": "auto"}, - "index": 0, - "type": "image_url", - } - ], - }, - {"role": "user", "content": "Can you make it more colorful?"}, - ] - - # Convert messages to Gemini format - contents = _gemini_convert_messages_with_history(messages=messages) - - # Verify structure: user -> model (with image) -> user - assert len(contents) == 3 - assert contents[0]["role"] == "user" - assert contents[1]["role"] == "model" - assert contents[2]["role"] == "user" - - # Verify assistant message has image in history - inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] - assert len(inline_data_parts) == 1 - assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" - - -def test_function_response_has_user_role(): - """ - Test that function response ContentType blocks include role="user". - - Gemini API only accepts two roles: "user" and "model". Function responses - must be sent with role="user". Previously, LiteLLM omitted the role field - entirely, causing 400 errors from the Gemini API. - - Fixes: https://github.com/BerriAI/litellm/issues/22003 - Fixes: https://github.com/BerriAI/litellm/issues/20690 - """ - messages = [ - {"role": "user", "content": "What is the weather in Berlin?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city": "Berlin"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_abc123", - "content": '{"temperature": "15°C", "condition": "Cloudy"}', - }, - ] - - contents = _gemini_convert_messages_with_history(messages=messages) - - # Expect: user -> model (functionCall) -> user (functionResponse) - assert len(contents) == 3 - - assert contents[0]["role"] == "user" - assert contents[1]["role"] == "model" - assert "function_call" in contents[1]["parts"][0] - - # The critical assertion: function response must have role="user" - assert contents[2]["role"] == "user" - assert "function_response" in contents[2]["parts"][0] - - -def test_multi_turn_function_calling_roles(): - """ - Test a full multi-turn function calling conversation produces correct roles. - - Simulates: user asks → model calls tool → tool responds → model answers → user asks again. - Every content block must have an explicit role of "user" or "model". - - Fixes: https://github.com/BerriAI/litellm/issues/22003 - """ - messages = [ - {"role": "user", "content": "What is the weather in Berlin?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_001", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city": "Berlin"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_001", - "content": '{"temperature": "15°C"}', - }, - { - "role": "assistant", - "content": "The weather in Berlin is 15°C.", - }, - {"role": "user", "content": "And in Paris?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_002", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city": "Paris"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_002", - "content": '{"temperature": "18°C"}', - }, - ] - - contents = _gemini_convert_messages_with_history(messages=messages) - - # Every content block must have a valid role - for i, content in enumerate(contents): - assert "role" in content, f"Content block {i} missing 'role' field" - assert content["role"] in ( - "user", - "model", - ), f"Content block {i} has invalid role: {content.get('role')}" - - # Verify the function response blocks specifically have role="user" - for i, content in enumerate(contents): - for part in content["parts"]: - if "function_response" in part: - assert ( - content["role"] == "user" - ), f"Content block {i} with function_response has role='{content['role']}', expected 'user'" - - -def test_gemini_thought_signature_preservation_real_response(): - """Test that thought signatures are preserved on the text part if originally there, without dropping or duplicating (real response case).""" - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - real_candidate = { - "content": { - "parts": [ - { - "text": "I will explain and then list files.", - "thoughtSignature": "mock_signature_from_text_part", - }, - { - "functionCall": { - "name": "list_files", - "args": {}, - } - }, - ] - } - } - - parts = real_candidate["content"]["parts"] - - content, reasoning_content = ( - VertexGeminiConfig().get_assistant_content_message(parts=parts) - ) - thought_signatures = ( - VertexGeminiConfig()._extract_thought_signatures_from_parts( - parts=parts - ) - ) - functions, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts, - cumulative_tool_call_idx=0, - is_function_call=False, - ) - - msg: dict = {"role": "assistant"} - if content is not None: - msg["content"] = content - if tools: - msg["tool_calls"] = tools - if functions is not None: - msg["function_call"] = functions - if thought_signatures is not None: - msg["provider_specific_fields"] = { - "thought_signatures": thought_signatures - } - - converted_real = _gemini_convert_messages_with_history( - messages=[msg], - model="gemini-2.5-pro", - ) - - assert len(converted_real) == 1 - assert "parts" in converted_real[0] - parts_out = converted_real[0]["parts"] - assert len(parts_out) == 2 - assert "text" in parts_out[0] - assert ( - parts_out[0]["thoughtSignature"] == "mock_signature_from_text_part" - ) - assert "function_call" in parts_out[1] - assert "thoughtSignature" not in parts_out[1] - - -def test_gemini_thought_signature_deduplication_assumed_response(): - """Test that thought signatures are deduplicated and not attached to the text part if already present in the tool call (assumed response case).""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - pr_assumed_msg = { - "role": "assistant", - "content": "I will list the directory.", - "provider_specific_fields": { - "thought_signatures": ["mock_signature_63k"] - }, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "list_files", "arguments": "{}"}, - "provider_specific_fields": { - "thought_signature": "mock_signature_63k" - }, - } - ], - } - - converted_pr = _gemini_convert_messages_with_history( - messages=[pr_assumed_msg], - model="gemini-2.5-pro", - ) - - assert len(converted_pr) == 1 - assert "parts" in converted_pr[0] - parts_out = converted_pr[0]["parts"] - assert len(parts_out) == 2 - assert "text" in parts_out[0] - assert "thoughtSignature" not in parts_out[0] - assert "function_call" in parts_out[1] - assert parts_out[1]["thoughtSignature"] == "mock_signature_63k" - - -def test_gemini_thought_signature_pure_text(): - """Test that thought signatures are preserved on the text part for responses with no tool calls.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": "Hello, I am a model.", - "provider_specific_fields": { - "thought_signatures": ["pure_text_signature"] - }, - } - - converted = _gemini_convert_messages_with_history( - messages=[msg], - model="gemini-2.5-pro", - ) - - assert len(converted) == 1 - assert "parts" in converted[0] - parts_out = converted[0]["parts"] - assert len(parts_out) == 1 - assert "text" in parts_out[0] - assert parts_out[0]["thoughtSignature"] == "pure_text_signature" - - -def test_gemini_thought_signature_pure_tool_call(): - """Test that thought signatures are preserved on the tool call for responses with no intermediate text.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": None, - "provider_specific_fields": { - "thought_signatures": ["pure_tool_signature"] - }, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "list_files", "arguments": "{}"}, - "provider_specific_fields": { - "thought_signature": "pure_tool_signature" - }, - } - ], - } - - converted = _gemini_convert_messages_with_history( - messages=[msg], - model="gemini-2.5-pro", - ) - - assert len(converted) == 1 - assert "parts" in converted[0] - parts_out = converted[0]["parts"] - assert len(parts_out) == 1 - assert "function_call" in parts_out[0] - assert parts_out[0]["thoughtSignature"] == "pure_tool_signature" - - -def test_gemini_distinct_text_and_tool_signatures_are_both_preserved(): - """A text-part signature that differs from the tool-call signature must stay on the text part.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": "Some analysis.", - "provider_specific_fields": { - "thought_signatures": ["text_signature", "tool_signature"] - }, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "list_files", "arguments": "{}"}, - "provider_specific_fields": {"thought_signature": "tool_signature"}, - } - ], - } - - parts = _gemini_convert_messages_with_history( - messages=[msg], model="gemini-2.5-pro" - )[0]["parts"] - - assert parts[0]["text"] == "Some analysis." - assert parts[0]["thoughtSignature"] == "text_signature" - assert "function_call" in parts[1] - assert parts[1]["thoughtSignature"] == "tool_signature" - - -def test_gemini_25_text_signature_survives_replay_to_gemini_3(): - """gemini-2.5 history (signed text, unsigned tool call) replayed to gemini-3 keeps the real - text signature; the dummy signature synthesized for the unsigned tool call must not suppress it.""" - from litellm.litellm_core_utils.prompt_templates.factory import ( - _get_dummy_thought_signature, - ) - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": "I will list the directory.", - "provider_specific_fields": {"thought_signatures": ["real_25_signature"]}, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "list_files", "arguments": "{}"}, - } - ], - } - - parts = _gemini_convert_messages_with_history(messages=[msg], model="gemini-3-pro")[ - 0 - ]["parts"] - - assert parts[0]["text"] == "I will list the directory." - assert parts[0]["thoughtSignature"] == "real_25_signature" - assert "function_call" in parts[1] - assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature() - - -def test_gemini_function_call_signature_round_trip_no_duplicate(): - """End to end: a gemini-3-style response (unsigned text + signed functionCall) parsed and - re-serialized sends the signature exactly once, on the function-call part.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - - response_parts = [ - {"text": "I will calculate the result for you."}, - { - "functionCall": {"name": "add_numbers", "args": {"a": 17, "b": 25}}, - "thoughtSignature": "signature_from_function_call", - }, - ] - - config = VertexGeminiConfig() - content, _ = config.get_assistant_content_message(parts=response_parts) - thought_signatures = config._extract_thought_signatures_from_parts( - parts=response_parts - ) - _, tools, _ = VertexGeminiConfig._transform_parts( - parts=response_parts, cumulative_tool_call_idx=0, is_function_call=False - ) - - msg = { - "role": "assistant", - "content": content, - "tool_calls": tools, - "provider_specific_fields": {"thought_signatures": thought_signatures}, - } - - parts = _gemini_convert_messages_with_history(messages=[msg], model="gemini-3-pro")[ - 0 - ]["parts"] - - signatures = [p["thoughtSignature"] for p in parts if "thoughtSignature" in p] - assert signatures == ["signature_from_function_call"] - assert "thoughtSignature" not in parts[0] - assert "function_call" in parts[1] - - -def test_gemini_server_side_tool_signature_not_duplicated_on_text(): - """A signature already re-injected on a server-side toolCall part is not attached to the text part again.""" - from litellm.llms.vertex_ai.gemini.transformation import ( - _gemini_convert_messages_with_history, - ) - - msg = { - "role": "assistant", - "content": "The weather in Buenos Aires is sunny.", - "provider_specific_fields": { - "thought_signatures": ["server_side_signature"], - "server_side_tool_invocations": [ - { - "tool_type": "GOOGLE_SEARCH_WEB", - "id": "abc123", - "args": {"queries": ["weather Buenos Aires"]}, - "response": {"weather": "Sunny"}, - "thought_signature": "server_side_signature", - } - ], - }, - } - - parts = _gemini_convert_messages_with_history( - messages=[msg], model="gemini-2.5-pro" - )[0]["parts"] - - text_part = next(p for p in parts if "text" in p) - assert "thoughtSignature" not in text_part - tool_call_part = next(p for p in parts if "toolCall" in p) - assert tool_call_part["thoughtSignature"] == "server_side_signature" diff --git a/tests/test_litellm/llms/vertex_ai/image_edit/__init__.py b/tests/test_litellm/llms/vertex_ai/image_edit/__init__.py deleted file mode 100644 index 50135ba1f92..00000000000 --- a/tests/test_litellm/llms/vertex_ai/image_edit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Vertex AI Image Edit Tests diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py index 54607cc5284..aeba9f0fa3c 100644 --- a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -1,13 +1,9 @@ import os -from unittest.mock import MagicMock, patch +from unittest.mock import patch -import httpx import pytest -from litellm.llms.vertex_ai.image_generation import ( - get_vertex_ai_image_generation_config, -) from litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation import ( VertexAIGeminiImageGenerationConfig, ) @@ -16,588 +12,6 @@ from litellm.llms.vertex_ai.image_generation.vertex_imagen_transformation import ) -class TestVertexAIGeminiImageGenerationConfig: - def setup_method(self): - """Set up test fixtures""" - self.config = VertexAIGeminiImageGenerationConfig() - - def test_get_supported_openai_params(self): - """Test get_supported_openai_params returns correct params""" - supported = self.config.get_supported_openai_params("gemini-2.5-flash-image") - assert "n" in supported - assert "size" in supported - - def test_map_openai_params_n(self): - """Test mapping n parameter to candidate_count""" - non_default_params = {"n": 3} - optional_params = {} - result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) - assert result.get("candidate_count") == 3 - - def test_map_openai_params_size(self): - """Test mapping size parameter to aspectRatio""" - non_default_params = {"size": "1024x1024"} - optional_params = {} - result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) - assert result.get("aspectRatio") == "1:1" - - def test_map_openai_params_size_16_9(self): - """Test mapping 16:9 size""" - non_default_params = {"size": "1792x1024"} - optional_params = {} - result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) - assert result.get("aspectRatio") == "16:9" - - def test_map_size_to_aspect_ratio(self): - """Test size to aspect ratio mapping""" - assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" - assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" - assert self.config._map_size_to_aspect_ratio("1024x1792") == "9:16" - assert self.config._map_size_to_aspect_ratio("1280x896") == "4:3" - assert self.config._map_size_to_aspect_ratio("896x1280") == "3:4" - assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default - - def test_get_supported_openai_params_includes_native_gemini_params(self): - """Test that native Gemini imageConfig params are supported""" - supported = self.config.get_supported_openai_params("gemini-3-pro-image-preview") - assert "aspectRatio" in supported - assert "aspect_ratio" in supported - assert "imageSize" in supported - assert "image_size" in supported - assert "imageConfig" in supported - - def test_map_openai_params_aspect_ratio_camel_case(self): - """Test mapping native aspectRatio parameter""" - result = self.config.map_openai_params({"aspectRatio": "9:16"}, {}, "gemini-3-pro-image-preview", False) - assert result["aspectRatio"] == "9:16" - - def test_map_openai_params_aspect_ratio_snake_case(self): - """Test mapping native aspect_ratio parameter""" - result = self.config.map_openai_params({"aspect_ratio": "16:9"}, {}, "gemini-3-pro-image-preview", False) - assert result["aspectRatio"] == "16:9" - - def test_map_openai_params_image_size_camel_case(self): - """Test mapping native imageSize parameter""" - result = self.config.map_openai_params({"imageSize": "4K"}, {}, "gemini-3-pro-image-preview", False) - assert result["imageSize"] == "4K" - - def test_map_openai_params_image_size_snake_case(self): - """Test mapping native image_size parameter""" - result = self.config.map_openai_params({"image_size": "2K"}, {}, "gemini-3-pro-image-preview", False) - assert result["imageSize"] == "2K" - - def test_map_openai_params_image_config_dict_stored_whole(self): - """imageConfig dict is stored as-is so all fields survive""" - result = self.config.map_openai_params( - {"imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}}, - {}, - "gemini-3.1-flash-image", - False, - ) - assert result["imageConfig"] == {"aspectRatio": "16:9", "imageSize": "2K"} - - def test_map_openai_params_image_config_all_fields(self): - """All ImageConfig fields (personGeneration, imageOutputOptions) pass through""" - payload = { - "imageConfig": { - "aspectRatio": "9:16", - "imageSize": "4K", - "personGeneration": "DONT_ALLOW", - "imageOutputOptions": { - "mimeType": "image/jpeg", - "compressionQuality": 80, - }, - } - } - result = self.config.map_openai_params(payload, {}, "gemini-3.1-flash-image", False) - assert result["imageConfig"] == payload["imageConfig"] - - def test_map_openai_params_image_config_non_dict_warns_and_drops(self): - """Non-dict imageConfig is dropped with a warning, not silently discarded""" - with patch("litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation.verbose_logger") as mock_log: - result = self.config.map_openai_params( - {"imageConfig": "bad-string-value"}, {}, "gemini-3.1-flash-image", False - ) - assert "imageConfig" not in result - mock_log.warning.assert_called_once() - - def test_transform_image_generation_request_from_image_config(self): - """Full imageConfig dict is forwarded verbatim into generationConfig""" - full_config = { - "aspectRatio": "16:9", - "imageSize": "2K", - "personGeneration": "DONT_ALLOW", - "imageOutputOptions": {"mimeType": "image/jpeg", "compressionQuality": 85}, - } - mapped = self.config.map_openai_params( - {"imageConfig": full_config}, - {}, - "gemini-3.1-flash-image", - False, - ) - request = self.config.transform_image_generation_request( - model="gemini-3.1-flash-image", - prompt="A nano banana on a desk", - optional_params=mapped, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["imageConfig"] == full_config - - def test_transform_image_generation_flat_params_override_image_config(self): - """Explicit flat params win over the same key inside imageConfig""" - request = self.config.transform_image_generation_request( - model="gemini-3.1-flash-image", - prompt="A nano banana", - optional_params={ - "imageConfig": {"aspectRatio": "1:1", "personGeneration": "DONT_ALLOW"}, - "aspectRatio": "16:9", # should win - }, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" - assert request["generationConfig"]["imageConfig"]["personGeneration"] == "DONT_ALLOW" - - def test_transform_image_generation_request_basic(self): - """Test basic request transformation""" - request = self.config.transform_image_generation_request( - model="gemini-2.5-flash-image", - prompt="A nano banana", - optional_params={}, - litellm_params={}, - headers={}, - ) - assert "contents" in request - assert "generationConfig" in request - assert request["generationConfig"]["responseModalities"] == ["IMAGE"] - assert request["contents"][0]["parts"][0]["text"] == "A nano banana" - - def test_transform_image_generation_request_with_aspect_ratio(self): - """Test request transformation with aspectRatio""" - request = self.config.transform_image_generation_request( - model="gemini-2.5-flash-image", - prompt="A nano banana", - optional_params={"aspectRatio": "16:9"}, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" - - def test_transform_image_generation_request_with_image_size(self): - """Test request transformation with imageSize (Gemini 3 Pro)""" - request = self.config.transform_image_generation_request( - model="gemini-3-pro-image-preview", - prompt="A nano banana", - optional_params={"imageSize": "4K"}, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["imageConfig"]["imageSize"] == "4K" - - def test_map_openai_params_web_search_options(self): - """Test web_search_options maps to googleSearch tool""" - result = self.config.map_openai_params({"web_search_options": {}}, {}, "gemini-3.1-flash-image-preview", False) - assert result["tools"] == [{"googleSearch": {}}] - - def test_transform_image_generation_request_with_web_search_tools(self): - """Test request transformation includes googleSearch tools""" - request = self.config.transform_image_generation_request( - model="gemini-3.1-flash-image-preview", - prompt="Generate an image of the latest iPhone", - optional_params={"tools": [{"googleSearch": {}}]}, - litellm_params={}, - headers={}, - ) - assert request["tools"] == [{"googleSearch": {}}] - - def test_transform_image_generation_request_forwards_tool_config(self): - """Test request transformation forwards toolConfig side-effects from tool mapping""" - mapped = self.config.map_openai_params( - {"tools": [{"googleMaps": {"latitude": 37.7, "longitude": -122.4}}]}, - {}, - "gemini-3.1-flash-image-preview", - False, - ) - request = self.config.transform_image_generation_request( - model="gemini-3.1-flash-image-preview", - prompt="Generate an image of a coffee shop nearby", - optional_params=mapped, - litellm_params={}, - headers={}, - ) - assert request["tools"] == [{"googleMaps": {}}] - assert request["toolConfig"] == {"retrievalConfig": {"latLng": {"latitude": 37.7, "longitude": -122.4}}} - - def test_transform_image_generation_request_with_candidate_count(self): - """Test request transformation with candidate_count""" - request = self.config.transform_image_generation_request( - model="gemini-2.5-flash-image", - prompt="A nano banana", - optional_params={"candidate_count": 2}, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["candidateCount"] == 2 - - def test_transform_image_generation_request_with_n(self): - """Test request transformation with n parameter""" - request = self.config.transform_image_generation_request( - model="gemini-2.5-flash-image", - prompt="A nano banana", - optional_params={"n": 2}, - litellm_params={}, - headers={}, - ) - assert request["generationConfig"]["candidateCount"] == 2 - - def test_transform_image_generation_response(self): - """Test response transformation""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [ - { - "inlineData": { - "mimeType": "image/png", - "data": "base64_encoded_image_data", - } - } - ] - } - } - ], - "usageMetadata": { - "promptTokenCount": 93, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 54, - }, - { - "modality": "IMAGE", - "tokenCount": 39, - }, - ], - "candidatesTokenCount": 17, - "totalTokenCount": 110, - }, - } - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - model_response = ImageResponse() - result = self.config.transform_image_generation_response( - model="gemini-2.5-flash-image", - raw_response=mock_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert len(result.data) == 1 - assert result.data[0].b64_json == "base64_encoded_image_data" - assert result.data[0].url is None - assert result.usage.input_tokens == 93 - assert result.usage.input_tokens_details.text_tokens == 54 - assert result.usage.input_tokens_details.image_tokens == 39 - assert result.usage.output_tokens == 17 - assert result.usage.total_tokens == 110 - - def test_transform_image_generation_response_multiple_images(self): - """Test response transformation with multiple images""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [ - { - "inlineData": { - "mimeType": "image/png", - "data": "image1", - } - }, - { - "inlineData": { - "mimeType": "image/png", - "data": "image2", - } - }, - ] - } - } - ] - } - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - model_response = ImageResponse() - result = self.config.transform_image_generation_response( - model="gemini-2.5-flash-image", - raw_response=mock_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert len(result.data) == 2 - assert result.data[0].b64_json == "image1" - assert result.data[1].b64_json == "image2" - - def test_transform_image_generation_response_signature(self): - """Test response transformation includes thoughtSignature for Gemini 3 Pro""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [ - { - "inlineData": { - "mimeType": "image/png", - "data": "base64_encoded_image_data", - }, - "thoughtSignature": "test_signature_abc123", - } - ] - } - } - ] - } - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - model_response = ImageResponse() - result = self.config.transform_image_generation_response( - model="gemini-3-pro-image-preview", - raw_response=mock_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert len(result.data) == 1 - assert result.data[0].b64_json == "base64_encoded_image_data" - assert result.data[0].provider_specific_fields["thought_signature"] == "test_signature_abc123" - - def test_transform_image_generation_response_tracks_web_search_requests(self): - """Grounding queries are carried onto usage so search spend can be billed""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [ - { - "inlineData": { - "mimeType": "image/png", - "data": "base64_encoded_image_data", - } - } - ] - }, - "groundingMetadata": {"webSearchQueries": ["eiffel tower", "paris skyline"]}, - } - ], - "usageMetadata": { - "promptTokenCount": 93, - "candidatesTokenCount": 17, - "totalTokenCount": 110, - }, - } - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - result = self.config.transform_image_generation_response( - model="gemini-2.5-flash-image", - raw_response=mock_response, - model_response=ImageResponse(), - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert result.usage.web_search_requests == 2 - - -class TestVertexAIImagenImageGenerationConfig: - def setup_method(self): - """Set up test fixtures""" - self.config = VertexAIImagenImageGenerationConfig() - - def test_get_supported_openai_params(self): - """Test get_supported_openai_params returns correct params""" - supported = self.config.get_supported_openai_params("imagegeneration@006") - assert "n" in supported - assert "size" in supported - - def test_map_openai_params_n(self): - """Test mapping n parameter to sampleCount""" - non_default_params = {"n": 3} - optional_params = {} - result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False) - assert result.get("sampleCount") == 3 - - def test_map_openai_params_size(self): - """Test mapping size parameter to aspectRatio""" - non_default_params = {"size": "1024x1024"} - optional_params = {} - result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False) - assert result.get("aspectRatio") == "1:1" - - def test_map_size_to_aspect_ratio(self): - """Test size to aspect ratio mapping""" - assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" - assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" - assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default - - def test_transform_image_generation_request_basic(self): - """Test basic request transformation""" - request = self.config.transform_image_generation_request( - model="imagegeneration@006", - prompt="A cat", - optional_params={}, - litellm_params={}, - headers={}, - ) - assert "instances" in request - assert "parameters" in request - assert request["instances"][0]["prompt"] == "A cat" - assert request["parameters"]["sampleCount"] == 1 - - def test_transform_image_generation_request_with_params(self): - """Test request transformation with parameters""" - request = self.config.transform_image_generation_request( - model="imagegeneration@006", - prompt="A cat", - optional_params={"sampleCount": 2, "aspectRatio": "16:9"}, - litellm_params={}, - headers={}, - ) - assert request["parameters"]["sampleCount"] == 2 - assert request["parameters"]["aspectRatio"] == "16:9" - - def test_transform_image_generation_request_labels_from_metadata(self): - """Billing labels from litellm_params.metadata.requester_metadata on predict body.""" - request = self.config.transform_image_generation_request( - model="imagegeneration@006", - prompt="A cat", - optional_params={}, - litellm_params={"metadata": {"requester_metadata": {"team": "platform", "env": "prod"}}}, - headers={}, - ) - assert request["labels"] == {"team": "platform", "env": "prod"} - assert "labels" not in request["parameters"] - - def test_transform_image_generation_response(self): - """Test response transformation""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = {"predictions": [{"bytesBase64Encoded": "base64_encoded_image_data"}]} - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - model_response = ImageResponse() - result = self.config.transform_image_generation_response( - model="imagegeneration@006", - raw_response=mock_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert len(result.data) == 1 - assert result.data[0].b64_json == "base64_encoded_image_data" - assert result.data[0].url is None - - def test_transform_image_generation_response_multiple_images(self): - """Test response transformation with multiple images""" - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = { - "predictions": [ - {"bytesBase64Encoded": "image1"}, - {"bytesBase64Encoded": "image2"}, - ] - } - mock_response.headers = {} - - from litellm.types.utils import ImageResponse - - model_response = ImageResponse() - result = self.config.transform_image_generation_response( - model="imagegeneration@006", - raw_response=mock_response, - model_response=model_response, - logging_obj=MagicMock(), - request_data={}, - optional_params={}, - litellm_params={}, - encoding=None, - ) - - assert len(result.data) == 2 - assert result.data[0].b64_json == "image1" - assert result.data[1].b64_json == "image2" - - -class TestGetVertexAIImageGenerationConfig: - """Test the router function that selects the correct config""" - - def test_get_gemini_model_config(self): - """Test that Gemini models return Gemini config""" - config = get_vertex_ai_image_generation_config("gemini-2.5-flash-image") - assert isinstance(config, VertexAIGeminiImageGenerationConfig) - - config = get_vertex_ai_image_generation_config("gemini-3-pro-image-preview") - assert isinstance(config, VertexAIGeminiImageGenerationConfig) - - config = get_vertex_ai_image_generation_config("vertex_ai/gemini-2.5-flash-image") - assert isinstance(config, VertexAIGeminiImageGenerationConfig) - - def test_get_imagen_model_config(self): - """Test that Imagen models return Imagen config""" - config = get_vertex_ai_image_generation_config("imagegeneration@006") - assert isinstance(config, VertexAIImagenImageGenerationConfig) - - config = get_vertex_ai_image_generation_config("imagen-4.0-generate-001") - assert isinstance(config, VertexAIImagenImageGenerationConfig) - - config = get_vertex_ai_image_generation_config("vertex_ai/imagegeneration@006") - assert isinstance(config, VertexAIImagenImageGenerationConfig) - - def test_get_non_gemini_model_config(self): - """Test that non-Gemini models default to Imagen config""" - config = get_vertex_ai_image_generation_config("some-other-model") - assert isinstance(config, VertexAIImagenImageGenerationConfig) - - class TestVertexAIImageGenerationIntegration: """Integration tests for Vertex AI image generation""" @@ -642,39 +56,3 @@ class TestVertexAIImageGenerationIntegration: litellm_params={}, ) assert "Authorization" in headers - - def test_gemini_get_complete_url(self): - """Test Gemini config URL generation""" - config = VertexAIGeminiImageGenerationConfig() - url = config.get_complete_url( - api_base=None, - api_key=None, - model="gemini-2.5-flash-image", - optional_params={}, - litellm_params={ - "vertex_project": "test-project", - "vertex_location": "us-central1", - }, - ) - assert "test-project" in url - assert "us-central1" in url - assert "gemini-2.5-flash-image" in url - assert "generateContent" in url - - def test_imagen_get_complete_url(self): - """Test Imagen config URL generation""" - config = VertexAIImagenImageGenerationConfig() - url = config.get_complete_url( - api_base=None, - api_key=None, - model="imagegeneration@006", - optional_params={}, - litellm_params={ - "vertex_project": "test-project", - "vertex_location": "us-central1", - }, - ) - assert "test-project" in url - assert "us-central1" in url - assert "imagegeneration@006" in url - assert "predict" in url diff --git a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/__init__.py b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/__init__.py deleted file mode 100644 index 8b41c5ab3f8..00000000000 --- a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for Vertex AI Gemma-AI models""" diff --git a/tests/test_litellm/llms/vertex_ai/videos/__init__.py b/tests/test_litellm/llms/vertex_ai/videos/__init__.py deleted file mode 100644 index f29c2a16fd5..00000000000 --- a/tests/test_litellm/llms/vertex_ai/videos/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Tests for Vertex AI video generation. -""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 1d3d7a452b6..45ad336368c 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -30,7 +30,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockTextContent, ) from litellm.types.utils import CallTypes, ModelResponse -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index bbc8fd539a3..5169d4c9ec6 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -23,7 +23,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrailResponse, ) from litellm.types.utils import Choices, Message, ModelResponse -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe CONTENT_FILTER_CHECKS = {"contentFilter": {"categories": [{"category": "VIOLENCE"}]}} diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 353ffadfa46..227921d6150 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -24,7 +24,7 @@ from starlette.datastructures import FormData import litellm from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 653b2c9914a..ecea4723bf4 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,11 +1,14 @@ import asyncio +import base64 import importlib import os from collections.abc import Coroutine, Iterator +from dataclasses import dataclass, field from pathlib import Path from typing import Final import boto3 +import httpx import pytest from pytest_socket import enable_socket, socket_allow_hosts @@ -15,9 +18,12 @@ import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at im import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency +from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency +from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency image_handling as image_handling_module, ) +from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency close_litellm_async_clients, ) @@ -89,6 +95,9 @@ RESTORED_GLOBALS: Final = ( ) MODULE_LEVEL_CLIENTS: Final = ("module_level_client", "module_level_aclient") SESSION_CLIENTS: Final = ("base_llm_aiohttp_handler", "httpx_client", "aclient", "client") +ONE_PIXEL_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +) def _allow_loopback_only() -> None: @@ -236,6 +245,47 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +@pytest.fixture +def local_beta_headers_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + reload_beta_headers_config() + yield + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + reload_beta_headers_config() + + +@dataclass(slots=True) +class AsyncOnlyImageFetch: + fetched: list[str] = field(default_factory=list) # mutable-ok: tests assert on the URLs fetched, in order + base64_png: str = base64.b64encode(ONE_PIXEL_PNG).decode() + data_url: str = "data:image/png;base64," + base64.b64encode(ONE_PIXEL_PNG).decode() + + +@pytest.fixture +def async_only_image_fetch(monkeypatch: pytest.MonkeyPatch) -> AsyncOnlyImageFetch: + fetch: Final = AsyncOnlyImageFetch() + + def forbid_sync_fetch(client: object, url: str, **kwargs: object) -> httpx.Response: + raise litellm.ImageFetchError(f"sync image fetch ran on the event loop: {url}") + + async def serve_png(client: object, url: str, **kwargs: object) -> httpx.Response: + fetch.fetched.append(url) + return httpx.Response( + 200, content=ONE_PIXEL_PNG, headers={"content-type": "image/png"}, request=httpx.Request("GET", url) + ) + + def forbid_sync_convert(url: str, *args: object, **kwargs: object) -> str: + if url.startswith(("http://", "https://")): + raise litellm.ImageFetchError(f"sync convert_url_to_base64 ran on the request path: {url}") + return url + + monkeypatch.setattr(image_handling_module, "safe_get", forbid_sync_fetch) + monkeypatch.setattr(image_handling_module, "async_safe_get", serve_png) + for module in (image_handling_module, prompt_factory_module, gemini_chat_transformation_module): + monkeypatch.setattr(module, "convert_url_to_base64", forbid_sync_convert) + return fetch + + @pytest.fixture def no_ambient_azure_credentials(monkeypatch: pytest.MonkeyPatch) -> None: for name in AMBIENT_AZURE_CREDENTIAL_ENV_VARS: diff --git a/tests/test_litellm/llms/anthropic/__init__.py b/tests/unit/expected_fine_tuning_api/__init__.py similarity index 100% rename from tests/test_litellm/llms/anthropic/__init__.py rename to tests/unit/expected_fine_tuning_api/__init__.py diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_cancel_expected_output.json b/tests/unit/expected_fine_tuning_api/azure_cancel_expected_output.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_cancel_expected_output.json rename to tests/unit/expected_fine_tuning_api/azure_cancel_expected_output.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_cancel_raw_response.json b/tests/unit/expected_fine_tuning_api/azure_cancel_raw_response.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_cancel_raw_response.json rename to tests/unit/expected_fine_tuning_api/azure_cancel_raw_response.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_cancel_request.json b/tests/unit/expected_fine_tuning_api/azure_cancel_request.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_cancel_request.json rename to tests/unit/expected_fine_tuning_api/azure_cancel_request.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_create_expected_output.json b/tests/unit/expected_fine_tuning_api/azure_create_expected_output.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_create_expected_output.json rename to tests/unit/expected_fine_tuning_api/azure_create_expected_output.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_create_raw_response.json b/tests/unit/expected_fine_tuning_api/azure_create_raw_response.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_create_raw_response.json rename to tests/unit/expected_fine_tuning_api/azure_create_raw_response.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_create_request.json b/tests/unit/expected_fine_tuning_api/azure_create_request.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_create_request.json rename to tests/unit/expected_fine_tuning_api/azure_create_request.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_list_raw_response.json b/tests/unit/expected_fine_tuning_api/azure_list_raw_response.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_list_raw_response.json rename to tests/unit/expected_fine_tuning_api/azure_list_raw_response.json diff --git a/tests/test_litellm/expected_fine_tuning_api/azure_list_request.json b/tests/unit/expected_fine_tuning_api/azure_list_request.json similarity index 100% rename from tests/test_litellm/expected_fine_tuning_api/azure_list_request.json rename to tests/unit/expected_fine_tuning_api/azure_list_request.json diff --git a/tests/test_litellm/llms/anthropic/batches/__init__.py b/tests/unit/llms/aiml/__init__.py similarity index 100% rename from tests/test_litellm/llms/anthropic/batches/__init__.py rename to tests/unit/llms/aiml/__init__.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/__init__.py b/tests/unit/llms/aiml/image_generation/__init__.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/__init__.py rename to tests/unit/llms/aiml/image_generation/__init__.py diff --git a/tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py b/tests/unit/llms/aiml/image_generation/test_aiml_image_generation_transformation.py similarity index 100% rename from tests/test_litellm/llms/aiml/image_generation/test_aiml_image_generation_transformation.py rename to tests/unit/llms/aiml/image_generation/test_aiml_image_generation_transformation.py diff --git a/tests/unit/llms/anthropic/batches/test_transformation.py b/tests/unit/llms/anthropic/batches/test_transformation.py index eacd2c9d03b..419fc7740eb 100644 --- a/tests/unit/llms/anthropic/batches/test_transformation.py +++ b/tests/unit/llms/anthropic/batches/test_transformation.py @@ -616,7 +616,7 @@ def test_transform_response_reraises_unexpected_error(config): # automatically. See base_batches_config_test.py. # --------------------------------------------------------------------------- # -from tests.test_litellm.llms.base_llm.batches.base_batches_config_test import ( # noqa: E402 +from tests.unit.llms.base_llm.batches.base_batches_config_test import ( # noqa: E402 BatchesConfigContractTests, ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/tests/unit/llms/anthropic/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py rename to tests/unit/llms/anthropic/chat/__init__.py diff --git a/tests/test_litellm/llms/anthropic/chat/conftest.py b/tests/unit/llms/anthropic/chat/conftest.py similarity index 100% rename from tests/test_litellm/llms/anthropic/chat/conftest.py rename to tests/unit/llms/anthropic/chat/conftest.py diff --git a/tests/test_litellm/llms/anthropic/files/__init__.py b/tests/unit/llms/anthropic/chat/guardrail_translation/__init__.py similarity index 100% rename from tests/test_litellm/llms/anthropic/files/__init__.py rename to tests/unit/llms/anthropic/chat/guardrail_translation/__init__.py diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py similarity index 100% rename from tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py rename to tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py similarity index 100% rename from tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py rename to tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py rename to tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/unit/llms/anthropic/chat/test_code_interpreter_results_extraction.py similarity index 100% rename from tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py rename to tests/unit/llms/anthropic/chat/test_code_interpreter_results_extraction.py diff --git a/tests/test_litellm/llms/azure/batches/__init__.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/__init__.py similarity index 100% rename from tests/test_litellm/llms/azure/batches/__init__.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/__init__.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_compaction.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_message_id.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_message_id.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_message_id.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_message_id.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_mid_stream_error.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_mid_stream_error.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_mid_stream_error.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_stop_reason.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py rename to tests/unit/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_tool_args.py diff --git a/tests/test_litellm/llms/azure/vector_stores/__init__.py b/tests/unit/llms/anthropic/experimental_pass_through/context_management/__init__.py similarity index 100% rename from tests/test_litellm/llms/azure/vector_stores/__init__.py rename to tests/unit/llms/anthropic/experimental_pass_through/context_management/__init__.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/experimental_pass_through/context_management/test_clear_tool_uses.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_clear_tool_uses.py rename to tests/unit/llms/anthropic/experimental_pass_through/context_management/test_clear_tool_uses.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/experimental_pass_through/context_management/test_compact.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py rename to tests/unit/llms/anthropic/experimental_pass_through/context_management/test_compact.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/experimental_pass_through/context_management/test_dispatcher.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_dispatcher.py rename to tests/unit/llms/anthropic/experimental_pass_through/context_management/test_dispatcher.py diff --git a/tests/test_litellm/llms/base_llm/__init__.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/__init__.py similarity index 100% rename from tests/test_litellm/llms/base_llm/__init__.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/__init__.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_advisor_integration.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_advisor_integration.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_advisor_integration.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_advisor_integration.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_effort.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_per_turn_control.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_per_turn_control.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_per_turn_control.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_per_turn_control.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_structured_outputs.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_content_after_stop_reason.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_mcp_handler.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_mid_conversation_system.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_parallel_tool_calls.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_reasoning_auto_summary_messages.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_reasoning_effort_translation.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_request_optional_param_utils.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_response_cache.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_response_cache.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_sse_wrapper.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py rename to tests/unit/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py diff --git a/tests/test_litellm/llms/base_llm/batches/__init__.py b/tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py similarity index 100% rename from tests/test_litellm/llms/base_llm/batches/__init__.py rename to tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py rename to tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py rename to tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py similarity index 100% rename from tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py rename to tests/unit/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py rename to tests/unit/llms/anthropic/test_anthropic_common_utils.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_count_tokens_transformation.py rename to tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py b/tests/unit/llms/anthropic/test_anthropic_files_and_batches.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_files_and_batches.py rename to tests/unit/llms/anthropic/test_anthropic_files_and_batches.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_output_format_filter.py b/tests/unit/llms/anthropic/test_anthropic_output_format_filter.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_output_format_filter.py rename to tests/unit/llms/anthropic/test_anthropic_output_format_filter.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py rename to tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_reasoning_effort.py rename to tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py b/tests/unit/llms/anthropic/test_anthropic_schema_filter.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_schema_filter.py rename to tests/unit/llms/anthropic/test_anthropic_schema_filter.py diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_structured_output.py b/tests/unit/llms/anthropic/test_anthropic_structured_output.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_anthropic_structured_output.py rename to tests/unit/llms/anthropic/test_anthropic_structured_output.py diff --git a/tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py b/tests/unit/llms/anthropic/test_azure_ai_cache_pricing.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_azure_ai_cache_pricing.py rename to tests/unit/llms/anthropic/test_azure_ai_cache_pricing.py diff --git a/tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py b/tests/unit/llms/anthropic/test_cost_calculation_dict_safety.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py rename to tests/unit/llms/anthropic/test_cost_calculation_dict_safety.py diff --git a/tests/test_litellm/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_count_tokens_oauth.py rename to tests/unit/llms/anthropic/test_count_tokens_oauth.py diff --git a/tests/test_litellm/llms/anthropic/test_message_sanitization.py b/tests/unit/llms/anthropic/test_message_sanitization.py similarity index 100% rename from tests/test_litellm/llms/anthropic/test_message_sanitization.py rename to tests/unit/llms/anthropic/test_message_sanitization.py diff --git a/tests/test_litellm/llms/base_llm/files/__init__.py b/tests/unit/llms/azure/batches/__init__.py similarity index 100% rename from tests/test_litellm/llms/base_llm/files/__init__.py rename to tests/unit/llms/azure/batches/__init__.py diff --git a/tests/test_litellm/llms/azure/batches/test_handler.py b/tests/unit/llms/azure/batches/test_handler.py similarity index 100% rename from tests/test_litellm/llms/azure/batches/test_handler.py rename to tests/unit/llms/azure/batches/test_handler.py diff --git a/tests/test_litellm/llms/base_llm/realtime/__init__.py b/tests/unit/llms/azure/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/base_llm/realtime/__init__.py rename to tests/unit/llms/azure/chat/__init__.py diff --git a/tests/test_litellm/llms/azure/chat/test_azure_base_model_routing.py b/tests/unit/llms/azure/chat/test_azure_base_model_routing.py similarity index 100% rename from tests/test_litellm/llms/azure/chat/test_azure_base_model_routing.py rename to tests/unit/llms/azure/chat/test_azure_base_model_routing.py diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py rename to tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py rename to tests/unit/llms/azure/chat/test_azure_chat_o_series_transformation.py diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py rename to tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py diff --git a/tests/test_litellm/llms/azure/realtime/test_handler.py b/tests/unit/llms/azure/realtime/test_handler.py similarity index 100% rename from tests/test_litellm/llms/azure/realtime/test_handler.py rename to tests/unit/llms/azure/realtime/test_handler.py diff --git a/tests/test_litellm/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py similarity index 100% rename from tests/test_litellm/llms/azure/test_audio_transcriptions.py rename to tests/unit/llms/azure/test_audio_transcriptions.py diff --git a/tests/test_litellm/llms/azure/test_azure.py b/tests/unit/llms/azure/test_azure.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure.py rename to tests/unit/llms/azure/test_azure.py diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_common_utils.py rename to tests/unit/llms/azure/test_azure_common_utils.py diff --git a/tests/test_litellm/llms/azure/test_azure_cost_calculation.py b/tests/unit/llms/azure/test_azure_cost_calculation.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_cost_calculation.py rename to tests/unit/llms/azure/test_azure_cost_calculation.py diff --git a/tests/test_litellm/llms/azure/test_azure_embedding.py b/tests/unit/llms/azure/test_azure_embedding.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_embedding.py rename to tests/unit/llms/azure/test_azure_embedding.py diff --git a/tests/test_litellm/llms/azure/test_azure_exception_mapping.py b/tests/unit/llms/azure/test_azure_exception_mapping.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_exception_mapping.py rename to tests/unit/llms/azure/test_azure_exception_mapping.py diff --git a/tests/test_litellm/llms/azure/test_azure_fine_tuning_api.py b/tests/unit/llms/azure/test_azure_fine_tuning_api.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_fine_tuning_api.py rename to tests/unit/llms/azure/test_azure_fine_tuning_api.py diff --git a/tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py b/tests/unit/llms/azure/test_azure_speech_audio_transcription.py similarity index 100% rename from tests/test_litellm/llms/azure/test_azure_speech_audio_transcription.py rename to tests/unit/llms/azure/test_azure_speech_audio_transcription.py diff --git a/tests/test_litellm/llms/bedrock/__init__.py b/tests/unit/llms/azure/videos/__init__.py similarity index 100% rename from tests/test_litellm/llms/bedrock/__init__.py rename to tests/unit/llms/azure/videos/__init__.py diff --git a/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py b/tests/unit/llms/azure/videos/test_azure_video_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py rename to tests/unit/llms/azure/videos/test_azure_video_transformation.py diff --git a/tests/test_litellm/llms/bedrock/batches/__init__.py b/tests/unit/llms/azure_ai/claude/__init__.py similarity index 100% rename from tests/test_litellm/llms/bedrock/batches/__init__.py rename to tests/unit/llms/azure_ai/claude/__init__.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py rename to tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py rename to tests/unit/llms/azure_ai/claude/test_azure_anthropic_handler.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py rename to tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_provider_routing.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_provider_routing.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_provider_routing.py rename to tests/unit/llms/azure_ai/claude/test_azure_anthropic_provider_routing.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py rename to tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py diff --git a/tests/test_litellm/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py b/tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py rename to tests/unit/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/__init__.py b/tests/unit/llms/azure_ai/image_generation/__init__.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/agentcore/__init__.py rename to tests/unit/llms/azure_ai/image_generation/__init__.py diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py rename to tests/unit/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/image_generation/test_mai_image_generation.py rename to tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_agents_handler.py b/tests/unit/llms/azure_ai/test_azure_ai_agents_handler.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_agents_handler.py rename to tests/unit/llms/azure_ai/test_azure_ai_agents_handler.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py rename to tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py b/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py rename to tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py rename to tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_fw_models_metadata.py rename to tests/unit/llms/azure_ai/test_azure_ai_fw_models_metadata.py diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py similarity index 100% rename from tests/test_litellm/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py rename to tests/unit/llms/azure_ai/test_azure_ai_kimi_k26_metadata.py diff --git a/tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py b/tests/unit/llms/base_llm/batches/base_batches_config_test.py similarity index 100% rename from tests/test_litellm/llms/base_llm/batches/base_batches_config_test.py rename to tests/unit/llms/base_llm/batches/base_batches_config_test.py diff --git a/tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/__init__.py b/tests/unit/llms/base_llm/files/__init__.py similarity index 100% rename from tests/test_litellm/llms/bedrock/passthrough/guardrail_translation/__init__.py rename to tests/unit/llms/base_llm/files/__init__.py diff --git a/tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py b/tests/unit/llms/base_llm/files/test_azure_blob_storage_backend.py similarity index 100% rename from tests/test_litellm/llms/base_llm/files/test_azure_blob_storage_backend.py rename to tests/unit/llms/base_llm/files/test_azure_blob_storage_backend.py diff --git a/tests/test_litellm/llms/base_llm/files/test_litellm_db_storage_backend.py b/tests/unit/llms/base_llm/files/test_litellm_db_storage_backend.py similarity index 100% rename from tests/test_litellm/llms/base_llm/files/test_litellm_db_storage_backend.py rename to tests/unit/llms/base_llm/files/test_litellm_db_storage_backend.py diff --git a/tests/test_litellm/llms/base_llm/files/test_storage_backend_factory.py b/tests/unit/llms/base_llm/files/test_storage_backend_factory.py similarity index 100% rename from tests/test_litellm/llms/base_llm/files/test_storage_backend_factory.py rename to tests/unit/llms/base_llm/files/test_storage_backend_factory.py diff --git a/tests/test_litellm/llms/black_forest_labs/__init__.py b/tests/unit/llms/base_llm/responses/__init__.py similarity index 100% rename from tests/test_litellm/llms/black_forest_labs/__init__.py rename to tests/unit/llms/base_llm/responses/__init__.py diff --git a/tests/test_litellm/llms/base_llm/responses/test_codex_compat.py b/tests/unit/llms/base_llm/responses/test_codex_compat.py similarity index 100% rename from tests/test_litellm/llms/base_llm/responses/test_codex_compat.py rename to tests/unit/llms/base_llm/responses/test_codex_compat.py diff --git a/tests/test_litellm/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py similarity index 100% rename from tests/test_litellm/llms/base_llm/responses/test_transformation.py rename to tests/unit/llms/base_llm/responses/test_transformation.py diff --git a/tests/test_litellm/llms/black_forest_labs/image_edit/__init__.py b/tests/unit/llms/base_llm/search/__init__.py similarity index 100% rename from tests/test_litellm/llms/black_forest_labs/image_edit/__init__.py rename to tests/unit/llms/base_llm/search/__init__.py diff --git a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py b/tests/unit/llms/base_llm/search/test_base_search_transformation.py similarity index 100% rename from tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py rename to tests/unit/llms/base_llm/search/test_base_search_transformation.py diff --git a/tests/test_litellm/llms/base_llm/test_base_managed_resource.py b/tests/unit/llms/base_llm/test_base_managed_resource.py similarity index 100% rename from tests/test_litellm/llms/base_llm/test_base_managed_resource.py rename to tests/unit/llms/base_llm/test_base_managed_resource.py diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/unit/llms/base_llm/test_base_model_iterator.py similarity index 100% rename from tests/test_litellm/llms/base_llm/test_base_model_iterator.py rename to tests/unit/llms/base_llm/test_base_model_iterator.py diff --git a/tests/test_litellm/llms/base_llm/test_managed_resource_isolation.py b/tests/unit/llms/base_llm/test_managed_resource_isolation.py similarity index 100% rename from tests/test_litellm/llms/base_llm/test_managed_resource_isolation.py rename to tests/unit/llms/base_llm/test_managed_resource_isolation.py diff --git a/tests/test_litellm/llms/base_llm/test_managed_resources_utils.py b/tests/unit/llms/base_llm/test_managed_resources_utils.py similarity index 100% rename from tests/test_litellm/llms/base_llm/test_managed_resources_utils.py rename to tests/unit/llms/base_llm/test_managed_resources_utils.py diff --git a/tests/test_litellm/llms/black_forest_labs/image_generation/__init__.py b/tests/unit/llms/bedrock/batches/__init__.py similarity index 100% rename from tests/test_litellm/llms/black_forest_labs/image_generation/__init__.py rename to tests/unit/llms/bedrock/batches/__init__.py diff --git a/tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py b/tests/unit/llms/bedrock/batches/test_batch_metadata_sanitization.py similarity index 100% rename from tests/test_litellm/llms/bedrock/batches/test_batch_metadata_sanitization.py rename to tests/unit/llms/bedrock/batches/test_batch_metadata_sanitization.py diff --git a/tests/test_litellm/llms/bedrock/batches/test_handler.py b/tests/unit/llms/bedrock/batches/test_handler.py similarity index 100% rename from tests/test_litellm/llms/bedrock/batches/test_handler.py rename to tests/unit/llms/bedrock/batches/test_handler.py diff --git a/tests/test_litellm/llms/bedrock/batches/test_transformation.py b/tests/unit/llms/bedrock/batches/test_transformation.py similarity index 99% rename from tests/test_litellm/llms/bedrock/batches/test_transformation.py rename to tests/unit/llms/bedrock/batches/test_transformation.py index 347c459a369..5e987239c54 100644 --- a/tests/test_litellm/llms/bedrock/batches/test_transformation.py +++ b/tests/unit/llms/bedrock/batches/test_transformation.py @@ -878,7 +878,7 @@ def test_validate_environment_passes_headers_through(config): # Shared BaseBatchesConfig contract suite. # --------------------------------------------------------------------------- # -from tests.test_litellm.llms.base_llm.batches.base_batches_config_test import ( # noqa: E402 +from tests.unit.llms.base_llm.batches.base_batches_config_test import ( # noqa: E402 BatchesConfigContractTests, ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py similarity index 99% rename from tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py rename to tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 67ffe7570a1..08bcac33a35 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -20,7 +20,7 @@ from litellm.llms.bedrock.chat.converse_handler import BedrockConverseLLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.rust_bridge import configuration from litellm.types.utils import ModelResponse -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe RESOLVED_CREDENTIALS = Credentials( access_key="AKIARESOLVED", diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py rename to tests/unit/llms/bedrock/chat/test_converse_transformation.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py b/tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_converse_transformation_nova_2.py rename to tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py rename to tests/unit/llms/bedrock/chat/test_invoke_handler.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_mistral_config.py b/tests/unit/llms/bedrock/chat/test_mistral_config.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_mistral_config.py rename to tests/unit/llms/bedrock/chat/test_mistral_config.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_service_tier.py b/tests/unit/llms/bedrock/chat/test_service_tier.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_service_tier.py rename to tests/unit/llms/bedrock/chat/test_service_tier.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_streaming_choice_index.py b/tests/unit/llms/bedrock/chat/test_streaming_choice_index.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_streaming_choice_index.py rename to tests/unit/llms/bedrock/chat/test_streaming_choice_index.py diff --git a/tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py b/tests/unit/llms/bedrock/chat/test_writer_palmyra.py similarity index 100% rename from tests/test_litellm/llms/bedrock/chat/test_writer_palmyra.py rename to tests/unit/llms/bedrock/chat/test_writer_palmyra.py diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py index 3622ce7f212..d67724f261d 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_count_tokens_handler.py @@ -7,7 +7,7 @@ from botocore.credentials import RefreshableCredentials from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe class _ProbedCountTokensHandler(BedrockCountTokensHandler): diff --git a/tests/test_litellm/llms/cerebras/__init__.py b/tests/unit/llms/bedrock/embed/__init__.py similarity index 100% rename from tests/test_litellm/llms/cerebras/__init__.py rename to tests/unit/llms/bedrock/embed/__init__.py diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py similarity index 99% rename from tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py rename to tests/unit/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py index 18f4b0f6ced..fbcbd0aaea6 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_async_invoke_embedding.py @@ -9,7 +9,7 @@ import respx import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.llms.base import HiddenParams -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe # Mock async invoke responses async_invoke_response = { diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py similarity index 99% rename from tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py rename to tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index e5a460e2f1a..ad21cadaa4b 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -11,7 +11,7 @@ import litellm from litellm.llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.bedrock.embed.embedding import BedrockEmbedding -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe # Mock responses for different embedding models titan_embedding_response = {"embedding": [0.1, 0.2, 0.3], "inputTextTokenCount": 10} diff --git a/tests/test_litellm/llms/bedrock/embed/test_embedding.py b/tests/unit/llms/bedrock/embed/test_embedding.py similarity index 100% rename from tests/test_litellm/llms/bedrock/embed/test_embedding.py rename to tests/unit/llms/bedrock/embed/test_embedding.py diff --git a/tests/test_litellm/llms/bedrock/embed/test_twelvelabs_marengo_3_transformation.py b/tests/unit/llms/bedrock/embed/test_twelvelabs_marengo_3_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock/embed/test_twelvelabs_marengo_3_transformation.py rename to tests/unit/llms/bedrock/embed/test_twelvelabs_marengo_3_transformation.py diff --git a/tests/test_litellm/llms/bedrock/event_loop_probe.py b/tests/unit/llms/bedrock/event_loop_probe.py similarity index 100% rename from tests/test_litellm/llms/bedrock/event_loop_probe.py rename to tests/unit/llms/bedrock/event_loop_probe.py diff --git a/tests/test_litellm/llms/chatgpt/__init__.py b/tests/unit/llms/bedrock/messages/__init__.py similarity index 100% rename from tests/test_litellm/llms/chatgpt/__init__.py rename to tests/unit/llms/bedrock/messages/__init__.py diff --git a/tests/test_litellm/llms/chatgpt/chat/__init__.py b/tests/unit/llms/bedrock/messages/invoke_transformations/__init__.py similarity index 100% rename from tests/test_litellm/llms/chatgpt/chat/__init__.py rename to tests/unit/llms/bedrock/messages/invoke_transformations/__init__.py diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py rename to tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py diff --git a/tests/test_litellm/llms/bedrock/rerank/transformation.py b/tests/unit/llms/bedrock/rerank/transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock/rerank/transformation.py rename to tests/unit/llms/bedrock/rerank/transformation.py diff --git a/tests/test_litellm/llms/crusoe/__init__.py b/tests/unit/llms/bedrock/responses/__init__.py similarity index 100% rename from tests/test_litellm/llms/crusoe/__init__.py rename to tests/unit/llms/bedrock/responses/__init__.py diff --git a/tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py similarity index 100% rename from tests/test_litellm/llms/bedrock/responses/test_bedrock_openai_responses.py rename to tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py diff --git a/tests/test_litellm/llms/databricks/chat/__init__.py b/tests/unit/llms/bedrock/search/__init__.py similarity index 100% rename from tests/test_litellm/llms/databricks/chat/__init__.py rename to tests/unit/llms/bedrock/search/__init__.py diff --git a/tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py b/tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock/search/test_agentcore_search_transformation.py rename to tests/unit/llms/bedrock/search/test_agentcore_search_transformation.py diff --git a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py b/tests/unit/llms/bedrock/test_anthropic_beta_support.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py rename to tests/unit/llms/bedrock/test_anthropic_beta_support.py diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/unit/llms/bedrock/test_base_aws_llm.py similarity index 99% rename from tests/test_litellm/llms/bedrock/test_base_aws_llm.py rename to tests/unit/llms/bedrock/test_base_aws_llm.py index 6b9450afed4..db144ab6d56 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/unit/llms/bedrock/test_base_aws_llm.py @@ -28,7 +28,7 @@ from litellm.llms.bedrock.base_aws_llm import ( run_aws_signing, sign_request_off_loop_if_aws, ) -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe # Global variable for the base_aws_llm.py file path diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py rename to tests/unit/llms/bedrock/test_bedrock_common_utils.py diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py b/tests/unit/llms/bedrock/test_bedrock_ssl_verify.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py rename to tests/unit/llms/bedrock/test_bedrock_ssl_verify.py diff --git a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py b/tests/unit/llms/bedrock/test_claude_platform_provider.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_claude_platform_provider.py rename to tests/unit/llms/bedrock/test_claude_platform_provider.py diff --git a/tests/test_litellm/llms/bedrock/test_converse_context_management.py b/tests/unit/llms/bedrock/test_converse_context_management.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_converse_context_management.py rename to tests/unit/llms/bedrock/test_converse_context_management.py diff --git a/tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py rename to tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/unit/llms/bedrock/test_mantle.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_mantle.py rename to tests/unit/llms/bedrock/test_mantle.py diff --git a/tests/test_litellm/llms/bedrock/test_nova_imported_models.py b/tests/unit/llms/bedrock/test_nova_imported_models.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_nova_imported_models.py rename to tests/unit/llms/bedrock/test_nova_imported_models.py diff --git a/tests/test_litellm/llms/bedrock/test_request_metadata.py b/tests/unit/llms/bedrock/test_request_metadata.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_request_metadata.py rename to tests/unit/llms/bedrock/test_request_metadata.py diff --git a/tests/test_litellm/llms/bedrock/test_web_identity_session_policy.py b/tests/unit/llms/bedrock/test_web_identity_session_policy.py similarity index 100% rename from tests/test_litellm/llms/bedrock/test_web_identity_session_policy.py rename to tests/unit/llms/bedrock/test_web_identity_session_policy.py diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py rename to tests/unit/llms/bedrock_mantle/test_bedrock_mantle_messages_transformation.py diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py similarity index 100% rename from tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py rename to tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py similarity index 99% rename from tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py rename to tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index 0cc3963358f..4bf3dd11fa1 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -19,7 +19,7 @@ import litellm from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws from litellm.types.utils import LlmProviders -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @pytest.fixture diff --git a/tests/test_litellm/llms/databricks/responses/__init__.py b/tests/unit/llms/cometapi/__init__.py similarity index 100% rename from tests/test_litellm/llms/databricks/responses/__init__.py rename to tests/unit/llms/cometapi/__init__.py diff --git a/tests/test_litellm/llms/deepseek/__init__.py b/tests/unit/llms/cometapi/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/deepseek/__init__.py rename to tests/unit/llms/cometapi/chat/__init__.py diff --git a/tests/unit/llms/cometapi/chat/test_cometapi_chat_transformation.py b/tests/unit/llms/cometapi/chat/test_cometapi_chat_transformation.py new file mode 100644 index 00000000000..607648dd6c9 --- /dev/null +++ b/tests/unit/llms/cometapi/chat/test_cometapi_chat_transformation.py @@ -0,0 +1,183 @@ +""" +Unit tests for CometAPI Chat Configuration + +Tests the CometAPIChatConfig class methods using mocks +""" + + +import pytest + + +from litellm.llms.cometapi.chat.transformation import ( + CometAPIChatCompletionStreamingHandler, + CometAPIConfig, +) +from litellm.llms.cometapi.common_utils import CometAPIException + + +class TestCometAPIChatCompletionStreamingHandler: + def test_chunk_parser_successful(self): + handler = CometAPIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + # Test input chunk + chunk = { + "id": "test_id", + "created": 1234567890, + "model": "gpt-3.5-turbo", + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + "choices": [ + {"delta": {"content": "test content", "reasoning": "test reasoning"}} + ], + } + + # Parse chunk + result = handler.chunk_parser(chunk) + + # Verify response + assert result.id == "test_id" + assert result.object == "chat.completion.chunk" + assert result.created == 1234567890 + assert result.model == "gpt-3.5-turbo" + assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] + assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] + assert result.usage.total_tokens == chunk["usage"]["total_tokens"] + assert len(result.choices) == 1 + assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" + + def test_chunk_parser_error_response(self): + handler = CometAPIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + # Test error chunk + error_chunk = { + "error": { + "message": "test error", + "code": 400, + } + } + + # Verify error handling + with pytest.raises(CometAPIException) as exc_info: + handler.chunk_parser(error_chunk) + + assert "CometAPI Error: test error" in str(exc_info.value) + assert exc_info.value.status_code == 400 + + def test_chunk_parser_key_error(self): + handler = CometAPIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + # Test invalid chunk missing required fields + invalid_chunk = {"incomplete": "data"} + + # Verify KeyError handling + with pytest.raises(CometAPIException) as exc_info: + handler.chunk_parser(invalid_chunk) + + assert "KeyError" in str(exc_info.value) + assert exc_info.value.status_code == 400 + + +class TestCometAPIConfig: + def test_transform_request_basic(self): + """Test basic request transformation""" + config = CometAPIConfig() + + transformed_request = config.transform_request( + model="cometapi/gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert transformed_request["model"] == "cometapi/gpt-3.5-turbo" + assert transformed_request["messages"] == [ + {"role": "user", "content": "Hello, world!"} + ] + + def test_transform_request_with_extra_body(self): + """Test request transformation with extra_body parameters""" + config = CometAPIConfig() + + transformed_request = config.transform_request( + model="cometapi/gpt-4", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={"extra_body": {"custom_param": "custom_value"}}, + litellm_params={}, + headers={}, + ) + + # Validate that extra_body parameters are merged into the request + assert transformed_request["custom_param"] == "custom_value" + assert transformed_request["messages"] == [ + {"role": "user", "content": "Hello, world!"} + ] + + def test_cache_control_flag_removal(self): + """Test cache control flag removal from messages""" + config = CometAPIConfig() + + transformed_request = config.transform_request( + model="cometapi/gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": "Hello, world!", + "cache_control": {"type": "ephemeral"}, + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + # CometAPI should remove cache_control flags by default + assert transformed_request["messages"][0].get("cache_control") is None + + def test_map_openai_params(self): + """Test OpenAI parameter mapping""" + config = CometAPIConfig() + + non_default_params = { + "temperature": 0.7, + "max_tokens": 100, + "top_p": 0.9, + } + + mapped_params = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model="cometapi/gpt-3.5-turbo", + drop_params=False, + ) + + assert mapped_params["temperature"] == 0.7 + assert mapped_params["max_tokens"] == 100 + assert mapped_params["top_p"] == 0.9 + + def test_get_error_class(self): + """Test error class creation""" + config = CometAPIConfig() + + error = config.get_error_class( + error_message="Test error", + status_code=400, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, CometAPIException) + assert error.message == "Test error" + assert error.status_code == 400 + + +# Integration test example (requires real API key) + + +if __name__ == "__main__": + # Quick test runner + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/deepseek/chat/__init__.py b/tests/unit/llms/compactifai/__init__.py similarity index 100% rename from tests/test_litellm/llms/deepseek/chat/__init__.py rename to tests/unit/llms/compactifai/__init__.py diff --git a/tests/test_litellm/llms/compactifai/test_compactifai.py b/tests/unit/llms/compactifai/test_compactifai.py similarity index 84% rename from tests/test_litellm/llms/compactifai/test_compactifai.py rename to tests/unit/llms/compactifai/test_compactifai.py index fd31049731a..1367c703fda 100644 --- a/tests/test_litellm/llms/compactifai/test_compactifai.py +++ b/tests/unit/llms/compactifai/test_compactifai.py @@ -104,56 +104,6 @@ def test_compactifai_completion_streaming(respx_mock): assert chunks[0].choices[0].delta.content == "Hello" -@pytest.mark.respx() -def test_compactifai_models_endpoint(respx_mock): - """Test CompactifAI models listing""" - litellm.disable_aiohttp_transport = True - - mock_response = { - "object": "list", - "data": [ - { - "id": "cai-llama-3-1-8b-slim", - "object": "model", - "created": 1677610602, - "owned_by": "compactifai", - }, - { - "id": "mistral-7b-compressed", - "object": "model", - "created": 1677610602, - "owned_by": "compactifai", - }, - ], - } - - respx_mock.post("https://api.compactif.ai/v1/chat/completions").respond( - json={ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1677652288, - "model": "cai-llama-3-1-8b-slim", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Test response"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15}, - }, - status_code=200, - ) - - # This would be tested if litellm had a models() function - # For now, we'll test that the provider is properly configured - response = litellm.completion( - model="compactifai/cai-llama-3-1-8b-slim", - messages=[{"role": "user", "content": "test"}], - api_key="test-key", - ) - - @pytest.mark.respx() def test_compactifai_authentication_error(respx_mock): """Test CompactifAI authentication error handling""" diff --git a/tests/test_litellm/llms/deepseek/messages/__init__.py b/tests/unit/llms/custom_httpx/__init__.py similarity index 100% rename from tests/test_litellm/llms/deepseek/messages/__init__.py rename to tests/unit/llms/custom_httpx/__init__.py diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_cleanup_closed.py b/tests/unit/llms/custom_httpx/test_aiohttp_cleanup_closed.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_aiohttp_cleanup_closed.py rename to tests/unit/llms/custom_httpx/test_aiohttp_cleanup_closed.py diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py b/tests/unit/llms/custom_httpx/test_aiohttp_handler.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_aiohttp_handler.py rename to tests/unit/llms/custom_httpx/test_aiohttp_handler.py diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_so_keepalive.py b/tests/unit/llms/custom_httpx/test_aiohttp_so_keepalive.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_aiohttp_so_keepalive.py rename to tests/unit/llms/custom_httpx/test_aiohttp_so_keepalive.py diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/unit/llms/custom_httpx/test_aiohttp_transport.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py rename to tests/unit/llms/custom_httpx/test_aiohttp_transport.py diff --git a/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py b/tests/unit/llms/custom_httpx/test_asgi_handler.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_asgi_handler.py rename to tests/unit/llms/custom_httpx/test_asgi_handler.py diff --git a/tests/test_litellm/llms/custom_httpx/test_async_client_cleanup.py b/tests/unit/llms/custom_httpx/test_async_client_cleanup.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_async_client_cleanup.py rename to tests/unit/llms/custom_httpx/test_async_client_cleanup.py diff --git a/tests/test_litellm/llms/custom_httpx/test_container_handler.py b/tests/unit/llms/custom_httpx/test_container_handler.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_container_handler.py rename to tests/unit/llms/custom_httpx/test_container_handler.py diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/unit/llms/custom_httpx/test_credential_leak_prevention.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py rename to tests/unit/llms/custom_httpx/test_credential_leak_prevention.py diff --git a/tests/test_litellm/llms/custom_httpx/test_gemini_session_leak.py b/tests/unit/llms/custom_httpx/test_gemini_session_leak.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_gemini_session_leak.py rename to tests/unit/llms/custom_httpx/test_gemini_session_leak.py diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_http_handler.py rename to tests/unit/llms/custom_httpx/test_http_handler.py diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py similarity index 99% rename from tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py rename to tests/unit/llms/custom_httpx/test_llm_http_handler.py index 0350ca74904..399e4dbf206 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -45,7 +45,7 @@ from litellm.llms.tinyfish.search.transformation import TinyfishSearchConfig from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse -from tests.test_litellm.llms.bedrock.event_loop_probe import EventLoopProbe +from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe _ACTIVE_KEY = "_code_interpreter_interception_active" _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key" diff --git a/tests/test_litellm/llms/custom_httpx/test_mock_transport.py b/tests/unit/llms/custom_httpx/test_mock_transport.py similarity index 100% rename from tests/test_litellm/llms/custom_httpx/test_mock_transport.py rename to tests/unit/llms/custom_httpx/test_mock_transport.py diff --git a/tests/test_litellm/llms/gemini/__init__.py b/tests/unit/llms/dashscope/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/__init__.py rename to tests/unit/llms/dashscope/__init__.py diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py b/tests/unit/llms/dashscope/test_dashscope_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/dashscope/test_dashscope_chat_transformation.py rename to tests/unit/llms/dashscope/test_dashscope_chat_transformation.py diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py rename to tests/unit/llms/dashscope/test_dashscope_cost_calculator.py diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py b/tests/unit/llms/dashscope/test_dashscope_embedding_transformation.py similarity index 100% rename from tests/test_litellm/llms/dashscope/test_dashscope_embedding_transformation.py rename to tests/unit/llms/dashscope/test_dashscope_embedding_transformation.py diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/unit/llms/dashscope/test_dashscope_rerank_transformation.py similarity index 100% rename from tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py rename to tests/unit/llms/dashscope/test_dashscope_rerank_transformation.py diff --git a/tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py b/tests/unit/llms/dashscope/test_qwen_brand_aliases.py similarity index 100% rename from tests/test_litellm/llms/dashscope/test_qwen_brand_aliases.py rename to tests/unit/llms/dashscope/test_qwen_brand_aliases.py diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 52bb89fed5a..9cd17bd3580 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -16,6 +16,9 @@ from litellm.llms.databricks.chat.transformation import ( DatabricksConfig, _sanitize_empty_content, ) +from typing import Final +import httpx +import respx @pytest.fixture() @@ -808,3 +811,75 @@ def test_chunk_parser_surfaces_top_level_reasoning_delta(reasoning_key: str) -> assert parsed.choices[0].delta.reasoning_content == "We need answer" assert parsed.choices[0].delta.content is None + + +def test_completion_merges_leading_system_and_developer_messages_for_chat_template_models( + respx_mock: respx.MockRouter, +): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "developer", "content": "Skills: none."}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse.\n\nSkills: none."}, + {"role": "user", "content": "Hello"}, + ] + assert response.choices[0].message.content == "Answer" + + +def test_completion_merges_system_messages_when_one_has_empty_content(respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": ""}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/test_litellm/llms/databricks/test_databricks_common_utils.py b/tests/unit/llms/databricks/test_databricks_common_utils.py similarity index 100% rename from tests/test_litellm/llms/databricks/test_databricks_common_utils.py rename to tests/unit/llms/databricks/test_databricks_common_utils.py diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/unit/llms/databricks/test_databricks_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py rename to tests/unit/llms/databricks/test_databricks_cost_calculator.py diff --git a/tests/test_litellm/llms/databricks/test_databricks_partner_integration.py b/tests/unit/llms/databricks/test_databricks_partner_integration.py similarity index 100% rename from tests/test_litellm/llms/databricks/test_databricks_partner_integration.py rename to tests/unit/llms/databricks/test_databricks_partner_integration.py diff --git a/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py b/tests/unit/llms/databricks/test_databricks_streaming_utils.py similarity index 100% rename from tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py rename to tests/unit/llms/databricks/test_databricks_streaming_utils.py diff --git a/tests/test_litellm/llms/gemini/audio_transcription/__init__.py b/tests/unit/llms/deepgram/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/audio_transcription/__init__.py rename to tests/unit/llms/deepgram/__init__.py diff --git a/tests/test_litellm/llms/gemini/google_genai/__init__.py b/tests/unit/llms/deepgram/audio_transcription/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/google_genai/__init__.py rename to tests/unit/llms/deepgram/audio_transcription/__init__.py diff --git a/tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py rename to tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/deepgram/test_deepgram_common_utils.py b/tests/unit/llms/deepgram/test_deepgram_common_utils.py similarity index 100% rename from tests/test_litellm/llms/deepgram/test_deepgram_common_utils.py rename to tests/unit/llms/deepgram/test_deepgram_common_utils.py diff --git a/tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py b/tests/unit/llms/deepgram/test_deepgram_mock_transcription.py similarity index 100% rename from tests/test_litellm/llms/deepgram/test_deepgram_mock_transcription.py rename to tests/unit/llms/deepgram/test_deepgram_mock_transcription.py diff --git a/tests/test_litellm/llms/gemini/google_genai/guardrail_translation/__init__.py b/tests/unit/llms/deepinfra/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/google_genai/guardrail_translation/__init__.py rename to tests/unit/llms/deepinfra/__init__.py diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py rename to tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py b/tests/unit/llms/deepinfra/test_deepinfra_rerank.py similarity index 100% rename from tests/test_litellm/llms/deepinfra/test_deepinfra_rerank.py rename to tests/unit/llms/deepinfra/test_deepinfra_rerank.py diff --git a/tests/unit/llms/deepinfra/test_deepinfra_rerank_integration.py b/tests/unit/llms/deepinfra/test_deepinfra_rerank_integration.py new file mode 100644 index 00000000000..8a2a1d09cb6 --- /dev/null +++ b/tests/unit/llms/deepinfra/test_deepinfra_rerank_integration.py @@ -0,0 +1,159 @@ +""" +Integration tests for DeepInfra rerank functionality. +Tests the full rerank flow following the repository patterns. +""" + +import asyncio +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") +@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") +def test_deepinfra_rerank_with_queries_param( + mock_sync_post, mock_async_post, sync_mode +): + """Test DeepInfra rerank with multiple queries parameter.""" + mock_response_data = { + "scores": [0.8, 0.6, 0.2], + "input_tokens": 35, + "request_id": "deepinfra-multi-query-123", + "inference_status": {"status": "success", "runtime_ms": 200}, + } + + def return_val(): + return mock_response_data + + if sync_mode: + mock_response = MagicMock() + mock_response.json = return_val + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.text = json.dumps(mock_response_data) + mock_sync_post.return_value = mock_response + + response = litellm.rerank( + model="deepinfra/Qwen/Qwen3-Reranker-4B", + query="hello", + documents=["hello", "world", "test"], + queries=["hello", "hi there"], # DeepInfra specific param + custom_llm_provider="deepinfra", + api_key="test_key", + api_base="https://api.deepinfra.com", + ) + + mock_sync_post.assert_called_once() + # Verify that queries parameter was passed in request + call_data = json.loads(mock_sync_post.call_args.kwargs["data"]) + assert "queries" in call_data + assert call_data["queries"] == ["hello", "hi there"] + else: + mock_response = AsyncMock() + mock_response.json = return_val + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.text = json.dumps(mock_response_data) + mock_async_post.return_value = mock_response + + response = asyncio.run( + litellm.arerank( + model="deepinfra/Qwen/Qwen3-Reranker-4B", + query="hello", + documents=["hello", "world", "test"], + queries=["hello", "hi there"], + custom_llm_provider="deepinfra", + api_key="test_key", + api_base="https://api.deepinfra.com", + ) + ) + + mock_async_post.assert_called_once() + call_data = json.loads(mock_async_post.call_args.kwargs["data"]) + assert "queries" in call_data + assert call_data["queries"] == ["hello", "hi there"] + + assert response.results is not None + assert len(response.results) == 3 + + +@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") +def test_deepinfra_rerank_with_env_vars(mock_post, monkeypatch): + """Test DeepInfra rerank with environment variable configuration.""" + monkeypatch.setenv("DEEPINFRA_API_KEY", "env_test_key") + monkeypatch.setenv("DEEPINFRA_API_BASE", "https://custom-deepinfra.com") + + mock_response_data = { + "scores": [0.88, 0.22], + "input_tokens": 28, + "request_id": "env-test-123", + } + + def return_val(): + return mock_response_data + + mock_response = MagicMock() + mock_response.json = return_val + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.text = json.dumps(mock_response_data) + mock_post.return_value = mock_response + + response = litellm.rerank( + model="deepinfra/Qwen/Qwen3-Reranker-0.6B", + query="hello", + documents=["hello", "world"], + custom_llm_provider="deepinfra", + ) + + mock_post.assert_called_once() + + # Verify headers contain env API key + headers = mock_post.call_args.kwargs.get("headers", {}) + assert "Bearer env_test_key" in headers.get("Authorization", "") + + assert response.results is not None + + +@patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") +def test_deepinfra_rerank_defaults_api_base_when_missing(mock_post, monkeypatch): + """With no api_base anywhere, the call still goes out against DeepInfra's own base.""" + monkeypatch.delenv("DEEPINFRA_API_BASE", raising=False) + + mock_response = MagicMock() + mock_response.json = lambda: {"scores": [0.9, 0.1], "input_tokens": 20} + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_post.return_value = mock_response + + response = litellm.rerank( + model="deepinfra/Qwen/Qwen3-Reranker-0.6B", + query="hello", + documents=["hello", "world"], + custom_llm_provider="deepinfra", + api_key="test_key", + # api_base is intentionally missing + ) + + assert "api.deepinfra.com" in mock_post.call_args.kwargs["url"] + assert [result["relevance_score"] for result in response.results] == [0.9, 0.1] + + +def test_deepinfra_rerank_models(): + """Test that DeepInfra Qwen rerank models are recognized.""" + # These should not raise errors during model validation + models = [ + "deepinfra/Qwen/Qwen3-Reranker-0.6B", + "deepinfra/Qwen/Qwen3-Reranker-4B", + "deepinfra/Qwen/Qwen3-Reranker-8B", + ] + + for model in models: + resolved_model, provider, _, api_base = litellm.get_llm_provider(model=model) + assert provider == "deepinfra" + assert resolved_model == model.removeprefix("deepinfra/") + assert api_base == "https://api.deepinfra.com/v1/openai" diff --git a/tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py b/tests/unit/llms/deepinfra/test_deepinfra_rerank_transformation.py similarity index 100% rename from tests/test_litellm/llms/deepinfra/test_deepinfra_rerank_transformation.py rename to tests/unit/llms/deepinfra/test_deepinfra_rerank_transformation.py diff --git a/tests/test_litellm/llms/gemini/image_edit/__init__.py b/tests/unit/llms/edenai/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/image_edit/__init__.py rename to tests/unit/llms/edenai/__init__.py diff --git a/tests/test_litellm/llms/gemini/realtime/__init__.py b/tests/unit/llms/edenai/audio_transcription/__init__.py similarity index 100% rename from tests/test_litellm/llms/gemini/realtime/__init__.py rename to tests/unit/llms/edenai/audio_transcription/__init__.py diff --git a/tests/test_litellm/llms/edenai/audio_transcription/test_edenai_audio_transcription_transformation.py b/tests/unit/llms/edenai/audio_transcription/test_edenai_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/audio_transcription/test_edenai_audio_transcription_transformation.py rename to tests/unit/llms/edenai/audio_transcription/test_edenai_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/gigachat/__init__.py b/tests/unit/llms/edenai/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/gigachat/__init__.py rename to tests/unit/llms/edenai/chat/__init__.py diff --git a/tests/test_litellm/llms/edenai/chat/test_edenai_chat_transformation.py b/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/chat/test_edenai_chat_transformation.py rename to tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py diff --git a/tests/test_litellm/llms/edenai/conftest.py b/tests/unit/llms/edenai/conftest.py similarity index 100% rename from tests/test_litellm/llms/edenai/conftest.py rename to tests/unit/llms/edenai/conftest.py diff --git a/tests/test_litellm/llms/gigachat/embedding/__init__.py b/tests/unit/llms/edenai/embedding/__init__.py similarity index 100% rename from tests/test_litellm/llms/gigachat/embedding/__init__.py rename to tests/unit/llms/edenai/embedding/__init__.py diff --git a/tests/test_litellm/llms/edenai/embedding/test_edenai_embedding_transformation.py b/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/embedding/test_edenai_embedding_transformation.py rename to tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py diff --git a/tests/test_litellm/llms/gigachat/passthrough/__init__.py b/tests/unit/llms/edenai/image_generation/__init__.py similarity index 100% rename from tests/test_litellm/llms/gigachat/passthrough/__init__.py rename to tests/unit/llms/edenai/image_generation/__init__.py diff --git a/tests/test_litellm/llms/edenai/image_generation/test_edenai_image_generation_transformation.py b/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/image_generation/test_edenai_image_generation_transformation.py rename to tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py diff --git a/tests/test_litellm/llms/github_copilot/messages/__init__.py b/tests/unit/llms/edenai/messages/__init__.py similarity index 100% rename from tests/test_litellm/llms/github_copilot/messages/__init__.py rename to tests/unit/llms/edenai/messages/__init__.py diff --git a/tests/test_litellm/llms/edenai/messages/test_edenai_anthropic_messages_transformation.py b/tests/unit/llms/edenai/messages/test_edenai_anthropic_messages_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/messages/test_edenai_anthropic_messages_transformation.py rename to tests/unit/llms/edenai/messages/test_edenai_anthropic_messages_transformation.py diff --git a/tests/test_litellm/llms/gradient_ai/__init__.py b/tests/unit/llms/edenai/responses/__init__.py similarity index 100% rename from tests/test_litellm/llms/gradient_ai/__init__.py rename to tests/unit/llms/edenai/responses/__init__.py diff --git a/tests/test_litellm/llms/edenai/responses/test_edenai_responses_transformation.py b/tests/unit/llms/edenai/responses/test_edenai_responses_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/responses/test_edenai_responses_transformation.py rename to tests/unit/llms/edenai/responses/test_edenai_responses_transformation.py diff --git a/tests/test_litellm/llms/edenai/test_edenai_common_utils.py b/tests/unit/llms/edenai/test_edenai_common_utils.py similarity index 100% rename from tests/test_litellm/llms/edenai/test_edenai_common_utils.py rename to tests/unit/llms/edenai/test_edenai_common_utils.py diff --git a/tests/test_litellm/llms/gradient_ai/chat/__init__.py b/tests/unit/llms/edenai/text_to_speech/__init__.py similarity index 100% rename from tests/test_litellm/llms/gradient_ai/chat/__init__.py rename to tests/unit/llms/edenai/text_to_speech/__init__.py diff --git a/tests/test_litellm/llms/edenai/text_to_speech/test_edenai_text_to_speech_transformation.py b/tests/unit/llms/edenai/text_to_speech/test_edenai_text_to_speech_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/text_to_speech/test_edenai_text_to_speech_transformation.py rename to tests/unit/llms/edenai/text_to_speech/test_edenai_text_to_speech_transformation.py diff --git a/tests/test_litellm/llms/groq/__init__.py b/tests/unit/llms/edenai/videos/__init__.py similarity index 100% rename from tests/test_litellm/llms/groq/__init__.py rename to tests/unit/llms/edenai/videos/__init__.py diff --git a/tests/test_litellm/llms/edenai/videos/test_edenai_video_transformation.py b/tests/unit/llms/edenai/videos/test_edenai_video_transformation.py similarity index 100% rename from tests/test_litellm/llms/edenai/videos/test_edenai_video_transformation.py rename to tests/unit/llms/edenai/videos/test_edenai_video_transformation.py diff --git a/tests/test_litellm/llms/groq/chat/__init__.py b/tests/unit/llms/fal_ai/__init__.py similarity index 100% rename from tests/test_litellm/llms/groq/chat/__init__.py rename to tests/unit/llms/fal_ai/__init__.py diff --git a/tests/test_litellm/llms/huggingface/__init__.py b/tests/unit/llms/fal_ai/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/huggingface/__init__.py rename to tests/unit/llms/fal_ai/chat/__init__.py diff --git a/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py rename to tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py diff --git a/tests/test_litellm/llms/inception/__init__.py b/tests/unit/llms/fal_ai/image_edit/__init__.py similarity index 100% rename from tests/test_litellm/llms/inception/__init__.py rename to tests/unit/llms/fal_ai/image_edit/__init__.py diff --git a/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py b/tests/unit/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py rename to tests/unit/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py diff --git a/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_image_edit_transformation.py b/tests/unit/llms/fal_ai/image_edit/test_fal_ai_image_edit_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_image_edit_transformation.py rename to tests/unit/llms/fal_ai/image_edit/test_fal_ai_image_edit_transformation.py diff --git a/tests/test_litellm/llms/mistral/batches/__init__.py b/tests/unit/llms/fal_ai/image_generation/__init__.py similarity index 100% rename from tests/test_litellm/llms/mistral/batches/__init__.py rename to tests/unit/llms/fal_ai/image_generation/__init__.py diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_flux_dev_transformation.py b/tests/unit/llms/fal_ai/image_generation/test_fal_ai_flux_dev_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_flux_dev_transformation.py rename to tests/unit/llms/fal_ai/image_generation/test_fal_ai_flux_dev_transformation.py diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_gpt_image_2_transformation.py b/tests/unit/llms/fal_ai/image_generation/test_fal_ai_gpt_image_2_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_gpt_image_2_transformation.py rename to tests/unit/llms/fal_ai/image_generation/test_fal_ai_gpt_image_2_transformation.py diff --git a/tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py b/tests/unit/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py rename to tests/unit/llms/fal_ai/image_generation/test_fal_ai_nano_banana_transformation.py diff --git a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py b/tests/unit/llms/fal_ai/test_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/test_cost_calculator.py rename to tests/unit/llms/fal_ai/test_cost_calculator.py diff --git a/tests/test_litellm/llms/mistral/files/__init__.py b/tests/unit/llms/fal_ai/videos/__init__.py similarity index 100% rename from tests/test_litellm/llms/mistral/files/__init__.py rename to tests/unit/llms/fal_ai/videos/__init__.py diff --git a/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py b/tests/unit/llms/fal_ai/videos/test_fal_ai_video_transformation.py similarity index 100% rename from tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py rename to tests/unit/llms/fal_ai/videos/test_fal_ai_video_transformation.py diff --git a/tests/test_litellm/llms/nvidia_riva/__init__.py b/tests/unit/llms/featherless_ai/__init__.py similarity index 100% rename from tests/test_litellm/llms/nvidia_riva/__init__.py rename to tests/unit/llms/featherless_ai/__init__.py diff --git a/tests/test_litellm/llms/oci/rerank/__init__.py b/tests/unit/llms/featherless_ai/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/oci/rerank/__init__.py rename to tests/unit/llms/featherless_ai/chat/__init__.py diff --git a/tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py b/tests/unit/llms/featherless_ai/chat/test_featherless_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/featherless_ai/chat/test_featherless_chat_transformation.py rename to tests/unit/llms/featherless_ai/chat/test_featherless_chat_transformation.py diff --git a/tests/test_litellm/llms/ocr/__init__.py b/tests/unit/llms/fireworks_ai/completion/__init__.py similarity index 100% rename from tests/test_litellm/llms/ocr/__init__.py rename to tests/unit/llms/fireworks_ai/completion/__init__.py diff --git a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py similarity index 100% rename from tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py rename to tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_completion_transformation.py diff --git a/tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py similarity index 100% rename from tests/test_litellm/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py rename to tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py diff --git a/tests/test_litellm/llms/openai_like/responses/__init__.py b/tests/unit/llms/gdc/__init__.py similarity index 100% rename from tests/test_litellm/llms/openai_like/responses/__init__.py rename to tests/unit/llms/gdc/__init__.py diff --git a/tests/test_litellm/llms/parallel_ai/__init__.py b/tests/unit/llms/gdc/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/parallel_ai/__init__.py rename to tests/unit/llms/gdc/chat/__init__.py diff --git a/tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py b/tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/gdc/chat/test_gdc_chat_transformation.py rename to tests/unit/llms/gdc/chat/test_gdc_chat_transformation.py diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/unit/llms/gemini/test_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/gemini/test_cost_calculator.py rename to tests/unit/llms/gemini/test_cost_calculator.py diff --git a/tests/test_litellm/llms/gemini/test_gemini_client_setup.py b/tests/unit/llms/gemini/test_gemini_client_setup.py similarity index 100% rename from tests/test_litellm/llms/gemini/test_gemini_client_setup.py rename to tests/unit/llms/gemini/test_gemini_client_setup.py diff --git a/tests/test_litellm/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py similarity index 100% rename from tests/test_litellm/llms/gemini/test_gemini_common_utils.py rename to tests/unit/llms/gemini/test_gemini_common_utils.py diff --git a/tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py b/tests/unit/llms/gemini/test_gemini_image_generation_transformation.py similarity index 100% rename from tests/test_litellm/llms/gemini/test_gemini_image_generation_transformation.py rename to tests/unit/llms/gemini/test_gemini_image_generation_transformation.py diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/unit/llms/gemini/test_gemini_tts.py similarity index 100% rename from tests/test_litellm/llms/gemini/test_gemini_tts.py rename to tests/unit/llms/gemini/test_gemini_tts.py diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py b/tests/unit/llms/github_copilot/test_github_copilot_authenticator.py similarity index 100% rename from tests/test_litellm/llms/github_copilot/test_github_copilot_authenticator.py rename to tests/unit/llms/github_copilot/test_github_copilot_authenticator.py diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py similarity index 100% rename from tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py rename to tests/unit/llms/github_copilot/test_github_copilot_transformation.py diff --git a/tests/test_litellm/llms/pass_through/__init__.py b/tests/unit/llms/heroku/__init__.py similarity index 100% rename from tests/test_litellm/llms/pass_through/__init__.py rename to tests/unit/llms/heroku/__init__.py diff --git a/tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py b/tests/unit/llms/heroku/test_heroku_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/heroku/test_heroku_chat_transformation.py rename to tests/unit/llms/heroku/test_heroku_chat_transformation.py diff --git a/tests/test_litellm/llms/pass_through/guardrail_translation/__init__.py b/tests/unit/llms/huggingface/embedding/__init__.py similarity index 100% rename from tests/test_litellm/llms/pass_through/guardrail_translation/__init__.py rename to tests/unit/llms/huggingface/embedding/__init__.py diff --git a/tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py similarity index 100% rename from tests/test_litellm/llms/huggingface/embedding/test_huggingface_embedding_handler.py rename to tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py diff --git a/tests/test_litellm/llms/langflow/test_langflow_a2a.py b/tests/unit/llms/langflow/test_langflow_a2a.py similarity index 100% rename from tests/test_litellm/llms/langflow/test_langflow_a2a.py rename to tests/unit/llms/langflow/test_langflow_a2a.py diff --git a/tests/test_litellm/llms/perplexity/__init__.py b/tests/unit/llms/lemonade/__init__.py similarity index 100% rename from tests/test_litellm/llms/perplexity/__init__.py rename to tests/unit/llms/lemonade/__init__.py diff --git a/tests/test_litellm/llms/lemonade/test_lemonade.py b/tests/unit/llms/lemonade/test_lemonade.py similarity index 100% rename from tests/test_litellm/llms/lemonade/test_lemonade.py rename to tests/unit/llms/lemonade/test_lemonade.py diff --git a/tests/test_litellm/llms/perplexity/embedding/__init__.py b/tests/unit/llms/lm_studio/__init__.py similarity index 100% rename from tests/test_litellm/llms/perplexity/embedding/__init__.py rename to tests/unit/llms/lm_studio/__init__.py diff --git a/tests/test_litellm/llms/lm_studio/test_lm_studio_chat_transformation.py b/tests/unit/llms/lm_studio/test_lm_studio_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/lm_studio/test_lm_studio_chat_transformation.py rename to tests/unit/llms/lm_studio/test_lm_studio_chat_transformation.py diff --git a/tests/test_litellm/llms/stability/__init__.py b/tests/unit/llms/mistral/audio_transcription/__init__.py similarity index 100% rename from tests/test_litellm/llms/stability/__init__.py rename to tests/unit/llms/mistral/audio_transcription/__init__.py diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py new file mode 100644 index 00000000000..68875ff6d32 --- /dev/null +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -0,0 +1,195 @@ +import os +from unittest.mock import MagicMock + +import httpx +import litellm + +from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig, +) +from litellm.llms.mistral.audio_transcription.transformation import ( + MistralAudioTranscriptionConfig, +) +from litellm.types.utils import TranscriptionResponse +from litellm.utils import ProviderConfigManager + + +def test_mistral_audio_transcription_config_installed(): + """Ensure Mistral audio transcription config is registered with ProviderConfigManager.""" + config = ProviderConfigManager.get_provider_audio_transcription_config( + model="mistral/voxtral-mini-latest", + provider=litellm.LlmProviders.MISTRAL, + ) + assert config is not None + assert isinstance(config, BaseAudioTranscriptionConfig) + assert isinstance(config, MistralAudioTranscriptionConfig) + + +def test_mistral_audio_transcription_get_complete_url(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.mistral.ai/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_get_complete_url_custom_base(): + config = MistralAudioTranscriptionConfig() + url = config.get_complete_url( + api_base="https://custom.api.example.com/v1/", + api_key="fake-key", + model="voxtral-mini-latest", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.api.example.com/v1/audio/transcriptions" + + +def test_mistral_audio_transcription_validate_environment(): + config = MistralAudioTranscriptionConfig() + headers = config.validate_environment( + headers={}, + model="voxtral-mini-latest", + messages=[], + optional_params={}, + litellm_params={}, + api_key="test-key-123", + ) + assert headers["Authorization"] == "Bearer test-key-123" + assert headers["accept"] == "application/json" + + +def test_mistral_audio_transcription_supported_params(): + config = MistralAudioTranscriptionConfig() + params = config.get_supported_openai_params("voxtral-mini-latest") + assert "language" in params + assert "temperature" in params + assert "response_format" in params + assert "timestamp_granularities" in params + + +def test_mistral_audio_transcription_request_transform(): + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), + "../../../../..", + "tests", + "llm_translation", + "gettysburg.wav", + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"language": "en", "temperature": 0.0}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["model"] == "voxtral-mini-latest" + assert result.data["language"] == "en" + assert result.data["temperature"] == 0.0 + assert result.files is not None + assert "file" in result.files + + +def test_mistral_audio_transcription_request_with_diarize(): + """Test that Mistral-specific params like diarize are passed through.""" + config = MistralAudioTranscriptionConfig() + + wav_path = os.path.join( + os.path.dirname(__file__), + "../../../../..", + "tests", + "llm_translation", + "gettysburg.wav", + ) + audio_file = open(wav_path, "rb") + + result = config.transform_audio_transcription_request( + model="voxtral-mini-latest", + audio_file=audio_file, + optional_params={"diarize": True}, + litellm_params={}, + ) + + audio_file.close() + + assert isinstance(result.data, dict) + assert result.data["diarize"] == "true" + + +def test_mistral_audio_transcription_response_transform(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = {"text": "Four score and seven years ago..."} + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "Four score and seven years ago..." + + +def test_mistral_audio_transcription_response_transform_diarized(): + """Test that diarized responses preserve segments and language.""" + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = { + "model": "voxtral-mini-latest", + "text": "Hello, how are you? I am fine.", + "language": None, + "segments": [ + { + "text": "Hello, how are you?", + "start": 0.3, + "end": 2.1, + "speaker_id": "speaker_1", + "type": "transcription_segment", + }, + { + "text": "I am fine.", + "start": 2.5, + "end": 3.8, + "speaker_id": "speaker_2", + "type": "transcription_segment", + }, + ], + "usage": { + "prompt_audio_seconds": 4, + "prompt_tokens": 5, + "total_tokens": 50, + "completion_tokens": 20, + }, + } + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "Hello, how are you? I am fine." + assert response["segments"] is not None + assert len(response["segments"]) == 2 + assert response["segments"][0]["speaker_id"] == "speaker_1" + assert response["segments"][1]["speaker_id"] == "speaker_2" + assert response["language"] is None + + +def test_mistral_audio_transcription_response_transform_empty(): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = {} + + response = config.transform_audio_transcription_response(mock_response) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "" diff --git a/tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/mistral/test_mistral_chat_transformation.py rename to tests/unit/llms/mistral/test_mistral_chat_transformation.py diff --git a/tests/test_litellm/llms/mistral/test_mistral_completion.py b/tests/unit/llms/mistral/test_mistral_completion.py similarity index 100% rename from tests/test_litellm/llms/mistral/test_mistral_completion.py rename to tests/unit/llms/mistral/test_mistral_completion.py diff --git a/tests/test_litellm/llms/stability/image_generation/__init__.py b/tests/unit/llms/modelscope/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/stability/image_generation/__init__.py rename to tests/unit/llms/modelscope/chat/__init__.py diff --git a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py b/tests/unit/llms/modelscope/chat/test_modelscope_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py rename to tests/unit/llms/modelscope/chat/test_modelscope_chat_transformation.py diff --git a/tests/test_litellm/llms/tencent/__init__.py b/tests/unit/llms/nadir/__init__.py similarity index 100% rename from tests/test_litellm/llms/tencent/__init__.py rename to tests/unit/llms/nadir/__init__.py diff --git a/tests/test_litellm/llms/nadir/test_nadir.py b/tests/unit/llms/nadir/test_nadir.py similarity index 100% rename from tests/test_litellm/llms/nadir/test_nadir.py rename to tests/unit/llms/nadir/test_nadir.py diff --git a/tests/test_litellm/llms/tencent/chat/__init__.py b/tests/unit/llms/nebius/__init__.py similarity index 100% rename from tests/test_litellm/llms/tencent/chat/__init__.py rename to tests/unit/llms/nebius/__init__.py diff --git a/tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py b/tests/unit/llms/nebius/test_nebius_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/nebius/test_nebius_chat_transformation.py rename to tests/unit/llms/nebius/test_nebius_chat_transformation.py diff --git a/tests/test_litellm/llms/nebius/test_nebius_embedding_transformation.py b/tests/unit/llms/nebius/test_nebius_embedding_transformation.py similarity index 100% rename from tests/test_litellm/llms/nebius/test_nebius_embedding_transformation.py rename to tests/unit/llms/nebius/test_nebius_embedding_transformation.py diff --git a/tests/test_litellm/llms/tencent/messages/__init__.py b/tests/unit/llms/oci/rerank/__init__.py similarity index 100% rename from tests/test_litellm/llms/tencent/messages/__init__.py rename to tests/unit/llms/oci/rerank/__init__.py diff --git a/tests/test_litellm/llms/oci/test_oci_common_utils.py b/tests/unit/llms/oci/test_oci_common_utils.py similarity index 100% rename from tests/test_litellm/llms/oci/test_oci_common_utils.py rename to tests/unit/llms/oci/test_oci_common_utils.py diff --git a/tests/test_litellm/llms/oci/test_oci_coverage_boost.py b/tests/unit/llms/oci/test_oci_coverage_boost.py similarity index 100% rename from tests/test_litellm/llms/oci/test_oci_coverage_boost.py rename to tests/unit/llms/oci/test_oci_coverage_boost.py diff --git a/tests/test_litellm/llms/vercel_ai_gateway/embedding/__init__.py b/tests/unit/llms/ollama/__init__.py similarity index 100% rename from tests/test_litellm/llms/vercel_ai_gateway/embedding/__init__.py rename to tests/unit/llms/ollama/__init__.py diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py rename to tests/unit/llms/ollama/test_ollama_chat_transformation.py diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py similarity index 100% rename from tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py rename to tests/unit/llms/ollama/test_ollama_completion_transformation.py diff --git a/tests/test_litellm/llms/ollama/test_ollama_embedding.py b/tests/unit/llms/ollama/test_ollama_embedding.py similarity index 100% rename from tests/test_litellm/llms/ollama/test_ollama_embedding.py rename to tests/unit/llms/ollama/test_ollama_embedding.py diff --git a/tests/test_litellm/llms/ollama/test_ollama_model_info.py b/tests/unit/llms/ollama/test_ollama_model_info.py similarity index 100% rename from tests/test_litellm/llms/ollama/test_ollama_model_info.py rename to tests/unit/llms/ollama/test_ollama_model_info.py diff --git a/tests/test_litellm/llms/openai/realtime/README.md b/tests/unit/llms/openai/realtime/README.md similarity index 100% rename from tests/test_litellm/llms/openai/realtime/README.md rename to tests/unit/llms/openai/realtime/README.md diff --git a/tests/test_litellm/llms/vertex_ai/agent_engine/__init__.py b/tests/unit/llms/openai/realtime/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/agent_engine/__init__.py rename to tests/unit/llms/openai/realtime/__init__.py diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py similarity index 100% rename from tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py rename to tests/unit/llms/openai/realtime/test_openai_realtime_handler.py diff --git a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py similarity index 100% rename from tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py rename to tests/unit/llms/openai/realtime/test_transcription_sessions.py diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/__init__.py b/tests/unit/llms/openai/responses/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/audio_transcription/__init__.py rename to tests/unit/llms/openai/responses/__init__.py diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py similarity index 100% rename from tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py rename to tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_data_residency.py b/tests/unit/llms/openai/responses/test_openai_responses_data_residency.py similarity index 100% rename from tests/test_litellm/llms/openai/responses/test_openai_responses_data_residency.py rename to tests/unit/llms/openai/responses/test_openai_responses_data_residency.py diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py similarity index 100% rename from tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py rename to tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py similarity index 100% rename from tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py rename to tests/unit/llms/openai/responses/test_openai_responses_guardrail_tool_merge.py diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py similarity index 100% rename from tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py rename to tests/unit/llms/openai/responses/test_openai_responses_transformation.py diff --git a/tests/test_litellm/llms/openai/test_cost_calculation.py b/tests/unit/llms/openai/test_cost_calculation.py similarity index 100% rename from tests/test_litellm/llms/openai/test_cost_calculation.py rename to tests/unit/llms/openai/test_cost_calculation.py diff --git a/tests/test_litellm/llms/openai/test_data_residency.py b/tests/unit/llms/openai/test_data_residency.py similarity index 100% rename from tests/test_litellm/llms/openai/test_data_residency.py rename to tests/unit/llms/openai/test_data_residency.py diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py similarity index 100% rename from tests/test_litellm/llms/openai/test_gpt5_transformation.py rename to tests/unit/llms/openai/test_gpt5_transformation.py diff --git a/tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py b/tests/unit/llms/openai/test_is_model_gpt_5_model.py similarity index 100% rename from tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py rename to tests/unit/llms/openai/test_is_model_gpt_5_model.py diff --git a/tests/test_litellm/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py similarity index 100% rename from tests/test_litellm/llms/openai/test_o_series_transformation.py rename to tests/unit/llms/openai/test_o_series_transformation.py diff --git a/tests/test_litellm/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai.py rename to tests/unit/llms/openai/test_openai.py diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai_common_utils.py rename to tests/unit/llms/openai/test_openai_common_utils.py diff --git a/tests/test_litellm/llms/openai/test_openai_empty_response.py b/tests/unit/llms/openai/test_openai_empty_response.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai_empty_response.py rename to tests/unit/llms/openai/test_openai_empty_response.py diff --git a/tests/test_litellm/llms/openai/test_openai_file_content_streaming.py b/tests/unit/llms/openai/test_openai_file_content_streaming.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai_file_content_streaming.py rename to tests/unit/llms/openai/test_openai_file_content_streaming.py diff --git a/tests/test_litellm/llms/openai/test_openai_image_edit_transformation.py b/tests/unit/llms/openai/test_openai_image_edit_transformation.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai_image_edit_transformation.py rename to tests/unit/llms/openai/test_openai_image_edit_transformation.py diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/unit/llms/openai/test_openai_workload_identity.py similarity index 100% rename from tests/test_litellm/llms/openai/test_openai_workload_identity.py rename to tests/unit/llms/openai/test_openai_workload_identity.py diff --git a/tests/test_litellm/llms/openai/test_organization_costs.py b/tests/unit/llms/openai/test_organization_costs.py similarity index 100% rename from tests/test_litellm/llms/openai/test_organization_costs.py rename to tests/unit/llms/openai/test_organization_costs.py diff --git a/tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py b/tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py similarity index 100% rename from tests/test_litellm/llms/openai/test_use_chat_completions_api_no_leak.py rename to tests/unit/llms/openai/test_use_chat_completions_api_no_leak.py diff --git a/tests/test_litellm/llms/openai/transcriptions/test_openai_transcriptions_handler.py b/tests/unit/llms/openai/transcriptions/test_openai_transcriptions_handler.py similarity index 100% rename from tests/test_litellm/llms/openai/transcriptions/test_openai_transcriptions_handler.py rename to tests/unit/llms/openai/transcriptions/test_openai_transcriptions_handler.py diff --git a/tests/test_litellm/llms/vertex_ai/batches/__init__.py b/tests/unit/llms/openai_like/responses/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/batches/__init__.py rename to tests/unit/llms/openai_like/responses/__init__.py diff --git a/tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py b/tests/unit/llms/openai_like/responses/test_openai_like_responses.py similarity index 100% rename from tests/test_litellm/llms/openai_like/responses/test_openai_like_responses.py rename to tests/unit/llms/openai_like/responses/test_openai_like_responses.py diff --git a/tests/test_litellm/llms/openai_like/test_abliteration_provider.py b/tests/unit/llms/openai_like/test_abliteration_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_abliteration_provider.py rename to tests/unit/llms/openai_like/test_abliteration_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_assemblyai_provider.py b/tests/unit/llms/openai_like/test_assemblyai_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_assemblyai_provider.py rename to tests/unit/llms/openai_like/test_assemblyai_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_charity_engine.py b/tests/unit/llms/openai_like/test_charity_engine.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_charity_engine.py rename to tests/unit/llms/openai_like/test_charity_engine.py diff --git a/tests/test_litellm/llms/openai_like/test_cognition_provider.py b/tests/unit/llms/openai_like/test_cognition_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_cognition_provider.py rename to tests/unit/llms/openai_like/test_cognition_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_dynamic_config.py b/tests/unit/llms/openai_like/test_dynamic_config.py similarity index 96% rename from tests/test_litellm/llms/openai_like/test_dynamic_config.py rename to tests/unit/llms/openai_like/test_dynamic_config.py index 55e1a1679de..de70f98c3f1 100644 --- a/tests/test_litellm/llms/openai_like/test_dynamic_config.py +++ b/tests/unit/llms/openai_like/test_dynamic_config.py @@ -20,9 +20,6 @@ def _isolate_generated_class_cache(): class TestClassCaching: - def test_same_slug_returns_the_identical_class_object(self): - provider = _provider("cache_same_slug") - assert create_responses_config_class(provider) is create_responses_config_class(provider) def test_cache_is_keyed_on_slug_not_on_the_provider_instance(self): first = create_responses_config_class(_provider("cache_by_slug")) diff --git a/tests/test_litellm/llms/openai_like/test_empiriolabs_provider.py b/tests/unit/llms/openai_like/test_empiriolabs_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_empiriolabs_provider.py rename to tests/unit/llms/openai_like/test_empiriolabs_provider.py diff --git a/tests/unit/llms/openai_like/test_json_providers.py b/tests/unit/llms/openai_like/test_json_providers.py new file mode 100644 index 00000000000..a56108ca9ac --- /dev/null +++ b/tests/unit/llms/openai_like/test_json_providers.py @@ -0,0 +1,317 @@ +""" +Tests for JSON-based provider configuration system. +""" + +import os +import sys +from unittest.mock import patch + +try: + import pytest +except ImportError: + # pytest not available, will run as standalone script + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + + + +class TestJSONProviderLoader: + """Test JSON provider loading and configuration""" + + def test_load_json_providers(self): + """Test that JSON providers load correctly""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + # Verify publicai is loaded + assert JSONProviderRegistry.exists("publicai") + + # Get publicai config + publicai = JSONProviderRegistry.get("publicai") + assert publicai is not None + assert publicai.base_url == "https://api.publicai.co/v1" + assert publicai.api_key_env == "PUBLICAI_API_KEY" + assert publicai.api_base_env == "PUBLICAI_API_BASE" + assert publicai.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_dynamic_config_generation(self): + """Test dynamic config class creation""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("publicai") + config_class = create_config_class(provider) + config = config_class() + + # Test API info resolution + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.publicai.co/v1" + + # Test with custom base + api_base, api_key = config._get_openai_compatible_provider_info( + "https://custom.api.com", "test-key" + ) + assert api_base == "https://custom.api.com" + assert api_key == "test-key" + + def test_parameter_mapping(self): + """Test parameter mapping works""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("publicai") + config_class = create_config_class(provider) + config = config_class() + + # Test parameter mapping + optional_params = {} + non_default_params = {"max_completion_tokens": 100, "temperature": 0.7} + result = config.map_openai_params( + non_default_params, optional_params, "gpt-4", False + ) + + # max_completion_tokens should be mapped to max_tokens + assert "max_tokens" in result + assert result["max_tokens"] == 100 + assert "max_completion_tokens" not in result + + # temperature should be passed through + assert result["temperature"] == 0.7 + + def test_supported_params(self): + """Test that config returns supported params""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("publicai") + config_class = create_config_class(provider) + config = config_class() + + # Get supported params + supported = config.get_supported_openai_params("gpt-4") + + # Should have standard OpenAI params + assert isinstance(supported, list) + assert len(supported) > 0 + + def test_tool_params_excluded_when_function_calling_not_supported(self): + """Test that tool-related params are excluded for models that don't support + function calling. Regression test for https://github.com/BerriAI/litellm/issues/21125 + """ + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("publicai") + config_class = create_config_class(provider) + config = config_class() + + # Mock supports_function_calling to return False + with patch("litellm.utils.supports_function_calling", return_value=False): + supported = config.get_supported_openai_params("some-model-without-fc") + + tool_params = [ + "tools", + "tool_choice", + "function_call", + "functions", + "parallel_tool_calls", + ] + for param in tool_params: + assert ( + param not in supported + ), f"'{param}' should not be in supported params when function calling is not supported" + + # Non-tool params should still be present + assert "temperature" in supported + assert "max_tokens" in supported + assert "stop" in supported + + def test_tool_params_included_when_function_calling_supported(self): + """Test that tool-related params are included for models that support function calling.""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("publicai") + config_class = create_config_class(provider) + config = config_class() + + # Mock supports_function_calling to return True + with patch("litellm.utils.supports_function_calling", return_value=True): + supported = config.get_supported_openai_params("some-model-with-fc") + + assert "tools" in supported + assert "tool_choice" in supported + + def test_provider_resolution(self): + """Test that provider resolution finds JSON providers""" + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + ) + + model, provider, api_key, api_base = get_llm_provider( + model="publicai/gpt-4", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "gpt-4" + assert provider == "publicai" + assert api_base == "https://api.publicai.co/v1" + + def test_provider_config_manager(self): + """Test that ProviderConfigManager returns JSON-based configs""" + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="gpt-4", provider=LlmProviders.PUBLICAI + ) + + assert config is not None + assert config.custom_llm_provider == "publicai" + + +class TestPinstripes: + """Tests for Pinstripes JSON-configured provider""" + + def test_pinstripes_json_config_exists(self): + """Test that pinstripes is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("pinstripes") + + pinstripes = JSONProviderRegistry.get("pinstripes") + assert pinstripes is not None + assert pinstripes.base_url == "https://pinstripes.io/v1" + assert pinstripes.api_key_env == "PINSTRIPES_API_KEY" + assert pinstripes.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_pinstripes_provider_resolution(self): + """Test that provider resolution finds pinstripes and returns the default base URL""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="pinstripes/ps/glm-4.5-air", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "ps/glm-4.5-air" + assert provider == "pinstripes" + assert api_base == "https://pinstripes.io/v1" + + def test_pinstripes_dynamic_config(self): + """Test dynamic config class creation for pinstripes""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("pinstripes") + config_class = create_config_class(provider) + config = config_class() + + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://pinstripes.io/v1" + + api_base, api_key = config._get_openai_compatible_provider_info( + "https://custom.pinstripes.io/v1", "test-key" + ) + assert api_base == "https://custom.pinstripes.io/v1" + assert api_key == "test-key" + + def test_pinstripes_parameter_mapping(self): + """Test that max_completion_tokens is mapped to max_tokens for pinstripes""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("pinstripes") + config_class = create_config_class(provider) + config = config_class() + + optional_params = {} + non_default_params = {"max_completion_tokens": 100, "temperature": 0.7} + result = config.map_openai_params( + non_default_params, optional_params, "ps/glm-4.5-air", False + ) + + assert "max_tokens" in result + assert result["max_tokens"] == 100 + assert "max_completion_tokens" not in result + assert result["temperature"] == 0.7 + + +class TestDarkbloom: + def test_darkbloom_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + darkbloom = JSONProviderRegistry.get("darkbloom") + assert darkbloom is not None + assert darkbloom.base_url == "https://api.darkbloom.dev/v1" + assert darkbloom.api_key_env == "DARKBLOOM_API_KEY" + assert darkbloom.api_base_env == "DARKBLOOM_API_BASE" + assert darkbloom.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_darkbloom_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="darkbloom/gemma-4-26b", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "gemma-4-26b" + assert provider == "darkbloom" + assert api_key is None + assert api_base == "https://api.darkbloom.dev/v1" + + def test_darkbloom_dynamic_config(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("darkbloom") + config_class = create_config_class(provider) + config = config_class() + + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.darkbloom.dev/v1" + + api_base, api_key = config._get_openai_compatible_provider_info( + "https://custom.darkbloom.dev/v1", "test-key" + ) + assert api_base == "https://custom.darkbloom.dev/v1" + assert api_key == "test-key" + + def test_darkbloom_complete_url_appends_endpoint(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("darkbloom") + config_class = create_config_class(provider) + config = config_class() + + url = config.get_complete_url( + api_base="https://api.darkbloom.dev/v1", + api_key="test-key", + model="darkbloom/gemma-4-26b", + optional_params={}, + litellm_params={}, + stream=True, + ) + + assert url == "https://api.darkbloom.dev/v1/chat/completions" + + def test_darkbloom_provider_config_manager(self): + from litellm import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="gemma-4-26b", provider=LlmProviders.DARKBLOOM + ) + + assert config is not None + assert config.custom_llm_provider == "darkbloom" diff --git a/tests/test_litellm/llms/openai_like/test_libertai_provider.py b/tests/unit/llms/openai_like/test_libertai_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_libertai_provider.py rename to tests/unit/llms/openai_like/test_libertai_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_meta_provider.py b/tests/unit/llms/openai_like/test_meta_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_meta_provider.py rename to tests/unit/llms/openai_like/test_meta_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_model_info.py b/tests/unit/llms/openai_like/test_model_info.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_model_info.py rename to tests/unit/llms/openai_like/test_model_info.py diff --git a/tests/test_litellm/llms/openai_like/test_pinstripes_provider.py b/tests/unit/llms/openai_like/test_pinstripes_provider.py similarity index 68% rename from tests/test_litellm/llms/openai_like/test_pinstripes_provider.py rename to tests/unit/llms/openai_like/test_pinstripes_provider.py index 70bb786b2e6..e7a2dfb92dc 100644 --- a/tests/test_litellm/llms/openai_like/test_pinstripes_provider.py +++ b/tests/unit/llms/openai_like/test_pinstripes_provider.py @@ -16,17 +16,6 @@ class TestPinstripeProviderConfig: assert LlmProviders.PINSTRIPES.value == "pinstripes" assert "pinstripes" in litellm.provider_list - def test_pinstripes_json_config_exists(self): - """Test that pinstripes is configured in providers.json""" - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - assert JSONProviderRegistry.exists("pinstripes") - - pinstripes = JSONProviderRegistry.get("pinstripes") - assert pinstripes is not None - assert pinstripes.base_url == "https://pinstripes.io/v1" - assert pinstripes.api_key_env == "PINSTRIPES_API_KEY" - assert pinstripes.param_mappings.get("max_completion_tokens") == "max_tokens" def test_pinstripes_in_openai_compatible_providers(self): """Test that pinstripes is in the openai_compatible_providers list""" @@ -34,20 +23,6 @@ class TestPinstripeProviderConfig: assert "pinstripes" in openai_compatible_providers - def test_pinstripes_provider_resolution(self): - """Test that provider resolution finds pinstripes and returns the default base URL""" - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="pinstripes/ps/glm-4.5-air", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "ps/glm-4.5-air" - assert provider == "pinstripes" - assert api_base == "https://pinstripes.io/v1" def test_pinstripes_api_base_override(self): """Test that an explicit api_base / api_key overrides the default""" diff --git a/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py b/tests/unit/llms/openai_like/test_provider_affinity_forwarding.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py rename to tests/unit/llms/openai_like/test_provider_affinity_forwarding.py diff --git a/tests/test_litellm/llms/openai_like/test_scx_ai_provider.py b/tests/unit/llms/openai_like/test_scx_ai_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_scx_ai_provider.py rename to tests/unit/llms/openai_like/test_scx_ai_provider.py diff --git a/tests/test_litellm/llms/openai_like/test_tensormesh_provider.py b/tests/unit/llms/openai_like/test_tensormesh_provider.py similarity index 100% rename from tests/test_litellm/llms/openai_like/test_tensormesh_provider.py rename to tests/unit/llms/openai_like/test_tensormesh_provider.py diff --git a/tests/unit/llms/openai_like/test_xiaomi_mimo.py b/tests/unit/llms/openai_like/test_xiaomi_mimo.py new file mode 100644 index 00000000000..a642cc91f90 --- /dev/null +++ b/tests/unit/llms/openai_like/test_xiaomi_mimo.py @@ -0,0 +1,84 @@ +""" +Tests for Xiaomi MiMo provider configuration and integration. +Related to issue #18794 +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +try: + import pytest +except ImportError: + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + +import litellm + + +class TestXiaomiMiMoProviderConfig: + """Test Xiaomi MiMo provider configuration""" + + def test_xiaomi_mimo_in_provider_list(self): + """Test that xiaomi_mimo is in the provider list (fixes #18794)""" + from litellm import LlmProviders + + # Verify xiaomi_mimo is in the enum + assert hasattr(LlmProviders, "XIAOMI_MIMO") + assert LlmProviders.XIAOMI_MIMO.value == "xiaomi_mimo" + + # Verify it's in the provider list + assert "xiaomi_mimo" in litellm.provider_list + + def test_xiaomi_mimo_json_config_exists(self): + """Test that xiaomi_mimo is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + # Verify xiaomi_mimo is loaded + assert JSONProviderRegistry.exists("xiaomi_mimo") + + # Get xiaomi_mimo config + xiaomi_mimo = JSONProviderRegistry.get("xiaomi_mimo") + assert xiaomi_mimo is not None + assert xiaomi_mimo.base_url == "https://api.xiaomimimo.com/v1" + assert xiaomi_mimo.api_key_env == "XIAOMI_MIMO_API_KEY" + assert xiaomi_mimo.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_xiaomi_mimo_provider_resolution(self): + """Test that provider resolution finds xiaomi_mimo""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="xiaomi_mimo/mimo-v2-flash", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "mimo-v2-flash" + assert provider == "xiaomi_mimo" + assert api_base == "https://api.xiaomimimo.com/v1" + + def test_xiaomi_mimo_router_config(self): + """Test that xiaomi_mimo can be used in Router configuration (fixes #18794)""" + from litellm import Router + + # This should not raise "Unsupported provider - xiaomi_mimo" + router = Router( + model_list=[ + { + "model_name": "mimo-v2-flash", + "litellm_params": { + "model": "xiaomi_mimo/mimo-v2-flash", + "api_key": "test-key", + }, + } + ] + ) + + # Verify the deployment was created successfully + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "mimo-v2-flash" diff --git a/tests/test_litellm/llms/vertex_ai/files/__init__.py b/tests/unit/llms/ovhcloud/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/files/__init__.py rename to tests/unit/llms/ovhcloud/__init__.py diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py new file mode 100644 index 00000000000..87e54dfba9b --- /dev/null +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -0,0 +1,58 @@ + + + + +class TestOVHCloudDurationFieldMigration: + """Tests for OVHCloud duration -> seconds field migration.""" + + def test_seconds_field_mapped_to_duration(self): + """New `seconds` field should be normalized to `duration`.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "seconds": 3.14, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 3.14 + + def test_legacy_duration_field_still_works(self): + """Legacy `duration` field should still be accepted.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "text": "Hello world", + "duration": 2.71, + } + + result = config.transform_audio_transcription_response(mock_response) + + assert result.text == "Hello world" + assert result._hidden_params["duration"] == 2.71 + + + def test_seconds_zero_mapped_to_duration(self): + """seconds=0.0 must not be treated as falsy and lost.""" + from litellm.llms.ovhcloud.audio_transcription.transformation import ( + OVHCloudAudioTranscriptionConfig, + ) + from unittest.mock import MagicMock + + config = OVHCloudAudioTranscriptionConfig() + mock_response = MagicMock() + mock_response.json.return_value = {"text": "silence", "seconds": 0.0} + result = config.transform_audio_transcription_response(mock_response) + assert result._hidden_params["duration"] == 0.0 diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_chat_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_chat_transformation.py new file mode 100644 index 00000000000..c2bc4ee4a4c --- /dev/null +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_chat_transformation.py @@ -0,0 +1,250 @@ +""" +Unit tests for OVHCloud AI Endpoints chat integration. +""" + + +import pytest + +from litellm.llms.ovhcloud.utils import OVHCloudException +from litellm.utils import get_optional_params + + +from litellm.llms.ovhcloud.chat.transformation import ( + OVHCloudChatCompletionStreamingHandler, + OVHCloudChatConfig, +) + +config = OVHCloudChatConfig() +model = "ovhcloud/Mistral-7B-Instruct-v0.3" + + +class TestOvhCloudChatCompletionStreamingHandler: + def test_chunk_parser_successful(self): + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + chunk = { + "id": "test_id", + "created": 1234567890, + "model": "gpt-oss-20b", + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + "choices": [ + {"delta": {"content": "test content", "reasoning": "test reasoning"}} + ], + } + + result = handler.chunk_parser(chunk) + + assert result.id == "test_id" + assert result.object == "chat.completion.chunk" + assert result.created == 1234567890 + assert result.model == "gpt-oss-20b" + assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] + assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] + assert result.usage.total_tokens == chunk["usage"]["total_tokens"] + assert len(result.choices) == 1 + assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" + + def test_chunk_parser_error_response(self): + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + error_chunk = { + "error": { + "message": "test error", + "code": 400, + } + } + + with pytest.raises(OVHCloudException) as exc_info: + handler.chunk_parser(error_chunk) + + assert "OVHCloud Error: test error" in str(exc_info.value) + assert exc_info.value.status_code == 400 + + def test_chunk_parser_key_error(self): + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + invalid_chunk = {"incomplete": "data"} + + with pytest.raises(OVHCloudException) as exc_info: + handler.chunk_parser(invalid_chunk) + + assert "KeyError" in str(exc_info.value) + assert exc_info.value.status_code == 400 + + +class TestOVHCloudConfig: + def test_transform_request_basic(self): + """Test basic request transformation""" + transformed_request = config.transform_request( + model, + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert transformed_request["model"] == model + assert transformed_request["messages"] == [ + {"role": "user", "content": "Hello, world!"} + ] + + def test_transform_request_with_extra_body(self): + """Test request transformation with extra_body parameters""" + transformed_request = config.transform_request( + model, + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={"extra_body": {"custom_param": "custom_value"}}, + litellm_params={}, + headers={}, + ) + + assert transformed_request["custom_param"] == "custom_value" + assert transformed_request["messages"] == [ + {"role": "user", "content": "Hello, world!"} + ] + + def test_map_openai_params(self): + """Test OpenAI parameter mapping""" + non_default_params = { + "temperature": 0.7, + "max_tokens": 100, + "top_p": 0.9, + } + + mapped_params = config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=False, + ) + + assert mapped_params["temperature"] == 0.7 + assert mapped_params["max_tokens"] == 100 + assert mapped_params["top_p"] == 0.9 + + def test_get_error_class(self): + """Test error class creation""" + error = config.get_error_class( + error_message="Test error", + status_code=400, + headers={"Content-Type": "application/json"}, + ) + + assert isinstance(error, OVHCloudException) + assert error.message == "Test error" + assert error.status_code == 400 + + @pytest.mark.parametrize( + "model", + [ + "Meta-Llama-3_3-70B-Instruct", + "Meta-Llama-3_1-70B-Instruct", + "Mixtral-8x7B-Instruct-v0.1", + "gpt-oss-120b", + "some-model-not-in-the-cost-map", + ], + ) + def test_tools_not_filtered_by_static_model_map(self, model): + """ + OVHCloud AI Endpoints are OpenAI-compatible; tools/tool_choice must pass + through for any model. The server is responsible for rejecting unsupported + tool calls — LiteLLM must not strip them based on a stale static catalog. + """ + + params = get_optional_params( + model=model, + custom_llm_provider="ovhcloud", + tools=[ + { + "type": "function", + "function": {"name": "x", "parameters": {}}, + } + ], + tool_choice="auto", + ) + + assert "tools" in params + assert "tool_choice" in params + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + + +class TestOVHCloudReasoningFieldMigration: + """Tests for OVHCloud reasoning_content -> reasoning field migration.""" + + def test_streaming_new_reasoning_field(self): + """New `reasoning` field should be mapped to `reasoning_content`.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning": "Let me think...", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Let me think..." + + def test_streaming_legacy_reasoning_content_unchanged(self): + """Legacy `reasoning_content` field should pass through untouched.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "role": "assistant", + "reasoning_content": "Already correct field.", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "Already correct field." + + def test_streaming_both_fields_legacy_wins(self): + """When both fields present, existing `reasoning_content` is not overwritten.""" + handler = OVHCloudChatCompletionStreamingHandler( + streaming_response=iter([]), + sync_stream=True, + ) + chunk = { + "id": "test-id", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "delta": { + "reasoning": "new field", + "reasoning_content": "legacy field", + }, + "index": 0, + } + ], + } + result = handler.chunk_parser(chunk) + assert result.choices[0]["delta"]["reasoning_content"] == "legacy field" diff --git a/tests/test_litellm/llms/ovhcloud/test_ovhcloud_embeddings_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_embeddings_transformation.py similarity index 100% rename from tests/test_litellm/llms/ovhcloud/test_ovhcloud_embeddings_transformation.py rename to tests/unit/llms/ovhcloud/test_ovhcloud_embeddings_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/__init__.py b/tests/unit/llms/pass_through/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini_embeddings/__init__.py rename to tests/unit/llms/pass_through/__init__.py diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/__init__.py b/tests/unit/llms/pass_through/guardrail_translation/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/text_to_speech/__init__.py rename to tests/unit/llms/pass_through/guardrail_translation/__init__.py diff --git a/tests/test_litellm/llms/perplexity/test_perplexity.py b/tests/unit/llms/perplexity/test_perplexity.py similarity index 100% rename from tests/test_litellm/llms/perplexity/test_perplexity.py rename to tests/unit/llms/perplexity/test_perplexity.py diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py rename to tests/unit/llms/perplexity/test_perplexity_cost_calculator.py diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/unit/llms/perplexity/test_perplexity_integration.py similarity index 100% rename from tests/test_litellm/llms/perplexity/test_perplexity_integration.py rename to tests/unit/llms/perplexity/test_perplexity_integration.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/__init__.py b/tests/unit/llms/pg_vector/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/__init__.py rename to tests/unit/llms/pg_vector/__init__.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/__init__.py b/tests/unit/llms/pg_vector/vector_stores/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/__init__.py rename to tests/unit/llms/pg_vector/vector_stores/__init__.py diff --git a/tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py b/tests/unit/llms/pg_vector/vector_stores/test_pg_vector_transformation.py similarity index 100% rename from tests/test_litellm/llms/pg_vector/vector_stores/test_pg_vector_transformation.py rename to tests/unit/llms/pg_vector/vector_stores/test_pg_vector_transformation.py diff --git a/tests/test_litellm/llms/azure/realtime/__init__.py b/tests/unit/llms/reducto/__init__.py similarity index 100% rename from tests/test_litellm/llms/azure/realtime/__init__.py rename to tests/unit/llms/reducto/__init__.py diff --git a/tests/test_litellm/llms/reducto/conftest.py b/tests/unit/llms/reducto/conftest.py similarity index 100% rename from tests/test_litellm/llms/reducto/conftest.py rename to tests/unit/llms/reducto/conftest.py diff --git a/tests/test_litellm/llms/reducto/test_cost.py b/tests/unit/llms/reducto/test_cost.py similarity index 100% rename from tests/test_litellm/llms/reducto/test_cost.py rename to tests/unit/llms/reducto/test_cost.py diff --git a/tests/test_litellm/llms/reducto/test_model_info.py b/tests/unit/llms/reducto/test_model_info.py similarity index 100% rename from tests/test_litellm/llms/reducto/test_model_info.py rename to tests/unit/llms/reducto/test_model_info.py diff --git a/tests/test_litellm/llms/reducto/test_parse_legacy.py b/tests/unit/llms/reducto/test_parse_legacy.py similarity index 100% rename from tests/test_litellm/llms/reducto/test_parse_legacy.py rename to tests/unit/llms/reducto/test_parse_legacy.py diff --git a/tests/test_litellm/llms/reducto/test_parse_v3.py b/tests/unit/llms/reducto/test_parse_v3.py similarity index 100% rename from tests/test_litellm/llms/reducto/test_parse_v3.py rename to tests/unit/llms/reducto/test_parse_v3.py diff --git a/tests/test_litellm/llms/reducto/test_upload.py b/tests/unit/llms/reducto/test_upload.py similarity index 100% rename from tests/test_litellm/llms/reducto/test_upload.py rename to tests/unit/llms/reducto/test_upload.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/__init__.py b/tests/unit/llms/sagemaker/__init__.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/__init__.py rename to tests/unit/llms/sagemaker/__init__.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_handler.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_chat_handler.py rename to tests/unit/llms/sagemaker/test_sagemaker_chat_handler.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_chat_transformation.py rename to tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py b/tests/unit/llms/sagemaker/test_sagemaker_common_utils.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py rename to tests/unit/llms/sagemaker/test_sagemaker_common_utils.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_completion_handler.py rename to tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py b/tests/unit/llms/sagemaker/test_sagemaker_embedding_role_assumption.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py rename to tests/unit/llms/sagemaker/test_sagemaker_embedding_role_assumption.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py b/tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_voyage.py rename to tests/unit/llms/sagemaker/test_sagemaker_embedding_voyage.py diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_nova_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py similarity index 100% rename from tests/test_litellm/llms/sagemaker/test_sagemaker_nova_transformation.py rename to tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py diff --git a/tests/test_litellm/llms/voyage/rerank/__init__.py b/tests/unit/llms/sambanova/__init__.py similarity index 100% rename from tests/test_litellm/llms/voyage/rerank/__init__.py rename to tests/unit/llms/sambanova/__init__.py diff --git a/tests/test_litellm/llms/sambanova/tests_sambanova_embedding_transformation.py b/tests/unit/llms/sambanova/tests_sambanova_embedding_transformation.py similarity index 100% rename from tests/test_litellm/llms/sambanova/tests_sambanova_embedding_transformation.py rename to tests/unit/llms/sambanova/tests_sambanova_embedding_transformation.py diff --git a/tests/test_litellm/llms/watsonx/__init__.py b/tests/unit/llms/sap/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/watsonx/__init__.py rename to tests/unit/llms/sap/chat/__init__.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py b/tests/unit/llms/sap/chat/test_sap_chat_calls.py similarity index 100% rename from tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py rename to tests/unit/llms/sap/chat/test_sap_chat_calls.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_langchain_strict_param.py b/tests/unit/llms/sap/chat/test_sap_langchain_strict_param.py similarity index 100% rename from tests/test_litellm/llms/sap/chat/test_sap_langchain_strict_param.py rename to tests/unit/llms/sap/chat/test_sap_langchain_strict_param.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_response_format.py b/tests/unit/llms/sap/chat/test_sap_response_format.py similarity index 100% rename from tests/test_litellm/llms/sap/chat/test_sap_response_format.py rename to tests/unit/llms/sap/chat/test_sap_response_format.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_tool_parameters.py b/tests/unit/llms/sap/chat/test_sap_tool_parameters.py similarity index 100% rename from tests/test_litellm/llms/sap/chat/test_sap_tool_parameters.py rename to tests/unit/llms/sap/chat/test_sap_tool_parameters.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/unit/llms/sap/chat/test_sap_transformation.py similarity index 100% rename from tests/test_litellm/llms/sap/chat/test_sap_transformation.py rename to tests/unit/llms/sap/chat/test_sap_transformation.py diff --git a/tests/test_litellm/llms/watsonx/audio_transcription/__init__.py b/tests/unit/llms/sap/embed/__init__.py similarity index 100% rename from tests/test_litellm/llms/watsonx/audio_transcription/__init__.py rename to tests/unit/llms/sap/embed/__init__.py diff --git a/tests/test_litellm/llms/sap/embed/test_sap_embed_transformation.py b/tests/unit/llms/sap/embed/test_sap_embed_transformation.py similarity index 100% rename from tests/test_litellm/llms/sap/embed/test_sap_embed_transformation.py rename to tests/unit/llms/sap/embed/test_sap_embed_transformation.py diff --git a/tests/test_litellm/llms/sap/embed/test_sap_embedding.py b/tests/unit/llms/sap/embed/test_sap_embedding.py similarity index 100% rename from tests/test_litellm/llms/sap/embed/test_sap_embedding.py rename to tests/unit/llms/sap/embed/test_sap_embedding.py diff --git a/tests/test_litellm/llms/watsonx/rerank/__init__.py b/tests/unit/llms/snowflake/chat/__init__.py similarity index 100% rename from tests/test_litellm/llms/watsonx/rerank/__init__.py rename to tests/unit/llms/snowflake/chat/__init__.py diff --git a/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py rename to tests/unit/llms/snowflake/chat/test_snowflake_chat_transformation.py diff --git a/tests/test_litellm/llms/you_com/__init__.py b/tests/unit/llms/snowflake/embedding/__init__.py similarity index 100% rename from tests/test_litellm/llms/you_com/__init__.py rename to tests/unit/llms/snowflake/embedding/__init__.py diff --git a/tests/test_litellm/llms/snowflake/embedding/test_snowflake_embedding.py b/tests/unit/llms/snowflake/embedding/test_snowflake_embedding.py similarity index 100% rename from tests/test_litellm/llms/snowflake/embedding/test_snowflake_embedding.py rename to tests/unit/llms/snowflake/embedding/test_snowflake_embedding.py diff --git a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py index 7970f7771fc..344b8e5573d 100644 --- a/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py +++ b/tests/unit/llms/snowflake/test_snowflake_native_endpoints.py @@ -7,7 +7,7 @@ Covers: - Claude models → /messages (Anthropic format) Run: - pytest tests/test_litellm/llms/snowflake/test_snowflake_native_endpoints.py -v + pytest tests/unit/llms/snowflake/test_snowflake_native_endpoints.py -v """ import json diff --git a/tests/test_litellm/llms/soniox/audio_transcription/__init__.py b/tests/unit/llms/soniox/audio_transcription/__init__.py similarity index 100% rename from tests/test_litellm/llms/soniox/audio_transcription/__init__.py rename to tests/unit/llms/soniox/audio_transcription/__init__.py diff --git a/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py b/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py similarity index 100% rename from tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py rename to tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py diff --git a/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py b/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py rename to tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/test_cache_control_and_reasoning.py b/tests/unit/llms/test_cache_control_and_reasoning.py similarity index 100% rename from tests/test_litellm/llms/test_cache_control_and_reasoning.py rename to tests/unit/llms/test_cache_control_and_reasoning.py diff --git a/tests/test_litellm/llms/test_file_content_block.py b/tests/unit/llms/test_file_content_block.py similarity index 100% rename from tests/test_litellm/llms/test_file_content_block.py rename to tests/unit/llms/test_file_content_block.py diff --git a/tests/test_litellm/llms/test_file_search_responses.py b/tests/unit/llms/test_file_search_responses.py similarity index 100% rename from tests/test_litellm/llms/test_file_search_responses.py rename to tests/unit/llms/test_file_search_responses.py diff --git a/tests/test_litellm/llms/test_lifecycle_fix.py b/tests/unit/llms/test_lifecycle_fix.py similarity index 100% rename from tests/test_litellm/llms/test_lifecycle_fix.py rename to tests/unit/llms/test_lifecycle_fix.py diff --git a/tests/test_litellm/llms/test_polling_url_origin_match.py b/tests/unit/llms/test_polling_url_origin_match.py similarity index 100% rename from tests/test_litellm/llms/test_polling_url_origin_match.py rename to tests/unit/llms/test_polling_url_origin_match.py diff --git a/tests/test_litellm/llms/test_predibase_transformation.py b/tests/unit/llms/test_predibase_transformation.py similarity index 100% rename from tests/test_litellm/llms/test_predibase_transformation.py rename to tests/unit/llms/test_predibase_transformation.py diff --git a/tests/unit/llms/tinyfish/__init__.py b/tests/unit/llms/tinyfish/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/tinyfish/test_tinyfish_search.py b/tests/unit/llms/tinyfish/test_tinyfish_search.py similarity index 100% rename from tests/test_litellm/llms/tinyfish/test_tinyfish_search.py rename to tests/unit/llms/tinyfish/test_tinyfish_search.py diff --git a/tests/test_litellm/llms/vercel_ai_gateway/test_vercel_ai_gateway.py b/tests/unit/llms/vercel_ai_gateway/test_vercel_ai_gateway.py similarity index 100% rename from tests/test_litellm/llms/vercel_ai_gateway/test_vercel_ai_gateway.py rename to tests/unit/llms/vercel_ai_gateway/test_vercel_ai_gateway.py diff --git a/tests/unit/llms/vertex_ai/audio_transcription/__init__.py b/tests/unit/llms/vertex_ai/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py rename to tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py rename to tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_backend.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_backend.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_backend.py rename to tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_backend.py diff --git a/tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_transformation.py rename to tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_realtime_transformation.py diff --git a/tests/unit/llms/vertex_ai/batches/__init__.py b/tests/unit/llms/vertex_ai/batches/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_handler.py b/tests/unit/llms/vertex_ai/batches/test_handler.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/batches/test_handler.py rename to tests/unit/llms/vertex_ai/batches/test_handler.py diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py b/tests/unit/llms/vertex_ai/batches/test_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/batches/test_transformation.py rename to tests/unit/llms/vertex_ai/batches/test_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/files/test_transformation.py b/tests/unit/llms/vertex_ai/files/test_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/files/test_transformation.py rename to tests/unit/llms/vertex_ai/files/test_transformation.py diff --git a/tests/unit/llms/vertex_ai/gemini/__init__.py b/tests/unit/llms/vertex_ai/gemini/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py b/tests/unit/llms/vertex_ai/gemini/test_context_circulation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_context_circulation.py rename to tests/unit/llms/vertex_ai/gemini/test_context_circulation.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py b/tests/unit/llms/vertex_ai/gemini/test_function_call_args_serialization.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_function_call_args_serialization.py rename to tests/unit/llms/vertex_ai/gemini/test_function_call_args_serialization.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_image_url_missing_field.py b/tests/unit/llms/vertex_ai/gemini/test_gemini_image_url_missing_field.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_gemini_image_url_missing_field.py rename to tests/unit/llms/vertex_ai/gemini/test_gemini_image_url_missing_field.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py b/tests/unit/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py rename to tests/unit/llms/vertex_ai/gemini/test_gemini_streaming_tool_call_finish_reason.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_grounding_requests.py b/tests/unit/llms/vertex_ai/gemini/test_grounding_requests.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_grounding_requests.py rename to tests/unit/llms/vertex_ai/gemini/test_grounding_requests.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py rename to tests/unit/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py b/tests/unit/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py rename to tests/unit/llms/vertex_ai/gemini/test_tool_call_followed_by_text_assistant.py diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_transformation.py rename to tests/unit/llms/vertex_ai/gemini/test_transformation.py diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py new file mode 100644 index 00000000000..4f23ac1773a --- /dev/null +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -0,0 +1,2729 @@ +import base64 + +import pytest + +from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_result, +) +from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + _transform_request_body, + check_if_part_exists_in_parts, + _get_highest_media_resolution, + _extract_max_media_resolution_from_messages, +) +from litellm.types.llms.vertex_ai import BlobType +from litellm.types.utils import Message + + +def test_check_if_part_exists_in_parts(): + parts = [ + {"text": "Hello", "thought": True}, + {"text": "World", "thought": False}, + ] + part = {"text": "Hello", "thought": True} + new_part = {"text": "Hello World", "thought": True} + assert check_if_part_exists_in_parts(parts, part) + assert not check_if_part_exists_in_parts(parts, new_part, ["thought"]) + assert check_if_part_exists_in_parts(parts, new_part, ["text"]) + + +def test_check_if_part_exists_in_parts_camel_case_snake_case(): + """Test that function handles both camelCase and snake_case key variations""" + # Test snake_case to camelCase matching + parts_with_snake_case = [ + { + "function_call": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + }, + {"text": "Some other content"}, + ] + + part_with_camel_case = { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "San Francisco, CA"}, + } + } + + # Should find match between function_call and functionCall + assert check_if_part_exists_in_parts(parts_with_snake_case, part_with_camel_case) + + # Test camelCase to snake_case matching + parts_with_camel_case = [ + {"functionCall": {"name": "calculate_sum", "args": {"a": 1, "b": 2}}} + ] + + part_with_snake_case = { + "function_call": {"name": "calculate_sum", "args": {"a": 1, "b": 2}} + } + + # Should find match between functionCall and function_call + assert check_if_part_exists_in_parts(parts_with_camel_case, part_with_snake_case) + + # Test no match when values differ + part_with_different_values = { + "function_call": {"name": "different_function", "args": {"x": 5}} + } + + assert not check_if_part_exists_in_parts( + parts_with_snake_case, part_with_different_values + ) + + # Test multiple keys with mixed casing + parts_mixed = [ + { + "function_call": {"name": "test"}, + "thoughtSignature": "reasoning", + "text": "content", + } + ] + + part_mixed_casing = { + "functionCall": {"name": "test"}, + "thought_signature": "reasoning", + "text": "content", + } + + assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) + + +def test_cached_content_respects_modify_params_for_cache_incompatible_fields(): + """Regression: cachedContent drops system/tools/toolConfig only when modify_params=True.""" + import litellm + + cache_name = "projects/p/locations/us-central1/cachedContents/abc123" + messages = [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "hi"}, + ] + optional_params = { + "tools": [ + { + "functionDeclarations": [ + {"name": "get_weather", "description": "Get weather"}, + ] + } + ], + "tool_choice": {"functionCallingConfig": {"mode": "AUTO"}}, + } + + original_modify_params = litellm.modify_params + try: + # With modify_params=False (default), keep fields even with cachedContent. + litellm.modify_params = False + result = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=cache_name, + ) + assert result.get("cachedContent") == cache_name + assert "system_instruction" in result + assert "tools" in result + assert "toolConfig" in result + assert "contents" in result + + # With modify_params=True, drop cache-incompatible fields. + litellm.modify_params = True + result_modify_true = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=cache_name, + ) + assert result_modify_true.get("cachedContent") == cache_name + assert "system_instruction" not in result_modify_true + assert "tools" not in result_modify_true + assert "toolConfig" not in result_modify_true + assert "contents" in result_modify_true + + # Without cache, fields are always included. + result_no_cache = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=None, + ) + assert "system_instruction" in result_no_cache + assert "tools" in result_no_cache + assert "toolConfig" in result_no_cache + finally: + litellm.modify_params = original_modify_params + + +# Tests for issue #14556: Labels field provider-aware filtering +def test_google_genai_excludes_labels(): + """Test that Google GenAI/AI Studio endpoints exclude labels when custom_llm_provider='gemini'""" + messages = [{"role": "user", "content": "test"}] + optional_params = {"labels": {"project": "test", "team": "ai"}} + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="gemini", + litellm_params=litellm_params, + cached_content=None, + ) + + # Google GenAI/AI Studio should NOT include labels + assert "labels" not in result + assert "contents" in result + + +def test_vertex_ai_includes_labels(): + """Test that Vertex AI endpoints include labels when custom_llm_provider='vertex_ai'""" + messages = [{"role": "user", "content": "test"}] + optional_params = {"labels": {"project": "test", "team": "ai"}} + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + # Vertex AI SHOULD include labels + assert "labels" in result + assert result["labels"] == {"project": "test", "team": "ai"} + + +def test_service_tier_forwarded_to_vertex_ai(): + """Test that service_tier in optional_params is mapped to serviceTier in request body.""" + messages = [{"role": "user", "content": "test"}] + optional_params = {"service_tier": "flex"} + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + assert "serviceTier" in result + assert result["serviceTier"] == "flex" + + +def test_extra_body_cache_not_forwarded_to_vertex_ai(): + """ + 'cache' inside extra_body is a LiteLLM-internal proxy caching control. + It must NOT be forwarded to the Vertex AI request body. + + Regression test for: "Invalid JSON payload received. Unknown name \"cache\": Cannot find field." + Vertex AI enforces a strict JSON schema and rejects any unknown field. + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "cache": {"use-cache": True, "ttl": 86400}, # LiteLLM-internal + "some_vertex_param": "value", # legitimate provider extra + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + # 'cache' must be stripped — Vertex AI has no such field + assert "cache" not in result, ( + "extra_body.cache must not be forwarded to Vertex AI. " + 'Vertex AI rejects it with 400: Unknown name "cache": Cannot find field.' + ) + + # Other legitimate extra_body keys should still pass through + assert "some_vertex_param" in result + assert result["some_vertex_param"] == "value" + + # Core request fields must be present + assert "contents" in result + + +def test_extra_body_tags_not_forwarded_to_vertex_ai(): + """ + 'tags' inside extra_body is a LiteLLM-internal param for logging/tracking. + It must NOT be forwarded to the Vertex AI request body. + Documented in litellm_proxy.md: "Send tags by including them in the extra_body parameter" + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "tags": ["user:alice", "env:prod"], + "custom_param": "allowed", + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + assert "tags" not in result + assert "custom_param" in result + assert result["custom_param"] == "allowed" + + +def test_extra_body_google_maps_rewrites_json_response_format(): + messages = [{"role": "user", "content": "test"}] + optional_params = { + "response_mime_type": "application/json", + "response_schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + }, + "extra_body": { + "tools": [{"googleMaps": {}}], + }, + } + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=None, + ) + + generation_config = result["generationConfig"] + assert "response_mime_type" not in generation_config + assert generation_config["responseFormat"] == { + "text": { + "mimeType": "APPLICATION_JSON", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + }, + } + } + + +def test_extra_body_generation_config_cannot_restore_google_maps_json_mime_type(): + messages = [{"role": "user", "content": "test"}] + optional_params = { + "tools": [{"googleMaps": {}}], + "response_mime_type": "application/json", + "extra_body": { + "generationConfig": { + "response_mime_type": "application/json", + "response_json_schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + }, + }, + }, + } + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=None, + ) + + generation_config = result["generationConfig"] + assert "response_mime_type" not in generation_config + assert "response_json_schema" not in generation_config + assert generation_config["responseFormat"] == { + "text": { + "mimeType": "APPLICATION_JSON", + "schema": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + }, + } + } + + +def test_metadata_to_labels_vertex_only(): + """Test that metadata->labels conversion only happens for Vertex AI""" + messages = [{"role": "user", "content": "test"}] + optional_params = {} + litellm_params = { + "metadata": { + "requester_metadata": {"user": "john_doe", "project": "test-project"} + } + } + + # Google GenAI/AI Studio should not include labels from metadata + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params.copy(), + custom_llm_provider="gemini", + litellm_params=litellm_params.copy(), + cached_content=None, + ) + assert "labels" not in result + + # Vertex AI should include labels from metadata + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params.copy(), + custom_llm_provider="vertex_ai", + litellm_params=litellm_params.copy(), + cached_content=None, + ) + assert "labels" in result + assert result["labels"] == {"user": "john_doe", "project": "test-project"} + + +def test_empty_content_handling(): + """Test that empty content strings are properly handled in Gemini message transformation""" + # Test with empty content in user message + messages = [{"content": "", "role": "user"}] + + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify that the content was properly transformed + assert len(contents) == 1 + assert contents[0]["role"] == "user" + assert len(contents[0]["parts"]) == 1 + assert "text" in contents[0]["parts"][0] + assert contents[0]["parts"][0]["text"] == "" + + +def test_thought_signature_extraction_from_response(): + """Test that thought signatures are extracted from Gemini response parts and stored in provider_specific_fields""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.vertex_ai import HttpxPartType + + # Test case: Single function call with thought signature + test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + + parts_with_signature = [ + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "Paris"}, + }, + thoughtSignature=test_signature, + ) + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_with_signature, + cumulative_tool_call_idx=0, + is_function_call=False, + ) + + # Verify thought signature is stored in provider_specific_fields + assert tools is not None + assert len(tools) == 1 + assert "provider_specific_fields" in tools[0] + assert tools[0]["provider_specific_fields"]["thought_signature"] == test_signature + + +def test_thought_signature_parallel_function_calls(): + """Test that only the first function call in parallel calls has thought signature""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.vertex_ai import HttpxPartType + + test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + + # Parallel function calls - only first has signature + parts_parallel = [ + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "Paris"}, + }, + thoughtSignature=test_signature, # First FC has signature + ), + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "London"}, + }, + # Second FC has no signature (parallel call) + ), + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_parallel, + cumulative_tool_call_idx=0, + is_function_call=False, + ) + + # Verify only first tool call has thought signature + assert tools is not None + assert len(tools) == 2 + assert "provider_specific_fields" in tools[0] + assert tools[0]["provider_specific_fields"]["thought_signature"] == test_signature + # Second tool call should not have thought signature + assert "provider_specific_fields" not in tools[ + 1 + ] or "thought_signature" not in tools[1].get("provider_specific_fields", {}) + + +def test_thought_signature_preservation_in_conversion(): + """Test that thought signatures are preserved when converting assistant messages back to Gemini format""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + + # Assistant message with tool calls containing thought signatures + assistant_message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + }, + "index": 0, + "provider_specific_fields": { + "thought_signature": test_signature, + }, + }, + { + "id": "call_def456", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "London"}', + }, + "index": 1, + # No thought signature for parallel call + }, + ], + } + + gemini_parts = convert_to_gemini_tool_call_invoke(assistant_message) + + # Verify thought signature is preserved in first function call part + assert len(gemini_parts) == 2 + assert "function_call" in gemini_parts[0] + assert "thoughtSignature" in gemini_parts[0] + assert gemini_parts[0]["thoughtSignature"] == test_signature + + # Verify second function call part does not have thought signature + assert "function_call" in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[1] + + +def test_thought_signature_sequential_function_calls(): + """Test that each sequential function call preserves its own thought signature""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + signature_1 = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + signature_2 = "DifferentSignatureForSecondCall1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + # Sequential function calls - each has its own signature + # This simulates a multi-step conversation where each step has a signature + assistant_message_step1 = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_step1", + "type": "function", + "function": { + "name": "check_flight", + "arguments": '{"flight": "AA100"}', + }, + "index": 0, + "provider_specific_fields": { + "thought_signature": signature_1, + }, + }, + ], + } + + assistant_message_step2 = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_step2", + "type": "function", + "function": { + "name": "book_taxi", + "arguments": '{"destination": "airport"}', + }, + "index": 0, + "provider_specific_fields": { + "thought_signature": signature_2, + }, + }, + ], + } + + gemini_parts_step1 = convert_to_gemini_tool_call_invoke(assistant_message_step1) + gemini_parts_step2 = convert_to_gemini_tool_call_invoke(assistant_message_step2) + + # Verify each step preserves its own signature + assert len(gemini_parts_step1) == 1 + assert gemini_parts_step1[0]["thoughtSignature"] == signature_1 + + assert len(gemini_parts_step2) == 1 + assert gemini_parts_step2[0]["thoughtSignature"] == signature_2 + + +def test_thought_signature_with_function_call_mode(): + """Test thought signature extraction in function_call mode (is_function_call=True)""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.types.llms.vertex_ai import HttpxPartType + + test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + + parts_with_signature = [ + HttpxPartType( + functionCall={ + "name": "get_current_weather", + "args": {"location": "Tokyo"}, + }, + thoughtSignature=test_signature, + ) + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_with_signature, + cumulative_tool_call_idx=0, + is_function_call=True, + ) + + # Verify thought signature is stored in function's provider_specific_fields + assert function is not None + # Function should be dict-like (TypedDict or dict) + assert hasattr(function, "__getitem__") or isinstance(function, dict) + assert "provider_specific_fields" in function + assert function["provider_specific_fields"]["thought_signature"] == test_signature + assert tools is None + + +def test_dummy_signature_added_for_gemini_3_conversation_history(): + """Test that dummy signatures are added when transferring conversation history from older models (like gemini-2.5-flash) to gemini-3.""" + import base64 + + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + # Simulate conversation history from gemini-2.5-flash (no thought signature) + assistant_message_from_older_model = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + }, + "index": 0, + # No provider_specific_fields - older model doesn't provide signatures + }, + ], + } + + # Convert to Gemini format for gemini-3-pro-preview (should add dummy signature) + gemini_parts = convert_to_gemini_tool_call_invoke( + assistant_message_from_older_model, model="gemini-3-pro-preview" + ) + + # Verify dummy signature is added + assert len(gemini_parts) == 1 + assert "function_call" in gemini_parts[0] + assert "thoughtSignature" in gemini_parts[0] + + # Verify it's the expected dummy signature (base64 encoded "skip_thought_signature_validator") + expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" + ) + assert gemini_parts[0]["thoughtSignature"] == expected_dummy + + +def test_dummy_signature_not_added_for_gemini_2_5(): + """Test that dummy signatures are NOT added when target model is not gemini-3.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + # Simulate conversation history from gemini-2.5-flash (no thought signature) + assistant_message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + }, + "index": 0, + # No provider_specific_fields + }, + ], + } + + # Convert to Gemini format for gemini-2.5-flash (should NOT add dummy signature) + gemini_parts = convert_to_gemini_tool_call_invoke( + assistant_message, model="gemini-2.5-flash" + ) + + # Verify no dummy signature is added for non-gemini-3 models + assert len(gemini_parts) == 1 + assert "function_call" in gemini_parts[0] + assert "thoughtSignature" not in gemini_parts[0] + + +def test_dummy_signature_not_added_when_signature_exists(): + """Test that dummy signatures are NOT added when a real signature already exists.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + real_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" + + # Assistant message with existing thought signature + assistant_message_with_signature = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + "provider_specific_fields": { + "thought_signature": real_signature, + }, + }, + "index": 0, + }, + ], + } + + # Convert to Gemini format for gemini-3-pro-preview + gemini_parts = convert_to_gemini_tool_call_invoke( + assistant_message_with_signature, model="gemini-3-pro-preview" + ) + + # Verify real signature is preserved, not replaced with dummy + assert len(gemini_parts) == 1 + assert "function_call" in gemini_parts[0] + assert "thoughtSignature" in gemini_parts[0] + assert gemini_parts[0]["thoughtSignature"] == real_signature + + +def test_dummy_signature_with_function_call_mode(): + """Test that dummy signatures are added for function_call mode when converting to gemini-3.""" + import base64 + + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + # Assistant message with function_call (not tool_calls) and no signature + assistant_message_function_call = { + "role": "assistant", + "content": None, + "function_call": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + # No provider_specific_fields + }, + } + + # Convert to Gemini format for gemini-3-pro-preview + gemini_parts = convert_to_gemini_tool_call_invoke( + assistant_message_function_call, model="gemini-3-pro-preview" + ) + + # Verify dummy signature is added + assert len(gemini_parts) == 1 + assert "function_call" in gemini_parts[0] + assert "thoughtSignature" in gemini_parts[0] + + # Verify it's the expected dummy signature + expected_dummy = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" + ) + assert gemini_parts[0]["thoughtSignature"] == expected_dummy + + +def _parallel_tool_calls(*signatures): + return [ + { + "id": f"call_{idx}", + "type": "function", + "function": { + "name": f"tool_{idx}", + "arguments": '{"location": "Paris"}', + **( + {"provider_specific_fields": {"thought_signature": signature}} + if signature is not None + else {} + ), + }, + "index": idx, + } + for idx, signature in enumerate(signatures) + ] + + +def _parallel_tool_calls_signed_via_id(*signatures): + """Parallel tool calls in the shape LiteLLM actually hands back to clients. + + The signature rides in the tool call id behind __thought__, which is what an + OpenAI-format client echoes back on the next turn. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _encode_tool_call_id_with_signature, + ) + + return [ + { + "id": _encode_tool_call_id_with_signature(f"call_{idx}", signature), + "type": "function", + "function": {"name": f"tool_{idx}", "arguments": '{"location": "Paris"}'}, + "index": idx, + } + for idx, signature in enumerate(signatures) + ] + + +REAL_THOUGHT_SIGNATURE = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n" +PLACEHOLDER_SIGNATURE = base64.b64encode(b"skip_thought_signature_validator").decode( + "utf-8" +) + + +def test_dummy_signature_only_on_first_parallel_tool_call(): + """Google documents the placeholder as a last resort that degrades quality, so an unsigned + parallel turn replayed to gemini-3 gets a budget of exactly one.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None, None), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_real_signature_on_first_parallel_tool_call_leaves_siblings_empty(): + """Gemini signs only the first of N parallel function calls, so a faithful replay has + nothing to attach to the siblings.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None, None), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_real_signature_on_later_parallel_tool_call_is_preserved(): + """Clients may reorder or drop calls, so a signature that lands on a non-first call is + still the model's own and must survive the round trip.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, REAL_THOUGHT_SIGNATURE), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE + assert gemini_parts[1]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + + +def test_no_signatures_on_parallel_tool_calls_for_gemini_2_5(): + """Non-gemini-3 models never get a placeholder signature, on any call.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None), + }, + model="gemini-2.5-flash", + ) + + assert len(gemini_parts) == 2 + assert all("thoughtSignature" not in part for part in gemini_parts) + + +def test_signature_embedded_in_tool_call_id_only_on_first_parallel_call(): + """The production shape: the signature arrives inside the first call's id, siblings have bare ids.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_tool_level_provider_specific_fields_signature_leaves_siblings_empty(): + """A signature on the tool call itself, rather than on its function, behaves the same way.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + tool_calls = _parallel_tool_calls(None, None) + tool_calls[0]["provider_specific_fields"] = { + "thought_signature": REAL_THOUGHT_SIGNATURE + } + + gemini_parts = convert_to_gemini_tool_call_invoke( + {"role": "assistant", "content": None, "tool_calls": tool_calls}, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + + +def test_placeholder_lands_on_first_emitted_part_not_first_tool_call_entry(): + """A non-function entry (e.g. an OpenAI custom tool call) emits no part, so it must not + consume the one placeholder slot and leave the real first function call bare.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + tool_calls = [ + {"id": "call_custom", "type": "custom", "custom": {"name": "noop", "input": ""}} + ] + _parallel_tool_calls(None, None) + + gemini_parts = convert_to_gemini_tool_call_invoke( + {"role": "assistant", "content": None, "tool_calls": tool_calls}, + model="gemini-3-pro-preview", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + + +def test_no_placeholder_when_model_is_unknown(): + """Without a model there is nothing to prove the target needs a placeholder, so none is added.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None), + }, + ) + + assert len(gemini_parts) == 2 + assert all("thoughtSignature" not in part for part in gemini_parts) + + +def test_real_signature_forwarded_to_gemini_2_5_without_placeholder_siblings(): + """Older models still receive a real signature that a client replays, and still get no placeholder.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(REAL_THOUGHT_SIGNATURE, None), + }, + model="gemini-2.5-flash", + ) + + assert len(gemini_parts) == 2 + assert gemini_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + + +def test_parallel_tool_call_history_replayed_through_full_message_conversion(): + """End to end through the message-history converter, the path a real /chat/completions replay takes.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + messages = [ + {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + ] + + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-3-pro-preview" + ) + + model_parts = contents[1]["parts"] + assert len(model_parts) == 3 + assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in model_parts[1] + assert "thoughtSignature" not in model_parts[2] + + +@pytest.mark.parametrize( + "model", + ["gemini-3.5-flash", "vertex_ai/gemini-3.5-flash", "gemini/gemini-3.5-flash"], +) +def test_natively_signed_parallel_turn_never_carries_a_placeholder(model): + """A native gemini-3.5 parallel turn replays with zero skip_thought_signature_validator parts. + + Fabricating the placeholder alongside a real signature is what produced empty text responses + on gemini-3.5 parallel function calling, so the whole payload has to stay placeholder-free. + """ + import json + + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + messages = [ + {"role": "user", "content": "Weather in Paris, London and Tokyo?"}, + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls_signed_via_id( + REAL_THOUGHT_SIGNATURE, None, None + ), + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages, model=model) + + model_parts = contents[1]["parts"] + assert len(model_parts) == 3 + assert model_parts[0]["thoughtSignature"] == REAL_THOUGHT_SIGNATURE + assert "thoughtSignature" not in model_parts[1] + assert "thoughtSignature" not in model_parts[2] + assert PLACEHOLDER_SIGNATURE not in json.dumps(contents) + + +@pytest.mark.parametrize( + "model", + [ + "gemini-3-pro-preview", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "vertex_ai/gemini-3.5-flash", + "vertex_ai/gemini-3.7-flash", + "vertex_ai/gemini-3.8-flash", + "gemini/gemini-3.5-flash", + "gemini/gemini-3.7-flash", + "gemini/gemini-3.8-flash", + ], +) +def test_placeholder_scoped_to_first_call_across_gemini_3_variants(model): + """The gemini-3 gate is a substring match, so every family member and prefix form has to + land on the same one-placeholder budget rather than only the versions we happened to try.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + gemini_parts = convert_to_gemini_tool_call_invoke( + { + "role": "assistant", + "content": None, + "tool_calls": _parallel_tool_calls(None, None, None), + }, + model=model, + ) + + assert len(gemini_parts) == 3 + assert gemini_parts[0]["thoughtSignature"] == PLACEHOLDER_SIGNATURE + assert "thoughtSignature" not in gemini_parts[1] + assert "thoughtSignature" not in gemini_parts[2] + + +def test_signed_text_part_survives_alongside_unsigned_parallel_tool_calls(): + """Text-part and function-call signatures are collected by separate code paths, so scoping the + placeholder must not disturb a real signature that arrived on the text part.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "Checking all three cities.", + "provider_specific_fields": {"thought_signatures": ["real_25_signature"]}, + "tool_calls": _parallel_tool_calls(None, None, None), + } + + parts = _gemini_convert_messages_with_history( + messages=[msg], model="gemini-3-pro-preview" + )[0]["parts"] + + assert parts[0]["text"] == "Checking all three cities." + assert parts[0]["thoughtSignature"] == "real_25_signature" + assert parts[1]["thoughtSignature"] == PLACEHOLDER_SIGNATURE + assert "thoughtSignature" not in parts[2] + assert "thoughtSignature" not in parts[3] + + +# Tests for media_resolution (detail parameter) handling - Issue #17084 +class TestMediaResolution: + """Tests for media_resolution handling in Gemini 2.x models""" + + def test_get_highest_media_resolution_high_wins(self): + """Test that 'high' resolution takes precedence over 'low'""" + assert _get_highest_media_resolution("low", "high") == "high" + assert _get_highest_media_resolution("high", "low") == "high" + assert _get_highest_media_resolution(None, "high") == "high" + assert _get_highest_media_resolution("high", None) == "high" + + def test_get_highest_media_resolution_low_over_none(self): + """Test that 'low' resolution takes precedence over None""" + assert _get_highest_media_resolution(None, "low") == "low" + assert _get_highest_media_resolution("low", None) == "low" + + def test_get_highest_media_resolution_same_values(self): + """Test handling of same resolution values""" + assert _get_highest_media_resolution("high", "high") == "high" + assert _get_highest_media_resolution("low", "low") == "low" + assert _get_highest_media_resolution(None, None) is None + + def test_get_highest_media_resolution_medium(self): + """Test that 'medium' resolution is correctly ranked between 'low' and 'high'""" + assert _get_highest_media_resolution("low", "medium") == "medium" + assert _get_highest_media_resolution("medium", "low") == "medium" + assert _get_highest_media_resolution("medium", "high") == "high" + assert _get_highest_media_resolution("high", "medium") == "high" + assert _get_highest_media_resolution(None, "medium") == "medium" + assert _get_highest_media_resolution("medium", None) == "medium" + + def test_get_highest_media_resolution_ultra_high(self): + """Test that 'ultra_high' resolution takes precedence over all others""" + assert _get_highest_media_resolution("high", "ultra_high") == "ultra_high" + assert _get_highest_media_resolution("ultra_high", "high") == "ultra_high" + assert _get_highest_media_resolution("medium", "ultra_high") == "ultra_high" + assert _get_highest_media_resolution("low", "ultra_high") == "ultra_high" + assert _get_highest_media_resolution(None, "ultra_high") == "ultra_high" + assert _get_highest_media_resolution("ultra_high", None) == "ultra_high" + + def test_extract_max_media_resolution_single_image_high(self): + """Test extraction of media resolution from single image with detail=high""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abc123", + "detail": "high", + }, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) == "high" + + def test_extract_max_media_resolution_single_image_low(self): + """Test extraction of media resolution from single image with detail=low""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abc123", + "detail": "low", + }, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) == "low" + + def test_extract_max_media_resolution_no_detail(self): + """Test extraction when no detail parameter is provided""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,abc123"}, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) is None + + def test_extract_max_media_resolution_multiple_images_mixed(self): + """Test that highest resolution is returned when multiple images have different details""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these images"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abc123", + "detail": "low", + }, + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,def456", + "detail": "high", + }, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) == "high" + + def test_extract_max_media_resolution_text_only(self): + """Test extraction from messages with no images""" + messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well!"}, + ] + assert _extract_max_media_resolution_from_messages(messages) is None + + def test_transform_request_body_gemini_2x_adds_media_resolution(self): + """Test that media_resolution is added to generationConfig for Gemini 2.x models""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + }, + ], + } + ] + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-flash", + optional_params={}, + custom_llm_provider="gemini", + litellm_params={}, + cached_content=None, + ) + + assert "generationConfig" in result + assert "mediaResolution" in result["generationConfig"] + assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_HIGH" + + def test_transform_request_body_gemini_2x_low_resolution(self): + """Test that low media_resolution is correctly added for Gemini 2.x""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "low", + }, + }, + ], + } + ] + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-flash", + optional_params={}, + custom_llm_provider="gemini", + litellm_params={}, + cached_content=None, + ) + + assert "generationConfig" in result + assert "mediaResolution" in result["generationConfig"] + assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_LOW" + + def test_transform_request_body_gemini_3_no_global_media_resolution(self): + """Test that Gemini 3 models don't add media_resolution to generationConfig (they use per-part)""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + }, + ], + } + ] + + result = _transform_request_body( + messages=messages, + model="gemini-3-pro-preview", + optional_params={}, + custom_llm_provider="gemini", + litellm_params={}, + cached_content=None, + ) + + # Gemini 3 should NOT have mediaResolution in generationConfig + # (it's handled per-part in the content transformation) + if "generationConfig" in result: + assert "mediaResolution" not in result["generationConfig"] + + def test_transform_request_body_no_detail_no_media_resolution(self): + """Test that no mediaResolution is added when detail is not specified""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ], + } + ] + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-flash", + optional_params={}, + custom_llm_provider="gemini", + litellm_params={}, + cached_content=None, + ) + + # When no detail is specified, mediaResolution should not be in generationConfig + if "generationConfig" in result: + assert "mediaResolution" not in result["generationConfig"] + + def test_extract_max_media_resolution_file_type_with_detail(self): + """Test that detail is extracted from file content type, not just image_url""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this file?"}, + { + "type": "file", + "file": { + "url": "data:image/png;base64,abc123", + "detail": "high", + }, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) == "high" + + def test_extract_max_media_resolution_mixed_image_and_file(self): + """Test that highest detail is returned across both image_url and file types""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,abc123", + "detail": "low", + }, + }, + { + "type": "file", + "file": { + "url": "data:image/png;base64,def456", + "detail": "high", + }, + }, + ], + } + ] + assert _extract_max_media_resolution_from_messages(messages) == "high" + + def test_transform_request_body_gemini_1x_no_media_resolution(self): + """Test that Gemini 1.x models don't get mediaResolution in generationConfig""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgo=", + "detail": "high", + }, + }, + ], + } + ] + + result = _transform_request_body( + messages=messages, + model="gemini-1.5-pro", + optional_params={}, + custom_llm_provider="gemini", + litellm_params={}, + cached_content=None, + ) + + # Gemini 1.x should NOT have mediaResolution (not supported) + if "generationConfig" in result: + assert "mediaResolution" not in result["generationConfig"] + + +# Tests for VideoMetadata support across all Gemini models (Issue #25474) +class TestVideoMetadataAllGeminiModels: + """Tests that video_metadata (fps, start_offset, end_offset) works for all Gemini models""" + + def _make_video_messages(self, video_metadata: dict) -> list: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Analyze this video"}, + { + "type": "file", + "file": { + "file_id": "gs://bucket/video.mp4", + "format": "video/mp4", + "video_metadata": video_metadata, + }, + }, + ], + } + ] + + def _get_file_part(self, contents: list) -> dict: + for part in contents[0]["parts"]: + if "file_data" in part: + return part + raise AssertionError("No file part found in contents") + + def test_video_metadata_fps_gemini_2_5_flash(self): + """Gemini 2.5 Flash: fps in video_metadata should be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 5}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 5 + + def test_video_metadata_fps_gemini_2_5_pro(self): + """Gemini 2.5 Pro: fps in video_metadata should be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 10}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-pro" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 10 + + def test_video_metadata_offsets_gemini_2_5_flash(self): + """Gemini 2.5 Flash: start_offset/end_offset converted to camelCase (Issue #25474)""" + messages = self._make_video_messages( + {"start_offset": "5s", "end_offset": "30s"} + ) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + vm = file_part["video_metadata"] + assert vm["startOffset"] == "5s" + assert vm["endOffset"] == "30s" + + def test_video_metadata_all_fields_gemini_2_5_flash(self): + """Gemini 2.5 Flash: all video_metadata fields forwarded correctly (Issue #25474)""" + messages = self._make_video_messages( + {"fps": 5, "start_offset": "10s", "end_offset": "60s"} + ) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-2.5-flash" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + vm = file_part["video_metadata"] + assert vm["fps"] == 5 + assert vm["startOffset"] == "10s" + assert vm["endOffset"] == "60s" + + def test_video_metadata_gemini_1_5_pro(self): + """Gemini 1.5 Pro: video_metadata should also be forwarded (Issue #25474)""" + messages = self._make_video_messages({"fps": 2}) + contents = _gemini_convert_messages_with_history( + messages=messages, model="gemini-1.5-pro" + ) + file_part = self._get_file_part(contents) + assert "video_metadata" in file_part + assert file_part["video_metadata"]["fps"] == 2 + + +def test_convert_tool_response_with_base64_image(): + """Test tool response with base64 data URI image.""" + # Create a small test image (1x1 red pixel PNG) + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + # Create tool message with image + tool_message = { + "role": "tool", + "tool_call_id": "call_test123", + "content": [ + { + "type": "text", + "text": '{"url": "https://example.com", "status": "success"}', + }, + {"type": "input_image", "image_url": image_data_uri}, + ], + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_test123", + "function": {"name": "click_at", "arguments": '{"x": 100, "y": 200}'}, + } + ] + } + + # Convert tool response with nested multimodal functionResponse.parts. + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] + assert function_response["name"] == "click_at" + assert "response" in function_response + # Verify JSON response is parsed correctly + assert "url" in function_response["response"] + assert function_response["response"]["url"] == "https://example.com" + + # Check inline_data is nested under functionResponse.parts. + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "image/png" + assert inline_data["data"] == test_image_base64 + + +def test_gemini_history_nests_multimodal_tool_response_parts(): + """Full history conversion should not emit sibling inline_data tool result parts.""" + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + messages = [ + {"role": "user", "content": "Get me an image"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_get_image", + "type": "function", + "function": {"name": "get_image", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_get_image", + "content": [ + {"type": "text", "text": '{"image_ref": "inline"}'}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": test_image_base64, + }, + }, + ], + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages) + + tool_response_parts = contents[-1]["parts"] + assert len(tool_response_parts) == 1 + assert "inline_data" not in tool_response_parts[0] + function_response = tool_response_parts[0]["function_response"] + assert function_response["parts"] == [ + { + "inline_data": { + "data": test_image_base64, + "mime_type": "image/png", + } + } + ] + + +def test_convert_tool_response_text_only(): + """Test tool response with only text (no image).""" + tool_message = { + "role": "tool", + "tool_call_id": "call_test789", + "content": [ + {"type": "text", "text": '{"status": "completed", "result": "success"}'} + ], + } + + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_test789", + "function": {"name": "wait_5_seconds", "arguments": "{}"}, + } + ] + } + + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Should be a single part (no list) when no image + assert not isinstance(result, list), "Should return single part when no image" + + # Check function_response exists + assert "function_response" in result + function_response = result["function_response"] + assert function_response["name"] == "wait_5_seconds" + # Verify JSON response is parsed correctly + assert "status" in function_response["response"] + assert function_response["response"]["status"] == "completed" + + # Check inline_data does NOT exist (no image provided) + assert "inline_data" not in result + + +def test_file_data_field_order(): + """ + Test that file_data fields are in the correct order (mime_type before file_uri). + + The Gemini API is sensitive to field order in the file_data object. + This test verifies that mime_type comes before file_uri in both: + 1. Dictionary key order + 2. JSON serialization + + Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order. + """ + import json + + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + # Test with HTTPS URL and explicit format (audio file) + file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123" + format = "audio/mpeg" + + result = _process_gemini_media(image_url=file_url, format=format) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + assert file_data["mime_type"] == "audio/mpeg" + assert file_data["file_uri"] == file_url + + # Verify field order by checking dictionary keys + # In Python 3.7+, dict maintains insertion order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index( + "file_uri" + ), "mime_type must come before file_uri in the file_data dict" + + # Also verify by serializing to JSON string + json_str = json.dumps(file_data) + mime_type_pos = json_str.find('"mime_type"') + file_uri_pos = json_str.find('"file_uri"') + assert ( + mime_type_pos < file_uri_pos + ), "mime_type must appear before file_uri in JSON serialization" + + +def test_file_data_field_order_gcs_urls(): + """Test that GCS URLs also maintain correct field order.""" + import json + + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + # Test with GCS URL + gcs_url = "gs://bucket/audio.mp3" + + result = _process_gemini_media(image_url=gcs_url) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + + # Verify field order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index( + "file_uri" + ), "mime_type must come before file_uri in the file_data dict" + + +def test_gemini_files_api_uri_without_format(): + """ + Test that Gemini Files API URIs work WITHOUT an explicit format/mime_type. + + When a user uploads a file via the Gemini Files API and then references it + by URI (https://generativelanguage.googleapis.com/v1beta/files/...), + the file is already on Google's servers. These URLs return 403 when + fetched directly, so _process_gemini_media must NOT try to resolve the + MIME type via HTTP. Instead it should pass the URI through as file_data + and let the Gemini API resolve the type from its stored metadata. + + Related issue: https://github.com/BerriAI/litellm/issues/24907 + """ + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + file_url = "https://generativelanguage.googleapis.com/v1beta/files/37eh7rsw1vfe" + + # Should NOT raise — previously this hit the generic https:// handler + # which called _get_image_mime_type_from_url() and got a 403. + result = _process_gemini_media(image_url=file_url) + + assert "file_data" in result + file_data = result["file_data"] + assert file_data["file_uri"] == file_url + # When no format is provided, mime_type should be absent so the + # Gemini API infers it from the stored file metadata. + assert "mime_type" not in file_data + + +def test_gemini_files_api_uri_with_format(): + """ + Test that Gemini Files API URIs correctly forward an explicit format. + + Related issue: https://github.com/BerriAI/litellm/issues/24907 + """ + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + file_url = "https://generativelanguage.googleapis.com/v1beta/files/n1vhxa28lyaw" + + result = _process_gemini_media(image_url=file_url, format="text/plain") + + assert "file_data" in result + file_data = result["file_data"] + assert file_data["file_uri"] == file_url + assert file_data["mime_type"] == "text/plain" + + +def test_extract_file_data_with_path_object(): + """ + Test that filename is correctly extracted from Path objects for MIME type detection. + + When uploading files using Path objects (e.g., Path("speech.mp3")), the filename + must be extracted to enable proper MIME type detection. Without this, files get + uploaded with 'application/octet-stream' instead of the correct MIME type. + + Related issue: Files uploaded with wrong MIME type cause Gemini API to reject + requests where the specified format doesn't match the uploaded file's MIME type. + """ + import os + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + + # Create a temporary MP3 file + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp: + tmp.write(b"fake mp3 content") + tmp_path = tmp.name + + try: + # Test with Path object + path_obj = Path(tmp_path) + extracted = extract_file_data(path_obj) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".mp3") + + # Verify MIME type was correctly detected + assert ( + extracted["content_type"] == "audio/mpeg" + ), f"Expected 'audio/mpeg' but got '{extracted['content_type']}'" + + # Verify content was read + assert extracted["content"] == b"fake mp3 content" + + finally: + # Clean up temporary file + os.unlink(tmp_path) + + +def test_extract_file_data_with_pathlib_path(): + """Test that filename is correctly extracted from pathlib.Path inputs. + Bare str paths are rejected — when this runs in a proxy request handler + the value is attacker-controlled and opening it as a path is an LFI.""" + import os + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(b"fake wav content") + tmp_path = Path(tmp.name) + + try: + extracted = extract_file_data(tmp_path) + + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".wav") + assert extracted["content_type"] in [ + "audio/wav", + "audio/x-wav", + ], f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'" + assert extracted["content"] == b"fake wav content" + finally: + os.unlink(str(tmp_path)) + + +def test_extract_file_data_with_tuple_format(): + """Test that tuple format (with explicit content_type) still works correctly.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + + # Test with tuple format: (filename, content, content_type) + filename = "test_audio.mp3" + content = b"test audio content" + content_type = "audio/mpeg" + + extracted = extract_file_data((filename, content, content_type)) + + # Verify all fields are correct + assert extracted["filename"] == filename + assert extracted["content"] == content + assert extracted["content_type"] == content_type + + +def test_extract_file_data_fallback_to_octet_stream(): + """Unknown file types fall back to application/octet-stream.""" + import os + import tempfile + from pathlib import Path + + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, + ) + + with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp: + tmp.write(b"unknown content") + tmp_path = Path(tmp.name) + + try: + extracted = extract_file_data(tmp_path) + + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".xyz123") + assert ( + extracted["content_type"] == "application/octet-stream" + ), f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" + finally: + os.unlink(str(tmp_path)) + + +def test_convert_tool_response_with_pdf_file(): + """Test tool response with PDF file content using file_data field.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with file + tool_message = { + "role": "tool", + "tool_call_id": "call_pdf_test", + "content": [ + {"type": "text", "text": '{"status": "success", "pages": 1}'}, + {"type": "file", "file_data": file_data_uri}, + ], + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_pdf_test", + "function": { + "name": "analyze_document", + "arguments": '{"path": "/tmp/doc.pdf"}', + }, + } + ] + } + + # Convert tool response with nested multimodal functionResponse.parts. + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] + assert function_response["name"] == "analyze_document" + assert "response" in function_response + # Verify JSON response is parsed correctly + assert "status" in function_response["response"] + assert function_response["response"]["status"] == "success" + + # Check inline_data is nested under functionResponse.parts. + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64 + + +def test_convert_tool_response_with_input_file_type(): + """Test tool response with input_file content type (Responses API format).""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with input_file type + tool_message = { + "role": "tool", + "tool_call_id": "call_input_file_test", + "content": [{"type": "input_file", "file_data": file_data_uri}], + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_input_file_test", + "function": {"name": "read_file", "arguments": "{}"}, + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Check inline_data is nested under functionResponse.parts. + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + function_response = result[0]["function_response"] + assert ( + function_response["parts"][0]["inline_data"]["mime_type"] == "application/pdf" + ) + + +def test_convert_tool_response_with_nested_file_object(): + """Test tool response with file content using nested file object format.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with nested file object (OpenAI Agents SDK format) + tool_message = { + "role": "tool", + "tool_call_id": "call_nested_test", + "content": [{"type": "file", "file": {"file_data": file_data_uri}}], + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_nested_test", + "function": {"name": "process_document", "arguments": "{}"}, + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Check inline_data is nested under functionResponse.parts. + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + function_response = result[0]["function_response"] + inline_data: BlobType = function_response["parts"][0]["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64 + + +def test_assistant_message_with_images_field(): + """ + Test that assistant messages with images field are properly converted to Gemini format. + + This handles the case where an assistant message contains generated images in the + `images` field (e.g., from image generation models like gemini-2.5-flash-image). + The images should be converted to inline_data parts in the Gemini format. + """ + # Create a small test image (1x1 red pixel PNG) + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + # Create messages with assistant message containing images field + messages = [ + { + "role": "user", + "content": "Generate an image of a banana wearing a costume that says LiteLLM", + }, + { + "role": "assistant", + "content": "Here's your banana in a LiteLLM costume!", + "images": [ + { + "image_url": {"url": image_data_uri, "detail": "auto"}, + "index": 0, + "type": "image_url", + } + ], + }, + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify structure + assert len(contents) == 2, f"Expected 2 content blocks, got {len(contents)}" + + # Verify user message + assert contents[0]["role"] == "user" + assert len(contents[0]["parts"]) == 1 + assert ( + contents[0]["parts"][0]["text"] + == "Generate an image of a banana wearing a costume that says LiteLLM" + ) + + # Verify assistant message + assert contents[1]["role"] == "model" + assert ( + len(contents[1]["parts"]) == 2 + ), f"Expected 2 parts (text + image), got {len(contents[1]['parts'])}" + + # Find text part and inline_data part + text_part = None + inline_data_part = None + for part in contents[1]["parts"]: + if "text" in part: + text_part = part + elif "inline_data" in part: + inline_data_part = part + + # Verify text part + assert text_part is not None, "Missing text part in assistant message" + assert text_part["text"] == "Here's your banana in a LiteLLM costume!" + + # Verify inline_data part (image) + assert inline_data_part is not None, "Missing inline_data part in assistant message" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "image/png" + assert inline_data["data"] == test_image_base64 + + +def test_assistant_message_with_multiple_images(): + """Test that assistant messages with multiple images are properly converted.""" + # Create two test images + test_image1_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + test_image2_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image1_data_uri = f"data:image/png;base64,{test_image1_base64}" + image2_data_uri = f"data:image/jpeg;base64,{test_image2_base64}" + + messages = [ + {"role": "user", "content": "Generate two images"}, + { + "role": "assistant", + "content": "Here are your images:", + "images": [ + { + "image_url": {"url": image1_data_uri, "detail": "auto"}, + "index": 0, + "type": "image_url", + }, + { + "image_url": {"url": image2_data_uri, "detail": "high"}, + "index": 1, + "type": "image_url", + }, + ], + }, + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify assistant message has 3 parts (1 text + 2 images) + assert contents[1]["role"] == "model" + assert ( + len(contents[1]["parts"]) == 3 + ), f"Expected 3 parts (text + 2 images), got {len(contents[1]['parts'])}" + + # Count inline_data parts + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert ( + len(inline_data_parts) == 2 + ), f"Expected 2 inline_data parts, got {len(inline_data_parts)}" + + # Verify first image + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_data_parts[0]["inline_data"]["data"] == test_image1_base64 + + # Verify second image + assert inline_data_parts[1]["inline_data"]["mime_type"] == "image/jpeg" + assert inline_data_parts[1]["inline_data"]["data"] == test_image2_base64 + + +def test_assistant_message_with_images_using_message_object(): + """Test that Message objects with images field are properly converted.""" + # Create a small test image + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + # Create messages using Message object (as returned by LiteLLM) + user_message = {"role": "user", "content": "Generate an image"} + + assistant_message = Message( + content="Here's your image!", + role="assistant", + tool_calls=None, + function_call=None, + images=[ + { + "image_url": {"url": image_data_uri, "detail": "auto"}, + "index": 0, + "type": "image_url", + } + ], + ) + + messages = [user_message, assistant_message] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify assistant message has both text and image + assert contents[1]["role"] == "model" + assert len(contents[1]["parts"]) == 2 + + # Verify image was converted + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert len(inline_data_parts) == 1 + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_data_parts[0]["inline_data"]["data"] == test_image_base64 + + +def test_assistant_message_with_images_in_conversation_history(): + """ + Test multi-turn conversation where assistant message with images is in history. + + This simulates the real use case where: + 1. User asks for image generation + 2. Assistant generates image (with images field) + 3. User asks follow-up question about the image + """ + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + image_data_uri = f"data:image/png;base64,{test_image_base64}" + + messages = [ + {"role": "user", "content": "Generate an image of a cat"}, + { + "role": "assistant", + "content": "Here's a cat image:", + "images": [ + { + "image_url": {"url": image_data_uri, "detail": "auto"}, + "index": 0, + "type": "image_url", + } + ], + }, + {"role": "user", "content": "Can you make it more colorful?"}, + ] + + # Convert messages to Gemini format + contents = _gemini_convert_messages_with_history(messages=messages) + + # Verify structure: user -> model (with image) -> user + assert len(contents) == 3 + assert contents[0]["role"] == "user" + assert contents[1]["role"] == "model" + assert contents[2]["role"] == "user" + + # Verify assistant message has image in history + inline_data_parts = [part for part in contents[1]["parts"] if "inline_data" in part] + assert len(inline_data_parts) == 1 + assert inline_data_parts[0]["inline_data"]["mime_type"] == "image/png" + + +def test_function_response_has_user_role(): + """ + Test that function response ContentType blocks include role="user". + + Gemini API only accepts two roles: "user" and "model". Function responses + must be sent with role="user". Previously, LiteLLM omitted the role field + entirely, causing 400 errors from the Gemini API. + + Fixes: https://github.com/BerriAI/litellm/issues/22003 + Fixes: https://github.com/BerriAI/litellm/issues/20690 + """ + messages = [ + {"role": "user", "content": "What is the weather in Berlin?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Berlin"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": '{"temperature": "15°C", "condition": "Cloudy"}', + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages) + + # Expect: user -> model (functionCall) -> user (functionResponse) + assert len(contents) == 3 + + assert contents[0]["role"] == "user" + assert contents[1]["role"] == "model" + assert "function_call" in contents[1]["parts"][0] + + # The critical assertion: function response must have role="user" + assert contents[2]["role"] == "user" + assert "function_response" in contents[2]["parts"][0] + + +def test_multi_turn_function_calling_roles(): + """ + Test a full multi-turn function calling conversation produces correct roles. + + Simulates: user asks → model calls tool → tool responds → model answers → user asks again. + Every content block must have an explicit role of "user" or "model". + + Fixes: https://github.com/BerriAI/litellm/issues/22003 + """ + messages = [ + {"role": "user", "content": "What is the weather in Berlin?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_001", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Berlin"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_001", + "content": '{"temperature": "15°C"}', + }, + { + "role": "assistant", + "content": "The weather in Berlin is 15°C.", + }, + {"role": "user", "content": "And in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_002", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_002", + "content": '{"temperature": "18°C"}', + }, + ] + + contents = _gemini_convert_messages_with_history(messages=messages) + + # Every content block must have a valid role + for i, content in enumerate(contents): + assert "role" in content, f"Content block {i} missing 'role' field" + assert content["role"] in ( + "user", + "model", + ), f"Content block {i} has invalid role: {content.get('role')}" + + # Verify the function response blocks specifically have role="user" + for i, content in enumerate(contents): + for part in content["parts"]: + if "function_response" in part: + assert ( + content["role"] == "user" + ), f"Content block {i} with function_response has role='{content['role']}', expected 'user'" + + +def test_gemini_thought_signature_preservation_real_response(): + """Test that thought signatures are preserved on the text part if originally there, without dropping or duplicating (real response case).""" + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + real_candidate = { + "content": { + "parts": [ + { + "text": "I will explain and then list files.", + "thoughtSignature": "mock_signature_from_text_part", + }, + { + "functionCall": { + "name": "list_files", + "args": {}, + } + }, + ] + } + } + + parts = real_candidate["content"]["parts"] + + content, reasoning_content = ( + VertexGeminiConfig().get_assistant_content_message(parts=parts) + ) + thought_signatures = ( + VertexGeminiConfig()._extract_thought_signatures_from_parts( + parts=parts + ) + ) + functions, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts, + cumulative_tool_call_idx=0, + is_function_call=False, + ) + + msg: dict = {"role": "assistant"} + if content is not None: + msg["content"] = content + if tools: + msg["tool_calls"] = tools + if functions is not None: + msg["function_call"] = functions + if thought_signatures is not None: + msg["provider_specific_fields"] = { + "thought_signatures": thought_signatures + } + + converted_real = _gemini_convert_messages_with_history( + messages=[msg], + model="gemini-2.5-pro", + ) + + assert len(converted_real) == 1 + assert "parts" in converted_real[0] + parts_out = converted_real[0]["parts"] + assert len(parts_out) == 2 + assert "text" in parts_out[0] + assert ( + parts_out[0]["thoughtSignature"] == "mock_signature_from_text_part" + ) + assert "function_call" in parts_out[1] + assert "thoughtSignature" not in parts_out[1] + + +def test_gemini_thought_signature_deduplication_assumed_response(): + """Test that thought signatures are deduplicated and not attached to the text part if already present in the tool call (assumed response case).""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + pr_assumed_msg = { + "role": "assistant", + "content": "I will list the directory.", + "provider_specific_fields": { + "thought_signatures": ["mock_signature_63k"] + }, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "list_files", "arguments": "{}"}, + "provider_specific_fields": { + "thought_signature": "mock_signature_63k" + }, + } + ], + } + + converted_pr = _gemini_convert_messages_with_history( + messages=[pr_assumed_msg], + model="gemini-2.5-pro", + ) + + assert len(converted_pr) == 1 + assert "parts" in converted_pr[0] + parts_out = converted_pr[0]["parts"] + assert len(parts_out) == 2 + assert "text" in parts_out[0] + assert "thoughtSignature" not in parts_out[0] + assert "function_call" in parts_out[1] + assert parts_out[1]["thoughtSignature"] == "mock_signature_63k" + + +def test_gemini_thought_signature_pure_text(): + """Test that thought signatures are preserved on the text part for responses with no tool calls.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "Hello, I am a model.", + "provider_specific_fields": { + "thought_signatures": ["pure_text_signature"] + }, + } + + converted = _gemini_convert_messages_with_history( + messages=[msg], + model="gemini-2.5-pro", + ) + + assert len(converted) == 1 + assert "parts" in converted[0] + parts_out = converted[0]["parts"] + assert len(parts_out) == 1 + assert "text" in parts_out[0] + assert parts_out[0]["thoughtSignature"] == "pure_text_signature" + + +def test_gemini_thought_signature_pure_tool_call(): + """Test that thought signatures are preserved on the tool call for responses with no intermediate text.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": None, + "provider_specific_fields": { + "thought_signatures": ["pure_tool_signature"] + }, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "list_files", "arguments": "{}"}, + "provider_specific_fields": { + "thought_signature": "pure_tool_signature" + }, + } + ], + } + + converted = _gemini_convert_messages_with_history( + messages=[msg], + model="gemini-2.5-pro", + ) + + assert len(converted) == 1 + assert "parts" in converted[0] + parts_out = converted[0]["parts"] + assert len(parts_out) == 1 + assert "function_call" in parts_out[0] + assert parts_out[0]["thoughtSignature"] == "pure_tool_signature" + + +def test_gemini_distinct_text_and_tool_signatures_are_both_preserved(): + """A text-part signature that differs from the tool-call signature must stay on the text part.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "Some analysis.", + "provider_specific_fields": { + "thought_signatures": ["text_signature", "tool_signature"] + }, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "list_files", "arguments": "{}"}, + "provider_specific_fields": {"thought_signature": "tool_signature"}, + } + ], + } + + parts = _gemini_convert_messages_with_history( + messages=[msg], model="gemini-2.5-pro" + )[0]["parts"] + + assert parts[0]["text"] == "Some analysis." + assert parts[0]["thoughtSignature"] == "text_signature" + assert "function_call" in parts[1] + assert parts[1]["thoughtSignature"] == "tool_signature" + + +def test_gemini_25_text_signature_survives_replay_to_gemini_3(): + """gemini-2.5 history (signed text, unsigned tool call) replayed to gemini-3 keeps the real + text signature; the dummy signature synthesized for the unsigned tool call must not suppress it.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _get_dummy_thought_signature, + ) + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "I will list the directory.", + "provider_specific_fields": {"thought_signatures": ["real_25_signature"]}, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "list_files", "arguments": "{}"}, + } + ], + } + + parts = _gemini_convert_messages_with_history(messages=[msg], model="gemini-3-pro")[ + 0 + ]["parts"] + + assert parts[0]["text"] == "I will list the directory." + assert parts[0]["thoughtSignature"] == "real_25_signature" + assert "function_call" in parts[1] + assert parts[1]["thoughtSignature"] == _get_dummy_thought_signature() + + +def test_gemini_function_call_signature_round_trip_no_duplicate(): + """End to end: a gemini-3-style response (unsigned text + signed functionCall) parsed and + re-serialized sends the signature exactly once, on the function-call part.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + response_parts = [ + {"text": "I will calculate the result for you."}, + { + "functionCall": {"name": "add_numbers", "args": {"a": 17, "b": 25}}, + "thoughtSignature": "signature_from_function_call", + }, + ] + + config = VertexGeminiConfig() + content, _ = config.get_assistant_content_message(parts=response_parts) + thought_signatures = config._extract_thought_signatures_from_parts( + parts=response_parts + ) + _, tools, _ = VertexGeminiConfig._transform_parts( + parts=response_parts, cumulative_tool_call_idx=0, is_function_call=False + ) + + msg = { + "role": "assistant", + "content": content, + "tool_calls": tools, + "provider_specific_fields": {"thought_signatures": thought_signatures}, + } + + parts = _gemini_convert_messages_with_history(messages=[msg], model="gemini-3-pro")[ + 0 + ]["parts"] + + signatures = [p["thoughtSignature"] for p in parts if "thoughtSignature" in p] + assert signatures == ["signature_from_function_call"] + assert "thoughtSignature" not in parts[0] + assert "function_call" in parts[1] + + +def test_gemini_server_side_tool_signature_not_duplicated_on_text(): + """A signature already re-injected on a server-side toolCall part is not attached to the text part again.""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _gemini_convert_messages_with_history, + ) + + msg = { + "role": "assistant", + "content": "The weather in Buenos Aires is sunny.", + "provider_specific_fields": { + "thought_signatures": ["server_side_signature"], + "server_side_tool_invocations": [ + { + "tool_type": "GOOGLE_SEARCH_WEB", + "id": "abc123", + "args": {"queries": ["weather Buenos Aires"]}, + "response": {"weather": "Sunny"}, + "thought_signature": "server_side_signature", + } + ], + }, + } + + parts = _gemini_convert_messages_with_history( + messages=[msg], model="gemini-2.5-pro" + )[0]["parts"] + + text_part = next(p for p in parts if "text" in p) + assert "thoughtSignature" not in text_part + tool_call_part = next(p for p in parts if "toolCall" in p) + assert tool_call_part["thoughtSignature"] == "server_side_signature" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py similarity index 99% rename from tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py rename to tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 88ba7fc37d9..739744336a1 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1894,42 +1894,6 @@ def test_vertex_ai_tool_call_id_format(): ), f"All 10 IDs should be unique, got {len(ids_generated)} unique IDs" -def test_vertex_ai_code_line_length(): - """ - Test that the specific code line generating tool call IDs is within character limit. - - This is a meta-test to ensure the code change meets the 40-character requirement. - """ - import inspect - - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - - # Get the source code of the _transform_parts method - source_lines = inspect.getsource(VertexGeminiConfig._transform_parts).split("\n") - - # Find the line that generates the ID - id_line = None - for line in source_lines: - if '"id": f"call_' in line and "uuid.uuid4().hex[:28]" in line: - id_line = line.strip() # Remove indentation for length check - break - - assert id_line is not None, "Could not find the ID generation line in source code" - - # Check that the line is 40 characters or less (excluding indentation) - line_length = len(id_line) - assert ( - line_length <= 40 - ), f"ID generation line is {line_length} characters, should be ≤40: {id_line}" - - # Verify it contains the expected UUID format - assert ( - "uuid.uuid4().hex[:28]" in id_line - ), f"Line should contain shortened UUID format: {id_line}" - - def test_vertex_ai_map_google_maps_tool_simple(): """ Test googleMaps tool transformation without location data. @@ -2530,8 +2494,6 @@ def test_fine_tuned_endpoint_and_gemma_get_no_gemini_3_default_temperature(model assert "temperature" not in mapped - - def _tool_call_messages(tool_call_id: str): return [ {"role": "user", "content": "hi"}, diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py rename to tests/unit/llms/vertex_ai/gemini/test_vertex_gemini_unbound_local_error.py diff --git a/tests/unit/llms/vertex_ai/image_generation/__init__.py b/tests/unit/llms/vertex_ai/image_generation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py rename to tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_cost_calculator.py diff --git a/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py new file mode 100644 index 00000000000..a72a570c2a2 --- /dev/null +++ b/tests/unit/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -0,0 +1,637 @@ +from unittest.mock import MagicMock, patch + +import httpx + + +from litellm.llms.vertex_ai.image_generation import ( + get_vertex_ai_image_generation_config, +) +from litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation import ( + VertexAIGeminiImageGenerationConfig, +) +from litellm.llms.vertex_ai.image_generation.vertex_imagen_transformation import ( + VertexAIImagenImageGenerationConfig, +) + + +class TestVertexAIGeminiImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIGeminiImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("gemini-2.5-flash-image") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to candidate_count""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) + assert result.get("candidate_count") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) + assert result.get("aspectRatio") == "1:1" + + def test_map_openai_params_size_16_9(self): + """Test mapping 16:9 size""" + non_default_params = {"size": "1792x1024"} + optional_params = {} + result = self.config.map_openai_params(non_default_params, optional_params, "gemini-2.5-flash-image", False) + assert result.get("aspectRatio") == "16:9" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("1024x1792") == "9:16" + assert self.config._map_size_to_aspect_ratio("1280x896") == "4:3" + assert self.config._map_size_to_aspect_ratio("896x1280") == "3:4" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_get_supported_openai_params_includes_native_gemini_params(self): + """Test that native Gemini imageConfig params are supported""" + supported = self.config.get_supported_openai_params("gemini-3-pro-image-preview") + assert "aspectRatio" in supported + assert "aspect_ratio" in supported + assert "imageSize" in supported + assert "image_size" in supported + assert "imageConfig" in supported + + def test_map_openai_params_aspect_ratio_camel_case(self): + """Test mapping native aspectRatio parameter""" + result = self.config.map_openai_params({"aspectRatio": "9:16"}, {}, "gemini-3-pro-image-preview", False) + assert result["aspectRatio"] == "9:16" + + def test_map_openai_params_aspect_ratio_snake_case(self): + """Test mapping native aspect_ratio parameter""" + result = self.config.map_openai_params({"aspect_ratio": "16:9"}, {}, "gemini-3-pro-image-preview", False) + assert result["aspectRatio"] == "16:9" + + def test_map_openai_params_image_size_camel_case(self): + """Test mapping native imageSize parameter""" + result = self.config.map_openai_params({"imageSize": "4K"}, {}, "gemini-3-pro-image-preview", False) + assert result["imageSize"] == "4K" + + def test_map_openai_params_image_size_snake_case(self): + """Test mapping native image_size parameter""" + result = self.config.map_openai_params({"image_size": "2K"}, {}, "gemini-3-pro-image-preview", False) + assert result["imageSize"] == "2K" + + def test_map_openai_params_image_config_dict_stored_whole(self): + """imageConfig dict is stored as-is so all fields survive""" + result = self.config.map_openai_params( + {"imageConfig": {"aspectRatio": "16:9", "imageSize": "2K"}}, + {}, + "gemini-3.1-flash-image", + False, + ) + assert result["imageConfig"] == {"aspectRatio": "16:9", "imageSize": "2K"} + + def test_map_openai_params_image_config_all_fields(self): + """All ImageConfig fields (personGeneration, imageOutputOptions) pass through""" + payload = { + "imageConfig": { + "aspectRatio": "9:16", + "imageSize": "4K", + "personGeneration": "DONT_ALLOW", + "imageOutputOptions": { + "mimeType": "image/jpeg", + "compressionQuality": 80, + }, + } + } + result = self.config.map_openai_params(payload, {}, "gemini-3.1-flash-image", False) + assert result["imageConfig"] == payload["imageConfig"] + + def test_map_openai_params_image_config_non_dict_warns_and_drops(self): + """Non-dict imageConfig is dropped with a warning, not silently discarded""" + with patch("litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation.verbose_logger") as mock_log: + result = self.config.map_openai_params( + {"imageConfig": "bad-string-value"}, {}, "gemini-3.1-flash-image", False + ) + assert "imageConfig" not in result + mock_log.warning.assert_called_once() + + def test_transform_image_generation_request_from_image_config(self): + """Full imageConfig dict is forwarded verbatim into generationConfig""" + full_config = { + "aspectRatio": "16:9", + "imageSize": "2K", + "personGeneration": "DONT_ALLOW", + "imageOutputOptions": {"mimeType": "image/jpeg", "compressionQuality": 85}, + } + mapped = self.config.map_openai_params( + {"imageConfig": full_config}, + {}, + "gemini-3.1-flash-image", + False, + ) + request = self.config.transform_image_generation_request( + model="gemini-3.1-flash-image", + prompt="A nano banana on a desk", + optional_params=mapped, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"] == full_config + + def test_transform_image_generation_flat_params_override_image_config(self): + """Explicit flat params win over the same key inside imageConfig""" + request = self.config.transform_image_generation_request( + model="gemini-3.1-flash-image", + prompt="A nano banana", + optional_params={ + "imageConfig": {"aspectRatio": "1:1", "personGeneration": "DONT_ALLOW"}, + "aspectRatio": "16:9", # should win + }, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" + assert request["generationConfig"]["imageConfig"]["personGeneration"] == "DONT_ALLOW" + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "contents" in request + assert "generationConfig" in request + assert request["generationConfig"]["responseModalities"] == ["IMAGE"] + assert request["contents"][0]["parts"][0]["text"] == "A nano banana" + + def test_transform_image_generation_request_with_aspect_ratio(self): + """Test request transformation with aspectRatio""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_request_with_image_size(self): + """Test request transformation with imageSize (Gemini 3 Pro)""" + request = self.config.transform_image_generation_request( + model="gemini-3-pro-image-preview", + prompt="A nano banana", + optional_params={"imageSize": "4K"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["imageSize"] == "4K" + + def test_map_openai_params_web_search_options(self): + """Test web_search_options maps to googleSearch tool""" + result = self.config.map_openai_params({"web_search_options": {}}, {}, "gemini-3.1-flash-image-preview", False) + assert result["tools"] == [{"googleSearch": {}}] + + def test_transform_image_generation_request_with_web_search_tools(self): + """Test request transformation includes googleSearch tools""" + request = self.config.transform_image_generation_request( + model="gemini-3.1-flash-image-preview", + prompt="Generate an image of the latest iPhone", + optional_params={"tools": [{"googleSearch": {}}]}, + litellm_params={}, + headers={}, + ) + assert request["tools"] == [{"googleSearch": {}}] + + def test_transform_image_generation_request_forwards_tool_config(self): + """Test request transformation forwards toolConfig side-effects from tool mapping""" + mapped = self.config.map_openai_params( + {"tools": [{"googleMaps": {"latitude": 37.7, "longitude": -122.4}}]}, + {}, + "gemini-3.1-flash-image-preview", + False, + ) + request = self.config.transform_image_generation_request( + model="gemini-3.1-flash-image-preview", + prompt="Generate an image of a coffee shop nearby", + optional_params=mapped, + litellm_params={}, + headers={}, + ) + assert request["tools"] == [{"googleMaps": {}}] + assert request["toolConfig"] == {"retrievalConfig": {"latLng": {"latitude": 37.7, "longitude": -122.4}}} + + def test_transform_image_generation_request_with_candidate_count(self): + """Test request transformation with candidate_count""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"candidate_count": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_request_with_n(self): + """Test request transformation with n parameter""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"n": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + } + } + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 93, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 54, + }, + { + "modality": "IMAGE", + "tokenCount": 39, + }, + ], + "candidatesTokenCount": 17, + "totalTokenCount": 110, + }, + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + assert result.usage.input_tokens == 93 + assert result.usage.input_tokens_details.text_tokens == 54 + assert result.usage.input_tokens_details.image_tokens == 39 + assert result.usage.output_tokens == 17 + assert result.usage.total_tokens == 110 + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "image1", + } + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "image2", + } + }, + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + def test_transform_image_generation_response_signature(self): + """Test response transformation includes thoughtSignature for Gemini 3 Pro""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + }, + "thoughtSignature": "test_signature_abc123", + } + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-3-pro-image-preview", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].provider_specific_fields["thought_signature"] == "test_signature_abc123" + + def test_transform_image_generation_response_tracks_web_search_requests(self): + """Grounding queries are carried onto usage so search spend can be billed""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + } + } + ] + }, + "groundingMetadata": {"webSearchQueries": ["eiffel tower", "paris skyline"]}, + } + ], + "usageMetadata": { + "promptTokenCount": 93, + "candidatesTokenCount": 17, + "totalTokenCount": 110, + }, + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert result.usage.web_search_requests == 2 + + +class TestVertexAIImagenImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIImagenImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("imagegeneration@006") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to sampleCount""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False) + assert result.get("sampleCount") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params(non_default_params, optional_params, "imagegeneration@006", False) + assert result.get("aspectRatio") == "1:1" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "instances" in request + assert "parameters" in request + assert request["instances"][0]["prompt"] == "A cat" + assert request["parameters"]["sampleCount"] == 1 + + def test_transform_image_generation_request_with_params(self): + """Test request transformation with parameters""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={"sampleCount": 2, "aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["parameters"]["sampleCount"] == 2 + assert request["parameters"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_request_labels_from_metadata(self): + """Billing labels from litellm_params.metadata.requester_metadata on predict body.""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={}, + litellm_params={"metadata": {"requester_metadata": {"team": "platform", "env": "prod"}}}, + headers={}, + ) + assert request["labels"] == {"team": "platform", "env": "prod"} + assert "labels" not in request["parameters"] + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = {"predictions": [{"bytesBase64Encoded": "base64_encoded_image_data"}]} + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + {"bytesBase64Encoded": "image1"}, + {"bytesBase64Encoded": "image2"}, + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + +class TestGetVertexAIImageGenerationConfig: + """Test the router function that selects the correct config""" + + def test_get_gemini_model_config(self): + """Test that Gemini models return Gemini config""" + config = get_vertex_ai_image_generation_config("gemini-2.5-flash-image") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("gemini-3-pro-image-preview") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("vertex_ai/gemini-2.5-flash-image") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + def test_get_imagen_model_config(self): + """Test that Imagen models return Imagen config""" + config = get_vertex_ai_image_generation_config("imagegeneration@006") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("imagen-4.0-generate-001") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("vertex_ai/imagegeneration@006") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + def test_get_non_gemini_model_config(self): + """Test that non-Gemini models default to Imagen config""" + config = get_vertex_ai_image_generation_config("some-other-model") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + +class TestVertexAIImageGenerationIntegration: + """Integration tests for Vertex AI image generation""" + + + def test_gemini_get_complete_url(self): + """Test Gemini config URL generation""" + config = VertexAIGeminiImageGenerationConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model="gemini-2.5-flash-image", + optional_params={}, + litellm_params={ + "vertex_project": "test-project", + "vertex_location": "us-central1", + }, + ) + assert "test-project" in url + assert "us-central1" in url + assert "gemini-2.5-flash-image" in url + assert "generateContent" in url + + def test_imagen_get_complete_url(self): + """Test Imagen config URL generation""" + config = VertexAIImagenImageGenerationConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model="imagegeneration@006", + optional_params={}, + litellm_params={ + "vertex_project": "test-project", + "vertex_location": "us-central1", + }, + ) + assert "test-project" in url + assert "us-central1" in url + assert "imagegeneration@006" in url + assert "predict" in url diff --git a/tests/unit/llms/vertex_ai/rerank/__init__.py b/tests/unit/llms/vertex_ai/rerank/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py rename to tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_integration.py diff --git a/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py rename to tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_userlabels_e2e.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_userlabels_e2e.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/rerank/test_vertex_ai_rerank_userlabels_e2e.py rename to tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_userlabels_e2e.py diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py b/tests/unit/llms/vertex_ai/test_bge_embedding.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_bge_embedding.py rename to tests/unit/llms/vertex_ai/test_bge_embedding.py diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py b/tests/unit/llms/vertex_ai/test_bge_response_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py rename to tests/unit/llms/vertex_ai/test_bge_response_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/test_gemini_batch_embeddings.py b/tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_gemini_batch_embeddings.py rename to tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py diff --git a/tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py b/tests/unit/llms/vertex_ai/test_gemini_empty_properties.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_gemini_empty_properties.py rename to tests/unit/llms/vertex_ai/test_gemini_empty_properties.py diff --git a/tests/test_litellm/llms/vertex_ai/test_gemini_header_forwarding.py b/tests/unit/llms/vertex_ai/test_gemini_header_forwarding.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_gemini_header_forwarding.py rename to tests/unit/llms/vertex_ai/test_gemini_header_forwarding.py diff --git a/tests/test_litellm/llms/vertex_ai/test_http_status_201.py b/tests/unit/llms/vertex_ai/test_http_status_201.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_http_status_201.py rename to tests/unit/llms/vertex_ai/test_http_status_201.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py similarity index 97% rename from tests/test_litellm/llms/vertex_ai/test_vertex.py rename to tests/unit/llms/vertex_ai/test_vertex.py index e3007bac7f3..ab8bf123ab2 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -1193,7 +1193,6 @@ def test_logprobs(): def test_process_gemini_media(): """Test the _process_gemini_media function for different image sources""" - from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media from litellm.types.llms.vertex_ai import FileDataType # Test GCS URI @@ -1271,7 +1270,6 @@ def test_process_gemini_media(): assert base64_result["inline_data"]["data"] == "/9j/4AAQSkZJRg..." - def test_get_image_mime_type_from_url(): """Test the _get_image_mime_type_from_url function for different image URLs""" from litellm.llms.vertex_ai.gemini.transformation import ( @@ -1372,46 +1370,6 @@ def encoded_images(): return [encode_image_to_base64(path) for path in image_paths] -@pytest.fixture -def mock_convert_url_to_base64(): - with patch( - "litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64", - ) as mock: - # Setup the mock to return a valid image object - mock.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg..." - yield mock - - -@pytest.fixture -def mock_blob(): - return Mock(spec=BlobType) - - -@pytest.mark.parametrize( - "http_url", - [ - "http://img1.etsystatic.com/260/0/7813604/il_fullxfull.4226713999_q86e.jpg", - "http://example.com/image.jpg", - "http://subdomain.domain.com/path/to/image.png", - ], -) -def test_process_gemini_media_http_url( - http_url: str, mock_convert_url_to_base64: Mock, mock_blob: Mock -) -> None: - """ - Test that _process_gemini_media correctly handles HTTP URLs. - - Args: - http_url: Test HTTP URL - mock_convert_to_anthropic: Mocked convert_to_anthropic_image_obj function - mock_blob: Mocked BlobType instance - - Vertex AI supports image urls. Ensure no network requests are made. - """ - expected_image_data = "data:image/jpeg;base64,/9j/4AAQSkZJRg..." - mock_convert_url_to_base64.return_value = expected_image_data - # Act - result = _process_gemini_media(http_url) # assert result["file_data"]["file_uri"] == http_url diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py rename to tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py rename to tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py b/tests/unit/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py rename to tests/unit/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py b/tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py rename to tests/unit/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py b/tests/unit/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py rename to tests/unit/llms/vertex_ai/test_vertex_gemini_gcs_uri_mime.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py b/tests/unit/llms/vertex_ai/test_vertex_global_url_support.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py rename to tests/unit/llms/vertex_ai/test_vertex_global_url_support.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py b/tests/unit/llms/vertex_ai/test_vertex_image_generation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_image_generation.py rename to tests/unit/llms/vertex_ai/test_vertex_image_generation.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py rename to tests/unit/llms/vertex_ai/test_vertex_llm_base.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_model_garden_openapi.py b/tests/unit/llms/vertex_ai/test_vertex_model_garden_openapi.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_model_garden_openapi.py rename to tests/unit/llms/vertex_ai/test_vertex_model_garden_openapi.py diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/unit/llms/vertex_ai/test_vertex_passthrough_logging_handler.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py rename to tests/unit/llms/vertex_ai/test_vertex_passthrough_logging_handler.py diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gemma/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gemma/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/gemma/test_vertex_ai_gemma_global_endpoint.py diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/test_vertex_ai_gpt_oss_transformation.py diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/qwen/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/qwen/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/qwen/test_vertex_ai_qwen_global_endpoint.py diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py similarity index 100% rename from tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py diff --git a/tests/test_litellm/llms/volcengine/embedding/__init__.py b/tests/unit/llms/volcengine/embedding/__init__.py similarity index 100% rename from tests/test_litellm/llms/volcengine/embedding/__init__.py rename to tests/unit/llms/volcengine/embedding/__init__.py diff --git a/tests/test_litellm/llms/volcengine/test_volcengine.py b/tests/unit/llms/volcengine/test_volcengine.py similarity index 100% rename from tests/test_litellm/llms/volcengine/test_volcengine.py rename to tests/unit/llms/volcengine/test_volcengine.py diff --git a/tests/unit/llms/wandb/__init__.py b/tests/unit/llms/wandb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/unit/llms/wandb/test_wandb_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py rename to tests/unit/llms/wandb/test_wandb_chat_transformation.py diff --git a/tests/test_litellm/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_audio_transcription_transformation.py rename to tests/unit/llms/xai/test_xai_audio_transcription_transformation.py diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_chat_transformation.py rename to tests/unit/llms/xai/test_xai_chat_transformation.py diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/unit/llms/xai/test_xai_cost_calculator.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_cost_calculator.py rename to tests/unit/llms/xai/test_xai_cost_calculator.py diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/unit/llms/xai/test_xai_key_fallback.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_key_fallback.py rename to tests/unit/llms/xai/test_xai_key_fallback.py diff --git a/tests/test_litellm/llms/xai/test_xai_model_registry.py b/tests/unit/llms/xai/test_xai_model_registry.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_model_registry.py rename to tests/unit/llms/xai/test_xai_model_registry.py diff --git a/tests/test_litellm/llms/xai/test_xai_oauth.py b/tests/unit/llms/xai/test_xai_oauth.py similarity index 100% rename from tests/test_litellm/llms/xai/test_xai_oauth.py rename to tests/unit/llms/xai/test_xai_oauth.py diff --git a/tests/unit/test_unit_shard_missing_paths.py b/tests/unit/test_unit_shard_missing_paths.py index 4fa9c5bd3c1..e464402c9d8 100644 --- a/tests/unit/test_unit_shard_missing_paths.py +++ b/tests/unit/test_unit_shard_missing_paths.py @@ -39,6 +39,7 @@ def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.Compl "GITHUB_OUTPUT": str(tmp_path / "github_output"), "TEST_PATH": test_path, "WORKERS": workers, + "UNIT_FLAG": "", }, capture_output=True, text=True,