Merge pull request #42110 from BerriAI/litellm_migrate_tests_p4

test(llms): migrate bedrock, baseten and base_llm batch tests to tests/unit

tests/test_litellm/llms/bedrock/files/test_bedrock_files_integration.py

@@ -1,115 +0,0 @@
-"""
-Test Bedrock files integration with main files API
-"""
-
-import base64
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-import litellm
-from litellm.types.llms.openai import HttpxBinaryResponseContent
-from litellm.types.utils import SpecialEnums
-
-
-class TestBedrockFilesIntegration:
-    """Test integration of Bedrock files with main litellm API"""
-
-    @pytest.mark.asyncio
-    async def test_litellm_afile_content_bedrock_provider_with_s3_uri(self):
-        """Test litellm.afile_content with bedrock provider using direct S3 URI"""
-        file_id = "s3://test-bucket/test-file.jsonl"
-        expected_content = (
-            b'{"recordId": "request-1", "modelInput": {}, "modelOutput": {}}'
-        )
-
-        # Create a mock HttpxBinaryResponseContent response
-        import httpx
-
-        mock_response = httpx.Response(
-            status_code=200,
-            content=expected_content,
-            headers={"content-type": "application/octet-stream"},
-            request=httpx.Request(method="GET", url="s3://test-bucket/test-file.jsonl"),
-        )
-        mock_result = HttpxBinaryResponseContent(response=mock_response)
-
-        # Mock the base_llm_http_handler.retrieve_file_content since the code
-        # now routes through ProviderConfigManager -> base_llm_http_handler
-        with patch(
-            "litellm.files.main.base_llm_http_handler.retrieve_file_content",
-            new_callable=MagicMock,
-        ) as mock_retrieve:
-            mock_retrieve.return_value = mock_result
-
-            # Call litellm.afile_content
-            result = await litellm.afile_content(
-                file_id=file_id,
-                custom_llm_provider="bedrock",
-                aws_region_name="us-west-2",
-            )
-
-            # Verify the result
-            assert isinstance(result, HttpxBinaryResponseContent)
-            assert result.response.content == expected_content
-            assert result.response.status_code == 200
-
-            # Verify the mock was called with correct parameters
-            mock_retrieve.assert_called_once()
-            call_kwargs = mock_retrieve.call_args.kwargs
-            assert call_kwargs["_is_async"] is True
-            assert call_kwargs["file_content_request"]["file_id"] == file_id
-
-    @pytest.mark.asyncio
-    async def test_litellm_afile_content_bedrock_provider_with_unified_file_id(self):
-        """Test litellm.afile_content with bedrock provider using unified file ID"""
-        # Create a unified file ID
-        s3_uri = "s3://test-bucket/batch-outputs/output.jsonl"
-        unified_id = "test-unified-id-123"
-        model_id = "test-model-id-456"
-
-        unified_file_id_str = f"litellm_proxy:application/json;unified_id,{unified_id};target_model_names,;llm_output_file_id,{s3_uri};llm_output_file_model_id,{model_id}"
-        encoded_file_id = (
-            base64.urlsafe_b64encode(unified_file_id_str.encode()).decode().rstrip("=")
-        )
-
-        expected_content = (
-            b'{"recordId": "request-1", "modelInput": {}, "modelOutput": {}}'
-        )
-
-        # Create a mock HttpxBinaryResponseContent response
-        import httpx
-
-        mock_response = httpx.Response(
-            status_code=200,
-            content=expected_content,
-            headers={"content-type": "application/octet-stream"},
-            request=httpx.Request(method="GET", url=s3_uri),
-        )
-        mock_result = HttpxBinaryResponseContent(response=mock_response)
-
-        # Mock the base_llm_http_handler.retrieve_file_content
-        with patch(
-            "litellm.files.main.base_llm_http_handler.retrieve_file_content",
-            new_callable=MagicMock,
-        ) as mock_retrieve:
-            mock_retrieve.return_value = mock_result
-
-            # Call litellm.afile_content with unified file ID
-            result = await litellm.afile_content(
-                file_id=encoded_file_id,
-                custom_llm_provider="bedrock",
-                aws_region_name="us-west-2",
-            )
-
-            # Verify the result
-            assert isinstance(result, HttpxBinaryResponseContent)
-            assert result.response.content == expected_content
-            assert result.response.status_code == 200
-
-            # Verify the mock was called
-            mock_retrieve.assert_called_once()
-            call_kwargs = mock_retrieve.call_args.kwargs
-            assert call_kwargs["_is_async"] is True
-            # The handler passes the encoded file_id as-is
-            assert call_kwargs["file_content_request"]["file_id"] == encoded_file_id

tests/unit/llms/base_llm/batches/test_transformation.py

@@ -129,22 +129,6 @@ def test_subclass_missing_any_abstract_member_cannot_instantiate(missing_member)
         Incomplete()
 
 
-def test_concrete_instance_methods_run():
-    """Sanity: the trivial overrides actually execute through the base contract."""
-    instance = _ConcreteBatchesConfig()
-    assert instance.custom_llm_provider == LlmProviders.OPENAI
-    assert instance.validate_environment(
-        headers={"x": "1"},
-        model="m",
-        messages=[],
-        optional_params={},
-        litellm_params={},
-    ) == {"x": "1"}
-    assert instance.transform_retrieve_batch_request(
-        batch_id="b-1", optional_params={}, litellm_params={}
-    ) == {"batch_id": "b-1"}
-
-
 # =========================================================================== #
 # get_config()
 # =========================================================================== #

tests/unit/llms/bedrock/chat/invoke_transformations/test_amazon_nova_transformation.py

@@ -1,5 +1,8 @@
 import json
 
+import pytest
+
+import litellm
 from litellm.llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import (
     AmazonInvokeNovaConfig,
 )
@@ -13,6 +16,25 @@ TOOL_CALL = {"id": "call_1", "type": "function", "function": {"name": "f", "argu
 PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
 
 
+@pytest.fixture
+def local_model_cost_map(monkeypatch):
+    """Force the bundled in-repo cost map so capability and pricing assertions do not
+    depend on the network-fetched ``main`` copy, which lags this branch until merge.
+
+    ``get_model_info`` is lru_cached, so swapping ``model_cost`` is not enough on its
+    own; clear on the way in and out so entries warmed against either map never leak
+    across tests."""
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
 def _transform_request(messages, optional_params, litellm_params=None):
     return AmazonInvokeNovaConfig().transform_request(
         model=MODEL,

tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py

@@ -1,6 +1,8 @@
 import asyncio
+import base64
 import json
 import uuid
+from types import SimpleNamespace
 from typing import Final
 from unittest.mock import patch
 
@@ -17,6 +19,77 @@ from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transfor
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 
 
+ONE_PIXEL_PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
+)
+
+
+@pytest.fixture
+def async_only_image_fetch(monkeypatch):
+    from litellm.litellm_core_utils.prompt_templates import factory, image_handling
+    from litellm.llms.gemini.chat import transformation as gemini_chat_transformation
+
+    fetch = SimpleNamespace(
+        fetched=[],
+        base64_png=base64.b64encode(ONE_PIXEL_PNG).decode(),
+        data_url="data:image/png;base64," + base64.b64encode(ONE_PIXEL_PNG).decode(),
+    )
+
+    def forbid_sync_fetch(client, url, **kwargs):
+        raise litellm.ImageFetchError(f"sync image fetch ran on the event loop: {url}")
+
+    async def serve_png(client, url, **kwargs):
+        fetch.fetched.append(url)
+        return httpx.Response(
+            200,
+            content=ONE_PIXEL_PNG,
+            headers={"content-type": "image/png"},
+            request=httpx.Request("GET", url),
+        )
+
+    def forbid_sync_convert(url, *args, **kwargs):
+        if url.startswith(("http://", "https://")):
+            raise litellm.ImageFetchError(f"sync convert_url_to_base64 ran on the request path: {url}")
+        return url
+
+    monkeypatch.setattr(image_handling, "safe_get", forbid_sync_fetch)
+    monkeypatch.setattr(image_handling, "async_safe_get", serve_png)
+    for module in (image_handling, factory, gemini_chat_transformation):
+        monkeypatch.setattr(module, "convert_url_to_base64", forbid_sync_convert)
+    return fetch
+
+
+@pytest.fixture
+def local_model_cost_map(monkeypatch):
+    """Force the bundled in-repo cost map so capability and pricing assertions do not
+    depend on the network-fetched ``main`` copy, which lags this branch until merge.
+
+    ``get_model_info`` is lru_cached, so swapping ``model_cost`` is not enough on its
+    own; clear on the way in and out so entries warmed against either map never leak
+    across tests."""
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
+@pytest.fixture
+def local_beta_headers_config(monkeypatch):
+    """Pin the bundled ``anthropic_beta_headers_config.json`` so beta header assertions
+    do not depend on the network-fetched copy or on what earlier tests left cached."""
+    from litellm.anthropic_beta_headers_manager import reload_beta_headers_config
+
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
+    reload_beta_headers_config()
+    yield
+    reload_beta_headers_config()
+
+
 def test_get_supported_params_thinking():
     config = AmazonAnthropicClaudeConfig()
     params = config.get_supported_openai_params(

tests/unit/llms/bedrock/chat/mantle/test_bedrock_chat_mantle_transformation.py

@@ -1,12 +1,55 @@
+import base64
 import json
 import uuid
+from types import SimpleNamespace
 
 import httpx
+import pytest
 
 import litellm
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 
 
+ONE_PIXEL_PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
+)
+
+
+@pytest.fixture
+def async_only_image_fetch(monkeypatch):
+    from litellm.litellm_core_utils.prompt_templates import factory, image_handling
+    from litellm.llms.gemini.chat import transformation as gemini_chat_transformation
+
+    fetch = SimpleNamespace(
+        fetched=[],
+        base64_png=base64.b64encode(ONE_PIXEL_PNG).decode(),
+        data_url="data:image/png;base64," + base64.b64encode(ONE_PIXEL_PNG).decode(),
+    )
+
+    def forbid_sync_fetch(client, url, **kwargs):
+        raise litellm.ImageFetchError(f"sync image fetch ran on the event loop: {url}")
+
+    async def serve_png(client, url, **kwargs):
+        fetch.fetched.append(url)
+        return httpx.Response(
+            200,
+            content=ONE_PIXEL_PNG,
+            headers={"content-type": "image/png"},
+            request=httpx.Request("GET", url),
+        )
+
+    def forbid_sync_convert(url, *args, **kwargs):
+        if url.startswith(("http://", "https://")):
+            raise litellm.ImageFetchError(f"sync convert_url_to_base64 ran on the request path: {url}")
+        return url
+
+    monkeypatch.setattr(image_handling, "safe_get", forbid_sync_fetch)
+    monkeypatch.setattr(image_handling, "async_safe_get", serve_png)
+    for module in (image_handling, factory, gemini_chat_transformation):
+        monkeypatch.setattr(module, "convert_url_to_base64", forbid_sync_convert)
+    return fetch
+
+
 async def test_bedrock_mantle_claude_async_completion_inlines_remote_images_off_the_event_loop(async_only_image_fetch):
     image_url = f"http://img.example/{uuid.uuid4()}.png"
     captured = {}