diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 00000000000..c6abc83f966 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,23 @@ +# tests/fixtures + +Media the test suite feeds to litellm, generated by us and committed here so no +test depends on a third-party host staying up or on someone else's copyright. + +- `test_image.png` / `test_image.jpg`: a 100x100 square with two diagonal lines, + drawn in code (see the snippet below). Any test that just needs valid image + bytes should use one of these. + +```python +from PIL import Image, ImageDraw + +img = Image.new("RGB", (100, 100), (255, 255, 255)) +d = ImageDraw.Draw(img) +d.rectangle([10, 10, 89, 89], outline=(17, 17, 17), width=4) +d.line([10, 10, 89, 89], fill=(200, 30, 30), width=4) +d.line([89, 10, 10, 89], fill=(30, 80, 200), width=4) +img.save("test_image.png", "PNG", optimize=True) +img.save("test_image.jpg", "JPEG", quality=85, optimize=True) +``` + +`asset_server.py` serves this directory over HTTP on loopback, for the tests that +have to exercise a real URL fetch rather than a `data:` URL diff --git a/tests/fixtures/asset_server.py b/tests/fixtures/asset_server.py new file mode 100644 index 00000000000..50c92fa6dc4 --- /dev/null +++ b/tests/fixtures/asset_server.py @@ -0,0 +1,52 @@ +"""Serves tests/fixtures over loopback HTTP. + +Some code paths under test (``convert_url_to_base64``, Bedrock's image +embedding, Gemini's tool-result media handling) only run when the image arrives +as a URL, so a ``data:`` URL would skip the very branch being tested. Those +tests point at this server instead of a third-party image host. + +litellm's SSRF guard rejects loopback by default, so the server also adds its +own host to ``litellm.user_url_allowed_hosts`` for as long as it is up: the +same setting an operator uses to reach an internal image host. +""" + +from __future__ import annotations + +import threading +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Iterator + +import litellm + +ASSETS_DIR = Path(__file__).parent +TEST_IMAGE_PNG = ASSETS_DIR / "test_image.png" +TEST_IMAGE_JPG = ASSETS_DIR / "test_image.jpg" +TEST_SPEECH_WAV = ASSETS_DIR / "test_speech.wav" +TEST_DOCUMENT_PDF = ASSETS_DIR / "test_document.pdf" +TEST_DOCUMENT_MD = ASSETS_DIR / "test_document.md" +TEST_TABLE_CSV = ASSETS_DIR / "test_table.csv" + + +class _QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, *args: object) -> None: + pass + + +def serve_assets() -> Iterator[str]: + server = ThreadingHTTPServer( + ("127.0.0.1", 0), partial(_QuietHandler, directory=str(ASSETS_DIR)) + ) + host = f"127.0.0.1:{server.server_address[1]}" + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + previously_allowed = litellm.user_url_allowed_hosts + litellm.user_url_allowed_hosts = [*previously_allowed, host] + try: + yield f"http://{host}" + finally: + litellm.user_url_allowed_hosts = previously_allowed + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/fixtures/test_document.md b/tests/fixtures/test_document.md new file mode 100644 index 00000000000..26e7ffccbf6 --- /dev/null +++ b/tests/fixtures/test_document.md @@ -0,0 +1,6 @@ +# LiteLLM test document + +This file exists so the test suite can send a Markdown document without +downloading one from someone else's web server. + +The quick brown fox jumps over the lazy dog. diff --git a/tests/fixtures/test_document.pdf b/tests/fixtures/test_document.pdf new file mode 100644 index 00000000000..249f61941b8 --- /dev/null +++ b/tests/fixtures/test_document.pdf @@ -0,0 +1,42 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >> +endobj +4 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj +5 0 obj +<< /Length 255 >> +stream +BT +/F1 14 Tf +72 720 Td +18 TL +(LiteLLM test document) Tj T* +() Tj T* +(This file exists so the test suite can send a PDF without) Tj T* +(downloading one from someone else's web server.) Tj T* +() Tj T* +(The quick brown fox jumps over the lazy dog.) Tj T* +ET +endstream +endobj +xref +0 6 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000115 00000 n +0000000241 00000 n +0000000311 00000 n +trailer +<< /Size 6 /Root 1 0 R >> +startxref +616 +%%EOF diff --git a/tests/fixtures/test_image.jpg b/tests/fixtures/test_image.jpg new file mode 100644 index 00000000000..51ec19f1f90 Binary files /dev/null and b/tests/fixtures/test_image.jpg differ diff --git a/tests/fixtures/test_image.png b/tests/fixtures/test_image.png new file mode 100644 index 00000000000..ae844ec3d48 Binary files /dev/null and b/tests/fixtures/test_image.png differ diff --git a/tests/fixtures/test_speech.wav b/tests/fixtures/test_speech.wav new file mode 100644 index 00000000000..09d04cfdee0 Binary files /dev/null and b/tests/fixtures/test_speech.wav differ diff --git a/tests/fixtures/test_table.csv b/tests/fixtures/test_table.csv new file mode 100644 index 00000000000..4c764ac9578 --- /dev/null +++ b/tests/fixtures/test_table.csv @@ -0,0 +1,5 @@ +region,quarter,revenue +north,Q1,1200 +north,Q2,1450 +south,Q1,980 +south,Q2,1130 diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 0ccfae55290..ee6b8723033 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -1124,14 +1124,10 @@ def test_usage_object_null_tokens(): def test_is_base64_encoded(): import base64 - import requests + from tests.fixtures.asset_server import TEST_IMAGE_PNG litellm.set_verbose = True - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - - encoded_file = base64.b64encode(file_data).decode("utf-8") + encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" from litellm.utils import is_base64_encoded diff --git a/tests/llm_translation/base_embedding_unit_tests.py b/tests/llm_translation/base_embedding_unit_tests.py index 1a88f0e9d6b..0f281e416f1 100644 --- a/tests/llm_translation/base_embedding_unit_tests.py +++ b/tests/llm_translation/base_embedding_unit_tests.py @@ -16,17 +16,13 @@ from litellm.utils import ( get_optional_params, get_optional_params_embeddings, ) -import requests import base64 -# test_example.py from abc import ABC, abstractmethod -url = "https://dummyimage.com/100/100/fff&text=Test+image" -response = requests.get(url) -file_data = response.content +from tests.fixtures.asset_server import TEST_IMAGE_PNG -encoded_file = base64.b64encode(file_data).decode("utf-8") +encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 1a33422a31c..c66023d507e 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -30,6 +30,7 @@ from openai import OpenAI sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) from tests._live_test_helpers import _skip_live_prompt_caching_test # noqa: E402 +from tests.fixtures.asset_server import TEST_SPEECH_WAV # noqa: E402 def _usage_format_tests(usage: litellm.Usage): @@ -758,11 +759,7 @@ class BaseLLMChatTest(ABC): f"Model={base_completion_call_args['model']} does not support audio input" ) - url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav" - response = httpx.get(url) - response.raise_for_status() - wav_data = response.content - encoded_string = base64.b64encode(wav_data).decode("utf-8") + encoded_string = base64.b64encode(TEST_SPEECH_WAV.read_bytes()).decode("utf-8") completion = self.completion_function( **base_completion_call_args, @@ -1274,12 +1271,8 @@ class BaseLLMChatTest(ABC): print("Model does not support audio input") pytest.skip("Model does not support audio input") - url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav" - response = httpx.get(url) - response.raise_for_status() - wav_data = response.content audio_format = "wav" - encoded_string = base64.b64encode(wav_data).decode("utf-8") + encoded_string = base64.b64encode(TEST_SPEECH_WAV.read_bytes()).decode("utf-8") audio_content = [{"type": "text", "text": "What is in this recording?"}] diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 8532af2851c..e858f3600ad 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -27,6 +27,12 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 vcr_config_dict, ) from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 +from tests.fixtures.asset_server import serve_assets # noqa: E402 + + +@pytest.fixture +def asset_base_url(): + yield from serve_assets() @pytest.fixture(scope="session", autouse=True) diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 550e82fb5bb..22bd3429036 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2441,7 +2441,7 @@ class TestBedrockEmbedding(BaseLLMEmbeddingTest): @pytest.mark.asyncio -async def test_bedrock_image_url_sync_client(): +async def test_bedrock_image_url_sync_client(asset_base_url): from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler import logging from litellm import verbose_logger @@ -2459,7 +2459,7 @@ async def test_bedrock_image_url_sync_client(): { "type": "image_url", "image_url": { - "url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png" + "url": f"{asset_base_url}/test_image.png" }, }, ], @@ -2523,18 +2523,12 @@ def test_bedrock_error_handling_streaming(exception_type, expected_status_code): @pytest.mark.parametrize( - "image_url", - [ - "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", - # "https://raw.githubusercontent.com/datasets/gdp/master/data/gdp.csv", - "https://www.cmu.edu/blackboard/files/evaluate/tests-example.xls", - # "https://raw.githubusercontent.com/datasets/sample-data/master/README.txt", # invalid url - "https://raw.githubusercontent.com/mdn/content/main/README.md", - ], + "asset_name", + ["test_document.pdf", "test_table.csv", "test_document.md"], ) @pytest.mark.flaky(retries=6, delay=2) @pytest.mark.asyncio -async def test_bedrock_document_understanding(image_url): +async def test_bedrock_document_understanding(asset_name, asset_base_url): from litellm import acompletion litellm._turn_on_debug() @@ -2544,7 +2538,7 @@ async def test_bedrock_document_understanding(image_url): {"type": "text", "text": f"What's this file about?"}, { "type": "image_url", - "image_url": image_url, + "image_url": f"{asset_base_url}/{asset_name}", }, ] diff --git a/tests/llm_translation/test_gpt4o_audio.py b/tests/llm_translation/test_gpt4o_audio.py index 0f20119e4ef..5e94d2ac142 100644 --- a/tests/llm_translation/test_gpt4o_audio.py +++ b/tests/llm_translation/test_gpt4o_audio.py @@ -11,7 +11,8 @@ import litellm from litellm import Choices, Message, ModelResponse from litellm.types.utils import StreamingChoices, ChatCompletionAudioResponse import base64 -import requests + +from tests.fixtures.asset_server import TEST_SPEECH_WAV def check_non_streaming_response(completion): @@ -88,17 +89,12 @@ async def test_audio_output_from_model(stream): @pytest.mark.parametrize("stream", [True, False]) @pytest.mark.parametrize("model", ["gpt-audio-1.5"]) async def test_audio_input_to_model(stream, model): - # Fetch the audio file and convert it to a base64 encoded string audio_format = "pcm16" if stream is False: audio_format = "wav" litellm._turn_on_debug() litellm.drop_params = True - url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav" - response = requests.get(url) - response.raise_for_status() - wav_data = response.content - encoded_string = base64.b64encode(wav_data).decode("utf-8") + encoded_string = base64.b64encode(TEST_SPEECH_WAV.read_bytes()).decode("utf-8") try: completion = await litellm.acompletion( model=model, diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 2b9abdec5d0..d8baa0a5c65 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -209,19 +209,16 @@ async def test_vision_with_custom_model(): """ import base64 - import requests from openai import AsyncOpenAI + from tests.fixtures.asset_server import TEST_IMAGE_PNG + client = AsyncOpenAI(api_key="fake-api-key") litellm.set_verbose = True api_base = "https://my-custom.api.openai.com" - # Fetch and encode a test image - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - encoded_file = base64.b64encode(file_data).decode("utf-8") + encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" with patch.object( @@ -261,9 +258,7 @@ async def test_vision_with_custom_model(): {"type": "text", "text": "What's in this image?"}, { "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkBAMAAACCzIhnAAAAG1BMVEURAAD///+ln5/h39/Dv79qX18uHx+If39MPz9oMSdmAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABDElEQVRYhe2SzWqEMBRGPyQTfQxJsc5jBKGzFmlslyFIZxsCQ7sUaWd87EanpdpIrbtC71mE/NyTm9wEIAiCIAiC+N/otQBxU2Sf/aeh4enqptHXri+/yxIq63jlKCw6cXssnr3ObdzdGYFYCJ2IzHKXLygHXCB98Gm4DE+ZZemu5EisQSyZTmyg+AuzQbkezCuIy7EI0k9Ig3FtruwydY+qniqtV5yQyo8qpUIl2fc90KVzJWohWf2qu75vlw52rdfjVDHg8vLWwixW7PChqLkSyUadwfSS0uQZhEvRuIkS53uJvrK8cGWYaPwpGt8efvw+vlo8TPMzcmP8w7lrNypc1RsNgiAIgiD+Iu/RyDYhCaWrgQAAAABJRU5ErkJggg==" - }, + "image_url": {"url": base64_image}, }, ], }, @@ -473,7 +468,7 @@ class TestOpenAIGPT4OAudioTranscription(BaseLLMAudioTranscriptionTest): @pytest.mark.asyncio @pytest.mark.parametrize("model", ["gpt-4o"]) -async def test_openai_pdf_url(model): +async def test_openai_pdf_url(model, asset_base_url): from litellm.utils import return_raw_request, CallTypes request = return_raw_request( @@ -487,7 +482,9 @@ async def test_openai_pdf_url(model): {"type": "text", "text": "What is the first page of the PDF?"}, { "type": "file", - "file": {"file_id": "https://arxiv.org/pdf/2303.08774"}, + "file": { + "file_id": f"{asset_base_url}/test_document.pdf" + }, }, ], } diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index a90a3df584e..f7d64fee065 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -181,10 +181,8 @@ def test_bedrock_tool_calling_pt(): print(converted_tools) -def test_convert_url_to_img(): - response_url = convert_url_to_base64( - url="https://images.pexels.com/photos/1319515/pexels-photo-1319515.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1" - ) +def test_convert_url_to_img(asset_base_url): + response_url = convert_url_to_base64(url=f"{asset_base_url}/test_image.jpg") assert "image/jpeg" in response_url @@ -1289,17 +1287,16 @@ def test_just_system_message(): assert "bedrock requires at least one non-system message" in str(e.value) -def test_convert_generic_image_chunk_to_openai_image_obj(): +def test_convert_generic_image_chunk_to_openai_image_obj(asset_base_url): from litellm.litellm_core_utils.prompt_templates.factory import ( convert_generic_image_chunk_to_openai_image_obj, convert_to_anthropic_image_obj, ) - url = "https://i.pinimg.com/736x/b4/b1/be/b4b1becad04d03a9071db2817fc9fe77.jpg" + url = f"{asset_base_url}/test_image.jpg" image_obj = convert_to_anthropic_image_obj(url, format=None) url_str = convert_generic_image_chunk_to_openai_image_obj(image_obj) - image_obj = convert_to_anthropic_image_obj(url_str, format=None) - print(image_obj) + assert convert_to_anthropic_image_obj(url_str, format=None) == image_obj def test_hf_chat_template(): diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 4f142664827..ae3e3c3db8b 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -49,6 +49,12 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 vcr_config_dict, ) from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 +from tests.fixtures.asset_server import serve_assets # noqa: E402 + + +@pytest.fixture +def asset_base_url(): + yield from serve_assets() @pytest.fixture(scope="session", autouse=True) diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 3d66064f5c0..def1d3bb9fa 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -2000,16 +2000,12 @@ async def test_vertexai_multimodal_embedding_image_in_input(): async def test_vertexai_multimodal_embedding_base64image_in_input(): import base64 - import requests + from tests.fixtures.asset_server import TEST_IMAGE_PNG load_vertex_ai_credentials() mock_response = AsyncMock() - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - - encoded_file = base64.b64encode(file_data).decode("utf-8") + encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" def return_val(): @@ -4061,7 +4057,7 @@ def test_vertex_ai_gemini_audio_ogg(): "content": [ { "file": { - "file_id": "https://upload.wikimedia.org/wikipedia/commons/5/5f/En-us-public.ogg" + "file_id": "https://example.com/audio.ogg" }, "type": "file", } diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index ef8d6c55148..c860319858b 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -616,14 +616,10 @@ def test_completion_base64(model): try: import base64 - import requests + from tests.fixtures.asset_server import TEST_IMAGE_PNG litellm.set_verbose = True - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - - encoded_file = base64.b64encode(file_data).decode("utf-8") + encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" resp = litellm.completion( model=model, diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index aed2849f056..0accba3cc27 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -162,14 +162,10 @@ async def test_together_ai_embedding(model, api_base, api_key, sync_mode): # test_openai_azure_embedding_simple() import base64 -import requests +from tests.fixtures.asset_server import TEST_IMAGE_PNG litellm.set_verbose = True -url = "https://dummyimage.com/100/100/fff&text=Test+image" -response = requests.get(url) -file_data = response.content - -encoded_file = base64.b64encode(file_data).decode("utf-8") +encoded_file = base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode("utf-8") base64_image = f"data:image/png;base64,{encoded_file}" diff --git a/tests/local_testing/test_ollama.py b/tests/local_testing/test_ollama.py index ad5d7d86501..1c05c062cbb 100644 --- a/tests/local_testing/test_ollama.py +++ b/tests/local_testing/test_ollama.py @@ -75,7 +75,7 @@ def test_ollama_json_mode(): # test_ollama_json_mode() -def test_ollama_vision_model(): +def test_ollama_vision_model(asset_base_url): from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() @@ -93,7 +93,7 @@ def test_ollama_vision_model(): { "type": "image_url", "image_url": { - "url": "https://dummyimage.com/100/100/fff&text=Test+image" + "url": f"{asset_base_url}/test_image.png" }, }, ], diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index 62c95cb100b..85b993e4674 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -39,6 +39,13 @@ from litellm.llms.custom_httpx.async_client_cleanup import ( ) from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module +from tests.fixtures.asset_server import serve_assets + + +@pytest.fixture +def asset_base_url(): + yield from serve_assets() + def _reset_module_level_aws_auth_caches(): """ diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..ccfd449aeac 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -380,7 +380,7 @@ def test_encoding_and_decoding(): # test_encoding_and_decoding() -def test_gpt_vision_token_counting(): +def test_gpt_vision_token_counting(asset_base_url): messages = [ { "role": "user", @@ -388,13 +388,17 @@ def test_gpt_vision_token_counting(): {"type": "text", "text": "What’s in this image?"}, { "type": "image_url", - "image_url": "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", + "image_url": f"{asset_base_url}/test_image.png", }, ], } ] + text_only_tokens = token_counter( + model="gpt-4-vision-preview", + messages=[{"role": "user", "content": "What’s in this image?"}], + ) tokens = token_counter(model="gpt-4-vision-preview", messages=messages) - print(f"tokens: {tokens}") + assert tokens > text_only_tokens # test_gpt_vision_token_counting() diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 8c1de12e7d9..534fc96dfc0 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -15,6 +15,8 @@ from litellm.llms.vertex_ai.gemini.transformation import ( from litellm.types.llms.vertex_ai import BlobType from litellm.types.utils import Message +from tests.fixtures.asset_server import TEST_IMAGE_PNG + def test_check_if_part_exists_in_parts(): parts = [ @@ -1669,12 +1671,9 @@ def test_gemini_history_nests_multimodal_tool_response_parts(): ] -def test_convert_tool_response_with_url_image(): +def test_convert_tool_response_with_url_image(asset_base_url): """Test tool response with HTTP URL image (will download and convert).""" - import pytest - - # Use a publicly accessible test image URL - test_image_url = "https://via.placeholder.com/1x1.png" + test_image_url = f"{asset_base_url}/test_image.png" tool_message = { "role": "tool", @@ -1697,30 +1696,23 @@ def test_convert_tool_response_with_url_image(): ] } - try: - result = convert_to_gemini_tool_call_result( - tool_message, last_message_with_tool_calls - ) + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) - assert isinstance( - result, list - ), "Should return a parts list when media is present" - assert len(result) == 1, "Should return one function_response part" - result_part = result[0] - assert "function_response" in result_part - assert "inline_data" not in result_part - function_response = result_part["function_response"] - assert function_response["name"] == "type_text_at" + assert isinstance(result, list), "Should return a parts list when media is present" + assert len(result) == 1, "Should return one function_response part" + result_part = result[0] + assert "function_response" in result_part + assert "inline_data" not in result_part + function_response = result_part["function_response"] + assert function_response["name"] == "type_text_at" - # Check inline_data is nested under functionResponse.parts. - assert "parts" in function_response - assert len(function_response["parts"]) == 1 - inline_data: BlobType = function_response["parts"][0]["inline_data"] - assert "data" in inline_data - assert "mime_type" in inline_data - except Exception as e: - # Skip test if URL download fails (no internet connection, etc.) - pytest.skip(f"Failed to download image from URL: {e}") + assert "parts" in function_response + assert len(function_response["parts"]) == 1 + inline_data: BlobType = function_response["parts"][0]["inline_data"] + assert inline_data["data"] == base64.b64encode(TEST_IMAGE_PNG.read_bytes()).decode() + assert inline_data["mime_type"] == "image/png" def test_convert_tool_response_text_only():