mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test(e2e): serve the vision image from our own fixture
The two vision tests pointed at a Wikipedia-hosted cat photo, so every run depended on upload.wikimedia.org staying up and unthrottled. It throttled, and the 429 surfaced as a bedrock APIConnectionError, which reads as a gateway failure rather than what it was. The image is now a fixture in the repo, passed as a data URL. That also puts the two providers on the same bytes: litellm downloads the image itself for bedrock, while openai is handed the link and fetches it from its own servers, so the hosted URL quietly meant the two tests were not testing the same thing. The image was generated for this repo rather than borrowed, so nothing here carries a third-party license. Also drops a stale comment about openai prompt caching that sat above the vision helper; no caching test uses it.
This commit is contained in:
parent
852cb3abbe
commit
ff418ffb9c
2 changed files with 18 additions and 4 deletions
BIN
tests/e2e/llm_translation/fixtures/cat.jpg
Normal file
BIN
tests/e2e/llm_translation/fixtures/cat.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
|
|
@ -16,7 +16,10 @@ via /model/new (Cohere, Gemini, hosted_vllm), each deleted on teardown.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -79,18 +82,29 @@ def _streamed_tool_call(events: list[str]) -> tuple[str, str]:
|
|||
return name, arguments
|
||||
|
||||
|
||||
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
|
||||
_FIXTURES_DIR: Final = Path(__file__).parent / "fixtures"
|
||||
CAT_IMAGE: Final = _FIXTURES_DIR / "cat.jpg"
|
||||
OPENAI_VISION_BACKEND = "openai/gpt-4o"
|
||||
|
||||
# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well
|
||||
# past that, so a repeat call reports cached prompt tokens.
|
||||
|
||||
def _cat_image_data_url() -> str:
|
||||
"""The vision image as a data URL, read from a fixture we own.
|
||||
|
||||
An https URL would make every vision run depend on a third-party host staying
|
||||
up and unthrottled, and a 429 from that host reads as a gateway failure. It also
|
||||
changes what is under test per provider: litellm downloads the image itself for
|
||||
bedrock, while openai is handed the link and fetches it from its own servers. A
|
||||
data URL removes the host and puts both providers on the same bytes."""
|
||||
return "data:image/jpeg;base64," + base64.b64encode(CAT_IMAGE.read_bytes()).decode()
|
||||
|
||||
|
||||
def _vision_messages() -> list[ChatMessage]:
|
||||
return [
|
||||
ChatMessage(
|
||||
role="user",
|
||||
content=[
|
||||
TextContentPart(text="What animal is in this image? Answer in one word."),
|
||||
ImageContentPart(image_url=ImageUrl(url=CAT_IMAGE_URL)),
|
||||
ImageContentPart(image_url=ImageUrl(url=_cat_image_data_url())),
|
||||
],
|
||||
)
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue