mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
3357ec8d34
* test: run the 30 test files stranded in the second mirror
tests/litellm sat beside tests/test_litellm, which is the mirror the repo
convention names, and no job collected it. The allowlist called the directory
unresolved and assumed it was a duplicate. It is not: 30 of its 34 files have no
counterpart in the real mirror, so they are tests nobody has run since they were
written, not copies of tests that run elsewhere.
Moving them in is byte-identical, and it is what makes them run. Every one is
now claimed by a shard's test-path rather than by an allowlist entry, and the
216 tests they hold pass. Directories that needed to become packages did, since
several files are named test_transformation.py and pytest cannot import two of
those from non-package directories in one session.
Never running is why three assertions had drifted away from the code:
* nvidia.nemotron-super-3-120b max_output_tokens, 32000 -> 32768
* sambanova/MiniMax-M2.7 max_input_tokens, 204800 -> 196608
* the Vertex text-to-speech handler moved from data= to json=, so the test
reads the decoded body off the json kwarg instead of parsing the data one
The first two follow model_prices_and_context_window.json, which the catalog
sync keeps current; the third follows the handler. In all three the test was the
stale side.
The lint workflow ran test_no_hardcoded_secrets.py by path and now points at the
new one.
Four files stay behind. Each shares a filename with a live test whose contents
are disjoint from it, so landing those means merging test bodies, which is a
content review rather than a move. The allowlist entry now names those four and
records how many tests each would bring, in place of calling the whole
directory unresolved.
* fix(ci): keep the secret scan out of the mirror's conftest
The secret-scan job runs pytest under uv run --no-project, so its environment
holds pytest and nothing else. That worked while the file sat in tests/litellm,
which has no conftest, and broke the moment it moved into tests/test_litellm,
whose conftest imports litellm on collection: ModuleNotFoundError: No module
named 'dotenv', before a single test ran.
The file is a repo-wide static scan that imports only base64, os, re and pytest,
so it belongs with the other repo-wide checks in tests/code_coverage_tests,
which has no conftest, rather than in the package mirror. Installing the full
dependency set into a 15-second job to satisfy a conftest it does not use would
be the wrong trade.
Verified with the job's exact command:
uv run --no-project --with 'pytest==9.0.2' pytest \
tests/code_coverage_tests/test_no_hardcoded_secrets.py -q
1 passed in 0.47s
251 lines
8.1 KiB
Python
251 lines
8.1 KiB
Python
"""
|
|
Test that stream_chunk_builder correctly preserves images from streaming chunks.
|
|
|
|
This tests the fix for https://github.com/BerriAI/litellm/issues/19478
|
|
where images from models like gemini-2.5-flash-image were lost when
|
|
rebuilding the response from streaming chunks.
|
|
"""
|
|
|
|
import pytest
|
|
import litellm
|
|
from litellm import stream_chunk_builder
|
|
|
|
|
|
def test_stream_chunk_builder_preserves_images():
|
|
"""
|
|
Test that stream_chunk_builder correctly preserves images from streaming chunks.
|
|
"""
|
|
# Simulate streaming chunks from an image generation model
|
|
init_chunks = [
|
|
{
|
|
"id": "chatcmpl-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"role": "assistant",
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"images": [
|
|
{
|
|
"image_url": {
|
|
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
|
|
"detail": "auto",
|
|
},
|
|
"index": 0,
|
|
"type": "image_url",
|
|
}
|
|
],
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
]
|
|
|
|
chunks = []
|
|
for chunk in init_chunks:
|
|
chunks.append(litellm.ModelResponseStream(**chunk))
|
|
|
|
response = stream_chunk_builder(chunks=chunks)
|
|
|
|
# Verify that images are preserved in the rebuilt response
|
|
assert (
|
|
response.choices[0].message.images is not None
|
|
), "Images should be preserved in stream_chunk_builder"
|
|
assert len(response.choices[0].message.images) == 1, "Should have exactly 1 image"
|
|
assert response.choices[0].message.images[0]["type"] == "image_url"
|
|
assert "base64" in response.choices[0].message.images[0]["image_url"]["url"]
|
|
|
|
|
|
def test_stream_chunk_builder_preserves_multiple_images():
|
|
"""
|
|
Test that stream_chunk_builder correctly preserves multiple images from different chunks.
|
|
"""
|
|
init_chunks = [
|
|
{
|
|
"id": "chatcmpl-multi-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"role": "assistant",
|
|
"content": "Here are your images:",
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-multi-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"images": [
|
|
{
|
|
"image_url": {
|
|
"url": "data:image/png;base64,image1data",
|
|
"detail": "auto",
|
|
},
|
|
"index": 0,
|
|
"type": "image_url",
|
|
}
|
|
],
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-multi-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"images": [
|
|
{
|
|
"image_url": {
|
|
"url": "data:image/png;base64,image2data",
|
|
"detail": "auto",
|
|
},
|
|
"index": 1,
|
|
"type": "image_url",
|
|
}
|
|
],
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-multi-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gemini/gemini-2.5-flash-image",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
]
|
|
|
|
chunks = []
|
|
for chunk in init_chunks:
|
|
chunks.append(litellm.ModelResponseStream(**chunk))
|
|
|
|
response = stream_chunk_builder(chunks=chunks)
|
|
|
|
# Verify content is preserved
|
|
assert response.choices[0].message.content == "Here are your images:"
|
|
|
|
# Verify all images are preserved
|
|
assert response.choices[0].message.images is not None, "Images should be preserved"
|
|
assert len(response.choices[0].message.images) == 2, "Should have exactly 2 images"
|
|
assert "image1data" in response.choices[0].message.images[0]["image_url"]["url"]
|
|
assert "image2data" in response.choices[0].message.images[1]["image_url"]["url"]
|
|
|
|
|
|
def test_stream_chunk_builder_no_images():
|
|
"""
|
|
Test that stream_chunk_builder works correctly when there are no images (regression test).
|
|
"""
|
|
init_chunks = [
|
|
{
|
|
"id": "chatcmpl-no-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"role": "assistant",
|
|
"content": "Hello, ",
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gpt-4",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-no-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {
|
|
"content": "world!",
|
|
},
|
|
"finish_reason": None,
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gpt-4",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
{
|
|
"id": "chatcmpl-no-image-test",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"delta": {},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"created": 1737654321,
|
|
"model": "gpt-4",
|
|
"object": "chat.completion.chunk",
|
|
},
|
|
]
|
|
|
|
chunks = []
|
|
for chunk in init_chunks:
|
|
chunks.append(litellm.ModelResponseStream(**chunk))
|
|
|
|
response = stream_chunk_builder(chunks=chunks)
|
|
|
|
# Verify content is preserved
|
|
assert response.choices[0].message.content == "Hello, world!"
|
|
|
|
# Verify images attribute doesn't exist or is None (no images in this stream)
|
|
images = getattr(response.choices[0].message, "images", None)
|
|
assert images is None, "Should not have images when none were in the stream"
|