mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(images): harden reference header measurement and cover pricing edge cases
This commit is contained in:
parent
93ebd1116e
commit
0bb9c54d11
4 changed files with 175 additions and 18 deletions
|
|
@ -13,7 +13,8 @@ from litellm.types.images.main import ImageEditOptionalRequestParams
|
|||
from litellm.types.llms.openai import FileTypes
|
||||
|
||||
_JPEG_SOF_MARKERS: Final = frozenset(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC}
|
||||
_MAX_JPEG_SEGMENTS: Final = 64
|
||||
_MAX_JPEG_SEGMENTS: Final = 1024
|
||||
_MAX_JPEG_HEADER_OFFSET: Final = 16 * 1024 * 1024
|
||||
_JPEG_FIRST_SEGMENT_OFFSET: Final = 2
|
||||
_JPEG_SOF_PAYLOAD_SIZE: Final = 5
|
||||
_HEADER_READ_SIZE: Final = 32
|
||||
|
|
@ -77,12 +78,14 @@ def _webp_dimensions(head: bytes) -> tuple[int, int] | None:
|
|||
def _jpeg_sof_dimensions(stream: IO[bytes], start: int, offset: int, segments_left: int) -> tuple[int, int] | None:
|
||||
segment_offset = offset # rebind-ok: advanced one JPEG segment per hop
|
||||
for _ in range(segments_left):
|
||||
if segment_offset > _MAX_JPEG_HEADER_OFFSET:
|
||||
return None
|
||||
stream.seek(start + segment_offset)
|
||||
marker = stream.read(4)
|
||||
if len(marker) < 4 or marker[0] != 0xFF:
|
||||
return None
|
||||
if marker[1] == 0xFF:
|
||||
segment_offset += 1
|
||||
segment_offset += 1 + int(marker[2] == 0xFF) + int(marker[3] == 0xFF)
|
||||
continue
|
||||
if marker[1] in _JPEG_SOF_MARKERS:
|
||||
sof = stream.read(_JPEG_SOF_PAYLOAD_SIZE)
|
||||
|
|
@ -114,17 +117,25 @@ def _header_dimensions(stream: IO[bytes], position: int) -> tuple[int, int] | No
|
|||
return None
|
||||
|
||||
|
||||
def measure_reference_image(image: FileTypes) -> tuple[int, int] | None:
|
||||
stream: Final = _content_stream(image)
|
||||
if stream is None:
|
||||
return None
|
||||
def _measure_stream(stream: IO[bytes]) -> tuple[int, int] | None:
|
||||
position: Final = stream.tell()
|
||||
try:
|
||||
return _header_dimensions(stream, position)
|
||||
except (OSError, ValueError, struct.error):
|
||||
return None
|
||||
dimensions: Final = _header_dimensions(stream, position)
|
||||
finally:
|
||||
stream.seek(position)
|
||||
if dimensions is None or dimensions[0] <= 0 or dimensions[1] <= 0:
|
||||
return None
|
||||
return dimensions
|
||||
|
||||
|
||||
def measure_reference_image(image: FileTypes) -> tuple[int, int] | None:
|
||||
try:
|
||||
stream: Final = _content_stream(image)
|
||||
if stream is None:
|
||||
return None
|
||||
return _measure_stream(stream)
|
||||
except (OSError, ValueError, AttributeError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None:
|
||||
|
|
@ -132,7 +143,8 @@ def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None:
|
|||
dimensions: Final = tuple(size for size in measured if size is not None)
|
||||
if len(dimensions) != len(measured):
|
||||
verbose_logger.debug(
|
||||
"Reference image %d has no readable PNG, JPEG, WebP, GIF or BMP header; billing generated pixels only",
|
||||
"Reference image %d could not be measured (path, non-seekable stream, or no readable PNG, JPEG, WebP, "
|
||||
"GIF or BMP header); billing generated pixels only",
|
||||
measured.index(None),
|
||||
)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import io
|
|||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Final, List, Optional
|
||||
from typing import Any, Dict, Final, List, Optional, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -79,7 +79,10 @@ def _webp_chunk(fourcc: bytes, body: bytes) -> bytes:
|
|||
|
||||
|
||||
JPEG_APP0: Final = 0xE0
|
||||
JPEG_APP2_MARKER: Final = 0xE2
|
||||
JPEG_DHT: Final = 0xC4
|
||||
JPEG_JPG_MARKER: Final = 0xC8
|
||||
JPEG_DAC_MARKER: Final = 0xCC
|
||||
JPEG_SOF0_MARKER: Final = 0xC0
|
||||
JPEG_SOF2_MARKER: Final = 0xC2
|
||||
|
||||
|
|
@ -92,6 +95,16 @@ SEGMENTS_63_JPEG: Final = (
|
|||
SEGMENTS_65_JPEG: Final = (
|
||||
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 65 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
|
||||
)
|
||||
SEGMENTS_300_JPEG: Final = (
|
||||
JPEG_SOI + _jpeg_segment(JPEG_APP2_MARKER, bytes(4)) * 300 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
|
||||
)
|
||||
SEGMENTS_1030_JPEG: Final = (
|
||||
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 1030 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
|
||||
)
|
||||
FILL_BYTES_JPEG: Final = JPEG_SOI + b"\xff" * 1100 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
|
||||
OVERSIZED_HEADER_JPEG: Final = (
|
||||
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(65533)) * 260 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
|
||||
)
|
||||
PROGRESSIVE_JPEG: Final = JPEG_SOI + _jpeg_sof_segment(JPEG_SOF2_MARKER, 111, 55)
|
||||
DHT_JPEG: Final = JPEG_SOI + _jpeg_segment(JPEG_DHT, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300)
|
||||
|
||||
|
|
@ -116,7 +129,11 @@ def _bmp(width: int, height: int) -> bytes:
|
|||
pytest.param(SHORT_LENGTH_JPEG, None, id="jpeg-segment-length-below-2"),
|
||||
pytest.param(TRUNCATED_SOF_JPEG, None, id="jpeg-truncated-sof-payload"),
|
||||
pytest.param(SEGMENTS_63_JPEG, (320, 200), id="jpeg-sof-at-segment-64-boundary"),
|
||||
pytest.param(SEGMENTS_65_JPEG, None, id="jpeg-more-than-64-segments"),
|
||||
pytest.param(SEGMENTS_65_JPEG, (320, 200), id="jpeg-65-segments-within-budget"),
|
||||
pytest.param(SEGMENTS_300_JPEG, (320, 200), id="jpeg-300-app2-segments"),
|
||||
pytest.param(SEGMENTS_1030_JPEG, None, id="jpeg-more-than-1024-segments"),
|
||||
pytest.param(FILL_BYTES_JPEG, (320, 200), id="jpeg-1100-fill-bytes-before-sof"),
|
||||
pytest.param(OVERSIZED_HEADER_JPEG, None, id="jpeg-sof-beyond-16mib"),
|
||||
pytest.param(PROGRESSIVE_JPEG, (111, 55), id="jpeg-sof2-progressive"),
|
||||
pytest.param(DHT_JPEG, (400, 300), id="jpeg-dht-skipped-before-sof0"),
|
||||
pytest.param(_webp_chunk(b"VP8 ", VP8_BODY), (500, 250), id="webp-vp8-lossy"),
|
||||
|
|
@ -130,6 +147,8 @@ def _bmp(width: int, height: int) -> bytes:
|
|||
pytest.param(b"GIF89a" + bytes(2), None, id="gif-shorter-than-10-bytes"),
|
||||
pytest.param(_bmp(1024, 768), (1024, 768), id="bmp-positive-height"),
|
||||
pytest.param(_bmp(1024, -768), (1024, 768), id="bmp-negative-height"),
|
||||
pytest.param(_bmp(-1024, 768), None, id="bmp-negative-width"),
|
||||
pytest.param(_bmp(0, 768), None, id="bmp-zero-width"),
|
||||
pytest.param(b"BM" + bytes(18), None, id="bmp-shorter-than-26-bytes"),
|
||||
pytest.param(
|
||||
b"BM" + bytes(12) + struct.pack("<I", 12) + struct.pack("<HH", 640, 480) + bytes(4),
|
||||
|
|
@ -139,7 +158,7 @@ def _bmp(width: int, height: int) -> bytes:
|
|||
pytest.param(b"II*\x00" + bytes(28), None, id="unknown-format"),
|
||||
),
|
||||
)
|
||||
def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected):
|
||||
def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected: tuple[int, int] | None):
|
||||
assert measure_reference_image(header) == expected
|
||||
|
||||
|
||||
|
|
@ -207,6 +226,38 @@ def test_measure_reference_image_returns_none_for_unreadable_bytes_paths_tuples_
|
|||
assert measure_reference_image(("ref.png", io.BytesIO(png), "image/png")) == (64, 64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sof_marker", (0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF))
|
||||
def test_measure_reference_image_reads_every_jpeg_sof_marker(sof_marker: int):
|
||||
assert measure_reference_image(JPEG_SOI + _jpeg_sof_segment(sof_marker, 320, 200)) == (320, 200)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("skipped_marker", (JPEG_JPG_MARKER, JPEG_DAC_MARKER))
|
||||
def test_measure_reference_image_skips_non_sof_c_range_markers(skipped_marker: int):
|
||||
jpeg: Final = JPEG_SOI + _jpeg_segment(skipped_marker, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300)
|
||||
|
||||
assert measure_reference_image(jpeg) == (400, 300)
|
||||
|
||||
|
||||
class _ReadOnlyObject:
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
return b""
|
||||
|
||||
|
||||
def test_measure_reference_pixels_returns_none_for_closed_and_read_only_streams():
|
||||
closed: Final = io.BytesIO(_png(64, 64))
|
||||
closed.close()
|
||||
|
||||
assert measure_reference_pixels([closed]) is None
|
||||
assert measure_reference_pixels([cast(Any, _ReadOnlyObject())]) is None
|
||||
|
||||
|
||||
def test_measure_reference_pixels_returns_none_when_a_reference_has_invalid_dimensions():
|
||||
negative_width_bmp: Final = _bmp(-1024, 768)
|
||||
|
||||
assert measure_reference_image(negative_width_bmp) is None
|
||||
assert measure_reference_pixels([negative_width_bmp, _png(1024, 1024)]) is None
|
||||
|
||||
|
||||
def test_measure_reference_pixels_returns_none_when_any_reference_is_unmeasurable():
|
||||
assert measure_reference_pixels([_png(64, 64), CAT_JPEG.read_bytes()]) == 64 * 64 + 512 * 512
|
||||
assert measure_reference_pixels([_png(64, 64), _gif(b"GIF89a", 100, 50)]) == 64 * 64 + 100 * 50
|
||||
|
|
|
|||
|
|
@ -336,6 +336,64 @@ def test_unlisted_azure_ai_model_bills_deployment_reference_rate() -> None:
|
|||
assert cost == pytest.approx(1e-07 * 1048576 * 2)
|
||||
|
||||
|
||||
def test_flux2_flex_reference_cost_honors_explicit_zero_deployment_rate() -> None:
|
||||
response: Final = ImageResponse(
|
||||
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
|
||||
)
|
||||
|
||||
cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator(
|
||||
model="FLUX.2-flex",
|
||||
completion_response=response,
|
||||
custom_llm_provider="azure_ai",
|
||||
size="1024x1024",
|
||||
call_type="image_edit",
|
||||
model_info={"input_cost_per_pixel": 2e-07, "input_cost_per_reference_pixel": 0.0},
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(2e-07 * 1048576)
|
||||
|
||||
|
||||
def test_flux2_flex_generated_cost_honors_explicit_zero_deployment_rate() -> None:
|
||||
response: Final = ImageResponse(
|
||||
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
|
||||
)
|
||||
|
||||
cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator(
|
||||
model="FLUX.2-flex",
|
||||
completion_response=response,
|
||||
custom_llm_provider="azure_ai",
|
||||
size="1024x1024",
|
||||
call_type="image_edit",
|
||||
model_info={"input_cost_per_pixel": 0.0, "input_cost_per_reference_pixel": 1e-07},
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(1e-07 * 1048576)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reference_pixels", (1048576.0, "1048576"))
|
||||
def test_flux2_flex_bills_only_integer_reference_pixels(reference_pixels: object) -> None:
|
||||
response: Final = ImageResponse(
|
||||
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": reference_pixels}, size="1024x1024"
|
||||
)
|
||||
|
||||
cost: Final = _flux2_flex_cost(response)
|
||||
|
||||
assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576)
|
||||
|
||||
|
||||
def test_flux2_flex_bills_zero_reference_cost_when_catalog_rate_is_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost["azure_ai/FLUX.2-flex"], "input_cost_per_reference_pixel", None)
|
||||
litellm.get_model_info.cache_clear()
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
response: Final = ImageResponse(
|
||||
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
|
||||
)
|
||||
|
||||
cost: Final = _flux2_flex_cost(response)
|
||||
|
||||
assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576)
|
||||
|
||||
|
||||
def test_unlisted_azure_ai_model_bills_deployment_input_cost_per_pixel() -> None:
|
||||
response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")])
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from litellm.llms.azure_ai.image_edit.flux2_transformation import (
|
|||
from litellm.llms.azure_ai.image_edit.transformation import (
|
||||
AzureFoundryFluxImageEditConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
CAT_JPEG: Final = Path(__file__).parents[4] / "e2e" / "llm_translation" / "fixtures" / "cat.jpg"
|
||||
CAT_JPEG_WIDTH: Final = 512
|
||||
|
|
@ -48,6 +48,16 @@ def _flux2_edit_client(bodies: list[dict[str, object]]) -> HTTPHandler:
|
|||
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
|
||||
|
||||
|
||||
def _flux2_edit_async_client(bodies: list[dict[str, object]]) -> AsyncHTTPHandler:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
bodies.append(json.loads(request.content))
|
||||
return httpx.Response(200, json={"data": [{"b64_json": "aW1n"}]})
|
||||
|
||||
client = AsyncHTTPHandler()
|
||||
client.client = httpx.AsyncClient(transport=httpx.MockTransport(respond))
|
||||
return client
|
||||
|
||||
|
||||
def test_azure_ai_validate_environment():
|
||||
"""Test Azure AI environment validation"""
|
||||
config = AzureFoundryFluxImageEditConfig()
|
||||
|
|
@ -175,7 +185,9 @@ def test_flux2_image_edit_rejects_too_many_references(model: str, reference_imag
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"}))
|
||||
@pytest.mark.parametrize(
|
||||
"dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"})
|
||||
)
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[str, int | str]):
|
||||
png: Final = _png(1024, 1024)
|
||||
|
|
@ -209,7 +221,8 @@ def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[
|
|||
assert stream.tell() == len(png)
|
||||
assert response._hidden_params["reference_pixels"] == 1024 * 1024
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2
|
||||
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 1024 * 1024
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -231,7 +244,10 @@ def test_flux2_image_edit_bills_jpeg_reference_and_sends_whole_file():
|
|||
assert [body["input_image"] for body in bodies] == [base64.b64encode(jpeg).decode()]
|
||||
assert response._hidden_params["reference_pixels"] == CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2
|
||||
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"]
|
||||
* CAT_JPEG_WIDTH
|
||||
* CAT_JPEG_HEIGHT
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -252,10 +268,30 @@ def test_flux2_image_edit_sums_ten_references():
|
|||
assert [body["input_image_10"] for body in bodies] == [base64.b64encode(pngs[-1]).decode()]
|
||||
assert response._hidden_params["reference_pixels"] == 10 * 64 * 64
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 10 * 64 * 64
|
||||
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2
|
||||
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 10 * 64 * 64
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
async def test_aimage_edit_measures_reference_pixels():
|
||||
png: Final = _png(64, 64)
|
||||
bodies: Final[list[dict[str, object]]] = []
|
||||
response: Final = await litellm.aimage_edit(
|
||||
model="azure_ai/FLUX.2-flex",
|
||||
image=[io.BytesIO(png), io.BytesIO(png)],
|
||||
prompt="Blend every reference",
|
||||
api_key="test-key",
|
||||
api_base="https://example.services.ai.azure.com",
|
||||
client=_flux2_edit_async_client(bodies),
|
||||
size="1024x1024",
|
||||
)
|
||||
|
||||
assert [body["input_image_2"] for body in bodies] == [base64.b64encode(png).decode()]
|
||||
assert response._hidden_params["reference_pixels"] == 2 * 64 * 64
|
||||
|
||||
|
||||
def test_flux2_image_edit_accepts_and_drops_openai_only_parameters():
|
||||
optional_params: Final = ImageEditRequestUtils.get_optional_params_image_edit(
|
||||
model="FLUX.2-pro",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue