From 0bb9c54d11a8a043c4563eef7db1f2f1e2cdbff4 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 23 Sep 2026 07:27:22 +0000 Subject: [PATCH] fix(images): harden reference header measurement and cover pricing edge cases --- litellm/images/utils.py | 32 ++++++---- .../images/test_image_edit_utils.py | 57 +++++++++++++++++- .../test_azure_ai_flux2_image_generation.py | 58 +++++++++++++++++++ ...test_azure_ai_image_edit_transformation.py | 46 +++++++++++++-- 4 files changed, 175 insertions(+), 18 deletions(-) diff --git a/litellm/images/utils.py b/litellm/images/utils.py index caefa8114ec..613020de899 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -13,7 +13,8 @@ from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import FileTypes _JPEG_SOF_MARKERS: Final = frozenset(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} -_MAX_JPEG_SEGMENTS: Final = 64 +_MAX_JPEG_SEGMENTS: Final = 1024 +_MAX_JPEG_HEADER_OFFSET: Final = 16 * 1024 * 1024 _JPEG_FIRST_SEGMENT_OFFSET: Final = 2 _JPEG_SOF_PAYLOAD_SIZE: Final = 5 _HEADER_READ_SIZE: Final = 32 @@ -77,12 +78,14 @@ def _webp_dimensions(head: bytes) -> tuple[int, int] | None: def _jpeg_sof_dimensions(stream: IO[bytes], start: int, offset: int, segments_left: int) -> tuple[int, int] | None: segment_offset = offset # rebind-ok: advanced one JPEG segment per hop for _ in range(segments_left): + if segment_offset > _MAX_JPEG_HEADER_OFFSET: + return None stream.seek(start + segment_offset) marker = stream.read(4) if len(marker) < 4 or marker[0] != 0xFF: return None if marker[1] == 0xFF: - segment_offset += 1 + segment_offset += 1 + int(marker[2] == 0xFF) + int(marker[3] == 0xFF) continue if marker[1] in _JPEG_SOF_MARKERS: sof = stream.read(_JPEG_SOF_PAYLOAD_SIZE) @@ -114,17 +117,25 @@ def _header_dimensions(stream: IO[bytes], position: int) -> tuple[int, int] | No return None -def measure_reference_image(image: FileTypes) -> tuple[int, int] | None: - stream: Final = _content_stream(image) - if stream is None: - return None +def _measure_stream(stream: IO[bytes]) -> tuple[int, int] | None: position: Final = stream.tell() try: - return _header_dimensions(stream, position) - except (OSError, ValueError, struct.error): - return None + dimensions: Final = _header_dimensions(stream, position) finally: stream.seek(position) + if dimensions is None or dimensions[0] <= 0 or dimensions[1] <= 0: + return None + return dimensions + + +def measure_reference_image(image: FileTypes) -> tuple[int, int] | None: + try: + stream: Final = _content_stream(image) + if stream is None: + return None + return _measure_stream(stream) + except (OSError, ValueError, AttributeError, struct.error): + return None def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None: @@ -132,7 +143,8 @@ def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None: dimensions: Final = tuple(size for size in measured if size is not None) if len(dimensions) != len(measured): verbose_logger.debug( - "Reference image %d has no readable PNG, JPEG, WebP, GIF or BMP header; billing generated pixels only", + "Reference image %d could not be measured (path, non-seekable stream, or no readable PNG, JPEG, WebP, " + "GIF or BMP header); billing generated pixels only", measured.index(None), ) return None diff --git a/tests/test_litellm/images/test_image_edit_utils.py b/tests/test_litellm/images/test_image_edit_utils.py index fadb5c362ec..a97766d34d3 100644 --- a/tests/test_litellm/images/test_image_edit_utils.py +++ b/tests/test_litellm/images/test_image_edit_utils.py @@ -2,7 +2,7 @@ import io import struct import zlib from pathlib import Path -from typing import Any, Dict, Final, List, Optional +from typing import Any, Dict, Final, List, Optional, cast from unittest.mock import MagicMock, patch import pytest @@ -79,7 +79,10 @@ def _webp_chunk(fourcc: bytes, body: bytes) -> bytes: JPEG_APP0: Final = 0xE0 +JPEG_APP2_MARKER: Final = 0xE2 JPEG_DHT: Final = 0xC4 +JPEG_JPG_MARKER: Final = 0xC8 +JPEG_DAC_MARKER: Final = 0xCC JPEG_SOF0_MARKER: Final = 0xC0 JPEG_SOF2_MARKER: Final = 0xC2 @@ -92,6 +95,16 @@ SEGMENTS_63_JPEG: Final = ( SEGMENTS_65_JPEG: Final = ( JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 65 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200) ) +SEGMENTS_300_JPEG: Final = ( + JPEG_SOI + _jpeg_segment(JPEG_APP2_MARKER, bytes(4)) * 300 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200) +) +SEGMENTS_1030_JPEG: Final = ( + JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 1030 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200) +) +FILL_BYTES_JPEG: Final = JPEG_SOI + b"\xff" * 1100 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200) +OVERSIZED_HEADER_JPEG: Final = ( + JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(65533)) * 260 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200) +) PROGRESSIVE_JPEG: Final = JPEG_SOI + _jpeg_sof_segment(JPEG_SOF2_MARKER, 111, 55) DHT_JPEG: Final = JPEG_SOI + _jpeg_segment(JPEG_DHT, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300) @@ -116,7 +129,11 @@ def _bmp(width: int, height: int) -> bytes: pytest.param(SHORT_LENGTH_JPEG, None, id="jpeg-segment-length-below-2"), pytest.param(TRUNCATED_SOF_JPEG, None, id="jpeg-truncated-sof-payload"), pytest.param(SEGMENTS_63_JPEG, (320, 200), id="jpeg-sof-at-segment-64-boundary"), - pytest.param(SEGMENTS_65_JPEG, None, id="jpeg-more-than-64-segments"), + pytest.param(SEGMENTS_65_JPEG, (320, 200), id="jpeg-65-segments-within-budget"), + pytest.param(SEGMENTS_300_JPEG, (320, 200), id="jpeg-300-app2-segments"), + pytest.param(SEGMENTS_1030_JPEG, None, id="jpeg-more-than-1024-segments"), + pytest.param(FILL_BYTES_JPEG, (320, 200), id="jpeg-1100-fill-bytes-before-sof"), + pytest.param(OVERSIZED_HEADER_JPEG, None, id="jpeg-sof-beyond-16mib"), pytest.param(PROGRESSIVE_JPEG, (111, 55), id="jpeg-sof2-progressive"), pytest.param(DHT_JPEG, (400, 300), id="jpeg-dht-skipped-before-sof0"), pytest.param(_webp_chunk(b"VP8 ", VP8_BODY), (500, 250), id="webp-vp8-lossy"), @@ -130,6 +147,8 @@ def _bmp(width: int, height: int) -> bytes: pytest.param(b"GIF89a" + bytes(2), None, id="gif-shorter-than-10-bytes"), pytest.param(_bmp(1024, 768), (1024, 768), id="bmp-positive-height"), pytest.param(_bmp(1024, -768), (1024, 768), id="bmp-negative-height"), + pytest.param(_bmp(-1024, 768), None, id="bmp-negative-width"), + pytest.param(_bmp(0, 768), None, id="bmp-zero-width"), pytest.param(b"BM" + bytes(18), None, id="bmp-shorter-than-26-bytes"), pytest.param( b"BM" + bytes(12) + struct.pack(" bytes: pytest.param(b"II*\x00" + bytes(28), None, id="unknown-format"), ), ) -def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected): +def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected: tuple[int, int] | None): assert measure_reference_image(header) == expected @@ -207,6 +226,38 @@ def test_measure_reference_image_returns_none_for_unreadable_bytes_paths_tuples_ assert measure_reference_image(("ref.png", io.BytesIO(png), "image/png")) == (64, 64) +@pytest.mark.parametrize("sof_marker", (0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF)) +def test_measure_reference_image_reads_every_jpeg_sof_marker(sof_marker: int): + assert measure_reference_image(JPEG_SOI + _jpeg_sof_segment(sof_marker, 320, 200)) == (320, 200) + + +@pytest.mark.parametrize("skipped_marker", (JPEG_JPG_MARKER, JPEG_DAC_MARKER)) +def test_measure_reference_image_skips_non_sof_c_range_markers(skipped_marker: int): + jpeg: Final = JPEG_SOI + _jpeg_segment(skipped_marker, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300) + + assert measure_reference_image(jpeg) == (400, 300) + + +class _ReadOnlyObject: + def read(self, size: int = -1) -> bytes: + return b"" + + +def test_measure_reference_pixels_returns_none_for_closed_and_read_only_streams(): + closed: Final = io.BytesIO(_png(64, 64)) + closed.close() + + assert measure_reference_pixels([closed]) is None + assert measure_reference_pixels([cast(Any, _ReadOnlyObject())]) is None + + +def test_measure_reference_pixels_returns_none_when_a_reference_has_invalid_dimensions(): + negative_width_bmp: Final = _bmp(-1024, 768) + + assert measure_reference_image(negative_width_bmp) is None + assert measure_reference_pixels([negative_width_bmp, _png(1024, 1024)]) is None + + def test_measure_reference_pixels_returns_none_when_any_reference_is_unmeasurable(): assert measure_reference_pixels([_png(64, 64), CAT_JPEG.read_bytes()]) == 64 * 64 + 512 * 512 assert measure_reference_pixels([_png(64, 64), _gif(b"GIF89a", 100, 50)]) == 64 * 64 + 100 * 50 diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index 7095ef643ce..5b5656c4fc2 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -336,6 +336,64 @@ def test_unlisted_azure_ai_model_bills_deployment_reference_rate() -> None: assert cost == pytest.approx(1e-07 * 1048576 * 2) +def test_flux2_flex_reference_cost_honors_explicit_zero_deployment_rate() -> None: + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024" + ) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="FLUX.2-flex", + completion_response=response, + custom_llm_provider="azure_ai", + size="1024x1024", + call_type="image_edit", + model_info={"input_cost_per_pixel": 2e-07, "input_cost_per_reference_pixel": 0.0}, + ) + + assert cost == pytest.approx(2e-07 * 1048576) + + +def test_flux2_flex_generated_cost_honors_explicit_zero_deployment_rate() -> None: + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024" + ) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="FLUX.2-flex", + completion_response=response, + custom_llm_provider="azure_ai", + size="1024x1024", + call_type="image_edit", + model_info={"input_cost_per_pixel": 0.0, "input_cost_per_reference_pixel": 1e-07}, + ) + + assert cost == pytest.approx(1e-07 * 1048576) + + +@pytest.mark.parametrize("reference_pixels", (1048576.0, "1048576")) +def test_flux2_flex_bills_only_integer_reference_pixels(reference_pixels: object) -> None: + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": reference_pixels}, size="1024x1024" + ) + + cost: Final = _flux2_flex_cost(response) + + assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576) + + +def test_flux2_flex_bills_zero_reference_cost_when_catalog_rate_is_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost["azure_ai/FLUX.2-flex"], "input_cost_per_reference_pixel", None) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024" + ) + + cost: Final = _flux2_flex_cost(response) + + assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576) + + def test_unlisted_azure_ai_model_bills_deployment_input_cost_per_pixel() -> None: response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")]) diff --git a/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py b/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py index dabcbf01208..7637ddcceba 100644 --- a/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py +++ b/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py @@ -18,7 +18,7 @@ from litellm.llms.azure_ai.image_edit.flux2_transformation import ( from litellm.llms.azure_ai.image_edit.transformation import ( AzureFoundryFluxImageEditConfig, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler CAT_JPEG: Final = Path(__file__).parents[4] / "e2e" / "llm_translation" / "fixtures" / "cat.jpg" CAT_JPEG_WIDTH: Final = 512 @@ -48,6 +48,16 @@ def _flux2_edit_client(bodies: list[dict[str, object]]) -> HTTPHandler: return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) +def _flux2_edit_async_client(bodies: list[dict[str, object]]) -> AsyncHTTPHandler: + def respond(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"data": [{"b64_json": "aW1n"}]}) + + client = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + return client + + def test_azure_ai_validate_environment(): """Test Azure AI environment validation""" config = AzureFoundryFluxImageEditConfig() @@ -175,7 +185,9 @@ def test_flux2_image_edit_rejects_too_many_references(model: str, reference_imag ) -@pytest.mark.parametrize("dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"})) +@pytest.mark.parametrize( + "dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"}) +) @pytest.mark.usefixtures("local_model_cost_map") def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[str, int | str]): png: Final = _png(1024, 1024) @@ -209,7 +221,8 @@ def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[ assert stream.tell() == len(png) assert response._hidden_params["reference_pixels"] == 1024 * 1024 assert response._hidden_params["response_cost"] == pytest.approx( - litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 1024 * 1024 ) @@ -231,7 +244,10 @@ def test_flux2_image_edit_bills_jpeg_reference_and_sends_whole_file(): assert [body["input_image"] for body in bodies] == [base64.b64encode(jpeg).decode()] assert response._hidden_params["reference_pixels"] == CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT assert response._hidden_params["response_cost"] == pytest.approx( - litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] + * CAT_JPEG_WIDTH + * CAT_JPEG_HEIGHT ) @@ -252,10 +268,30 @@ def test_flux2_image_edit_sums_ten_references(): assert [body["input_image_10"] for body in bodies] == [base64.b64encode(pngs[-1]).decode()] assert response._hidden_params["reference_pixels"] == 10 * 64 * 64 assert response._hidden_params["response_cost"] == pytest.approx( - litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 10 * 64 * 64 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2 + + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 10 * 64 * 64 ) +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_model_cost_map") +async def test_aimage_edit_measures_reference_pixels(): + png: Final = _png(64, 64) + bodies: Final[list[dict[str, object]]] = [] + response: Final = await litellm.aimage_edit( + model="azure_ai/FLUX.2-flex", + image=[io.BytesIO(png), io.BytesIO(png)], + prompt="Blend every reference", + api_key="test-key", + api_base="https://example.services.ai.azure.com", + client=_flux2_edit_async_client(bodies), + size="1024x1024", + ) + + assert [body["input_image_2"] for body in bodies] == [base64.b64encode(png).decode()] + assert response._hidden_params["reference_pixels"] == 2 * 64 * 64 + + def test_flux2_image_edit_accepts_and_drops_openai_only_parameters(): optional_params: Final = ImageEditRequestUtils.get_optional_params_image_edit( model="FLUX.2-pro",