fix(images): harden reference header measurement and cover pricing edge cases

This commit is contained in:
shrey kharbanda 2026-09-23 07:27:22 +00:00
parent 93ebd1116e
commit 0bb9c54d11
4 changed files with 175 additions and 18 deletions

View file

@ -13,7 +13,8 @@ from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import FileTypes
_JPEG_SOF_MARKERS: Final = frozenset(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC}
_MAX_JPEG_SEGMENTS: Final = 64
_MAX_JPEG_SEGMENTS: Final = 1024
_MAX_JPEG_HEADER_OFFSET: Final = 16 * 1024 * 1024
_JPEG_FIRST_SEGMENT_OFFSET: Final = 2
_JPEG_SOF_PAYLOAD_SIZE: Final = 5
_HEADER_READ_SIZE: Final = 32
@ -77,12 +78,14 @@ def _webp_dimensions(head: bytes) -> tuple[int, int] | None:
def _jpeg_sof_dimensions(stream: IO[bytes], start: int, offset: int, segments_left: int) -> tuple[int, int] | None:
segment_offset = offset # rebind-ok: advanced one JPEG segment per hop
for _ in range(segments_left):
if segment_offset > _MAX_JPEG_HEADER_OFFSET:
return None
stream.seek(start + segment_offset)
marker = stream.read(4)
if len(marker) < 4 or marker[0] != 0xFF:
return None
if marker[1] == 0xFF:
segment_offset += 1
segment_offset += 1 + int(marker[2] == 0xFF) + int(marker[3] == 0xFF)
continue
if marker[1] in _JPEG_SOF_MARKERS:
sof = stream.read(_JPEG_SOF_PAYLOAD_SIZE)
@ -114,17 +117,25 @@ def _header_dimensions(stream: IO[bytes], position: int) -> tuple[int, int] | No
return None
def measure_reference_image(image: FileTypes) -> tuple[int, int] | None:
stream: Final = _content_stream(image)
if stream is None:
return None
def _measure_stream(stream: IO[bytes]) -> tuple[int, int] | None:
position: Final = stream.tell()
try:
return _header_dimensions(stream, position)
except (OSError, ValueError, struct.error):
return None
dimensions: Final = _header_dimensions(stream, position)
finally:
stream.seek(position)
if dimensions is None or dimensions[0] <= 0 or dimensions[1] <= 0:
return None
return dimensions
def measure_reference_image(image: FileTypes) -> tuple[int, int] | None:
try:
stream: Final = _content_stream(image)
if stream is None:
return None
return _measure_stream(stream)
except (OSError, ValueError, AttributeError, struct.error):
return None
def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None:
@ -132,7 +143,8 @@ def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None:
dimensions: Final = tuple(size for size in measured if size is not None)
if len(dimensions) != len(measured):
verbose_logger.debug(
"Reference image %d has no readable PNG, JPEG, WebP, GIF or BMP header; billing generated pixels only",
"Reference image %d could not be measured (path, non-seekable stream, or no readable PNG, JPEG, WebP, "
"GIF or BMP header); billing generated pixels only",
measured.index(None),
)
return None

View file

@ -2,7 +2,7 @@ import io
import struct
import zlib
from pathlib import Path
from typing import Any, Dict, Final, List, Optional
from typing import Any, Dict, Final, List, Optional, cast
from unittest.mock import MagicMock, patch
import pytest
@ -79,7 +79,10 @@ def _webp_chunk(fourcc: bytes, body: bytes) -> bytes:
JPEG_APP0: Final = 0xE0
JPEG_APP2_MARKER: Final = 0xE2
JPEG_DHT: Final = 0xC4
JPEG_JPG_MARKER: Final = 0xC8
JPEG_DAC_MARKER: Final = 0xCC
JPEG_SOF0_MARKER: Final = 0xC0
JPEG_SOF2_MARKER: Final = 0xC2
@ -92,6 +95,16 @@ SEGMENTS_63_JPEG: Final = (
SEGMENTS_65_JPEG: Final = (
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 65 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
)
SEGMENTS_300_JPEG: Final = (
JPEG_SOI + _jpeg_segment(JPEG_APP2_MARKER, bytes(4)) * 300 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
)
SEGMENTS_1030_JPEG: Final = (
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(4)) * 1030 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
)
FILL_BYTES_JPEG: Final = JPEG_SOI + b"\xff" * 1100 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
OVERSIZED_HEADER_JPEG: Final = (
JPEG_SOI + _jpeg_segment(JPEG_APP0, bytes(65533)) * 260 + _jpeg_sof_segment(JPEG_SOF0_MARKER, 320, 200)
)
PROGRESSIVE_JPEG: Final = JPEG_SOI + _jpeg_sof_segment(JPEG_SOF2_MARKER, 111, 55)
DHT_JPEG: Final = JPEG_SOI + _jpeg_segment(JPEG_DHT, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300)
@ -116,7 +129,11 @@ def _bmp(width: int, height: int) -> bytes:
pytest.param(SHORT_LENGTH_JPEG, None, id="jpeg-segment-length-below-2"),
pytest.param(TRUNCATED_SOF_JPEG, None, id="jpeg-truncated-sof-payload"),
pytest.param(SEGMENTS_63_JPEG, (320, 200), id="jpeg-sof-at-segment-64-boundary"),
pytest.param(SEGMENTS_65_JPEG, None, id="jpeg-more-than-64-segments"),
pytest.param(SEGMENTS_65_JPEG, (320, 200), id="jpeg-65-segments-within-budget"),
pytest.param(SEGMENTS_300_JPEG, (320, 200), id="jpeg-300-app2-segments"),
pytest.param(SEGMENTS_1030_JPEG, None, id="jpeg-more-than-1024-segments"),
pytest.param(FILL_BYTES_JPEG, (320, 200), id="jpeg-1100-fill-bytes-before-sof"),
pytest.param(OVERSIZED_HEADER_JPEG, None, id="jpeg-sof-beyond-16mib"),
pytest.param(PROGRESSIVE_JPEG, (111, 55), id="jpeg-sof2-progressive"),
pytest.param(DHT_JPEG, (400, 300), id="jpeg-dht-skipped-before-sof0"),
pytest.param(_webp_chunk(b"VP8 ", VP8_BODY), (500, 250), id="webp-vp8-lossy"),
@ -130,6 +147,8 @@ def _bmp(width: int, height: int) -> bytes:
pytest.param(b"GIF89a" + bytes(2), None, id="gif-shorter-than-10-bytes"),
pytest.param(_bmp(1024, 768), (1024, 768), id="bmp-positive-height"),
pytest.param(_bmp(1024, -768), (1024, 768), id="bmp-negative-height"),
pytest.param(_bmp(-1024, 768), None, id="bmp-negative-width"),
pytest.param(_bmp(0, 768), None, id="bmp-zero-width"),
pytest.param(b"BM" + bytes(18), None, id="bmp-shorter-than-26-bytes"),
pytest.param(
b"BM" + bytes(12) + struct.pack("<I", 12) + struct.pack("<HH", 640, 480) + bytes(4),
@ -139,7 +158,7 @@ def _bmp(width: int, height: int) -> bytes:
pytest.param(b"II*\x00" + bytes(28), None, id="unknown-format"),
),
)
def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected):
def test_measure_reference_image_parses_hand_built_headers(header: bytes, expected: tuple[int, int] | None):
assert measure_reference_image(header) == expected
@ -207,6 +226,38 @@ def test_measure_reference_image_returns_none_for_unreadable_bytes_paths_tuples_
assert measure_reference_image(("ref.png", io.BytesIO(png), "image/png")) == (64, 64)
@pytest.mark.parametrize("sof_marker", (0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF))
def test_measure_reference_image_reads_every_jpeg_sof_marker(sof_marker: int):
assert measure_reference_image(JPEG_SOI + _jpeg_sof_segment(sof_marker, 320, 200)) == (320, 200)
@pytest.mark.parametrize("skipped_marker", (JPEG_JPG_MARKER, JPEG_DAC_MARKER))
def test_measure_reference_image_skips_non_sof_c_range_markers(skipped_marker: int):
jpeg: Final = JPEG_SOI + _jpeg_segment(skipped_marker, bytes(8)) + _jpeg_sof_segment(JPEG_SOF0_MARKER, 400, 300)
assert measure_reference_image(jpeg) == (400, 300)
class _ReadOnlyObject:
def read(self, size: int = -1) -> bytes:
return b""
def test_measure_reference_pixels_returns_none_for_closed_and_read_only_streams():
closed: Final = io.BytesIO(_png(64, 64))
closed.close()
assert measure_reference_pixels([closed]) is None
assert measure_reference_pixels([cast(Any, _ReadOnlyObject())]) is None
def test_measure_reference_pixels_returns_none_when_a_reference_has_invalid_dimensions():
negative_width_bmp: Final = _bmp(-1024, 768)
assert measure_reference_image(negative_width_bmp) is None
assert measure_reference_pixels([negative_width_bmp, _png(1024, 1024)]) is None
def test_measure_reference_pixels_returns_none_when_any_reference_is_unmeasurable():
assert measure_reference_pixels([_png(64, 64), CAT_JPEG.read_bytes()]) == 64 * 64 + 512 * 512
assert measure_reference_pixels([_png(64, 64), _gif(b"GIF89a", 100, 50)]) == 64 * 64 + 100 * 50

View file

@ -336,6 +336,64 @@ def test_unlisted_azure_ai_model_bills_deployment_reference_rate() -> None:
assert cost == pytest.approx(1e-07 * 1048576 * 2)
def test_flux2_flex_reference_cost_honors_explicit_zero_deployment_rate() -> None:
response: Final = ImageResponse(
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
)
cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator(
model="FLUX.2-flex",
completion_response=response,
custom_llm_provider="azure_ai",
size="1024x1024",
call_type="image_edit",
model_info={"input_cost_per_pixel": 2e-07, "input_cost_per_reference_pixel": 0.0},
)
assert cost == pytest.approx(2e-07 * 1048576)
def test_flux2_flex_generated_cost_honors_explicit_zero_deployment_rate() -> None:
response: Final = ImageResponse(
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
)
cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator(
model="FLUX.2-flex",
completion_response=response,
custom_llm_provider="azure_ai",
size="1024x1024",
call_type="image_edit",
model_info={"input_cost_per_pixel": 0.0, "input_cost_per_reference_pixel": 1e-07},
)
assert cost == pytest.approx(1e-07 * 1048576)
@pytest.mark.parametrize("reference_pixels", (1048576.0, "1048576"))
def test_flux2_flex_bills_only_integer_reference_pixels(reference_pixels: object) -> None:
response: Final = ImageResponse(
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": reference_pixels}, size="1024x1024"
)
cost: Final = _flux2_flex_cost(response)
assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576)
def test_flux2_flex_bills_zero_reference_cost_when_catalog_rate_is_unset(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(litellm.model_cost["azure_ai/FLUX.2-flex"], "input_cost_per_reference_pixel", None)
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
response: Final = ImageResponse(
data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024"
)
cost: Final = _flux2_flex_cost(response)
assert cost == pytest.approx(litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1048576)
def test_unlisted_azure_ai_model_bills_deployment_input_cost_per_pixel() -> None:
response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")])

View file

@ -18,7 +18,7 @@ from litellm.llms.azure_ai.image_edit.flux2_transformation import (
from litellm.llms.azure_ai.image_edit.transformation import (
AzureFoundryFluxImageEditConfig,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
CAT_JPEG: Final = Path(__file__).parents[4] / "e2e" / "llm_translation" / "fixtures" / "cat.jpg"
CAT_JPEG_WIDTH: Final = 512
@ -48,6 +48,16 @@ def _flux2_edit_client(bodies: list[dict[str, object]]) -> HTTPHandler:
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
def _flux2_edit_async_client(bodies: list[dict[str, object]]) -> AsyncHTTPHandler:
def respond(request: httpx.Request) -> httpx.Response:
bodies.append(json.loads(request.content))
return httpx.Response(200, json={"data": [{"b64_json": "aW1n"}]})
client = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(respond))
return client
def test_azure_ai_validate_environment():
"""Test Azure AI environment validation"""
config = AzureFoundryFluxImageEditConfig()
@ -175,7 +185,9 @@ def test_flux2_image_edit_rejects_too_many_references(model: str, reference_imag
)
@pytest.mark.parametrize("dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"}))
@pytest.mark.parametrize(
"dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"})
)
@pytest.mark.usefixtures("local_model_cost_map")
def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[str, int | str]):
png: Final = _png(1024, 1024)
@ -209,7 +221,8 @@ def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[
assert stream.tell() == len(png)
assert response._hidden_params["reference_pixels"] == 1024 * 1024
assert response._hidden_params["response_cost"] == pytest.approx(
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 1024 * 1024
)
@ -231,7 +244,10 @@ def test_flux2_image_edit_bills_jpeg_reference_and_sends_whole_file():
assert [body["input_image"] for body in bodies] == [base64.b64encode(jpeg).decode()]
assert response._hidden_params["reference_pixels"] == CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT
assert response._hidden_params["response_cost"] == pytest.approx(
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 2048 * 1024 * 2
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"]
* CAT_JPEG_WIDTH
* CAT_JPEG_HEIGHT
)
@ -252,10 +268,30 @@ def test_flux2_image_edit_sums_ten_references():
assert [body["input_image_10"] for body in bodies] == [base64.b64encode(pngs[-1]).decode()]
assert response._hidden_params["reference_pixels"] == 10 * 64 * 64
assert response._hidden_params["response_cost"] == pytest.approx(
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2 + litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 10 * 64 * 64
litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_pixel"] * 1024 * 1024 * 2
+ litellm.model_cost["azure_ai/FLUX.2-flex"]["input_cost_per_reference_pixel"] * 10 * 64 * 64
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_model_cost_map")
async def test_aimage_edit_measures_reference_pixels():
png: Final = _png(64, 64)
bodies: Final[list[dict[str, object]]] = []
response: Final = await litellm.aimage_edit(
model="azure_ai/FLUX.2-flex",
image=[io.BytesIO(png), io.BytesIO(png)],
prompt="Blend every reference",
api_key="test-key",
api_base="https://example.services.ai.azure.com",
client=_flux2_edit_async_client(bodies),
size="1024x1024",
)
assert [body["input_image_2"] for body in bodies] == [base64.b64encode(png).decode()]
assert response._hidden_params["reference_pixels"] == 2 * 64 * 64
def test_flux2_image_edit_accepts_and_drops_openai_only_parameters():
optional_params: Final = ImageEditRequestUtils.get_optional_params_image_edit(
model="FLUX.2-pro",