diff --git a/litellm/images/utils.py b/litellm/images/utils.py index 24454954714..5107fec82bd 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -1,12 +1,116 @@ -from collections.abc import Collection, Mapping +import os +import struct +from collections.abc import Collection, Mapping, Sequence from io import BufferedReader, BytesIO -from typing import Any, Final, cast, get_type_hints +from typing import IO, Any, Final, cast, get_type_hints import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.token_counter import get_image_type from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.types.files import FILE_MIME_TYPES, FileType from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.llms.openai import FileTypes + +_JPEG_SOF_MARKERS: Final = frozenset(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} +_MAX_JPEG_SEGMENTS: Final = 64 +_JPEG_FIRST_SEGMENT_OFFSET: Final = 2 +_JPEG_SOF_PAYLOAD_SIZE: Final = 5 +_HEADER_READ_SIZE: Final = 32 +_MIN_PNG_HEADER_SIZE: Final = 24 +_MIN_WEBP_HEADER_SIZE: Final = 30 + + +def _content_stream(image: FileTypes) -> IO[bytes] | None: + content: Final = image[1] if isinstance(image, tuple) else image + if isinstance(content, bytes): + return BytesIO(content) + if isinstance(content, (str, os.PathLike)): + return None + return content if content.seekable() else None + + +def _png_dimensions(head: bytes) -> tuple[int, int] | None: + if len(head) < _MIN_PNG_HEADER_SIZE: + return None + width, height = struct.unpack(">II", head[16:24]) + return width, height + + +def _webp_dimensions(head: bytes) -> tuple[int, int] | None: + if len(head) < _MIN_WEBP_HEADER_SIZE: + return None + match head[12:16]: + case b"VP8X": + return int.from_bytes(head[24:27], "little") + 1, int.from_bytes(head[27:30], "little") + 1 + case b"VP8 ": + width, height = struct.unpack("> 14) & 0x3FFF) + 1 + case _: + return None + + +def _jpeg_sof_dimensions(stream: IO[bytes], start: int, offset: int, segments_left: int) -> tuple[int, int] | None: + if segments_left == 0: + return None + stream.seek(start + offset) + marker: Final = stream.read(4) + if len(marker) < 4 or marker[0] != 0xFF: + return None + if marker[1] == 0xFF: + return _jpeg_sof_dimensions(stream, start, offset + 1, segments_left - 1) + if marker[1] in _JPEG_SOF_MARKERS: + sof: Final = stream.read(_JPEG_SOF_PAYLOAD_SIZE) + if len(sof) < _JPEG_SOF_PAYLOAD_SIZE: + return None + _precision, height, width = struct.unpack(">BHH", sof) + return width, height + segment_length: Final = int.from_bytes(marker[2:4], "big") + if segment_length < 2: + return None + return _jpeg_sof_dimensions(stream, start, offset + 2 + segment_length, segments_left - 1) + + +def _header_dimensions(stream: IO[bytes], position: int) -> tuple[int, int] | None: + stream.seek(position) + head: Final = stream.read(_HEADER_READ_SIZE) + match get_image_type(head): + case "png": + return _png_dimensions(head) + case "webp": + return _webp_dimensions(head) + case "jpeg": + return _jpeg_sof_dimensions(stream, position, _JPEG_FIRST_SEGMENT_OFFSET, _MAX_JPEG_SEGMENTS) + case _: + return None + + +def measure_reference_image(image: FileTypes) -> tuple[int, int] | None: + stream: Final = _content_stream(image) + if stream is None: + return None + position: Final = stream.tell() + try: + return _header_dimensions(stream, position) + except (OSError, ValueError, struct.error): + return None + finally: + stream.seek(position) + + +def measure_reference_pixels(images: Sequence[FileTypes]) -> int | None: + measured: Final = tuple(measure_reference_image(image) for image in images) + dimensions: Final = tuple(size for size in measured if size is not None) + if len(dimensions) != len(measured): + verbose_logger.debug( + "Reference image %d has no readable PNG, JPEG or WebP header; billing generated pixels only", + measured.index(None), + ) + return None + return sum(width * height for width, height in dimensions) class ImageEditRequestUtils: diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index 08b732197f3..89dad451604 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -21,6 +21,12 @@ def _input_cost_per_pixel(resolved: ModelInfo) -> float: return shared_entry.get("input_cost_per_pixel") or 0.0 +def _reference_cost(model_cost: Mapping[str, object], image_response: ImageResponse) -> float: + pixels: Final = image_response._hidden_params.get("reference_pixels") + rate: Final = model_cost.get("input_cost_per_reference_pixel") or 0.0 + return float(rate) * pixels if isinstance(pixels, int) and isinstance(rate, (int, float)) else 0.0 + + def cost_calculator( model: str, image_response: Any, @@ -48,28 +54,32 @@ def cost_calculator( if token_based_cost is not None: return token_based_cost + from litellm.cost_calculator import default_image_cost_calculator + num_images: Final = n if n is not None else len(image_response.data or ()) + model_cost: Final = litellm.model_cost[_model_info["key"]] output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 - if output_cost_per_image: - return output_cost_per_image * num_images - - if _input_cost_per_pixel(_model_info): - from litellm.cost_calculator import default_image_cost_calculator - - width: Final = optional_params.get("width") if optional_params else None - height: Final = optional_params.get("height") if optional_params else None - pixel_size: Final = ( - f"{width}x{height}" - if type(width) is int and type(height) is int and width > 0 and height > 0 - else size or image_response.size - ) - return default_image_cost_calculator( + input_cost_per_pixel: Final[float] = _input_cost_per_pixel(_model_info) + width: Final = optional_params.get("width") if optional_params else None + height: Final = optional_params.get("height") if optional_params else None + pixel_size: Final = ( + f"{width}x{height}" + if type(width) is int and type(height) is int and width > 0 and height > 0 + else size or image_response.size + ) + generated_cost: Final[float] = ( + output_cost_per_image * num_images + if output_cost_per_image + else default_image_cost_calculator( model=_model_info.get("key", model), custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, size=pixel_size, n=num_images, model_info=model_info, ) - return 0.0 + if input_cost_per_pixel + else 0.0 + ) + return generated_cost + _reference_cost(model_cost, image_response) raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8cbc28362a8..2229632dd93 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -33,6 +33,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult +from litellm.images.utils import measure_reference_pixels from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, @@ -382,6 +383,12 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]: ) +def _with_reference_pixels(response: ImageResponse, reference_pixels: int | None) -> ImageResponse: + if reference_pixels is not None: + response._hidden_params["reference_pixels"] = reference_pixels # rebind-ok: cost calculators read hidden params + return response + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -6881,6 +6888,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) + reference_pixels: Final = measure_reference_pixels(image if isinstance(image, list) else [image]) data, files = image_edit_provider_config.transform_image_edit_request( model=model, image=image, @@ -6928,10 +6936,13 @@ class BaseLLMHTTPHandler: provider_config=image_edit_provider_config, ) - return image_edit_provider_config.transform_image_edit_response( - model=model, - raw_response=response, - logging_obj=logging_obj, + return _with_reference_pixels( + image_edit_provider_config.transform_image_edit_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ), + reference_pixels, ) async def async_image_edit_handler( @@ -6980,6 +6991,7 @@ class BaseLLMHTTPHandler: litellm_params=dict(litellm_params), ) + reference_pixels: Final = measure_reference_pixels(image if isinstance(image, list) else [image]) data, files = await image_edit_provider_config.async_transform_image_edit_request( model=model, image=image, @@ -7027,10 +7039,13 @@ class BaseLLMHTTPHandler: provider_config=image_edit_provider_config, ) - return image_edit_provider_config.transform_image_edit_response( - model=model, - raw_response=response, - logging_obj=logging_obj, + return _with_reference_pixels( + image_edit_provider_config.transform_image_edit_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ), + reference_pixels, ) def image_generation_handler( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ad66cb76f8b..9afa75022e0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10541,6 +10541,7 @@ }, "azure_ai/FLUX.2-flex": { "input_cost_per_pixel": 5e-08, + "input_cost_per_reference_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_tokens": 32000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ad66cb76f8b..9afa75022e0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10541,6 +10541,7 @@ }, "azure_ai/FLUX.2-flex": { "input_cost_per_pixel": 5e-08, + "input_cost_per_reference_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_tokens": 32000, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 737a9b7fa60..1a93cdb504a 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -318,6 +318,10 @@ "type": "number", "minimum": 0 }, + "input_cost_per_reference_pixel": { + "type": "number", + "minimum": 0 + }, "input_cost_per_request": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/images/test_image_edit_utils.py b/tests/test_litellm/images/test_image_edit_utils.py index 2146c1fab01..fa42ad8e0ea 100644 --- a/tests/test_litellm/images/test_image_edit_utils.py +++ b/tests/test_litellm/images/test_image_edit_utils.py @@ -1,14 +1,126 @@ -from typing import Any, Dict, List, Optional +import io +import struct +import zlib +from pathlib import Path +from typing import Any, Dict, Final, List, Optional from unittest.mock import MagicMock, patch import pytest +from PIL import Image import litellm -from litellm.images.utils import ImageEditRequestUtils +from litellm.images.utils import ( + ImageEditRequestUtils, + measure_reference_image, + measure_reference_pixels, +) from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.types.images.main import ImageEditOptionalRequestParams +CAT_JPEG: Final = Path(__file__).parents[2] / "e2e" / "llm_translation" / "fixtures" / "cat.jpg" +JPEG_SOI: Final = b"\xff\xd8" +JPEG_APP1: Final = b"\xff\xe1" +JPEG_APP2: Final = b"\xff\xe2" +JPEG_SOF0: Final = b"\xff\xc0" +JPEG_MARKER_AND_LENGTH_SIZE: Final = 4 + + +def _png_chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) + + +def _png(width: int, height: int) -> bytes: + header: Final = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) + rows: Final = b"".join(b"\x00" + bytes(width) for _ in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", header) + + _png_chunk(b"IDAT", zlib.compress(rows)) + + _png_chunk(b"IEND", b"") + ) + + +def _pillow_bytes(width: int, height: int, mode: str, image_format: str, **save_options: object) -> bytes: + buffer: Final = io.BytesIO() + Image.new(mode, (width, height)).save(buffer, format=image_format, **save_options) + return buffer.getvalue() + + +def _jpeg_app_segment(marker: bytes, payload_size: int) -> bytes: + return marker + struct.pack(">H", payload_size + 2) + bytes(payload_size) + + +def _jpeg_with_leading_segments(jpeg: bytes, *segments: bytes) -> bytes: + return JPEG_SOI + b"".join(segments) + jpeg[len(JPEG_SOI) :] + + +class _NonSeekableStream(io.BytesIO): + def seekable(self) -> bool: + return False + + +@pytest.mark.parametrize("initial_position", (0, 3)) +def test_measure_reference_image_reads_png_jpeg_webp_headers_and_keeps_position(initial_position: int): + pillow_jpeg: Final = _pillow_bytes(1024, 768, "RGB", "JPEG") + large_app1_jpeg: Final = _jpeg_with_leading_segments(pillow_jpeg, _jpeg_app_segment(JPEG_APP1, 60_000)) + far_sof_jpeg: Final = _jpeg_with_leading_segments( + pillow_jpeg, _jpeg_app_segment(JPEG_APP2, 40_000), _jpeg_app_segment(JPEG_APP2, 40_000) + ) + assert far_sof_jpeg.index(JPEG_SOF0) > 70_000 + cases: Final = ( + (_png(1024, 1024), (1024, 1024)), + (CAT_JPEG.read_bytes(), (512, 512)), + (_pillow_bytes(640, 480, "RGB", "WEBP"), (640, 480)), + (_pillow_bytes(640, 480, "RGB", "WEBP", lossless=True), (640, 480)), + (_pillow_bytes(640, 480, "RGBA", "WEBP"), (640, 480)), + (large_app1_jpeg, (1024, 768)), + (far_sof_jpeg, (1024, 768)), + ) + layouts: Final = tuple(image[12:16] for image, _ in cases[2:5]) + assert layouts == (b"VP8 ", b"VP8L", b"VP8X") + + for image, expected in cases: + stream: Final = io.BytesIO(bytes(initial_position) + image) + stream.seek(initial_position) + assert measure_reference_image(stream) == expected, image[:16] + assert stream.tell() == initial_position, image[:16] + + +def test_measure_reference_image_returns_none_on_truncated_jpeg_and_keeps_position(): + jpeg: Final = _pillow_bytes(1024, 768, "RGB", "JPEG") + sof_offset: Final = jpeg.index(JPEG_SOF0) + prefixes: Final = (32, 160, sof_offset, sof_offset + JPEG_MARKER_AND_LENGTH_SIZE) + + for prefix in prefixes: + stream: Final = io.BytesIO(jpeg[:prefix]) + assert measure_reference_image(stream) is None, prefix + assert stream.tell() == 0, prefix + + +def test_measure_reference_image_returns_none_for_unreadable_bytes_paths_tuples_and_non_seekable_streams( + tmp_path: Path, +): + png: Final = _png(64, 64) + path: Final = tmp_path / "ref.png" + path.write_bytes(png) + non_seekable: Final = _NonSeekableStream(png) + + assert measure_reference_image(io.BytesIO(b"image")) is None + assert measure_reference_image(b"GIF89a" + bytes(26)) is None + assert measure_reference_image(path) is None + assert measure_reference_image(("ref.png", path, "image/png", {})) is None + assert measure_reference_image(non_seekable) is None + assert non_seekable.tell() == 0 + assert measure_reference_image(("ref.png", png)) == (64, 64) + assert measure_reference_image(("ref.png", io.BytesIO(png), "image/png")) == (64, 64) + + +def test_measure_reference_pixels_returns_none_when_any_reference_is_unmeasurable(): + assert measure_reference_pixels([_png(64, 64), CAT_JPEG.read_bytes()]) == 64 * 64 + 512 * 512 + assert measure_reference_pixels([_png(64, 64), io.BytesIO(b"image"), CAT_JPEG.read_bytes()]) is None + assert measure_reference_pixels([]) == 0 + class MockImageEditConfig(BaseImageEditConfig): def get_supported_openai_params(self, model: str) -> List[str]: diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index fdd21c87732..548c964decc 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -14,7 +14,7 @@ from litellm.llms.azure.image_generation.http_utils import azure_deployment_imag from litellm.llms.azure_ai.image_generation.flux_transformation import ( AzureFoundryFluxImageGenerationConfig, ) -from litellm.types.utils import ImageObject, ImageResponse +from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params_image_gen @@ -151,6 +151,83 @@ def test_flux2_flex_cost_uses_generated_megapixels(): assert cost == pytest.approx(5e-08 * 2048 * 1024 * 2) +def _patch_flux2_flex_pricing(monkeypatch: pytest.MonkeyPatch, **pricing: float) -> None: + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/FLUX.2-flex", + { + **{ + key: value + for key, value in litellm.model_cost["azure_ai/FLUX.2-flex"].items() + if not key.startswith(("input_cost_per", "output_cost_per")) + }, + **pricing, + }, + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + + +def _flux2_flex_cost(response: ImageResponse) -> float: + return CostCalculatorUtils.route_image_generation_cost_calculator( + model="FLUX.2-flex", + completion_response=response, + custom_llm_provider="azure_ai", + size="1024x1024", + call_type="image_edit", + ) + + +def test_flux2_flex_cost_adds_reference_pixels_to_generated_pixels(monkeypatch: pytest.MonkeyPatch): + _patch_flux2_flex_pricing(monkeypatch, input_cost_per_pixel=5e-08, input_cost_per_reference_pixel=5e-08) + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024" + ) + + assert _flux2_flex_cost(response) == pytest.approx(5e-08 * 1048576 * 2) + + +def test_flux2_cost_adds_reference_pixels_to_flat_price(monkeypatch: pytest.MonkeyPatch): + _patch_flux2_flex_pricing(monkeypatch, output_cost_per_image=0.04, input_cost_per_reference_pixel=1.5e-08) + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], hidden_params={"reference_pixels": 1048576}, size="1024x1024" + ) + + assert _flux2_flex_cost(response) == pytest.approx(0.04 + 1.5e-08 * 1048576) + + +def test_flux2_cost_ignores_reference_pixels_when_provider_reports_usage(monkeypatch: pytest.MonkeyPatch): + _patch_flux2_flex_pricing( + monkeypatch, + input_cost_per_pixel=5e-08, + input_cost_per_reference_pixel=5e-08, + input_cost_per_token=1e-05, + input_cost_per_image_token=2e-05, + output_cost_per_image_token=4e-05, + output_cost_per_token=4e-05, + ) + response: Final = ImageResponse( + data=[ImageObject(b64_json="aW1n")], + hidden_params={"reference_pixels": 1048576}, + size="1024x1024", + usage=ImageUsage( + input_tokens=150, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=100, text_tokens=50), + output_tokens=1000, + total_tokens=1150, + ), + ) + + assert _flux2_flex_cost(response) == pytest.approx(50 * 1e-05 + 100 * 2e-05 + 1000 * 4e-05) + + +def test_flux2_cost_is_unchanged_without_reference_measurement(monkeypatch: pytest.MonkeyPatch): + _patch_flux2_flex_pricing(monkeypatch, input_cost_per_pixel=5e-08, input_cost_per_reference_pixel=5e-08) + response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n")], size="1024x1024") + + assert _flux2_flex_cost(response) == pytest.approx(5e-08 * 1048576) + + @pytest.mark.parametrize("model", ("FLUX-1.1-pro", "FLUX.1-Kontext-pro")) def test_flux1_preserves_existing_openai_parameters(model: str): params: Final = {"n": 2, "size": "1536x1024", "quality": "high", "user": "test-user"} diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 052278631e2..5b713655bb4 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -408,6 +408,44 @@ def test_deepseek_rows_bill_half_rate_outside_weekday_peak_hours(path: Path): assert all(name in rows for name in DEEPSEEK_PRICED_ROWS) +AZURE_AI_PER_PIXEL_EDIT_ROWS_WITHOUT_A_REFERENCE_METER: Final = frozenset[str]() + + +def is_azure_ai_per_pixel_edit_row(name: str, entry: Mapping[str, object]) -> bool: + endpoints: Final = entry.get("supported_endpoints") + return ( + entry.get("litellm_provider") == "azure_ai" + and "input_cost_per_pixel" in entry + and isinstance(endpoints, list) + and "/v1/images/edits" in endpoints + and name not in AZURE_AI_PER_PIXEL_EDIT_ROWS_WITHOUT_A_REFERENCE_METER + ) + + +def has_non_negative_reference_rate(entry: Mapping[str, object]) -> bool: + rate: Final = entry.get("input_cost_per_reference_pixel") + return isinstance(rate, (int, float)) and rate >= 0 + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_azure_ai_per_pixel_edit_entries_declare_a_reference_rate(path: Path): + """The Azure AI image cost calculator bills reference images through input_cost_per_reference_pixel, + so every azure_ai row priced per generated pixel that serves /v1/images/edits carries that rate.""" + rows: Mapping[str, object] = json.loads(path.read_text()) + per_pixel_edit_rows: Final = { + name: entry + for name, entry in rows.items() + if isinstance(entry, dict) and is_azure_ai_per_pixel_edit_row(name, entry) + } + missing: Final = { + name: entry.get("input_cost_per_reference_pixel") + for name, entry in per_pixel_edit_rows.items() + if not has_non_negative_reference_rate(entry) + } + assert per_pixel_edit_rows != {} + assert missing == {} + + PROVIDER_LABELS_WITHOUT_A_MODEL_SET: Final = frozenset({"sagemaker", "bedrock_converse"}) MODES_SERVED_OUTSIDE_THE_LLM_PROVIDER_REGISTRY: Final = frozenset({"search", "evaluation"}) VERTEX_FAMILIES_A_VERTEX_WILDCARD_GRANT_DOES_NOT_LIST: Final = frozenset( diff --git a/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py b/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py index 51ba2c34cd7..a05f1c3b1b4 100644 --- a/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py +++ b/tests/unit/llms/azure_ai/image_edit/test_azure_ai_image_edit_transformation.py @@ -1,6 +1,10 @@ import base64 +import io import json +import struct +import zlib from collections.abc import Mapping +from pathlib import Path from typing import Final import httpx @@ -16,6 +20,34 @@ from litellm.llms.azure_ai.image_edit.transformation import ( ) from litellm.llms.custom_httpx.http_handler import HTTPHandler +CAT_JPEG: Final = Path(__file__).parents[4] / "e2e" / "llm_translation" / "fixtures" / "cat.jpg" +CAT_JPEG_WIDTH: Final = 512 +CAT_JPEG_HEIGHT: Final = 512 +FLUX2_FLEX_PIXEL_RATE: Final = 5e-08 + + +def _png_chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) + + +def _png(width: int, height: int) -> bytes: + header: Final = struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0) + rows: Final = b"".join(b"\x00" + bytes(width) for _ in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", header) + + _png_chunk(b"IDAT", zlib.compress(rows)) + + _png_chunk(b"IEND", b"") + ) + + +def _flux2_edit_client(bodies: list[dict[str, object]]) -> HTTPHandler: + def respond(request: httpx.Request) -> httpx.Response: + bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"data": [{"b64_json": "aW1n"}, {"b64_json": "aW1n"}]}) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + def test_azure_ai_validate_environment(): """Test Azure AI environment validation""" @@ -147,35 +179,82 @@ def test_flux2_image_edit_rejects_too_many_references(model: str, reference_imag @pytest.mark.parametrize("dimensions", ({"size": "2048x1024"}, {"width": 2048, "height": 1024}, {"width": "2048", "height": "1024"})) @pytest.mark.usefixtures("local_model_cost_map") def test_flux2_image_edit_preserves_controls_and_pixel_cost(dimensions: Mapping[str, int | str]): - def respond(request: httpx.Request) -> httpx.Response: - body: Final = json.loads(request.content) - assert body == { - "model": "FLUX.2-flex", - "prompt": "Add a hat", - "input_image": base64.b64encode(b"image").decode(), - "num_images": 2, - "width": 2048, - "height": 1024, - "guidance": 4.5, - "steps": 32, - } - return httpx.Response(200, json={"data": [{"b64_json": "aW1n"}, {"b64_json": "aW1n"}]}) - - client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + png: Final = _png(1024, 1024) + stream: Final = io.BytesIO(png) + bodies: Final[list[dict[str, object]]] = [] response: Final = litellm.image_edit( model="azure_ai/FLUX.2-flex", - image=b"image", + image=stream, prompt="Add a hat", api_key="test-key", api_base="https://example.services.ai.azure.com", - client=client, + client=_flux2_edit_client(bodies), n=2, guidance="4.5", steps="32", **dimensions, ) - assert response._hidden_params["response_cost"] == pytest.approx(5e-08 * 2048 * 1024 * 2) + assert bodies == [ + { + "model": "FLUX.2-flex", + "prompt": "Add a hat", + "input_image": base64.b64encode(png).decode(), + "num_images": 2, + "width": 2048, + "height": 1024, + "guidance": 4.5, + "steps": 32, + } + ] + assert stream.tell() == len(png) + assert response._hidden_params["reference_pixels"] == 1024 * 1024 + assert response._hidden_params["response_cost"] == pytest.approx( + FLUX2_FLEX_PIXEL_RATE * 2048 * 1024 * 2 + FLUX2_FLEX_PIXEL_RATE * 1024 * 1024 + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_flux2_image_edit_bills_jpeg_reference_and_sends_whole_file(): + jpeg: Final = CAT_JPEG.read_bytes() + bodies: Final[list[dict[str, object]]] = [] + response: Final = litellm.image_edit( + model="azure_ai/FLUX.2-flex", + image=io.BytesIO(jpeg), + prompt="Add a hat", + api_key="test-key", + api_base="https://example.services.ai.azure.com", + client=_flux2_edit_client(bodies), + n=2, + size="2048x1024", + ) + + assert [body["input_image"] for body in bodies] == [base64.b64encode(jpeg).decode()] + assert response._hidden_params["reference_pixels"] == CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT + assert response._hidden_params["response_cost"] == pytest.approx( + FLUX2_FLEX_PIXEL_RATE * 2048 * 1024 * 2 + FLUX2_FLEX_PIXEL_RATE * CAT_JPEG_WIDTH * CAT_JPEG_HEIGHT + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_flux2_image_edit_sums_ten_references(): + pngs: Final = tuple(_png(64, 64) for _ in range(10)) + bodies: Final[list[dict[str, object]]] = [] + response: Final = litellm.image_edit( + model="azure_ai/FLUX.2-flex", + image=[io.BytesIO(png) for png in pngs], + prompt="Blend every reference", + api_key="test-key", + api_base="https://example.services.ai.azure.com", + client=_flux2_edit_client(bodies), + size="1024x1024", + ) + + assert [body["input_image_10"] for body in bodies] == [base64.b64encode(pngs[-1]).decode()] + assert response._hidden_params["reference_pixels"] == 10 * 64 * 64 + assert response._hidden_params["response_cost"] == pytest.approx( + FLUX2_FLEX_PIXEL_RATE * 1024 * 1024 * 2 + FLUX2_FLEX_PIXEL_RATE * 10 * 64 * 64 + ) def test_flux2_image_edit_accepts_and_drops_openai_only_parameters():