From 1f84f188e7bbcf227fbf5a43bfd5cff09b4b3c33 Mon Sep 17 00:00:00 2001 From: CrystalVibe28 <82704512+CrystalVibe28@users.noreply.github.com> Date: Fri, 15 May 2026 14:34:04 +0800 Subject: [PATCH] feat(chatgpt): support image edits --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 4 + .../llms/chatgpt/image_generation/__init__.py | 4 +- .../image_generation/transformation.py | 190 +++++++++++++++++- litellm/utils.py | 4 + .../test_chatgpt_image_generation.py | 70 +++++++ 6 files changed, 270 insertions(+), 3 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index bd239a4c684..8f6cb0fd137 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1872,6 +1872,7 @@ if TYPE_CHECKING: ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig, ) from .llms.chatgpt.image_generation.transformation import ( + ChatGPTImageEditConfig as ChatGPTImageEditConfig, ChatGPTImageGenerationConfig as ChatGPTImageGenerationConfig, ) from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 3a6b72bc0cb..1d0ec103e0c 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1134,6 +1134,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.chatgpt.image_generation.transformation", "ChatGPTImageGenerationConfig", ), + "ChatGPTImageEditConfig": ( + ".llms.chatgpt.image_generation.transformation", + "ChatGPTImageEditConfig", + ), "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), diff --git a/litellm/llms/chatgpt/image_generation/__init__.py b/litellm/llms/chatgpt/image_generation/__init__.py index 0aff5eba9cb..041ed1611e5 100644 --- a/litellm/llms/chatgpt/image_generation/__init__.py +++ b/litellm/llms/chatgpt/image_generation/__init__.py @@ -1,3 +1,3 @@ -from .transformation import ChatGPTImageGenerationConfig +from .transformation import ChatGPTImageEditConfig, ChatGPTImageGenerationConfig -__all__ = ["ChatGPTImageGenerationConfig"] +__all__ = ["ChatGPTImageEditConfig", "ChatGPTImageGenerationConfig"] diff --git a/litellm/llms/chatgpt/image_generation/transformation.py b/litellm/llms/chatgpt/image_generation/transformation.py index 1c27464f807..72e69eeb337 100644 --- a/litellm/llms/chatgpt/image_generation/transformation.py +++ b/litellm/llms/chatgpt/image_generation/transformation.py @@ -1,20 +1,29 @@ import json import re +import base64 +from io import BufferedReader, BytesIO +from os import PathLike from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx +from httpx._types import RequestFiles from litellm.constants import STREAM_SSE_DONE_STRING from litellm.exceptions import AuthenticationError +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.llms.openai.common_utils import OpenAIError from litellm.types.llms.openai import ( AllMessageValues, + FileTypes, OpenAIImageGenerationOptionalParams, ResponsesAPIStreamEvents, ) +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ImageObject, ImageResponse, @@ -143,6 +152,21 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig): ) -> dict: self._validate_openai_image_generation_params(model, optional_params) + return self._build_responses_image_request( + model=model, + prompt=prompt, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + def _build_responses_image_request( + self, + model: str, + prompt: Optional[str], + optional_params: dict, + litellm_params: dict, + input_images: Optional[List[Dict[str, Any]]] = None, + ) -> dict: # Intentionally pinned fallback for ChatGPT image generation through the # Codex Responses API. Users can override this per request or via # litellm_params. @@ -151,12 +175,18 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig): or litellm_params.get("chatgpt_responses_model") or "gpt-5.5" ) + content: List[Dict[str, Any]] = [] + if prompt: + content.append({"type": "input_text", "text": prompt}) + if input_images: + content.extend(input_images) + request: Dict[str, Any] = { "model": responses_model, "input": [ { "role": "user", - "content": [{"type": "input_text", "text": prompt}], + "content": content, } ], "instructions": get_chatgpt_default_instructions(), @@ -524,3 +554,161 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig): status_code=status_code, headers=headers, ) + + +class ChatGPTImageEditConfig(BaseImageEditConfig): + """ + Bridge OpenAI-style Images Edits calls to ChatGPT/Codex Responses image generation. + """ + + def __init__(self) -> None: + self.image_generation_config = ChatGPTImageGenerationConfig() + + def get_supported_openai_params(self, model: str) -> List[str]: + return ["size"] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + return { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + litellm_params: Optional[dict] = None, + api_base: Optional[str] = None, + ) -> dict: + return self.image_generation_config.validate_environment( + headers=headers, + model=model, + messages=[], + optional_params={}, + litellm_params=litellm_params or {}, + api_key=api_key, + api_base=api_base, + ) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + return self.image_generation_config.get_complete_url( + api_base=api_base, + api_key=litellm_params.get("api_key"), + model=model, + optional_params={}, + litellm_params=litellm_params, + ) + + def use_multipart_form_data(self) -> bool: + return False + + def transform_image_edit_request( + self, + model: str, + prompt: Optional[str], + image: Optional[FileTypes], + image_edit_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], RequestFiles]: + optional_params = dict(image_edit_optional_request_params) + self.image_generation_config._validate_openai_image_generation_params( + model, optional_params + ) + + input_images = self._prepare_input_images(image) + if not input_images: + raise ValueError("ChatGPT image edit requires at least one image.") + + request = self.image_generation_config._build_responses_image_request( + model=model, + prompt=prompt, + optional_params=optional_params, + litellm_params=dict(litellm_params), + input_images=input_images, + ) + return request, [] + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + return self.image_generation_config.transform_image_generation_response( + model=model, + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + def _prepare_input_images( + self, image: Optional[Union[FileTypes, List[FileTypes]]] + ) -> List[Dict[str, Any]]: + if image is None: + return [] + + images = image if isinstance(image, list) else [image] + input_images: List[Dict[str, Any]] = [] + for img in images: + if img is None: + continue + mime_type = ImageEditRequestUtils.get_image_content_type(img) + image_bytes = self._read_image_bytes(img) + b64_data = base64.b64encode(image_bytes).decode("utf-8") + input_images.append( + { + "type": "input_image", + "image_url": f"data:{mime_type};base64,{b64_data}", + } + ) + return input_images + + @staticmethod + def _read_image_bytes(image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, BufferedReader): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, tuple): + return ChatGPTImageEditConfig._read_image_bytes(image[1]) + if isinstance(image, PathLike): + with open(image, "rb") as image_file: + return image_file.read() + raise ValueError("Unsupported image type for ChatGPT image edit.") + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> OpenAIError: + return self.image_generation_config.get_error_class( + error_message=error_message, + status_code=status_code, + headers=headers, + ) diff --git a/litellm/utils.py b/litellm/utils.py index e855a823766..8074c6c4ea0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9122,6 +9122,10 @@ class ProviderConfigManager: from litellm.llms.openai.image_edit import get_openai_image_edit_config return get_openai_image_edit_config(model=model) + elif LlmProviders.CHATGPT == provider: + from litellm.llms.chatgpt.image_generation import ChatGPTImageEditConfig + + return ChatGPTImageEditConfig() elif LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, diff --git a/tests/image_gen_tests/test_chatgpt_image_generation.py b/tests/image_gen_tests/test_chatgpt_image_generation.py index 7cfffda5cb4..66bda02f053 100644 --- a/tests/image_gen_tests/test_chatgpt_image_generation.py +++ b/tests/image_gen_tests/test_chatgpt_image_generation.py @@ -6,6 +6,7 @@ import pytest from litellm.exceptions import AuthenticationError from litellm.llms.chatgpt.common_utils import GetAccessTokenError from litellm.llms.chatgpt.image_generation.transformation import ( + ChatGPTImageEditConfig, ChatGPTImageGenerationConfig, ) from litellm.llms.openai.common_utils import OpenAIError @@ -138,6 +139,75 @@ def test_chatgpt_image_generation_config_registered(monkeypatch, tmp_path): assert isinstance(config, ChatGPTImageGenerationConfig) +def test_chatgpt_image_edit_config_registered(monkeypatch, tmp_path): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + config = ProviderConfigManager.get_provider_image_edit_config( + model="gpt-image-2", + provider=LlmProviders.CHATGPT, + ) + + assert isinstance(config, ChatGPTImageEditConfig) + + +def test_chatgpt_image_edit_transforms_request(monkeypatch, tmp_path): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + config = ChatGPTImageEditConfig() + png_bytes = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01" + b"\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89" + ) + + request, files = config.transform_image_edit_request( + model="gpt-image-2", + prompt="replace the background with a warm sunset", + image=png_bytes, + image_edit_optional_request_params={"size": "1024x1024"}, + litellm_params=cast(Any, {"chatgpt_responses_model": "gpt-5.5"}), + headers={}, + ) + + assert files == [] + assert request["model"] == "gpt-5.5" + assert request["input"][0]["content"][0] == { + "type": "input_text", + "text": "replace the background with a warm sunset", + } + assert request["input"][0]["content"][1]["type"] == "input_image" + assert request["input"][0]["content"][1]["image_url"].startswith( + "data:image/png;base64," + ) + assert request["tools"] == [ + { + "type": "image_generation", + "model": "gpt-image-2", + "size": "1024x1024", + } + ] + assert request["tool_choice"] == {"type": "image_generation"} + + +def test_chatgpt_image_edit_uses_json_requests(monkeypatch, tmp_path): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + config = ChatGPTImageEditConfig() + + assert config.use_multipart_form_data() is False + + +def test_chatgpt_image_edit_requires_image(monkeypatch, tmp_path): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + config = ChatGPTImageEditConfig() + + with pytest.raises(ValueError, match="requires at least one image"): + config.transform_image_edit_request( + model="gpt-image-2", + prompt="edit this image", + image=None, + image_edit_optional_request_params={}, + litellm_params=cast(Any, {}), + headers={}, + ) + + def test_chatgpt_image_generation_only_supports_prompt_output_format_and_size( monkeypatch, tmp_path ):