scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 98

feat(chatgpt): support image edits

Browse source
This commit is contained in:
CrystalVibe28 2026-05-15 14:34:04 +08:00
parent 4b168229b5
commit 1f84f188e7
6 changed files with 270 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/__init__.py
View file

@ -1872,6 +1872,7 @@ if TYPE_CHECKING:
ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig,
)
from .llms.chatgpt.image_generation.transformation import (
ChatGPTImageEditConfig as ChatGPTImageEditConfig,
ChatGPTImageGenerationConfig as ChatGPTImageGenerationConfig,
)
from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig

4
litellm/_lazy_imports_registry.py
View file

@ -1134,6 +1134,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.chatgpt.image_generation.transformation",
"ChatGPTImageGenerationConfig",
),
"ChatGPTImageEditConfig": (
".llms.chatgpt.image_generation.transformation",
"ChatGPTImageEditConfig",
),
"NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
"WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
"GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"),

4
litellm/llms/chatgpt/image_generation/__init__.py
View file

@ -1,3 +1,3 @@
from .transformation import ChatGPTImageGenerationConfig
from .transformation import ChatGPTImageEditConfig, ChatGPTImageGenerationConfig
__all__ = ["ChatGPTImageGenerationConfig"]
__all__ = ["ChatGPTImageEditConfig", "ChatGPTImageGenerationConfig"]

190
litellm/llms/chatgpt/image_generation/transformation.py
View file

@ -1,20 +1,29 @@
import json
import re
import base64
from io import BufferedReader, BytesIO
from os import PathLike
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from httpx._types import RequestFiles
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.exceptions import AuthenticationError
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.llms.openai.common_utils import OpenAIError
from litellm.types.llms.openai import (
AllMessageValues,
FileTypes,
OpenAIImageGenerationOptionalParams,
ResponsesAPIStreamEvents,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
ImageObject,
ImageResponse,
@ -143,6 +152,21 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig):
) -> dict:
self._validate_openai_image_generation_params(model, optional_params)
return self._build_responses_image_request(
model=model,
prompt=prompt,
optional_params=optional_params,
litellm_params=litellm_params,
)
def _build_responses_image_request(
self,
model: str,
prompt: Optional[str],
optional_params: dict,
litellm_params: dict,
input_images: Optional[List[Dict[str, Any]]] = None,
) -> dict:
# Intentionally pinned fallback for ChatGPT image generation through the
# Codex Responses API. Users can override this per request or via
# litellm_params.
@ -151,12 +175,18 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig):
or litellm_params.get("chatgpt_responses_model")
or "gpt-5.5"
)
content: List[Dict[str, Any]] = []
if prompt:
content.append({"type": "input_text", "text": prompt})
if input_images:
content.extend(input_images)
request: Dict[str, Any] = {
"model": responses_model,
"input": [
{
"role": "user",
"content": [{"type": "input_text", "text": prompt}],
"content": content,
}
],
"instructions": get_chatgpt_default_instructions(),
@ -524,3 +554,161 @@ class ChatGPTImageGenerationConfig(BaseImageGenerationConfig):
status_code=status_code,
headers=headers,
)
class ChatGPTImageEditConfig(BaseImageEditConfig):
"""
Bridge OpenAI-style Images Edits calls to ChatGPT/Codex Responses image generation.
"""
def __init__(self) -> None:
self.image_generation_config = ChatGPTImageGenerationConfig()
def get_supported_openai_params(self, model: str) -> List[str]:
return ["size"]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict[str, Any]:
supported_params = self.get_supported_openai_params(model)
return {
key: value
for key, value in image_edit_optional_params.items()
if key in supported_params
}
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[dict] = None,
api_base: Optional[str] = None,
) -> dict:
return self.image_generation_config.validate_environment(
headers=headers,
model=model,
messages=[],
optional_params={},
litellm_params=litellm_params or {},
api_key=api_key,
api_base=api_base,
)
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
return self.image_generation_config.get_complete_url(
api_base=api_base,
api_key=litellm_params.get("api_key"),
model=model,
optional_params={},
litellm_params=litellm_params,
)
def use_multipart_form_data(self) -> bool:
return False
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],
image: Optional[FileTypes],
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict[str, Any], RequestFiles]:
optional_params = dict(image_edit_optional_request_params)
self.image_generation_config._validate_openai_image_generation_params(
model, optional_params
)
input_images = self._prepare_input_images(image)
if not input_images:
raise ValueError("ChatGPT image edit requires at least one image.")
request = self.image_generation_config._build_responses_image_request(
model=model,
prompt=prompt,
optional_params=optional_params,
litellm_params=dict(litellm_params),
input_images=input_images,
)
return request, []
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
return self.image_generation_config.transform_image_generation_response(
model=model,
raw_response=raw_response,
model_response=ImageResponse(),
logging_obj=logging_obj,
request_data={},
optional_params={},
litellm_params={},
encoding=None,
)
def _prepare_input_images(
self, image: Optional[Union[FileTypes, List[FileTypes]]]
) -> List[Dict[str, Any]]:
if image is None:
return []
images = image if isinstance(image, list) else [image]
input_images: List[Dict[str, Any]] = []
for img in images:
if img is None:
continue
mime_type = ImageEditRequestUtils.get_image_content_type(img)
image_bytes = self._read_image_bytes(img)
b64_data = base64.b64encode(image_bytes).decode("utf-8")
input_images.append(
{
"type": "input_image",
"image_url": f"data:{mime_type};base64,{b64_data}",
}
)
return input_images
@staticmethod
def _read_image_bytes(image: FileTypes) -> bytes:
if isinstance(image, bytes):
return image
if isinstance(image, BytesIO):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
if isinstance(image, BufferedReader):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
if isinstance(image, tuple):
return ChatGPTImageEditConfig._read_image_bytes(image[1])
if isinstance(image, PathLike):
with open(image, "rb") as image_file:
return image_file.read()
raise ValueError("Unsupported image type for ChatGPT image edit.")
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> OpenAIError:
return self.image_generation_config.get_error_class(
error_message=error_message,
status_code=status_code,
headers=headers,
)

4
litellm/utils.py
View file

@ -9122,6 +9122,10 @@ class ProviderConfigManager:
from litellm.llms.openai.image_edit import get_openai_image_edit_config
return get_openai_image_edit_config(model=model)
elif LlmProviders.CHATGPT == provider:
from litellm.llms.chatgpt.image_generation import ChatGPTImageEditConfig
return ChatGPTImageEditConfig()
elif LlmProviders.AZURE == provider:
from litellm.llms.azure.image_edit.transformation import (
AzureImageEditConfig,

70
tests/image_gen_tests/test_chatgpt_image_generation.py
View file

@ -6,6 +6,7 @@ import pytest
from litellm.exceptions import AuthenticationError
from litellm.llms.chatgpt.common_utils import GetAccessTokenError
from litellm.llms.chatgpt.image_generation.transformation import (
ChatGPTImageEditConfig,
ChatGPTImageGenerationConfig,
)
from litellm.llms.openai.common_utils import OpenAIError
@ -138,6 +139,75 @@ def test_chatgpt_image_generation_config_registered(monkeypatch, tmp_path):
assert isinstance(config, ChatGPTImageGenerationConfig)
def test_chatgpt_image_edit_config_registered(monkeypatch, tmp_path):
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
config = ProviderConfigManager.get_provider_image_edit_config(
model="gpt-image-2",
provider=LlmProviders.CHATGPT,
)
assert isinstance(config, ChatGPTImageEditConfig)
def test_chatgpt_image_edit_transforms_request(monkeypatch, tmp_path):
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
config = ChatGPTImageEditConfig()
png_bytes = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01"
b"\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89"
)
request, files = config.transform_image_edit_request(
model="gpt-image-2",
prompt="replace the background with a warm sunset",
image=png_bytes,
image_edit_optional_request_params={"size": "1024x1024"},
litellm_params=cast(Any, {"chatgpt_responses_model": "gpt-5.5"}),
headers={},
)
assert files == []
assert request["model"] == "gpt-5.5"
assert request["input"][0]["content"][0] == {
"type": "input_text",
"text": "replace the background with a warm sunset",
}
assert request["input"][0]["content"][1]["type"] == "input_image"
assert request["input"][0]["content"][1]["image_url"].startswith(
"data:image/png;base64,"
)
assert request["tools"] == [
{
"type": "image_generation",
"model": "gpt-image-2",
"size": "1024x1024",
}
]
assert request["tool_choice"] == {"type": "image_generation"}
def test_chatgpt_image_edit_uses_json_requests(monkeypatch, tmp_path):
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
config = ChatGPTImageEditConfig()
assert config.use_multipart_form_data() is False
def test_chatgpt_image_edit_requires_image(monkeypatch, tmp_path):
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
config = ChatGPTImageEditConfig()
with pytest.raises(ValueError, match="requires at least one image"):
config.transform_image_edit_request(
model="gpt-image-2",
prompt="edit this image",
image=None,
image_edit_optional_request_params={},
litellm_params=cast(Any, {}),
headers={},
)
def test_chatgpt_image_generation_only_supports_prompt_output_format_and_size(
monkeypatch, tmp_path
):