mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
feat(openai/image_generation): add OpenAICompatibleImageGenerationConfig for community endpoints
Before this change, any model routed through the `openai/` provider whose name does not start with `dall-e-2` or `dall-e-3` was dispatched to `GPTImageGenerationConfig` (gpt-image-1's config). That config rejects `response_format` and doesn't know about non-OpenAI-standard params, so community OpenAI-compatible image endpoints — third-party aggregators (apiyi, openrouter, together.ai), and services that expose an OpenAI-shaped `/v1/images/generations` like Volcengine ark's `doubao-seedream-*` — would 400 on the very first request. `get_openai_image_generation_config` now: - matches `gpt-image*` explicitly for the GPT variant - falls back to a new `OpenAICompatibleImageGenerationConfig` for everything else, which accepts the union of dall-e-3 and gpt-image-1 standard params (so `response_format`, `style`, `background`, `moderation`, etc. all pass through). Vendor-specific params (`watermark`, `seed`, `guidance_scale`, …) are already forwarded automatically via `extra_body` by `add_provider_specific_params_to_optional_params`; the new config deliberately does *not* list them so that pathway stays in charge. Added example cost-map entries for `openai/doubao-seedream-4-5-251128` and `-5-0-260128` to demonstrate the use case. ## Tests - `tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_init.py` 15 new unit tests covering dispatcher routing, the supported-params superset, response_format acceptance, extra_body forwarding of vendor params, and response-shape passthrough. - `make test-unit` for the image_generation tree: 109 passed.
This commit is contained in:
VeechiYip
veechi
parent
f69b9d6564
commit
596bc9b20b
4 changed files with 298 additions and 1 deletions
|
|
@ -9,20 +9,38 @@ from .guardrail_translation import (
|
|||
OpenAIImageGenerationHandler,
|
||||
guardrail_translation_mappings,
|
||||
)
|
||||
from .openai_compatible_transformation import (
|
||||
OpenAICompatibleImageGenerationConfig,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DallE2ImageGenerationConfig",
|
||||
"DallE3ImageGenerationConfig",
|
||||
"GPTImageGenerationConfig",
|
||||
"OpenAICompatibleImageGenerationConfig",
|
||||
"OpenAIImageGenerationHandler",
|
||||
"guardrail_translation_mappings",
|
||||
]
|
||||
|
||||
|
||||
def get_openai_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
||||
"""
|
||||
Return the OpenAI image-generation transformation config for the given model.
|
||||
|
||||
- ``dall-e-2`` (and empty string) → :class:`DallE2ImageGenerationConfig`
|
||||
- ``dall-e-3*`` → :class:`DallE3ImageGenerationConfig`
|
||||
- ``gpt-image-1*`` → :class:`GPTImageGenerationConfig`
|
||||
- everything else routed through the ``openai/`` provider →
|
||||
:class:`OpenAICompatibleImageGenerationConfig` (generic fallback for
|
||||
community OpenAI-compatible image endpoints, e.g. third-party
|
||||
aggregators and services that expose an OpenAI-shaped
|
||||
``/v1/images/generations``).
|
||||
"""
|
||||
if model.startswith("dall-e-2") or model == "": # empty model is dall-e-2
|
||||
return DallE2ImageGenerationConfig()
|
||||
elif model.startswith("dall-e-3"):
|
||||
return DallE3ImageGenerationConfig()
|
||||
else:
|
||||
elif model.startswith("gpt-image"):
|
||||
return GPTImageGenerationConfig()
|
||||
else:
|
||||
return OpenAICompatibleImageGenerationConfig()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
from typing import TYPE_CHECKING, Any, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
||||
class OpenAICompatibleImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Generic OpenAI-compatible image generation config.
|
||||
|
||||
Used as the default fallback for models routed through the ``openai/``
|
||||
provider that are not OpenAI's own dall-e-* or gpt-image-* models.
|
||||
Covers community OpenAI-compatible image endpoints (e.g. third-party
|
||||
aggregators and self-hosted services whose /v1/images/generations is
|
||||
shaped like OpenAI's).
|
||||
|
||||
Accepts the union of standard OpenAI image-generation params so that the
|
||||
same config works whether the upstream follows dall-e-3 semantics
|
||||
(``response_format``, ``style``) or gpt-image-1 semantics
|
||||
(``background``, ``moderation``, ``output_format``, ``output_compression``).
|
||||
|
||||
Non-standard params passed by the caller (e.g. ``watermark``, ``seed``,
|
||||
``guidance_scale`` on Volcengine ark's doubao-seedream models) are not
|
||||
listed here; they are forwarded verbatim to the upstream via ``extra_body``
|
||||
by :func:`litellm.utils.add_provider_specific_params_to_optional_params`.
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
) -> List[OpenAIImageGenerationOptionalParams]:
|
||||
return [
|
||||
"background",
|
||||
"moderation",
|
||||
"n",
|
||||
"output_compression",
|
||||
"output_format",
|
||||
"quality",
|
||||
"response_format",
|
||||
"size",
|
||||
"style",
|
||||
"user",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params = self.get_supported_openai_params(model)
|
||||
for k in non_default_params.keys():
|
||||
if k not in optional_params.keys():
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ImageResponse:
|
||||
response = raw_response.json()
|
||||
|
||||
stringified_response = response
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
original_response=stringified_response,
|
||||
)
|
||||
image_response: ImageResponse = convert_to_model_response_object( # type: ignore
|
||||
response_object=stringified_response,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
|
||||
# passthrough whatever the caller asked for; defaults mirror the
|
||||
# dall-e / gpt-image response shape so downstream consumers don't break.
|
||||
image_response.size = optional_params.get("size", "1024x1024")
|
||||
image_response.quality = optional_params.get("quality")
|
||||
image_response.output_format = optional_params.get(
|
||||
"output_format", optional_params.get("response_format")
|
||||
)
|
||||
|
||||
return image_response
|
||||
|
|
@ -12259,6 +12259,28 @@
|
|||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 2560
|
||||
},
|
||||
"openai/doubao-seedream-4-5-251128": {
|
||||
"input_cost_per_image": 0.0345,
|
||||
"litellm_provider": "openai",
|
||||
"metadata": {
|
||||
"notes": "Volcengine ark doubao-seedream-4-5 image generation, served through an OpenAI-compatible endpoint (https://ark.cn-beijing.volces.com/api/v3). Price converted from 0.25 CNY/image."
|
||||
},
|
||||
"mode": "image_generation",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"openai/doubao-seedream-5-0-260128": {
|
||||
"input_cost_per_image": 0.2272727273,
|
||||
"litellm_provider": "openai",
|
||||
"metadata": {
|
||||
"notes": "Volcengine ark doubao-seedream-5-0 image generation, served through an OpenAI-compatible endpoint (https://ark.cn-beijing.volces.com/api/v3)."
|
||||
},
|
||||
"mode": "image_generation",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"doubao-embedding-large": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "volcengine",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
"""
|
||||
Unit tests for the openai/ image_generation config dispatcher.
|
||||
|
||||
Covers :func:`litellm.llms.openai.image_generation.get_openai_image_generation_config`
|
||||
and the :class:`OpenAICompatibleImageGenerationConfig` fallback used for
|
||||
community OpenAI-compatible image endpoints (e.g. third-party aggregators
|
||||
and ark-style services).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../../.."))
|
||||
|
||||
from litellm.llms.openai.image_generation import (
|
||||
DallE2ImageGenerationConfig,
|
||||
DallE3ImageGenerationConfig,
|
||||
GPTImageGenerationConfig,
|
||||
OpenAICompatibleImageGenerationConfig,
|
||||
get_openai_image_generation_config,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import get_optional_params_image_gen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expected_config",
|
||||
[
|
||||
("dall-e-2", DallE2ImageGenerationConfig),
|
||||
("dall-e-2-vtest", DallE2ImageGenerationConfig),
|
||||
("", DallE2ImageGenerationConfig), # empty string defaults to dall-e-2
|
||||
("dall-e-3", DallE3ImageGenerationConfig),
|
||||
("dall-e-3-preview", DallE3ImageGenerationConfig),
|
||||
("gpt-image-1", GPTImageGenerationConfig),
|
||||
("gpt-image-1-preview", GPTImageGenerationConfig),
|
||||
# unknown / community models fall through to the generic config
|
||||
("doubao-seedream-4-5-251128", OpenAICompatibleImageGenerationConfig),
|
||||
("doubao-seedream-5-0-260128", OpenAICompatibleImageGenerationConfig),
|
||||
("some-self-hosted-image-model", OpenAICompatibleImageGenerationConfig),
|
||||
],
|
||||
)
|
||||
def test_get_openai_image_generation_config(model, expected_config):
|
||||
"""Dispatcher returns the right transformer for each model family."""
|
||||
assert isinstance(get_openai_image_generation_config(model), expected_config)
|
||||
|
||||
|
||||
def test_openai_compatible_supported_params_superset():
|
||||
"""
|
||||
The generic config accepts the union of standard OpenAI image params so
|
||||
it works with both dall-e-style and gpt-image-style upstreams.
|
||||
"""
|
||||
config = OpenAICompatibleImageGenerationConfig()
|
||||
supported = config.get_supported_openai_params(model="doubao-seedream-4-5-251128")
|
||||
|
||||
# dall-e-3 style
|
||||
for k in ["n", "response_format", "quality", "size", "user", "style"]:
|
||||
assert k in supported
|
||||
|
||||
# gpt-image-1 style
|
||||
for k in ["background", "moderation", "output_compression", "output_format"]:
|
||||
assert k in supported
|
||||
|
||||
|
||||
def test_openai_compatible_accepts_response_format():
|
||||
"""
|
||||
Regression: before this config existed, passing ``response_format="url"``
|
||||
to an ``openai/<non-dall-e>`` model raised UnsupportedParamsError because
|
||||
it routed to GPTImageGenerationConfig (which doesn't list response_format).
|
||||
"""
|
||||
config = OpenAICompatibleImageGenerationConfig()
|
||||
mapped = config.map_openai_params(
|
||||
non_default_params={"response_format": "url"},
|
||||
optional_params={},
|
||||
model="doubao-seedream-4-5-251128",
|
||||
drop_params=False,
|
||||
)
|
||||
assert mapped["response_format"] == "url"
|
||||
|
||||
|
||||
def test_openai_compatible_rejects_truly_unknown_param_without_drop_params():
|
||||
"""
|
||||
Guards against accidentally flagging every extension as supported: a
|
||||
parameter that's not in the union (nor a valid OpenAI image param)
|
||||
should still raise unless ``drop_params=True``.
|
||||
"""
|
||||
config = OpenAICompatibleImageGenerationConfig()
|
||||
with pytest.raises(ValueError):
|
||||
config.map_openai_params(
|
||||
non_default_params={"definitely_not_an_openai_param": True},
|
||||
optional_params={},
|
||||
model="doubao-seedream-4-5-251128",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
def test_openai_compatible_forwards_vendor_params_via_extra_body():
|
||||
"""
|
||||
End-to-end: params outside OpenAI's standard image-generation set (e.g.
|
||||
Volcengine ark's ``watermark`` / ``seed``) should be transparently
|
||||
forwarded to the upstream via ``extra_body`` (LiteLLM's existing
|
||||
mechanism for openai-compatible providers). The generic config does
|
||||
not need to know about these params by name.
|
||||
"""
|
||||
optional_params = get_optional_params_image_gen(
|
||||
model="doubao-seedream-4-5-251128",
|
||||
response_format="url",
|
||||
size="2048x2048",
|
||||
custom_llm_provider="openai",
|
||||
watermark=False, # volcengine-specific
|
||||
seed=42, # volcengine-specific
|
||||
)
|
||||
# Standard params land on the top level
|
||||
assert optional_params.get("response_format") == "url"
|
||||
assert optional_params.get("size") == "2048x2048"
|
||||
# Vendor extras are preserved inside extra_body
|
||||
extra_body = optional_params.get("extra_body", {})
|
||||
assert extra_body.get("watermark") is False
|
||||
assert extra_body.get("seed") == 42
|
||||
|
||||
|
||||
def test_openai_compatible_transform_response_passthrough():
|
||||
"""transform_image_generation_response leaves the response shape intact."""
|
||||
config = OpenAICompatibleImageGenerationConfig()
|
||||
raw = MagicMock(spec=httpx.Response)
|
||||
raw.json.return_value = {
|
||||
"created": 1,
|
||||
"data": [{"url": "https://example.com/x.jpeg"}],
|
||||
}
|
||||
logging_obj = MagicMock()
|
||||
resp: ImageResponse = config.transform_image_generation_response(
|
||||
model="doubao-seedream-4-5-251128",
|
||||
raw_response=raw,
|
||||
model_response=ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
request_data={"prompt": "hi"},
|
||||
optional_params={"size": "2048x2048", "response_format": "url"},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
assert resp.data[0].url == "https://example.com/x.jpeg"
|
||||
assert resp.size == "2048x2048"
|
||||
assert resp.output_format == "url"
|
||||
Loading…
Add table
Reference in a new issue